harnex 0.8.0 → 0.10.0

data/lib/harnex/core.rb

@@ -1,5 +1,6 @@
 require "digest"
 require "fileutils"
+require "open3"
 require "optparse"
 require "securerandom"
 require "set"
@@ -19,6 +20,8 @@ module Harnex
   DEFAULT_PORT_SPAN = Integer(env_value("HARNEX_PORT_SPAN", default: "4000"))
   DEFAULT_ID = "default"
   WATCH_DEBOUNCE_SECONDS = 1.0
+  GIT_FINGERPRINT_FULL_BYTES = 4 * 1024 * 1024
+  GIT_FINGERPRINT_SAMPLE_BYTES = 64 * 1024
   STATE_DIR = File.expand_path(env_value("HARNEX_STATE_DIR", default: "~/.local/state/harnex"))
   SESSIONS_DIR = File.join(STATE_DIR, "sessions")
   WatchConfig = Struct.new(:absolute_path, :display_path, :hook_message, :debounce_seconds, keyword_init: true)
@@ -114,49 +117,204 @@ module Harnex
     text.to_s.gsub(/\e\[[0-9;]*[a-zA-Z]/, "")
   end
 
-  def default_summary_out_path(repo_root)
-    root = repo_root.to_s
-    return nil if root.empty?
-
-    File.join(root, ".harnex", "dispatch.jsonl")
-  end
-
-  def git_capture_start(repo_root)
+  def git_capture_start(repo_root, exclude_paths: [])
     sha = git_output(repo_root, "rev-parse", "HEAD")
     branch = git_output(repo_root, "rev-parse", "--abbrev-ref", "HEAD")
     return {} if sha.empty? || branch.empty?
 
+    excluded_paths = normalize_git_excluded_paths(exclude_paths)
+    worktree = git_worktree_snapshot(repo_root, exclude_paths: excluded_paths)
     {
       sha: sha,
-      branch: branch
+      branch: branch,
+      worktree: worktree,
+      excluded_paths: excluded_paths
     }
   rescue StandardError
     {}
   end
 
-  def git_capture_end(repo_root, start_sha)
-    start_sha = start_sha.to_s.strip
+  # Captures committed and in-progress work relative to the state observed at
+  # session start. Passing the full git_capture_start hash avoids treating an
+  # unchanged pre-existing dirty tree as worker activity; a SHA string remains
+  # supported for older callers.
+  def git_capture_end(repo_root, start, exclude_paths: nil)
+    start_capture = start.is_a?(Hash) ? start : {}
+    start_sha = (start_capture[:sha] || start_capture["sha"] || start).to_s.strip
     return {} if start_sha.empty?
 
+    excluded_paths ||= start_capture[:excluded_paths] || start_capture["excluded_paths"] || []
+    excluded_paths = normalize_git_excluded_paths(excluded_paths)
     end_sha = git_output(repo_root, "rev-parse", "HEAD")
     range = "#{start_sha}..#{end_sha}"
-    shortstat = git_output(repo_root, "diff", "--shortstat", range)
-    changed_paths = git_output(repo_root, "diff", "--name-only", range).lines.map(&:strip).reject(&:empty?).first(200)
     commits = Integer(git_output(repo_root, "rev-list", "--count", range))
-    stats = parse_git_shortstat(shortstat)
+    committed_paths = git_null_paths(repo_root, "diff", "--name-only", "-z", range)
+      .reject { |path| excluded_paths.include?(path) }
+    committed_stats = if committed_paths.empty?
+                        { loc_added: 0, loc_removed: 0 }
+                      else
+                        parse_git_numstat(
+                          git_output(repo_root, "diff", "--numstat", range, "--", *committed_paths.first(200))
+                        )
+                      end
+
+    start_worktree = start_capture[:worktree] || start_capture["worktree"]
+    end_worktree = git_worktree_snapshot(repo_root, exclude_paths: excluded_paths)
+    worktree_paths = changed_worktree_paths(start_worktree, end_worktree)
+    tracked_worktree_paths = worktree_paths.select do |path|
+      entry = end_worktree.fetch(:entries).fetch(path, nil)
+      entry && entry.fetch(:tracked)
+    end
+    worktree_stats = if tracked_worktree_paths.empty?
+                       { loc_added: 0, loc_removed: 0 }
+                     else
+                       parse_git_numstat(
+                         git_output(repo_root, "diff", "--numstat", end_sha, "--", *tracked_worktree_paths.first(200))
+                       )
+                     end
+    untracked_stats = untracked_worktree_stats(start_worktree, end_worktree, worktree_paths)
+    all_paths = (committed_paths + worktree_paths).uniq
 
     {
       sha: end_sha,
-      loc_added: stats.fetch(:loc_added),
-      loc_removed: stats.fetch(:loc_removed),
-      files_changed: stats.fetch(:files_changed),
-      changed_paths: changed_paths,
-      commits: commits
+      loc_added: committed_stats.fetch(:loc_added) + worktree_stats.fetch(:loc_added) + untracked_stats.fetch(:loc_added),
+      loc_removed: committed_stats.fetch(:loc_removed) + worktree_stats.fetch(:loc_removed) + untracked_stats.fetch(:loc_removed),
+      files_changed: all_paths.length,
+      changed_paths: all_paths.first(200),
+      commits: commits,
+      start_dirty: start_worktree ? !start_worktree.fetch(:entries).empty? : nil,
+      end_dirty: !end_worktree.fetch(:entries).empty?,
+      worktree_changed: !worktree_paths.empty?
     }
   rescue StandardError
     {}
   end
 
+  def git_worktree_snapshot(repo_root, exclude_paths: [])
+    excluded_paths = normalize_git_excluded_paths(exclude_paths)
+    tracked_paths = (
+      git_null_paths(repo_root, "diff", "--name-only", "-z") +
+      git_null_paths(repo_root, "diff", "--cached", "--name-only", "-z")
+    ).uniq.reject { |path| excluded_paths.include?(path) }
+    untracked_paths = git_null_paths(repo_root, "ls-files", "--others", "--exclude-standard", "-z")
+      .reject { |path| excluded_paths.include?(path) }
+    entries = {}
+    tracked_paths.each do |path|
+      entries[path] = git_worktree_entry(repo_root, path, tracked: true)
+    end
+    untracked_paths.each do |path|
+      entries[path] = git_worktree_entry(repo_root, path, tracked: false)
+    end
+    { entries: entries }
+  end
+
+  def normalize_git_excluded_paths(paths)
+    Array(paths).filter_map do |path|
+      text = path.to_s.sub(%r{\A\./}, "")
+      text unless text.empty?
+    end.uniq
+  end
+
+  def git_worktree_entry(repo_root, relative_path, tracked:)
+    path = File.join(repo_root.to_s, relative_path)
+    stat = File.lstat(path)
+    if stat.symlink?
+      content = File.readlink(path)
+      return {
+        tracked: tracked,
+        fingerprint: "symlink:#{Digest::SHA256.hexdigest(content)}",
+        lines: 0,
+        binary: false
+      }
+    end
+
+    return { tracked: tracked, fingerprint: "missing", lines: 0, binary: false } unless stat.file?
+
+    if stat.size > GIT_FINGERPRINT_FULL_BYTES
+      first = File.binread(path, GIT_FINGERPRINT_SAMPLE_BYTES)
+      last = File.open(path, "rb") do |file|
+        file.seek(-[stat.size, GIT_FINGERPRINT_SAMPLE_BYTES].min, IO::SEEK_END)
+        file.read(GIT_FINGERPRINT_SAMPLE_BYTES).to_s
+      end
+      digest = Digest::SHA256.new
+      digest.update("#{stat.mode}:#{stat.size}:#{(stat.mtime.to_r * 1_000_000_000).to_i}")
+      digest.update(first)
+      digest.update(last)
+      return {
+        tracked: tracked,
+        fingerprint: "large:#{digest.hexdigest}",
+        lines: 0,
+        binary: first.include?("\0") || last.include?("\0")
+      }
+    end
+
+    digest = Digest::SHA256.new
+    lines = 0
+    binary = false
+    last_byte = nil
+    File.open(path, "rb") do |file|
+      buffer = +""
+      while file.read(16 * 1024, buffer)
+        digest.update(buffer)
+        binary ||= buffer.include?("\0")
+        lines += buffer.count("\n")
+        last_byte = buffer.byteslice(-1)
+      end
+    end
+    lines += 1 if stat.size.positive? && last_byte != "\n"
+    {
+      tracked: tracked,
+      fingerprint: "file:#{stat.mode}:#{digest.hexdigest}",
+      lines: binary ? 0 : lines,
+      binary: binary
+    }
+  rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR
+    { tracked: tracked, fingerprint: "missing", lines: 0, binary: false }
+  end
+
+  def changed_worktree_paths(start_worktree, end_worktree)
+    end_entries = end_worktree.fetch(:entries)
+    return end_entries.keys.sort unless start_worktree.is_a?(Hash)
+
+    start_entries = start_worktree[:entries] || start_worktree["entries"] || {}
+    (start_entries.keys + end_entries.keys).uniq.select do |path|
+      start_entries[path] != end_entries[path]
+    end.sort
+  end
+
+  def untracked_worktree_stats(start_worktree, end_worktree, changed_paths)
+    start_entries = if start_worktree.is_a?(Hash)
+                      start_worktree[:entries] || start_worktree["entries"] || {}
+                    else
+                      {}
+                    end
+    end_entries = end_worktree.fetch(:entries)
+    changed_paths.each_with_object({ loc_added: 0, loc_removed: 0 }) do |path, stats|
+      before = start_entries[path]
+      after = end_entries[path]
+      next if after && after.fetch(:tracked)
+      next if before && before.fetch(:tracked)
+
+      stats[:loc_removed] += before.fetch(:lines) if before
+      stats[:loc_added] += after.fetch(:lines) if after
+    end
+  end
+
+  def git_null_paths(repo_root, *args)
+    output, _stderr, status = Open3.capture3("git", "-C", repo_root.to_s, *args)
+    raise "git #{args.join(' ')} failed" unless status.success?
+
+    output.split("\0").reject(&:empty?)
+  end
+
+  def parse_git_numstat(text)
+    text.to_s.lines.each_with_object({ loc_added: 0, loc_removed: 0 }) do |line, stats|
+      added, removed, = line.split("\t", 3)
+      stats[:loc_added] += Integer(added, exception: false).to_i
+      stats[:loc_removed] += Integer(removed, exception: false).to_i
+    end
+  end
+
   # Canonical path resolution shared by every registry/exit/events writer and
   # reader. realpath collapses symlinked prefixes so a session registered from
   # a symlinked cwd stays visible to checkers resolving the physical path.
@@ -292,6 +450,12 @@ module Harnex
     false
   rescue Errno::EPERM
     true
+  rescue ArgumentError, TypeError
+    # A non-numeric pid means a truncated, hand-edited, or foreign-written
+    # registry file. Report it dead so active_sessions prunes it, the same
+    # self-healing it already applies to unparseable JSON. Raising here would
+    # take down every command that scans sessions -- status, send, pane.
+    false
   end
 
   def read_registry(repo_root, id = DEFAULT_ID, cli: nil)
@@ -359,10 +523,25 @@ module Harnex
     nil
   end
 
-  def write_registry(path, payload)
-    tmp = "#{path}.tmp.#{Process.pid}"
+  # Atomically replace a JSON state file.
+  #
+  # The temp name must be unique per write, not just per process. Several
+  # threads in one session write the same registry path — the startup persist,
+  # the inbox delivery thread, and one thread per API client — and a shared
+  # temp name lets one thread rename the file another is still writing, which
+  # surfaces as ENOENT on the loser's rename. mkdir_p covers a state directory
+  # that was reaped (tmpfs, operator cleanup) while the process was live.
+  def atomic_write_json(path, payload)
+    FileUtils.mkdir_p(File.dirname(path))
+    tmp = "#{path}.tmp.#{Process.pid}.#{SecureRandom.hex(6)}"
     File.write(tmp, JSON.pretty_generate(payload))
     File.rename(tmp, path)
+  ensure
+    FileUtils.rm_f(tmp) if tmp
+  end
+
+  def write_registry(path, payload)
+    atomic_write_json(path, payload)
   end
 
   def allocate_port(repo_root, id, requested_port = nil, host: DEFAULT_HOST)

data/lib/harnex/dispatch_history.rb

@@ -9,6 +9,12 @@ module Harnex
 
     MAX_REPO_WALK_LEVELS = 10
 
+    # v2 marks the unified era: one rich dispatch_end row per dispatch
+    # carrying both the thin envelope and the summary sections. Readers
+    # key on record_type, not this stamp; legacy clauses keep accepting
+    # v1 and envelope-less rows mixed in the same file.
+    SCHEMA_VERSION = 2
+
     def global_path
       File.join(STATE_DIR, "dispatch.jsonl")
     end
@@ -138,7 +144,7 @@ module Harnex
     # trace; the dispatch_end row written in finalize_session! completes it.
     def build_start_record(session)
       {
-        schema_version: 1,
+        schema_version: SCHEMA_VERSION,
         record_type: "dispatch_start",
         id: session.id,
         session_id: session.session_id,
@@ -150,16 +156,21 @@ module Harnex
         repo_root: session.repo_root,
         tier: session.__send__(:meta_hash)["tier"],
         meta: session.__send__(:meta_hash),
-        summary_out_path: session.summary_out,
-        events_log_path: session.events_log_path
+        events_log_path: session.events_log_path,
+        artifact_report_path: session.artifact_report_path,
+        artifact_claims_path: session.artifact_claims_path
       }
     end
 
+    # The v2 end row: the thin envelope merged with the rich summary
+    # sections. The envelope carries no raw meta passthrough — the summary's
+    # meta section (a superset with provenance) rides in its place; top-level
+    # tier stays for the history renderer.
     def build_record(session)
       ended_at = session.ended_at || Time.now
       status, terminal_event = classify(session)
       {
-        schema_version: 1,
+        schema_version: SCHEMA_VERSION,
         record_type: "dispatch_end",
         id: session.id,
         session_id: session.session_id,
@@ -172,11 +183,9 @@ module Harnex
         terminal_event: terminal_event,
         commit_sha: commit_sha(session.git_start, session.git_end),
         tier: session.__send__(:meta_hash)["tier"],
-        meta: session.__send__(:meta_hash),
-        summary_out_path: session.summary_out,
         events_log_path: session.events_log_path,
         tmux_state: tmux_state(session.__send__(:summary_tmux_session))
-      }
+      }.merge(session.__send__(:build_summary_record))
     end
 
     def classify(session)

data/lib/harnex/pricing.rb

@@ -0,0 +1,135 @@
+module Harnex
+  # Static price table for computing usage.cost_usd when an adapter reports
+  # tokens but no cost (plan 33 Phase 2, locked decision 6). Cost is computed,
+  # never guessed: an unknown provider/model/service tier or a missing token
+  # component leaves cost_usd null.
+  #
+  # ## Update procedure
+  #
+  # Rates are USD per 1M tokens, copied by hand from the provider pricing
+  # pages at the URLs below. To update:
+  #
+  # 1. Fetch the provider's pricing page and read the per-1M-token rates for
+  #    input, cached input (cache read), and output.
+  # 2. Replace the entry's rates and set `as_of` to the date you read them.
+  #    Add new models as new entries; never delete an entry solely because a
+  #    model left the pricing page (old rows in dispatch.jsonl were priced
+  #    against it — the `as_of` on each row records which table vintage
+  #    applied).
+  # 3. Update the expected costs in test/harnex/pricing_test.rb — rate
+  #    changes are meant to be conscious, test-visible edits.
+  # 4. Never backfill: rows written before a table change keep the cost they
+  #    were written with.
+  #
+  # Models whose pricing varies by service tier use a nested `service_tiers`
+  # table. For those models a nil or unknown service tier is unpriceable. A
+  # `max_context_tokens_exclusive` entry also requires an observed context
+  # high-water below that boundary; Harnex leaves cost null when context is
+  # missing or entered a differently-priced long-context tier. Do not infer
+  # standard/flex/fast or context tier outside the recorded measurements.
+  # OpenAI documents `priority` as the fast alias for gpt-5.5 short-context
+  # pricing; Harnex maps that alias only where the source supports it.
+  #
+  # Known scheduled change: Anthropic's claude-sonnet-5 entry below carries
+  # introductory pricing ($2/$10) that ends 2026-08-31; standard pricing is
+  # $3/$15 (cache read $0.30) from 2026-09-01. Refresh the entry then.
+  #
+  # ## Token semantics (verified 2026-08-02, plan 33 Phase 2)
+  #
+  # What input_tokens contains depends on the CAPTURE PATH, not just the
+  # provider, so the caller passes `input_includes_cached:` (sourced from
+  # `Adapters::Base#usage_input_includes_cached?`):
+  #
+  # - codex app-server (input_includes_cached: true): the JSON
+  #   TokenUsageBreakdown reports cachedInputTokens as a subset of
+  #   inputTokens and reasoningOutputTokens as a subset of outputTokens
+  #   (verified against test/fixtures/codex_schema/v2 and a live captured
+  #   row: input 3,084,697 + output 17,705 == total 3,102,402 exactly).
+  #   Billable input = input - cached.
+  # - codex PTY (false): the scraped TUI line `input=X (+ Y cached)` shows
+  #   NON-cached input with cached as a separate additive count (fixture:
+  #   total 106,867 == input 104,158 + output 2,709, cached 250,880 > input).
+  #   Billable input = input; cached prices additively at the cached rate.
+  # - anthropic (false): the Messages API reports input_tokens EXCLUSIVE of
+  #   cache reads (cache_read_input_tokens is a separate field). Cache
+  #   WRITES (cache_creation_input_tokens, billed at 1.25x input) have no
+  #   harnex usage field yet — the Phase 3 Claude usage producer must either
+  #   fold them into input_tokens or extend this formula before pricing rows
+  #   that include cache writes.
+  #
+  # Reasoning tokens are never priced separately on any path — they are
+  # already inside output_tokens.
+  module Pricing
+    PRICES = {
+      "openai" => {
+        # Source: https://developers.openai.com/api/docs/pricing
+        "gpt-5.3-codex" => { input: 1.75, cached_input: 0.175, output: 14.00, as_of: "2026-08-02" },
+        # Short-context service-tier rates read 2026-08-03. Long-context
+        # rates are deliberately absent until exact source pricing is known.
+        "gpt-5.5" => {
+          max_context_tokens_exclusive: 272_000,
+          service_tier_aliases: { "priority" => "fast" }.freeze,
+          service_tiers: {
+            "standard" => { input: 5.00, cached_input: 0.50, output: 30.00, as_of: "2026-08-03" },
+            "flex" => { input: 2.50, cached_input: 0.25, output: 15.00, as_of: "2026-08-03" },
+            "fast" => { input: 12.50, cached_input: 1.25, output: 75.00, as_of: "2026-08-03" }
+          }.freeze
+        }.freeze,
+        "gpt-5.2" => { input: 1.75, cached_input: 0.175, output: 14.00, as_of: "2026-08-02" },
+        "gpt-5.1" => { input: 1.25, cached_input: 0.125, output: 10.00, as_of: "2026-08-02" },
+        "gpt-5" => { input: 1.25, cached_input: 0.125, output: 10.00, as_of: "2026-08-02" },
+        "gpt-5-mini" => { input: 0.25, cached_input: 0.025, output: 2.00, as_of: "2026-08-02" }
+      }.freeze,
+      "anthropic" => {
+        # Source: https://claude.com/platform/api (cache read = 0.1x input)
+        "claude-fable-5" => { input: 10.00, cached_input: 1.00, output: 50.00, as_of: "2026-08-02" },
+        "claude-opus-5" => { input: 5.00, cached_input: 0.50, output: 25.00, as_of: "2026-08-02" },
+        "claude-sonnet-5" => { input: 2.00, cached_input: 0.20, output: 10.00, as_of: "2026-08-02" },
+        "claude-haiku-4-5" => { input: 1.00, cached_input: 0.10, output: 5.00, as_of: "2026-08-02" }
+      }.freeze
+    }.freeze
+
+    module_function
+
+    # Returns { cost_usd:, as_of: } or nil when the cost cannot be computed
+    # (unknown provider/model/service tier, or input/output token counts missing).
+    def compute(provider:, model:, input_tokens:, output_tokens:, cached_tokens: nil,
+                service_tier: nil, context_tokens: nil,
+                input_includes_cached: false)
+      model_entry = PRICES.dig(provider.to_s, model.to_s)
+      return nil unless model_entry
+      return nil unless context_priceable?(model_entry, context_tokens)
+
+      entry = rates_for_service_tier(model_entry, service_tier)
+      return nil unless entry
+      return nil unless input_tokens.is_a?(Numeric) && output_tokens.is_a?(Numeric)
+
+      cached = cached_tokens.is_a?(Numeric) ? cached_tokens : 0
+      billable_input = input_includes_cached ? input_tokens - cached : input_tokens
+      billable_input = 0 if billable_input.negative?
+
+      cost = (billable_input * entry[:input] +
+              cached * entry[:cached_input] +
+              output_tokens * entry[:output]) / 1_000_000.0
+      { cost_usd: cost.round(6), as_of: entry[:as_of] }
+    end
+
+    def context_priceable?(entry, context_tokens)
+      limit = entry[:max_context_tokens_exclusive]
+      return true unless limit
+
+      context_tokens.is_a?(Numeric) && context_tokens >= 0 && context_tokens < limit
+    end
+
+    def rates_for_service_tier(entry, service_tier)
+      tiers = entry[:service_tiers]
+      return entry unless tiers
+
+      tier = service_tier.to_s
+      return nil if tier.empty?
+
+      aliases = entry[:service_tier_aliases] || {}
+      tiers[aliases.fetch(tier, tier)]
+    end
+  end
+end