harnex 0.12.0 → 0.29.0

data/lib/harnex/owner_stream.rb

@@ -0,0 +1,399 @@
+require "json"
+require "net/http"
+require "time"
+require "uri"
+
+module Harnex
+  # #77: `harnex watch --owner`. One compact JSON line per lifecycle change of
+  # every dispatch an owner started, for a Claude Code `Monitor` or a panel.
+  # Sources are the #72 owner records (registered `running`, then settled),
+  # the dispatch row's #57 class for `error` settlements, and live status via
+  # Harnex::Stall. Only event lines reach stdout; nothing is nudged or stopped.
+  class OwnerStream
+    SCHEMA = "harnex.owner_event.v1"
+    EVENTS = %w[started done rejected failed stalled stopped timeout].freeze
+    POLL_INTERVAL_S = 2.0
+    # An `error` settlement precedes its dispatch row by milliseconds; give the
+    # row this long to land before classifying without it.
+    ROW_GRACE_S = 5.0
+    STALL_POLL_MAX_INTERVAL_S = 15.0
+    MAX_WAIT_EXIT = 124
+    CURSOR_NAME = /\A[A-Za-z0-9._-]{1,64}\z/
+
+    # once: (#84) exit 0 after the first pass that delivers anything (results
+    # or stalls; every one ready in that pass), so one blocking call covers
+    # every lane. `started` lines are skipped there: they repeat on every
+    # start and would end each call.
+    # cursor: a named cursor, so a parent and its subagent (which share one
+    # Claude session id) do not take each other's events.
+    # match: lane-id globs; other lanes of the owner are ignored (#85).
+    # until_idle: like once, but keep collecting until no matching lane is
+    # still running, so one call waits for a whole batch of lanes.
+    def initialize(owner_id:, stall_after: nil, max_wait: nil, once: false, cursor: nil, match: nil,
+                   until_idle: false, out: $stdout, err: $stderr, clock: nil, wall_clock: nil, sleeper: nil,
+                   status_fetcher: nil)
+      @owner_id = owner_id.to_s
+      raise ArgumentError, "owner id is required" if @owner_id.strip.empty?
+      if cursor && !cursor.to_s.match?(CURSOR_NAME)
+        raise ArgumentError, "cursor name must be 1-64 letters, digits, '.', '_' or '-'"
+      end
+
+      @stall_after = stall_after
+      @max_wait = max_wait
+      @until_idle = until_idle
+      @once = once || until_idle
+      @busy_lanes = 0
+      @match = Array(match).flat_map { |glob| glob.to_s.split(",") }.map(&:strip).reject(&:empty?)
+      @emitted = 0
+      @out = out
+      @err = err
+      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @wall_clock = wall_clock || -> { Time.now }
+      @sleeper = sleeper || ->(seconds) { sleep(seconds) }
+      @status_fetcher = status_fetcher || method(:fetch_status)
+      @dir = Harnex.owned_notifications_dir(@owner_id)
+      @cursor_path = File.join(@dir, "watch", cursor ? "cursor-#{cursor}.json" : "cursor.json")
+      @running = {}
+    end
+
+    def run
+      deadline = @max_wait && @clock.call + @max_wait
+      load_cursor
+      loop do
+        scan
+        return 0 if @until_idle ? @busy_lanes.zero? : once_satisfied?
+        if deadline && @clock.call >= deadline
+          @err.puts("harnex watch: no owner event within #{duration(@max_wait)}") if @once
+          return MAX_WAIT_EXIT
+        end
+
+        @sleeper.call(POLL_INTERVAL_S)
+      end
+    rescue Interrupt
+      0
+    end
+
+    # One pass over the owner's records; public for deterministic tests.
+    def scan
+      load_cursor unless @delivered
+      list = records
+      seen = list.to_h { |record| [record["session_id"], true] }
+      mine = list.select { |record| matches?(record) }
+      delivered = deliver_results(mine)
+      @busy_lanes = 0
+      mine.each do |record|
+        if record["outcome"].to_s.empty?
+          next unless turn_of(record) > delivered.fetch(record["session_id"])
+
+          observe_running(record)
+          @busy_lanes += 1 if @running.key?(record["session_id"])
+        else
+          @running.delete(record["session_id"])
+          # A settled lane whose result still waits for its dispatch row.
+          @busy_lanes += 1 if delivered.fetch(record["session_id"]) < results_of(record).map { |r| turn_of(r) }.max.to_i
+        end
+      end
+      @running.select! { |session_id, _| seen[session_id] }
+      prune_cursor(seen)
+    end
+
+    private
+
+    def once_satisfied?
+      @once && @emitted.positive?
+    end
+
+    def matches?(record)
+      @match.empty? || @match.any? { |glob| File.fnmatch?(glob, record["id"].to_s) }
+    end
+
+    # Every turn result not yet delivered, across all lanes in the order
+    # they settled, so a watcher that was away while several turns ended
+    # still gets each one (#85). Returns each dispatch's last delivered turn.
+    def deliver_results(records)
+      delivered = records.to_h { |record| [record["session_id"], delivered_turn(record["session_id"])] }
+      pending = records.flat_map do |record|
+        results = results_of(record)
+        results.filter_map do |result|
+          next if turn_of(result) <= delivered[record["session_id"]]
+
+          final = result.equal?(results.last) && !record["outcome"].to_s.empty?
+          [result["notified_at"].to_s, turn_of(result), record, result, final]
+        end
+      end
+      blocked = {}
+      pending.sort_by { |at, turn, record, _result, _final| [at, record["session_id"], turn] }.each do |_at, turn, record, result, final|
+        session_id = record["session_id"]
+        next if blocked[session_id]
+        next blocked[session_id] = true unless settle(record.merge(result), final: final)
+
+        delivered[session_id] = turn
+      end
+      delivered
+    end
+
+    # Owner records from 0.18.1 keep a bounded `results` history; older ones
+    # hold only their latest settlement at the top level.
+    def results_of(record)
+      history = record["results"]
+      list = history.is_a?(Array) ? history.select { |item| item.is_a?(Hash) && item["outcome"] } : []
+      list = [record] if list.empty? && !record["outcome"].to_s.empty?
+      list.sort_by { |item| turn_of(item) }
+    end
+
+    # The cursor keeps one value per dispatch: the key of its last delivered
+    # turn result ("<notified_at>#N"; turn 1 is the bare notified_at, as
+    # before #81) or the runner_gone report "gone#N" for running turn N.
+    def delivered_turn(session_id)
+      value = @delivered[session_id].to_s
+      return 0 if value.empty?
+
+      number = value[/#(\d+)\z/, 1]
+      number ? number.to_i : 1
+    end
+
+    def turn_of(record)
+      [record["turn"].to_i, 1].max
+    end
+
+    # A first turn keys on notified_at alone, as cursors before #81 did.
+    def settled_key(record)
+      key = record["notified_at"].to_s
+      key = "settled" if key.empty?
+      turn_of(record) > 1 ? "#{key}##{turn_of(record)}" : key
+    end
+
+    def gone_key(record)
+      "gone##{turn_of(record)}"
+    end
+
+    def records
+      return [] unless File.directory?(@dir)
+
+      Dir.glob(File.join(@dir, "*.json")).sort.filter_map do |path|
+        record = JSON.parse(File.read(path))
+        next unless record.is_a?(Hash) && record["owner_id"] == @owner_id
+        next if record["session_id"].to_s.empty? || record["id"].to_s.empty?
+
+        record
+      rescue JSON::ParserError, Errno::ENOENT
+        nil
+      end
+    end
+
+    def observe_running(record)
+      session_id = record["session_id"]
+      state = @running[session_id]
+      unless state
+        state = @running[session_id] = { stalled: @stalled.key?(session_id), next_check: nil, status: nil }
+        # A later turn (a nudge) is watched silently; its result is reported.
+        if turn_of(record) <= 1 && !@once
+          emit("started", record, msg: "#{record['id']} started in #{portable(record['repo_root'])}")
+        end
+      end
+
+      # A runner that died without settling will never write an outcome.
+      pid = record["runner_pid"]
+      if pid && !Harnex.alive_pid?(pid)
+        emit("failed", record, reason: "runner_gone",
+                               msg: "#{record['id']} failed: its harnex runner exited without a result")
+        deliver(record, gone_key(record))
+        @running.delete(session_id)
+        return
+      end
+
+      check_stall(record, state) if @stall_after
+    end
+
+    def check_stall(record, state)
+      now = @clock.call
+      return if state[:next_check] && now < state[:next_check]
+
+      state[:next_check] = now + [[@stall_after / 4.0, 0.5].max, STALL_POLL_MAX_INTERVAL_S].min
+      status = @status_fetcher.call(record)
+      stall = Harnex::Stall.stalled(status, @stall_after)
+      if stall && !state[:stalled]
+        state[:stalled] = true
+        remember_stall(record["session_id"], true)
+        running = status.is_a?(Hash) ? Array(status["running_commands"]).first(3) : []
+        stall = stall.merge("running_commands" => running) unless running.empty?
+        detail = running.empty? ? "" : " while running `#{running.first}`"
+        emit("stalled", record.merge("elapsed_s" => running_elapsed(record)), stall: stall,
+             msg: "#{record['id']} stalled: no progress for #{duration(stall['idle_s'])} (#{stall['clock']})#{detail}")
+      elsif stall.nil?
+        state[:stalled] = false # progress resumed (or nothing observable): re-arm
+        remember_stall(record["session_id"], false)
+      end
+    end
+
+    # Seconds since the running turn began (or the dispatch, for turn 1).
+    def running_elapsed(record)
+      since = Time.iso8601((record["turn_started_at"] || record["started_at"]).to_s)
+      [(@wall_clock.call - since).to_i, 0].max
+    rescue ArgumentError
+      nil
+    end
+
+    # --once runs one process per event, so a stall episode must outlive the
+    # process or every call would end on the same stall.
+    def remember_stall(session_id, stalled)
+      return unless @once
+      return if @stalled.key?(session_id) == stalled
+
+      stalled ? @stalled[session_id] = true : @stalled.delete(session_id)
+      save_cursor
+    end
+
+    # final: the dispatch's last result once it settled for good; only that
+    # one may take its cause from the dispatch row. Returns false while it
+    # waits for that row.
+    def settle(record, final: true)
+      outcome = record["outcome"].to_s
+      outcome_class = record["outcome_class"]
+      event = { "completed" => "done", "rejected" => "rejected" }[outcome]
+      if event.nil?
+        # "failed"/"error": a stop or deadline can surface as either (Pi
+        # settles an interrupted turn as failed). The dispatch row's #57
+        # class names the cause; it lands moments after an auto-stop
+        # settlement, so wait briefly for it before falling back.
+        row = final ? dispatch_row(record) : nil
+        return false if final && row.nil? && within_row_grace?(record)
+
+        outcome_class = row&.dig("outcome", "class") || outcome_class
+        event = { "timeout" => "timeout", "stopped_by_caller" => "stopped" }.fetch(outcome_class.to_s, "failed")
+      end
+
+      emit(event, record, outcome_class: outcome_class, msg: settled_message(event, record, outcome_class))
+      deliver(record, settled_key(record))
+      true
+    end
+
+    def settled_message(event, record, outcome_class)
+      id = record["id"]
+      id = "#{id} turn #{turn_of(record)}" if turn_of(record) > 1
+      after = record["elapsed_s"] ? " after #{duration(record['elapsed_s'])}" : ""
+      case event
+      when "done" then "#{id} done (#{outcome_class || 'completed'})#{after}"
+      when "rejected" then "#{id} rejected: #{outcome_class || 'proof rejected'}#{after}"
+      when "timeout" then "#{id} hit its runtime budget#{after}"
+      when "stopped" then "#{id} stopped by its caller#{after}"
+      else "#{id} failed (#{outcome_class || 'unknown'})#{after}"
+      end
+    end
+
+    def emit(event, record, msg:, outcome_class: nil, stall: nil, reason: nil)
+      line = {
+        "schema" => SCHEMA,
+        "event" => event,
+        "id" => record["id"],
+        "session_id" => record["session_id"],
+        "turn" => turn_of(record),
+        "repo" => portable(record["repo_root"]),
+        "ts" => @wall_clock.call.utc.iso8601(3),
+        "work_state" => { "started" => "running", "stalled" => "running", "done" => "completed" }.fetch(event, "failed"),
+        "outcome_class" => outcome_class,
+        "artifact_report_status" => record["artifact_report_status"],
+        "receipt_path" => portable(record["receipt_path"]),
+        "end_sha" => record["end_sha"],
+        "turn_commits" => (record["turn_commits"] if record["outcome"]),
+        "model" => (record["model"] if record["outcome"]),
+        "elapsed_s" => record["elapsed_s"],
+        "settled_at" => (record["notified_at"] if record["outcome"]),
+        "follow_ups" => record["follow_ups"],
+        "final_message" => (record["final_message"] if record["outcome"]),
+        "final_message_path" => (portable(record["final_message_path"]) if record["outcome"]),
+        "stall" => stall,
+        "reason" => reason,
+        "msg" => msg
+      }.compact
+      @out.puts(JSON.generate(line))
+      @out.flush
+      @emitted += 1
+    end
+
+    def deliver(record, key)
+      @delivered[record["session_id"]] = key
+      save_cursor
+    end
+
+    def within_row_grace?(record)
+      notified = Time.iso8601(record["notified_at"].to_s)
+      @wall_clock.call - notified < ROW_GRACE_S
+    rescue ArgumentError
+      false
+    end
+
+    def dispatch_row(record)
+      path = Harnex::DispatchHistory.path_for(record["repo_root"].to_s)
+      return nil unless File.file?(path)
+
+      File.readlines(path).reverse_each do |line|
+        row = JSON.parse(line)
+        return row if Harnex::DispatchHistory.end_record?(row) && row["session_id"] == record["session_id"]
+      rescue JSON::ParserError
+        next
+      end
+      nil
+    end
+
+    # Live status for this exact attempt: a reused id must not be mistaken
+    # for the dispatch the record describes.
+    def fetch_status(record)
+      registry = Harnex.read_registry(record["repo_root"].to_s, record["id"])
+      return nil unless registry && registry["session_id"] == record["session_id"]
+
+      uri = URI("http://#{registry['host']}:#{registry['port']}/status")
+      request = Net::HTTP::Get.new(uri)
+      request["Authorization"] = "Bearer #{registry['token']}" if registry["token"]
+      response = Net::HTTP.start(uri.host, uri.port, open_timeout: 1, read_timeout: 2) { |http| http.request(request) }
+      response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body) : nil
+    rescue StandardError
+      nil
+    end
+
+    def load_cursor
+      data = JSON.parse(File.read(@cursor_path))
+      @delivered = data["delivered"].is_a?(Hash) ? data["delivered"] : {}
+      @stalled = data["stalled"].is_a?(Hash) ? data["stalled"] : {}
+    rescue Errno::ENOENT, JSON::ParserError
+      @delivered = {}
+      @stalled = {}
+    end
+
+    def save_cursor
+      FileUtils.mkdir_p(File.dirname(@cursor_path), mode: 0o700)
+      state = { schema_version: 1, delivered: @delivered }
+      state[:stalled] = @stalled unless @stalled.empty?
+      Harnex.atomic_write_json(@cursor_path, state)
+    rescue StandardError => e
+      @err.puts("harnex watch: could not save the owner cursor (#{e.class}); results may repeat after a restart")
+    end
+
+    # Records outlive delivery; drop cursor entries only for vanished records.
+    def prune_cursor(seen)
+      stale = @delivered.keys.reject { |session_id| seen[session_id] }
+      gone_stalls = @stalled.keys.reject { |session_id| seen[session_id] }
+      return if stale.empty? && gone_stalls.empty?
+
+      stale.each { |session_id| @delivered.delete(session_id) }
+      gone_stalls.each { |session_id| @stalled.delete(session_id) }
+      save_cursor
+    end
+
+    def portable(path)
+      return nil if path.nil? || path.to_s.empty?
+
+      Harnex::DispatchHistory.portable_path(path.to_s)
+    end
+
+    def duration(seconds)
+      value = seconds.to_f.round
+      return "#{value}s" if value < 60
+      return format("%dm%02ds", value / 60, value % 60) if value < 600
+      return "#{(value / 60.0).round}m" if value < 3600
+
+      hours, rest = value.divmod(3600)
+      format("%dh%02dm", hours, rest / 60)
+    end
+  end
+end

data/lib/harnex/pricing.rb

@@ -40,16 +40,11 @@ module Harnex
   # provider, so the caller passes `input_includes_cached:` (sourced from
   # `Adapters::Base#usage_input_includes_cached?`):
   #
-  # - codex app-server (input_includes_cached: true): the JSON
-  #   TokenUsageBreakdown reports cachedInputTokens as a subset of
-  #   inputTokens and reasoningOutputTokens as a subset of outputTokens
-  #   (verified against test/fixtures/codex_schema/v2 and a live captured
-  #   row: input 3,084,697 + output 17,705 == total 3,102,402 exactly).
+  # - input_includes_cached: true: cached tokens are a subset of input.
   #   Billable input = input - cached.
-  # - codex PTY (false): the scraped TUI line `input=X (+ Y cached)` shows
-  #   NON-cached input with cached as a separate additive count (fixture:
-  #   total 106,867 == input 104,158 + output 2,709, cached 250,880 > input).
-  #   Billable input = input; cached prices additively at the cached rate.
+  # - false: input is NON-cached; cached prices additively at the cached
+  #   rate. (Pi reports provider cost directly, so this table is only the
+  #   fallback for rows without a provider-reported cost.)
   # - anthropic (false): the Messages API reports input_tokens EXCLUSIVE of
   #   cache reads (cache_read_input_tokens is a separate field). Cache
   #   WRITES (cache_creation_input_tokens, billed at 1.25x input) have no

data/lib/harnex/runtime/activity_tracker.rb

@@ -0,0 +1,107 @@
+require "time"
+
+module Harnex
+  # Observed protocol activity, not output/log freshness. Retains timestamps,
+  # tool names and a tool count only: thinking, messages, tool arguments and
+  # results never enter this object.
+  class ActivityTracker
+    def initialize(transport:, monotonic_clock: nil, wall_clock: nil)
+      @source = { stdio_jsonl_rpc: "pi_rpc" }[transport]
+      @clock = monotonic_clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @wall_clock = wall_clock || -> { Time.now }
+      @mutex = Mutex.new
+      @turn_active = false
+      @model_active = false
+    end
+
+    def start_turn
+      @mutex.synchronize { start_turn_locked } if @source
+    end
+
+    def settle
+      @mutex.synchronize { @turn_active = @model_active = false } if @source
+    end
+
+    def observe_pi(message)
+      return unless @source == "pi_rpc"
+
+      @mutex.synchronize do
+        case message["type"]
+        when "agent_start", "turn_start"
+          start_turn_locked
+        when "agent_settled"
+          @turn_active = @model_active = false
+        when "message_start", "message_end"
+          if message["message"].is_a?(Hash) && message["message"]["role"] == "assistant"
+            model_activity_locked(finished: message["type"] == "message_end")
+          end
+        when "message_update"
+          event = message["assistantMessageEvent"] || {}
+          if %w[text_delta thinking_delta toolcall_delta].include?(event["type"]) && !event["delta"].to_s.empty?
+            model_activity_locked
+          end
+        when "tool_execution_start"
+          @last_tool = stamp
+          @tool_calls = @tool_calls.to_i + 1
+          @current_tool = [message["toolName"].to_s, @last_tool]
+        when "tool_execution_update"
+          @last_tool = stamp
+        when "tool_execution_end"
+          @last_tool = stamp
+          @current_tool = nil
+        end
+      end
+    end
+
+    def snapshot
+      @mutex.synchronize do
+        now = @clock.call
+        {
+          status: @source ? "observed" : "unknown",
+          source: @source,
+          turn_active: @source ? @turn_active : nil,
+          turn_started_at: @turn_active ? @turn_start&.last : nil,
+          turn_age_s: @turn_active ? age(@turn_start, now) : nil,
+          model_active: @source ? @model_active : nil,
+          model_started_at: @model_active ? @model_start&.last : nil,
+          model_age_s: @model_active ? age(@model_start, now) : nil,
+          last_model_activity_at: @last_model&.last,
+          model_idle_s: age(@last_model, now),
+          last_tool_activity_at: @last_tool&.last,
+          tool_idle_s: age(@last_tool, now),
+          # #87: what the worker is doing now, for progress lines.
+          current_tool: @turn_active ? @current_tool&.first : nil,
+          current_tool_s: @turn_active && @current_tool ? age(@current_tool.last, now) : nil,
+          tool_calls: @source ? @tool_calls.to_i : nil
+        }
+      end
+    end
+
+    private
+
+    def start_turn_locked
+      return if @turn_active
+
+      @turn_active = true
+      @turn_start = stamp
+      @model_active = false
+      @model_start = @last_model = @last_tool = @current_tool = nil
+      @tool_calls = 0
+    end
+
+    def model_activity_locked(finished: false)
+      current = stamp
+      @model_start = current unless @model_active || finished
+      @last_model = current
+      @model_active = !finished
+    end
+
+    def stamp
+      [@clock.call, @wall_clock.call.getutc.iso8601(3)]
+    end
+
+    def age(value, now)
+      value ? [now - value.first, 0.0].max.round(3) : nil
+    end
+  end
+end

data/lib/harnex/runtime/api_server.rb

@@ -66,7 +66,7 @@ module Harnex
       when ["POST", "/stop"]
         return unauthorized(client) unless authorized?(headers)
 
-        json(client, 200, @session.inject_stop)
+        json(client, 200, @session.inject_stop(**parse_stop_body(body)))
       when ["POST", "/send"]
         return unauthorized(client) unless authorized?(headers)
 
@@ -126,6 +126,16 @@ module Harnex
       client.close unless client.closed?
     end
 
+    def parse_stop_body(body)
+      parsed = body.to_s.empty? ? {} : JSON.parse(body)
+      raise ArgumentError, "stop body must be a JSON object" unless parsed.is_a?(Hash)
+      raise ArgumentError, "stop accepts only reason and origin" unless (parsed.keys - %w[reason origin]).empty?
+
+      context = { reason: parsed.fetch("reason", "manual"), origin: parsed.fetch("origin", "api") }
+      StopRequest.validate!(**context)
+      context
+    end
+
     def parse_send_body(headers, body)
       if headers["content-type"].to_s.include?("application/json")
         parsed = JSON.parse(body.empty? ? "{}" : body)