harnex 0.7.13 → 0.8.0

data/lib/harnex/commands/wait.rb

@@ -14,6 +14,19 @@ module Harnex
     EVENT_PREDICATES = %w[task_complete task_failed].freeze
     LEGACY_EVENT_TYPES = %w[agent_state exited task_complete task_failed].freeze
 
+    # Exit-code contract for `--until done` (documented in guides/04_monitoring.md).
+    DONE_EXIT_DONE = 0
+    DONE_EXIT_FAILED = 1
+    DONE_EXIT_REJECTED_PROOF = 2
+    DONE_EXIT_NO_SESSION = 3
+    DONE_EXIT_TIMEOUT = 124
+
+    # Outcome classes where the process completed but the work was not
+    # accepted: proof was missing, invalid, rejected, or nothing happened.
+    REJECTED_PROOF_CLASSES = %w[
+      completed_no_activity report_missing report_invalid report_rejected
+    ].freeze
+
     def self.usage(program_name = "harnex wait")
       <<~TEXT
         Usage: #{program_name} [options]
@@ -41,8 +54,18 @@ module Harnex
           #{program_name} --id cx-i-42 --until prompt --timeout 120
           #{program_name} --id cx-i-42
 
+        Exit codes for --until done:
+          0    completed with accepted work
+          1    failed (work failed, process failed, or killed)
+          2    completed but proof rejected (completed_no_activity,
+               report_missing, report_invalid, report_rejected)
+          3    no such session (no live, start, event, or terminal signal)
+          124  --timeout elapsed while the session was still running
+
         Gotchas:
           done is the safest work-level fence for monitors.
+          While the session's pid is alive, --until done blocks (up to
+          --timeout); "no signal yet" from a live worker is never terminal.
           task_complete/task_failed are event predicates; prompt/busy are live state polls.
           Prompt state alone does not prove work acceptance. Verify artifacts/tests.
           Exit waits can resolve from terminal summary rows when live registry/
@@ -134,7 +157,9 @@ module Harnex
       end
     end
 
-    def scan_events(path, offset, predicate, task_complete_seen, start_time)
+    # min_ts: with a live session in view, events written before it started
+    # belong to an earlier run that reused the id — never match on them.
+    def scan_events(path, offset, predicate, task_complete_seen, start_time, min_ts: nil)
       return [nil, offset, task_complete_seen] unless File.exist?(path) && File.size(path) > offset
 
       File.open(path, "r") do |f|
@@ -142,6 +167,7 @@ module Harnex
         f.each_line do |line|
           event = parse_event(line)
           next unless event
+          next if stale_event?(event, min_ts)
 
           task_complete_seen = true if %w[task_complete task_failed].include?(event_type(event))
           if matches?(event, predicate, task_complete_seen)
@@ -154,6 +180,17 @@ module Harnex
       [nil, offset, task_complete_seen]
     end
 
+    def stale_event?(event, min_ts)
+      return false unless min_ts
+
+      # Events without a parseable ts (legacy formats) cannot be dated;
+      # treat them as fresh rather than silently dropping completion signals.
+      ts = parse_wait_time(event["ts"])
+      return false unless ts
+
+      ts < min_ts
+    end
+
     def parse_event(line)
       event = JSON.parse(line)
       event.is_a?(Hash) ? event : nil
@@ -206,94 +243,157 @@ module Harnex
         seq: event["seq"],
         waited_seconds: waited
       }
+      exit_code = 0
       if predicate == "done"
         failed = done_event_failed?(event)
+        exit_code = failed ? done_failure_exit_code(event["outcome_class"]) : DONE_EXIT_DONE
         payload.merge!(
           ok: !failed,
           status: failed ? "failed" : "done",
+          wait_result: done_wait_result(exit_code),
           state: "running",
           process_state: "running",
           terminal: false,
           task_complete: !failed,
+          task_failed: failed,
           done: !failed,
-          work_state: failed ? "failed" : "completed"
+          work_state: failed ? "failed" : "completed",
+          outcome_class: event["outcome_class"],
+          artifact_report_status: event["artifact_report_status"]
         )
         payload[:last_error] = event["message"] || event["error"] if failed
       end
       puts JSON.generate(payload)
-      return 1 if predicate == "done" && done_event_failed?(event)
+      exit_code
+    end
+
+    def done_failure_exit_code(outcome_class)
+      REJECTED_PROOF_CLASSES.include?(outcome_class.to_s) ? DONE_EXIT_REJECTED_PROOF : DONE_EXIT_FAILED
+    end
 
-      0
+    def done_wait_result(exit_code)
+      case exit_code
+      when DONE_EXIT_DONE then "done"
+      when DONE_EXIT_REJECTED_PROOF then "rejected_proof"
+      else "failed"
+      end
     end
 
+    # Contract: while the session's pid is alive, block (up to --timeout).
+    # Liveness is re-checked every poll from the registry, falling back to an
+    # uncompleted dispatch-start row, so a running worker is never classified
+    # as dead just because one signal source is missing. "No signal yet"
+    # while alive is not terminal.
     def wait_until_done
       repo_root = Harnex.resolve_repo_root(@options[:repo_path])
       events_path = Harnex.events_log_path(repo_root, @options[:id])
       exit_path = Harnex.exit_status_path(repo_root, @options[:id])
-      registry = Harnex.read_registry(repo_root, @options[:id])
       start_time = Time.now
       deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
 
       offset = 0
       task_complete_seen = false
-      final_event_deadline = nil
-
-      status, offset, task_complete_seen = scan_events(events_path, offset, "done", task_complete_seen, start_time)
-      return status if status
+      observed_live = nil
+      session_started_at = nil
 
-      unless registry
-        terminal = done_status(repo_root)
-        return emit_done_terminal_status(terminal) if terminal
-        return emit_done_exit_status(exit_path, @options[:id]) if File.exist?(exit_path)
-
-        unless File.exist?(events_path)
-          warn("harnex wait: no session found with id #{@options[:id].inspect}")
-          puts JSON.generate(ok: false, id: @options[:id], state: "unknown", process_state: "unknown", terminal: false,
-                             task_complete: false, done: false, work_state: "unknown", status: "unknown")
-          return 1
+      loop do
+        live = live_session(repo_root)
+        if live
+          observed_live = live
+          session_started_at ||= parse_wait_time(live["started_at"])
         end
-      end
 
-      target_pid = registry && registry["pid"]
-
-      loop do
-        status, offset, task_complete_seen = scan_events(events_path, offset, "done", task_complete_seen, start_time)
+        status, offset, task_complete_seen =
+          scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: session_started_at)
         return status if status
 
-        unless registry
+        unless live
+          if observed_live
+            return resolve_after_exit(repo_root, events_path, exit_path, offset,
+                                      task_complete_seen, start_time, observed_live, session_started_at)
+          end
+
+          # Historical query: never observed the session alive.
+          return emit_done_exit_status(exit_path, @options[:id]) if File.exist?(exit_path)
+
           terminal = done_status(repo_root)
           return emit_done_terminal_status(terminal) if terminal
-          return emit_done_exit_status(exit_path, @options[:id]) if File.exist?(exit_path)
+
+          unless File.exist?(events_path)
+            warn("harnex wait: no session found with id #{@options[:id].inspect}")
+            puts JSON.generate(ok: false, id: @options[:id], status: "no_such_session",
+                               wait_result: "no_such_session", state: "unknown", process_state: "unknown",
+                               terminal: false, task_complete: false, done: false, work_state: "unknown")
+            return DONE_EXIT_NO_SESSION
+          end
         end
 
         if deadline && Time.now >= deadline
           waited = (Time.now - start_time).round(1)
-          puts JSON.generate(ok: false, id: @options[:id], status: "timeout", waited_seconds: waited,
-                             done: false, work_state: "running")
-          return 124
+          puts JSON.generate(ok: false, id: @options[:id], status: "timeout", wait_result: "timeout",
+                             waited_seconds: waited, done: false,
+                             work_state: live ? "running" : "unknown")
+          return DONE_EXIT_TIMEOUT
         end
 
-        if target_pid && !Harnex.alive_pid?(target_pid)
-          final_event_deadline ||= Time.now + FINAL_EVENT_GRACE_SECONDS
-          if Time.now >= final_event_deadline
-            await_exit_status(exit_path)
-            return emit_done_exit_status(exit_path, @options[:id]) if File.exist?(exit_path)
+        sleep EVENT_POLL_INTERVAL
+      end
+    end
 
-            terminal = done_status(repo_root)
-            return emit_done_terminal_status(terminal) if terminal
+    # Registry row first; when it is not visible from this context, an
+    # uncompleted dispatch-start row with an alive pid still proves liveness.
+    def live_session(repo_root)
+      registry = Harnex.read_registry(repo_root, @options[:id])
+      return registry if registry
 
-            waited = (Time.now - start_time).round(1)
-            puts JSON.generate(ok: false, id: @options[:id], state: "exited", process_state: "exited",
-                               terminal: true, task_complete: false, done: false, work_state: "unknown",
-                               waited_seconds: waited)
-            return 1
-          end
-        else
-          final_event_deadline = nil
-        end
+      Harnex::DispatchHistory.live_start_record(repo_root: repo_root, id: @options[:id])
+    end
 
-        sleep EVENT_POLL_INTERVAL
+    # The session was alive and its pid is now gone: give teardown a bounded
+    # grace to land the final events, summary row, and exit-status file, then
+    # classify from the freshest signal that belongs to the observed session.
+    def resolve_after_exit(repo_root, events_path, exit_path, offset, task_complete_seen,
+                           start_time, observed_live, session_started_at)
+      await_exit_status(exit_path)
+
+      status, _offset, _seen =
+        scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: session_started_at)
+      return status if status
+
+      if File.exist?(exit_path) && fresh_exit_status?(exit_path, observed_live, session_started_at)
+        return emit_done_exit_status(exit_path, @options[:id])
       end
+
+      terminal = done_status(repo_root, min_started_at: session_started_at)
+      return emit_done_terminal_status(terminal) if terminal
+
+      waited = (Time.now - start_time).round(1)
+      puts JSON.generate(ok: false, id: @options[:id], state: "exited", process_state: "exited",
+                         terminal: true, task_complete: false, done: false, work_state: "unknown",
+                         wait_result: "failed", waited_seconds: waited)
+      DONE_EXIT_FAILED
+    end
+
+    # Guards against classifying from an exit-status file left behind by an
+    # earlier dispatch that reused the same id.
+    def fresh_exit_status?(exit_path, observed_live, session_started_at)
+      data = JSON.parse(File.read(exit_path))
+      observed_session_id = observed_live["session_id"].to_s
+      exit_session_id = data["session_id"].to_s
+      return exit_session_id == observed_session_id unless exit_session_id.empty? || observed_session_id.empty?
+
+      exited_at = parse_wait_time(data["exited_at"])
+      return true unless exited_at && session_started_at
+
+      exited_at >= session_started_at
+    rescue StandardError
+      true
+    end
+
+    def parse_wait_time(value)
+      Time.iso8601(value.to_s)
+    rescue ArgumentError
+      nil
     end
 
     def wait_until_state
@@ -438,11 +538,16 @@ module Harnex
       status
     end
 
-    def done_status(repo_root)
+    def done_status(repo_root, min_started_at: nil)
       status = Harnex::TerminalStatus.resolve(id: @options[:id], repo_root: repo_root)
       return nil unless status
       return nil unless status["done"] || status["terminal"]
 
+      if min_started_at
+        row_started = parse_wait_time(status["started_at"])
+        return nil if row_started && row_started < min_started_at
+      end
+
       status
     end
 
@@ -454,6 +559,7 @@ module Harnex
       exit_success = !task_failed && (exit_code.nil? || exit_code.to_i == 0)
       state = exit_success ? "completed" : "failed"
       done = task_complete || exit_success
+      result_code = done ? DONE_EXIT_DONE : done_failure_exit_code(data["outcome_class"])
       payload = data.merge(
         "ok" => done,
         "id" => id,
@@ -463,32 +569,26 @@ module Harnex
         "task_complete" => task_complete,
         "task_failed" => task_failed,
         "done" => done,
-        "work_state" => Harnex.work_state_for(state, task_complete: task_complete)
+        "work_state" => Harnex.work_state_for(state, task_complete: task_complete),
+        "wait_result" => done_wait_result(result_code)
       )
-      success = done
       puts JSON.generate(payload)
-      return 0 if success
-
-      exit_code.is_a?(Integer) && exit_code.positive? ? exit_code : 1
+      result_code
     rescue JSON::ParserError
       puts JSON.generate(ok: false, id: id, state: "failed", process_state: "exited", terminal: true,
-                         task_complete: false, done: false, work_state: "failed", status: "invalid_exit_status")
-      1
+                         task_complete: false, done: false, work_state: "failed",
+                         status: "invalid_exit_status", wait_result: "failed")
+      DONE_EXIT_FAILED
     end
 
     def emit_done_terminal_status(status)
       payload = terminal_payload(status)
       payload[:ok] = !!payload[:done]
       payload[:status] = payload[:done] ? "done" : status["state"]
+      result_code = payload[:done] ? DONE_EXIT_DONE : done_failure_exit_code(status["outcome_class"])
+      payload[:wait_result] = done_wait_result(result_code)
       puts JSON.generate(payload)
-
-      if payload[:ok]
-        0
-      elsif status["exit_code"].is_a?(Integer) && status["exit_code"] > 0
-        status["exit_code"]
-      else
-        1
-      end
+      result_code
     end
 
     def emit_terminal_status(status)
@@ -520,6 +620,8 @@ module Harnex
         task_failed: task_failed,
         done: done,
         work_state: work_state,
+        outcome_class: status["outcome_class"],
+        artifact_report_status: status["artifact_report_status"],
         exit: status["exit"],
         exit_code: status["exit_code"],
         summary_out: status["summary_out"],

data/lib/harnex/commands/watch.rb

@@ -304,6 +304,8 @@ module Harnex
         exit_code: exit_code,
         status: payload["status"],
         work_state: payload["work_state"],
+        outcome_class: payload["outcome_class"],
+        artifact_report_status: payload["artifact_report_status"],
         task_complete: payload["task_complete"] || payload["event"] == "task_complete",
         task_failed: payload["task_failed"] || payload["event"] == "task_failed",
         done: payload["done"],
@@ -348,8 +350,9 @@ module Harnex
           -h, --help           Show this help
 
         `harnex watch` is the safe watcher for existing --tmux or detached
-        dispatches. It exits 0 for task_complete/done, non-zero for task_failed
-        or failed terminal summaries, and 124 for --max-wait timeouts.
+        dispatches. Exit codes follow the `harnex wait --until done` contract:
+        0 accepted work, 1 failed, 2 completed-but-proof-rejected, 3 no such
+        session, 124 for --max-wait timeouts.
 
         For launch-and-babysit stall recovery, use `harnex run --watch`.
       TEXT

data/lib/harnex/core.rb

@@ -157,8 +157,18 @@ module Harnex
     {}
   end
 
+  # Canonical path resolution shared by every registry/exit/events writer and
+  # reader. realpath collapses symlinked prefixes so a session registered from
+  # a symlinked cwd stays visible to checkers resolving the physical path.
+  def canonical_repo_root(path)
+    expanded = File.expand_path(path)
+    File.realpath(expanded)
+  rescue StandardError
+    expanded
+  end
+
   def repo_key(repo_root)
-    Digest::SHA256.hexdigest(repo_root)[0, 16]
+    Digest::SHA256.hexdigest(canonical_repo_root(repo_root))[0, 16]
   end
 
   def normalize_id(id)

data/lib/harnex/dispatch_history.rb

@@ -62,12 +62,107 @@ module Harnex
       end
     end
 
+    def start_record?(record)
+      record.is_a?(Hash) && record["record_type"] == "dispatch_start"
+    end
+
+    def end_record?(record)
+      return false unless record.is_a?(Hash)
+      return true if record["record_type"] == "dispatch_end"
+
+      # Legacy end rows predate record_type.
+      record["schema_version"] == 1 && record.key?("status") && !record.key?("record_type")
+    end
+
+    def end_matches_start?(end_record, start_record)
+      end_session = end_record["session_id"].to_s
+      start_session = start_record["session_id"].to_s
+      return end_session == start_session unless end_session.empty? || start_session.empty?
+
+      end_record["id"].to_s == start_record["id"].to_s &&
+        end_record["started_at"].to_s == start_record["started_at"].to_s
+    end
+
+    # Latest start row for `id` and any end row that completes it.
+    def latest_rows(path, id)
+      latest_start = nil
+      matching_end = nil
+
+      return { start: nil, end: nil } unless File.file?(path)
+
+      File.foreach(path) do |line|
+        record = JSON.parse(line)
+        next unless record.is_a?(Hash)
+        next unless record["id"].to_s == id
+
+        if start_record?(record)
+          latest_start = record
+          matching_end = nil
+        elsif end_record?(record)
+          matching_end = record if latest_start && end_matches_start?(record, latest_start)
+        end
+      rescue JSON::ParserError
+        next
+      end
+
+      { start: latest_start, end: matching_end }
+    end
+
+    # A dispatch_start row with no completing end row whose pid is alive on
+    # this host — evidence of a running session even when the live registry
+    # is not visible from the caller's context.
+    def live_start_record(repo_root:, id:)
+      normalized_id = Harnex.normalize_id(id)
+      rows = latest_rows(path_for(repo_root), normalized_id)
+      start = rows[:start]
+      return nil unless start
+      return nil if rows[:end]
+      return nil unless same_host?(start)
+
+      pid = start["pid"]
+      return nil unless pid && Harnex.alive_pid?(pid)
+
+      start
+    rescue StandardError
+      nil
+    end
+
+    def same_host?(record)
+      host = record["host"].to_s
+      return true if host.empty?
+
+      host == Harnex.host_info[:host].to_s
+    end
+
+    # Appended at registration so a running dispatch always has a durable
+    # trace; the dispatch_end row written in finalize_session! completes it.
+    def build_start_record(session)
+      {
+        schema_version: 1,
+        record_type: "dispatch_start",
+        id: session.id,
+        session_id: session.session_id,
+        pid: session.pid,
+        host: Harnex.host_info[:host],
+        cli: session.adapter.key,
+        description: session.description,
+        started_at: session.started_at.utc.iso8601,
+        repo_root: session.repo_root,
+        tier: session.__send__(:meta_hash)["tier"],
+        meta: session.__send__(:meta_hash),
+        summary_out_path: session.summary_out,
+        events_log_path: session.events_log_path
+      }
+    end
+
     def build_record(session)
       ended_at = session.ended_at || Time.now
       status, terminal_event = classify(session)
       {
         schema_version: 1,
+        record_type: "dispatch_end",
         id: session.id,
+        session_id: session.session_id,
         description: session.description,
         cli: session.adapter.key,
         started_at: session.started_at.utc.iso8601,