harnex 0.12.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +592 -0
  3. data/GUIDE.md +222 -68
  4. data/README.md +78 -48
  5. data/TECHNICAL.md +235 -98
  6. data/agents/gpt.md +60 -0
  7. data/docs/agents.md +119 -0
  8. data/docs/dispatch-telemetry.md +184 -50
  9. data/docs/events.md +20 -7
  10. data/docs/lanes.md +202 -0
  11. data/docs/mcp.md +102 -0
  12. data/docs/pi-rpc.md +81 -7
  13. data/docs/upgrading-to-0.29.md +87 -0
  14. data/guides/01_dispatch.md +101 -34
  15. data/guides/04_monitoring.md +223 -25
  16. data/guides/05_naming.md +0 -1
  17. data/guides/06_claude.md +96 -0
  18. data/lib/harnex/adapters/base.rb +4 -8
  19. data/lib/harnex/adapters/pi.rb +92 -12
  20. data/lib/harnex/adapters.rb +1 -16
  21. data/lib/harnex/artifact_report.rb +206 -12
  22. data/lib/harnex/cli.rb +68 -50
  23. data/lib/harnex/commands/ask.rb +278 -0
  24. data/lib/harnex/commands/delegate.rb +202 -0
  25. data/lib/harnex/commands/doctor.rb +34 -36
  26. data/lib/harnex/commands/events.rb +3 -3
  27. data/lib/harnex/commands/history.rb +2 -8
  28. data/lib/harnex/commands/land.rb +150 -0
  29. data/lib/harnex/commands/lanes.rb +84 -0
  30. data/lib/harnex/commands/logs.rb +3 -3
  31. data/lib/harnex/commands/mcp.rb +208 -0
  32. data/lib/harnex/commands/orchestration.rb +88 -9
  33. data/lib/harnex/commands/pane.rb +3 -3
  34. data/lib/harnex/commands/preflight.rb +566 -0
  35. data/lib/harnex/commands/reply.rb +125 -0
  36. data/lib/harnex/commands/result.rb +108 -0
  37. data/lib/harnex/commands/run.rb +248 -64
  38. data/lib/harnex/commands/send.rb +121 -16
  39. data/lib/harnex/commands/status.rb +32 -4
  40. data/lib/harnex/commands/stop.rb +14 -4
  41. data/lib/harnex/commands/telemetry.rb +19 -1
  42. data/lib/harnex/commands/wait.rb +421 -56
  43. data/lib/harnex/commands/watch.rb +134 -21
  44. data/lib/harnex/context_file.rb +85 -0
  45. data/lib/harnex/core.rb +100 -15
  46. data/lib/harnex/dispatch_history.rb +157 -10
  47. data/lib/harnex/lanes.rb +513 -0
  48. data/lib/harnex/mcp.rb +362 -0
  49. data/lib/harnex/orchestration.rb +128 -5
  50. data/lib/harnex/owned_process_group.rb +50 -0
  51. data/lib/harnex/owner_stream.rb +399 -0
  52. data/lib/harnex/pricing.rb +4 -9
  53. data/lib/harnex/runtime/activity_tracker.rb +107 -0
  54. data/lib/harnex/runtime/api_server.rb +11 -1
  55. data/lib/harnex/runtime/completion_notifier.rb +90 -11
  56. data/lib/harnex/runtime/inbox.rb +26 -7
  57. data/lib/harnex/runtime/message.rb +5 -2
  58. data/lib/harnex/runtime/runtime_budget.rb +96 -0
  59. data/lib/harnex/runtime/session.rb +684 -294
  60. data/lib/harnex/runtime/stop_request.rb +39 -0
  61. data/lib/harnex/stall.rb +46 -0
  62. data/lib/harnex/telemetry_reconciler.rb +42 -20
  63. data/lib/harnex/terminal_status.rb +5 -5
  64. data/lib/harnex/version.rb +2 -2
  65. data/lib/harnex.rb +16 -0
  66. data/recipes/01_fire_and_watch.md +14 -14
  67. data/recipes/02_chain_implement.md +35 -31
  68. data/recipes/03_buddy.md +1 -1
  69. metadata +34 -12
  70. data/docs/codex-appserver.md +0 -175
  71. data/lib/harnex/adapters/codex.rb +0 -160
  72. data/lib/harnex/adapters/codex_appserver.rb +0 -407
  73. data/lib/harnex/codex/app_server/client.rb +0 -347
@@ -1,6 +1,7 @@
1
1
  require "json"
2
2
  require "net/http"
3
3
  require "optparse"
4
+ require "stringio"
4
5
  require "uri"
5
6
 
6
7
  module Harnex
@@ -19,7 +20,10 @@ module Harnex
19
20
  DONE_EXIT_FAILED = 1
20
21
  DONE_EXIT_REJECTED_PROOF = 2
21
22
  DONE_EXIT_NO_SESSION = 3
23
+ DONE_EXIT_STALLED = 4
22
24
  DONE_EXIT_TIMEOUT = 124
25
+ # Stall checks poll the live status API; detection lags by at most this.
26
+ STALL_POLL_MAX_INTERVAL = 15.0
23
27
 
24
28
  # Outcome classes where the process completed but the work was not
25
29
  # accepted: proof was missing, invalid, rejected, or nothing happened.
@@ -45,14 +49,30 @@ module Harnex
45
49
  "prompt", "busy")
46
50
  Without --until, waits for session exit (default).
47
51
  --repo PATH Resolve session using PATH's repo root (default: current repo)
48
- --timeout SECS Maximum time to wait in seconds (default: unlimited)
52
+ --timeout DUR Hard observer cap (seconds, or suffix s/m/h; default: unlimited)
53
+ --max-wait DUR Alias for --timeout
54
+ --heartbeat DUR Flush observed progress to stderr at this interval (default: off)
55
+ --after-seq N With --until done/task_complete/task_failed: ignore every
56
+ event at or before events-log seq N (the after_seq that
57
+ `harnex send` reports), so the wait covers the turn that
58
+ message started or joined, not an earlier result.
59
+ --since TIME With --until done: ignore events before TIME (ISO 8601),
60
+ e.g. an earlier launch of a lane relaunched under the
61
+ same id (each launch restarts the events-log seq)
62
+ --stall-after DUR
63
+ With --until done: exit 4 when the live session shows no
64
+ progress for DUR (Pi/structured: model, tool or turn
65
+ events during an active turn; PTY: transcript idle).
66
+ Never nudges or stops the worker.
49
67
  -h, --help Show this help
50
68
 
51
69
  Common patterns:
52
- #{program_name} --id cx-i-42 --until done --timeout 900
53
- #{program_name} --id cx-i-42 --until task_complete --timeout 900
54
- #{program_name} --id cx-i-42 --until prompt --timeout 120
55
- #{program_name} --id cx-i-42
70
+ #{program_name} --id pi-i-42 --until done --timeout 900
71
+ #{program_name} --id pi-i-42 --until done --stall-after 25m --timeout 4h
72
+ #{program_name} --id pi-i-42 --until done --after-seq 812 --timeout 9m
73
+ #{program_name} --id pi-i-42 --until task_complete --timeout 900
74
+ #{program_name} --id pi-i-42 --until prompt --timeout 120
75
+ #{program_name} --id pi-i-42
56
76
 
57
77
  Exit codes for --until done:
58
78
  0 completed with accepted work
@@ -60,10 +80,13 @@ module Harnex
60
80
  2 completed but proof rejected (completed_no_activity,
61
81
  report_missing, report_invalid, report_rejected)
62
82
  3 no such session (no live, start, event, or terminal signal)
63
- 124 --timeout elapsed while the session was still running
83
+ 4 stalled: no progress for --stall-after (worker left running)
84
+ 124 observer deadline elapsed (does not stop the worker)
64
85
 
65
86
  Gotchas:
66
87
  done is the safest work-level fence for monitors.
88
+ A bare wait returns only on process exit; sessions without
89
+ --auto-stop stay alive after their turn (stderr notes say so).
67
90
  While the session's pid is alive, --until done blocks (up to
68
91
  --timeout); "no signal yet" from a live worker is never terminal.
69
92
  task_complete/task_failed are event predicates; prompt/busy are live state polls.
@@ -74,26 +97,72 @@ module Harnex
74
97
  TEXT
75
98
  end
76
99
 
77
- def initialize(argv)
100
+ # Reuse the shared duration grammar while preserving wait's numeric
101
+ # timeout syntax and guarding numeric fallback/forwarded Float values.
102
+ def self.duration(value, option_name:)
103
+ seconds = begin
104
+ Harnex.parse_duration_seconds(value, option_name: option_name)
105
+ rescue OptionParser::InvalidArgument => invalid_duration
106
+ # Preserve wait --timeout's former Float syntax (e.g. .5 or 1e-3),
107
+ # also used by Float#to_s when watch forwards very small intervals.
108
+ begin
109
+ Float(value)
110
+ rescue ArgumentError, TypeError
111
+ raise invalid_duration
112
+ end
113
+ end
114
+ unless seconds.finite? && seconds.positive?
115
+ raise OptionParser::InvalidArgument, "#{option_name} must be a finite positive duration"
116
+ end
117
+ seconds
118
+ end
119
+
120
+ def initialize(argv, out: nil, err: nil)
78
121
  @argv = argv.dup
122
+ @out = out
123
+ @err = err
79
124
  @options = {
80
125
  id: nil,
81
126
  until_state: nil,
82
127
  repo_path: Dir.pwd,
83
128
  timeout: nil,
129
+ heartbeat: nil,
130
+ stall_after: nil,
131
+ after_seq: nil,
132
+ since: nil,
84
133
  help: false
85
134
  }
86
135
  end
87
136
 
88
137
  def run
138
+ @out ||= $stdout
139
+ @err ||= $stderr
89
140
  parser.parse!(@argv)
90
141
  if @options[:help]
91
- puts self.class.usage
142
+ @out.puts self.class.usage
92
143
  return 0
93
144
  end
94
145
 
95
146
  raise "--id is required for harnex wait" unless @options[:id]
147
+ if @options[:stall_after] && @options[:until_state] != "done"
148
+ raise OptionParser::InvalidOption, "--stall-after requires --until done"
149
+ end
150
+ if @options[:after_seq] && !(["done"] + EVENT_PREDICATES).include?(@options[:until_state])
151
+ raise OptionParser::InvalidOption, "--after-seq requires --until done, task_complete, or task_failed"
152
+ end
96
153
 
154
+ @started_at = monotonic
155
+ @observation = { "state" => "unknown", "last_event" => "unknown", "seq" => nil }
156
+ if @options[:timeout] || @options[:heartbeat]
157
+ observe_in_child
158
+ else
159
+ wait
160
+ end
161
+ end
162
+
163
+ private
164
+
165
+ def wait
97
166
  if @options[:until_state]
98
167
  case @options[:until_state]
99
168
  when "done"
@@ -108,14 +177,180 @@ module Harnex
108
177
  end
109
178
  end
110
179
 
111
- private
180
+ # Only the read-side observer runs in this child, never the session or any
181
+ # stop/marker action. A process boundary lets the parent enforce its cap
182
+ # during blocking Ruby/HTTP probes, even probes with slow ensure clauses or
183
+ # broad rescue handlers. No asynchronous Ruby exception crosses a probe.
184
+ # The private process group also contains repo-resolution subprocesses.
185
+ def observe_in_child
186
+ reader, writer = IO.pipe
187
+ writer.sync = true
188
+ pid = fork do
189
+ exit_code = 1
190
+ begin
191
+ Process.setpgid(0, 0)
192
+ reader.close
193
+ @observation_writer = writer
194
+ @out = StringIO.new
195
+ @err = DiagnosticStream.new { |text| send_observation_message(["warning", text]) }
196
+ code = wait
197
+ send_observation_message(["result", @out.string, code])
198
+ exit_code = 0
199
+ rescue Exception => error # transport errors too; never run inherited at_exit hooks
200
+ begin
201
+ send_observation_message(["error", error.class.name, error.message, error.backtrace])
202
+ rescue IOError, SystemCallError
203
+ nil # the caller may already have closed the pipe
204
+ end
205
+ ensure
206
+ exit! exit_code
207
+ end
208
+ end
209
+ writer.close
210
+ deadline = @options[:timeout] && @started_at + @options[:timeout]
211
+ next_heartbeat = @options[:heartbeat] && @started_at + @options[:heartbeat]
212
+ buffer = +""
213
+
214
+ loop do
215
+ now = monotonic
216
+ return emit_timeout if deadline && now >= deadline
217
+
218
+ # Drain complete records without blocking on a partial pipe write.
219
+ if (newline = buffer.index("\n"))
220
+ message = JSON.parse(buffer.slice!(0..newline))
221
+ case message[0]
222
+ when "observation" then @observation = message[1]
223
+ when "warning" then @err.write(message[1])
224
+ when "result"
225
+ return emit_timeout if deadline && monotonic >= deadline
226
+
227
+ @out.write(message[1])
228
+ return message[2]
229
+ when "error"
230
+ raise RuntimeError, "harnex wait observer: #{message[1]}: #{message[2]}", message[3]
231
+ end
232
+ next
233
+ end
234
+
235
+ if next_heartbeat && now >= next_heartbeat
236
+ emit_heartbeat
237
+ # Do not burst missed heartbeats after a scheduler pause.
238
+ next_heartbeat = now + @options[:heartbeat]
239
+ end
240
+ wake_at = [deadline, next_heartbeat].compact.min
241
+ # Clamp even finite enormous durations to an OS-safe select timeout.
242
+ pause = [[wake_at - monotonic, 0].max, POLL_INTERVAL].min
243
+ IO.select([reader], nil, nil, pause)
244
+ chunk = reader.read_nonblock(16_384, exception: false)
245
+ raise "harnex wait: observer exited without a result" if chunk.nil?
246
+
247
+ buffer << chunk unless chunk == :wait_readable
248
+ end
249
+ ensure
250
+ reader&.close unless reader&.closed?
251
+ writer&.close unless writer&.closed?
252
+ reap_observer(pid) if pid
253
+ end
254
+
255
+ # SIGKILL bypasses slow Ruby ensure handlers. Reap without an unbounded
256
+ # wait/join. Uninterruptible kernel I/O or a descheduled OS cannot be given
257
+ # a universal preemption guarantee; no helper thread is left waiting on it.
258
+ def reap_observer(pid)
259
+ begin
260
+ Process.kill("KILL", -pid)
261
+ rescue Errno::ESRCH
262
+ nil
263
+ end
264
+ begin
265
+ Process.kill("KILL", pid)
266
+ rescue Errno::ESRCH
267
+ nil
268
+ end
269
+ cleanup_deadline = monotonic + EVENT_POLL_INTERVAL
270
+ loop do
271
+ return if Process.waitpid(pid, Process::WNOHANG)
272
+ break if monotonic >= cleanup_deadline
273
+
274
+ sleep 0.005
275
+ end
276
+ @err.puts("harnex wait: OS has not reaped the cancelled observer yet")
277
+ rescue Errno::ECHILD
278
+ nil
279
+ end
280
+
281
+ class DiagnosticStream
282
+ def initialize(&write)
283
+ @write = write
284
+ end
285
+
286
+ def write(text)
287
+ @write.call(text)
288
+ end
289
+
290
+ def puts(text)
291
+ write("#{text}\n")
292
+ end
293
+ end
294
+
295
+ def send_observation_message(message)
296
+ @observation_writer.write(JSON.generate(message) + "\n")
297
+ end
298
+
299
+ def observe(**fields)
300
+ updated = @observation.merge(fields.transform_keys(&:to_s))
301
+ return if updated == @observation
302
+
303
+ @observation = updated
304
+ send_observation_message(["observation", updated]) if @observation_writer
305
+ end
306
+
307
+ def emit_heartbeat
308
+ # These are observations, not model/tool activity or a log-age proxy.
309
+ values = %w[state last_event seq].map do |key|
310
+ value = @observation[key] || "unknown"
311
+ "#{key}=#{value.to_s.gsub(/[[:space:][:cntrl:]]/, '_')[0, 128]}"
312
+ end
313
+ @err.puts("harnex wait: id=#{@options[:id]} waited=#{(monotonic - @started_at).round(1)}s #{values.join(' ')}")
314
+ @err.flush
315
+ end
316
+
317
+ def emit_timeout
318
+ payload = { ok: false, id: @options[:id], status: "timeout",
319
+ waited_seconds: (monotonic - @started_at).round(1) }
320
+ case @options[:until_state]
321
+ when "done"
322
+ payload.merge!(wait_result: "timeout", done: false,
323
+ work_state: @observation["live"] ? "running" : "unknown")
324
+ when *EVENT_PREDICATES
325
+ # Event waits retain their existing timeout payload shape.
326
+ when nil
327
+ payload[:pid] = @observation["pid"] if @observation["pid"]
328
+ else
329
+ payload[:state] = @observation["state"]
330
+ end
331
+ @out.puts JSON.generate(payload)
332
+ DONE_EXIT_TIMEOUT
333
+ end
334
+
335
+ # All output goes through per-instance streams, including legacy wait
336
+ # diagnostics. Resolve defaults in run so callers can still capture IO.
337
+ def puts(text)
338
+ @out.puts(text)
339
+ end
340
+
341
+ def warn(text)
342
+ @err.puts(text)
343
+ end
344
+
345
+ def monotonic
346
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
347
+ end
112
348
 
113
349
  def wait_until_event(predicate)
114
350
  repo_root = Harnex.resolve_repo_root(@options[:repo_path])
115
351
  events_path = Harnex.events_log_path(repo_root, @options[:id])
116
352
  registry = Harnex.read_registry(repo_root, @options[:id])
117
- start_time = Time.now
118
- deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
353
+ start_time = @started_at
119
354
 
120
355
  unless registry || File.exist?(events_path)
121
356
  warn("harnex wait: no session found with id #{@options[:id].inspect}")
@@ -131,21 +366,17 @@ module Harnex
131
366
  return status if status
132
367
 
133
368
  target_pid = registry && registry["pid"]
369
+ observe(state: "running", pid: target_pid) if target_pid
134
370
 
135
371
  loop do
136
372
  status, offset, task_complete_seen = scan_events(events_path, offset, predicate, task_complete_seen, start_time)
137
373
  return status if status
138
374
 
139
- if deadline && Time.now >= deadline
140
- waited = (Time.now - start_time).round(1)
141
- puts JSON.generate(ok: false, id: @options[:id], status: "timeout", waited_seconds: waited)
142
- return 124
143
- end
144
-
145
375
  if target_pid && !Harnex.alive_pid?(target_pid)
146
- final_event_deadline ||= Time.now + FINAL_EVENT_GRACE_SECONDS
147
- if Time.now >= final_event_deadline
148
- waited = (Time.now - start_time).round(1)
376
+ observe(state: "exited")
377
+ final_event_deadline ||= monotonic + FINAL_EVENT_GRACE_SECONDS
378
+ if monotonic >= final_event_deadline
379
+ waited = (monotonic - start_time).round(1)
149
380
  puts JSON.generate(ok: false, id: @options[:id], state: "exited", waited_seconds: waited)
150
381
  return 1
151
382
  end
@@ -168,7 +399,10 @@ module Harnex
168
399
  event = parse_event(line)
169
400
  next unless event
170
401
  next if stale_event?(event, min_ts)
402
+ next if before_after_seq?(event)
171
403
 
404
+ observe(last_event: event_type(event), seq: event["seq"])
405
+ observe(state: event["state"]) if event_type(event) == "agent_state" && event["state"]
172
406
  task_complete_seen = true if %w[task_complete task_failed].include?(event_type(event))
173
407
  if matches?(event, predicate, task_complete_seen)
174
408
  return [emit_event_match(event, start_time, predicate), f.pos, task_complete_seen]
@@ -180,6 +414,15 @@ module Harnex
180
414
  [nil, offset, task_complete_seen]
181
415
  end
182
416
 
417
+ # #81: events at or before --after-seq belong to an earlier turn. An event
418
+ # without a seq cannot be placed after it, so it never matches either.
419
+ def before_after_seq?(event)
420
+ return false unless @options[:after_seq]
421
+
422
+ seq = event["seq"]
423
+ !seq.is_a?(Integer) || seq <= @options[:after_seq]
424
+ end
425
+
183
426
  def stale_event?(event, min_ts)
184
427
  return false unless min_ts
185
428
 
@@ -235,7 +478,7 @@ module Harnex
235
478
  end
236
479
 
237
480
  def emit_event_match(event, start_time, predicate)
238
- waited = (Time.now - start_time).round(1)
481
+ waited = (monotonic - start_time).round(1)
239
482
  payload = {
240
483
  ok: true,
241
484
  id: @options[:id],
@@ -243,30 +486,48 @@ module Harnex
243
486
  seq: event["seq"],
244
487
  waited_seconds: waited
245
488
  }
489
+ payload[:turn] = event["turn"] if event["turn"]
490
+ payload[:follow_ups] = event["follow_ups"] if event["follow_ups"]
491
+ %w[end_sha commits turn_commits model final_message final_message_path final_message_truncated].each do |key|
492
+ payload[key.to_sym] = event[key] if event.key?(key)
493
+ end
246
494
  exit_code = 0
247
495
  if predicate == "done"
248
496
  failed = done_event_failed?(event)
249
- exit_code = failed ? done_failure_exit_code(event["outcome_class"]) : DONE_EXIT_DONE
497
+ outcome_class = event["outcome_class"] || stop_outcome_class(event["stop"])
498
+ exit_code = failed ? done_failure_exit_code(outcome_class) : DONE_EXIT_DONE
499
+ # A session seen gone by now has exited; one never observed stays
500
+ # "running", the historical event-match default.
501
+ exited = @observation["live"] == false
250
502
  payload.merge!(
251
503
  ok: !failed,
252
504
  status: failed ? "failed" : "done",
253
505
  wait_result: done_wait_result(exit_code),
254
- state: "running",
255
- process_state: "running",
256
- terminal: false,
506
+ state: exited ? "exited" : "running",
507
+ process_state: exited ? "exited" : "running",
508
+ terminal: exited,
257
509
  task_complete: !failed,
258
510
  task_failed: failed,
259
511
  done: !failed,
260
512
  work_state: failed ? "failed" : "completed",
261
- outcome_class: event["outcome_class"],
513
+ outcome_class: outcome_class,
262
514
  artifact_report_status: event["artifact_report_status"]
263
515
  )
264
516
  payload[:last_error] = event["message"] || event["error"] if failed
517
+ payload[:stop] = event["stop"] if event["stop"].is_a?(Hash)
265
518
  end
266
519
  puts JSON.generate(payload)
267
520
  exit_code
268
521
  end
269
522
 
523
+ # A turn failed by a caller's stop or the runtime budget carries the
524
+ # stop provenance, not a class; name it as the dispatch row will.
525
+ def stop_outcome_class(stop)
526
+ return nil unless stop.is_a?(Hash)
527
+
528
+ { "manual" => "stopped_by_caller", "runtime_budget" => "timeout" }[stop["reason"].to_s]
529
+ end
530
+
270
531
  def done_failure_exit_code(outcome_class)
271
532
  REJECTED_PROOF_CLASSES.include?(outcome_class.to_s) ? DONE_EXIT_REJECTED_PROOF : DONE_EXIT_FAILED
272
533
  end
@@ -288,25 +549,32 @@ module Harnex
288
549
  repo_root = Harnex.resolve_repo_root(@options[:repo_path])
289
550
  events_path = Harnex.events_log_path(repo_root, @options[:id])
290
551
  exit_path = Harnex.exit_status_path(repo_root, @options[:id])
291
- start_time = Time.now
292
- deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
552
+ start_time = @started_at
293
553
 
294
554
  offset = 0
295
555
  task_complete_seen = false
296
556
  observed_live = nil
297
557
  session_started_at = nil
558
+ next_stall_check = nil
298
559
 
299
560
  loop do
300
561
  live = live_session(repo_root)
562
+ observe(state: live ? "running" : "unknown", pid: live && live["pid"], live: !!live)
301
563
  if live
302
564
  observed_live = live
303
565
  session_started_at ||= parse_wait_time(live["started_at"])
304
566
  end
305
567
 
306
568
  status, offset, task_complete_seen =
307
- scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: session_started_at)
569
+ scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: done_min_ts(session_started_at))
308
570
  return status if status
309
571
 
572
+ if @options[:stall_after] && live && (next_stall_check.nil? || monotonic >= next_stall_check)
573
+ next_stall_check = monotonic + stall_poll_interval
574
+ stall = Harnex::Stall.stalled(fetch_status_payload(live), @options[:stall_after])
575
+ return emit_stall(stall, start_time) if stall
576
+ end
577
+
310
578
  unless live
311
579
  if observed_live
312
580
  return resolve_after_exit(repo_root, events_path, exit_path, offset,
@@ -328,18 +596,45 @@ module Harnex
328
596
  end
329
597
  end
330
598
 
331
- if deadline && Time.now >= deadline
332
- waited = (Time.now - start_time).round(1)
333
- puts JSON.generate(ok: false, id: @options[:id], status: "timeout", wait_result: "timeout",
334
- waited_seconds: waited, done: false,
335
- work_state: live ? "running" : "unknown")
336
- return DONE_EXIT_TIMEOUT
337
- end
338
-
339
599
  sleep EVENT_POLL_INTERVAL
340
600
  end
341
601
  end
342
602
 
603
+ def done_min_ts(session_started_at)
604
+ [session_started_at, @options[:since]].compact.max
605
+ end
606
+
607
+ def stall_poll_interval
608
+ [[@options[:stall_after] / 4.0, 0.5].max, STALL_POLL_MAX_INTERVAL].min
609
+ end
610
+
611
+ # Only a registry row has the live API; start-record liveness cannot be
612
+ # probed, so a stall stays unobserved there rather than guessed.
613
+ def fetch_status_payload(live)
614
+ return nil unless live["host"] && live["port"]
615
+
616
+ uri = URI("http://#{live['host']}:#{live['port']}/status")
617
+ request = Net::HTTP::Get.new(uri)
618
+ request["Authorization"] = "Bearer #{live['token']}" if live["token"]
619
+ response = Net::HTTP.start(uri.host, uri.port, open_timeout: 1, read_timeout: 2) { |http| http.request(request) }
620
+ return nil unless response.is_a?(Net::HTTPSuccess)
621
+
622
+ JSON.parse(response.body)
623
+ rescue StandardError
624
+ nil
625
+ end
626
+
627
+ # The worker is left running: this is an observer verdict, like a timeout.
628
+ def emit_stall(stall, start_time)
629
+ waited = (monotonic - start_time).round(1)
630
+ warn("harnex wait: #{@options[:id]} shows no progress for #{stall['idle_s'].round}s " \
631
+ "(clock=#{stall['clock']}, threshold #{stall['threshold_s'].round}s); worker left running")
632
+ puts JSON.generate(ok: false, id: @options[:id], status: "stalled", wait_result: "stalled",
633
+ state: "running", process_state: "running", terminal: false, task_complete: false,
634
+ done: false, work_state: "running", stall: stall, waited_seconds: waited)
635
+ DONE_EXIT_STALLED
636
+ end
637
+
343
638
  # Registry row first; when it is not visible from this context, an
344
639
  # uncompleted dispatch-start row with an alive pid still proves liveness.
345
640
  def live_session(repo_root)
@@ -357,7 +652,7 @@ module Harnex
357
652
  await_exit_status(exit_path)
358
653
 
359
654
  status, _offset, _seen =
360
- scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: session_started_at)
655
+ scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: done_min_ts(session_started_at))
361
656
  return status if status
362
657
 
363
658
  if File.exist?(exit_path) && fresh_exit_status?(exit_path, observed_live, session_started_at)
@@ -367,7 +662,7 @@ module Harnex
367
662
  terminal = done_status(repo_root, min_started_at: session_started_at)
368
663
  return emit_done_terminal_status(terminal) if terminal
369
664
 
370
- waited = (Time.now - start_time).round(1)
665
+ waited = (monotonic - start_time).round(1)
371
666
  puts JSON.generate(ok: false, id: @options[:id], state: "exited", process_state: "exited",
372
667
  terminal: true, task_complete: false, done: false, work_state: "unknown",
373
668
  wait_result: "failed", waited_seconds: waited)
@@ -399,8 +694,7 @@ module Harnex
399
694
  def wait_until_state
400
695
  repo_root = Harnex.resolve_repo_root(@options[:repo_path])
401
696
  target_state = @options[:until_state]
402
- start_time = Time.now
403
- deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
697
+ start_time = @started_at
404
698
 
405
699
  registry = Harnex.read_registry(repo_root, @options[:id])
406
700
  unless registry
@@ -417,31 +711,25 @@ module Harnex
417
711
 
418
712
  loop do
419
713
  unless Harnex.alive_pid?(target_pid)
420
- waited = (Time.now - start_time).round(1)
714
+ waited = (monotonic - start_time).round(1)
421
715
  puts JSON.generate(ok: false, id: @options[:id], state: "exited", waited_seconds: waited)
422
716
  return 1
423
717
  end
424
718
 
425
719
  state = fetch_agent_state(host, port, token)
720
+ observe(state: state || "unknown", pid: target_pid)
426
721
  if state == target_state
427
- waited = (Time.now - start_time).round(1)
722
+ waited = (monotonic - start_time).round(1)
428
723
  puts JSON.generate(ok: true, id: @options[:id], state: state, waited_seconds: waited)
429
724
  return 0
430
725
  end
431
726
 
432
- if deadline && Time.now >= deadline
433
- waited = (Time.now - start_time).round(1)
434
- puts JSON.generate(ok: false, id: @options[:id], state: state || "unknown", waited_seconds: waited, status: "timeout")
435
- return 124
436
- end
437
-
438
727
  sleep POLL_INTERVAL
439
728
  end
440
729
  end
441
730
 
442
731
  def wait_until_exit
443
732
  repo_root = Harnex.resolve_repo_root(@options[:repo_path])
444
- deadline = @options[:timeout] ? Time.now + @options[:timeout] : nil
445
733
  exit_path = Harnex.exit_status_path(repo_root, @options[:id])
446
734
 
447
735
  registry = Harnex.read_registry(repo_root, @options[:id])
@@ -458,7 +746,9 @@ module Harnex
458
746
  end
459
747
 
460
748
  target_pid = registry["pid"]
749
+ observe(state: "running", pid: target_pid)
461
750
  warn("harnex wait: watching session #{@options[:id]} (pid #{target_pid})")
751
+ hint = exit_wait_hint(repo_root, registry)
462
752
 
463
753
  loop do
464
754
  unless Harnex.alive_pid?(target_pid)
@@ -473,22 +763,75 @@ module Harnex
473
763
  return 1
474
764
  end
475
765
 
476
- if deadline && Time.now >= deadline
477
- puts JSON.generate(ok: false, id: @options[:id], status: "timeout", pid: target_pid)
478
- return 124
766
+ if hint && (settled = settled_event_type(hint))
767
+ warn("harnex wait: note: #{@options[:id]} reached #{settled} but is still alive; " \
768
+ "this wait still returns only on process exit. Use --until done for a work-level fence.")
769
+ hint = nil
479
770
  end
480
771
 
481
772
  sleep POLL_INTERVAL
482
773
  end
483
774
  end
484
775
 
776
+ # A bare wait fences process exit, but an interactive session stays alive
777
+ # at its prompt after a turn. Say so on entry, and once more if the turn
778
+ # settles mid-wait, so a waiter on the wrong predicate learns it before
779
+ # its timeout (#49). --auto-stop sessions exit on their own: no hint.
780
+ # Returns the event-scan cursor for the mid-wait check, or nil when done.
781
+ def exit_wait_hint(repo_root, registry)
782
+ return nil if registry["auto_stop"] == true
783
+
784
+ cursor = {
785
+ path: Harnex.events_log_path(repo_root, @options[:id]),
786
+ offset: 0,
787
+ min_ts: parse_wait_time(registry["started_at"])
788
+ }
789
+ if (settled = settled_event_type(cursor))
790
+ warn("harnex wait: note: #{@options[:id]} already reached #{settled} and stays alive at its prompt; " \
791
+ "this wait returns only on process exit. Use --until done for a work-level fence.")
792
+ return nil
793
+ end
794
+
795
+ reason =
796
+ if registry["auto_stop"] == false
797
+ "was not started with --auto-stop, so it stays alive after its turn"
798
+ else
799
+ "may stay alive at its prompt after its turn unless started with --auto-stop"
800
+ end
801
+ warn("harnex wait: note: #{@options[:id]} #{reason}; this wait returns only on process exit. " \
802
+ "Use --until done for a work-level fence.")
803
+ cursor
804
+ end
805
+
806
+ def settled_event_type(cursor)
807
+ path = cursor[:path]
808
+ return nil unless File.exist?(path) && File.size(path) > cursor[:offset]
809
+
810
+ File.open(path, "r") do |f|
811
+ f.seek(cursor[:offset])
812
+ f.each_line do |line|
813
+ # Only whole lines advance the cursor; a partial tail is re-read.
814
+ break unless line.end_with?("\n")
815
+
816
+ cursor[:offset] = f.pos
817
+ event = parse_event(line)
818
+ next unless event
819
+ next if stale_event?(event, cursor[:min_ts])
820
+
821
+ type = event_type(event)
822
+ return type if %w[task_complete task_failed].include?(type)
823
+ end
824
+ end
825
+ nil
826
+ end
827
+
485
828
  # Subprocess death races the parent's DISPATCH-row write; the exit-status
486
829
  # file is written *after* the row, so polling it bounds the race.
487
830
  def await_exit_status(exit_path)
488
831
  return if File.exist?(exit_path)
489
832
 
490
- grace_deadline = Time.now + exit_status_grace_seconds
491
- until File.exist?(exit_path) || Time.now >= grace_deadline
833
+ grace_deadline = monotonic + exit_status_grace_seconds
834
+ until File.exist?(exit_path) || monotonic >= grace_deadline
492
835
  sleep EXIT_STATUS_GRACE_POLL_INTERVAL
493
836
  end
494
837
  end
@@ -622,6 +965,8 @@ module Harnex
622
965
  work_state: work_state,
623
966
  outcome_class: status["outcome_class"],
624
967
  artifact_report_status: status["artifact_report_status"],
968
+ stop: status["stop"],
969
+ runtime_budget: status["runtime_budget"],
625
970
  exit: status["exit"],
626
971
  exit_code: status["exit_code"],
627
972
  ended_at: status["ended_at"],
@@ -635,7 +980,27 @@ module Harnex
635
980
  opts.on("--id ID", "Session ID to wait for") { |value| @options[:id] = Harnex.normalize_id(value) }
636
981
  opts.on("--until STATE", "Wait until session reaches STATE") { |value| @options[:until_state] = value }
637
982
  opts.on("--repo PATH", "Resolve session using PATH's repo root") { |value| @options[:repo_path] = value }
638
- opts.on("--timeout SECONDS", Float, "Maximum time to wait") { |value| @options[:timeout] = value }
983
+ opts.on("--timeout DUR", "Hard observer cap") do |value|
984
+ @options[:timeout] = self.class.duration(value, option_name: "--timeout")
985
+ end
986
+ opts.on("--max-wait DUR", "Alias for --timeout") do |value|
987
+ @options[:timeout] = self.class.duration(value, option_name: "--max-wait")
988
+ end
989
+ opts.on("--heartbeat DUR", "Flush progress to stderr") do |value|
990
+ @options[:heartbeat] = self.class.duration(value, option_name: "--heartbeat")
991
+ end
992
+ opts.on("--after-seq N", Integer, "Ignore events at or before events-log seq N") do |value|
993
+ raise OptionParser::InvalidArgument, "--after-seq must be 0 or greater" if value.negative?
994
+
995
+ @options[:after_seq] = value
996
+ end
997
+ opts.on("--since TIME", "With --until done: ignore events before TIME") do |value|
998
+ @options[:since] = parse_wait_time(value) ||
999
+ raise(OptionParser::InvalidArgument, "--since must be an ISO 8601 time")
1000
+ end
1001
+ opts.on("--stall-after DUR", "With --until done: exit 4 after DUR without progress") do |value|
1002
+ @options[:stall_after] = self.class.duration(value, option_name: "--stall-after")
1003
+ end
639
1004
  opts.on("-h", "--help", "Show help") { @options[:help] = true }
640
1005
  end
641
1006
  end