harnex 0.12.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +592 -0
- data/GUIDE.md +222 -68
- data/README.md +78 -48
- data/TECHNICAL.md +235 -98
- data/agents/gpt.md +60 -0
- data/docs/agents.md +119 -0
- data/docs/dispatch-telemetry.md +184 -50
- data/docs/events.md +20 -7
- data/docs/lanes.md +202 -0
- data/docs/mcp.md +102 -0
- data/docs/pi-rpc.md +81 -7
- data/docs/upgrading-to-0.29.md +87 -0
- data/guides/01_dispatch.md +101 -34
- data/guides/04_monitoring.md +223 -25
- data/guides/05_naming.md +0 -1
- data/guides/06_claude.md +96 -0
- data/lib/harnex/adapters/base.rb +4 -8
- data/lib/harnex/adapters/pi.rb +92 -12
- data/lib/harnex/adapters.rb +1 -16
- data/lib/harnex/artifact_report.rb +206 -12
- data/lib/harnex/cli.rb +68 -50
- data/lib/harnex/commands/ask.rb +278 -0
- data/lib/harnex/commands/delegate.rb +202 -0
- data/lib/harnex/commands/doctor.rb +34 -36
- data/lib/harnex/commands/events.rb +3 -3
- data/lib/harnex/commands/history.rb +2 -8
- data/lib/harnex/commands/land.rb +150 -0
- data/lib/harnex/commands/lanes.rb +84 -0
- data/lib/harnex/commands/logs.rb +3 -3
- data/lib/harnex/commands/mcp.rb +208 -0
- data/lib/harnex/commands/orchestration.rb +88 -9
- data/lib/harnex/commands/pane.rb +3 -3
- data/lib/harnex/commands/preflight.rb +566 -0
- data/lib/harnex/commands/reply.rb +125 -0
- data/lib/harnex/commands/result.rb +108 -0
- data/lib/harnex/commands/run.rb +248 -64
- data/lib/harnex/commands/send.rb +121 -16
- data/lib/harnex/commands/status.rb +32 -4
- data/lib/harnex/commands/stop.rb +14 -4
- data/lib/harnex/commands/telemetry.rb +19 -1
- data/lib/harnex/commands/wait.rb +421 -56
- data/lib/harnex/commands/watch.rb +134 -21
- data/lib/harnex/context_file.rb +85 -0
- data/lib/harnex/core.rb +100 -15
- data/lib/harnex/dispatch_history.rb +157 -10
- data/lib/harnex/lanes.rb +513 -0
- data/lib/harnex/mcp.rb +362 -0
- data/lib/harnex/orchestration.rb +128 -5
- data/lib/harnex/owned_process_group.rb +50 -0
- data/lib/harnex/owner_stream.rb +399 -0
- data/lib/harnex/pricing.rb +4 -9
- data/lib/harnex/runtime/activity_tracker.rb +107 -0
- data/lib/harnex/runtime/api_server.rb +11 -1
- data/lib/harnex/runtime/completion_notifier.rb +90 -11
- data/lib/harnex/runtime/inbox.rb +26 -7
- data/lib/harnex/runtime/message.rb +5 -2
- data/lib/harnex/runtime/runtime_budget.rb +96 -0
- data/lib/harnex/runtime/session.rb +684 -294
- data/lib/harnex/runtime/stop_request.rb +39 -0
- data/lib/harnex/stall.rb +46 -0
- data/lib/harnex/telemetry_reconciler.rb +42 -20
- data/lib/harnex/terminal_status.rb +5 -5
- data/lib/harnex/version.rb +2 -2
- data/lib/harnex.rb +16 -0
- data/recipes/01_fire_and_watch.md +14 -14
- data/recipes/02_chain_implement.md +35 -31
- data/recipes/03_buddy.md +1 -1
- metadata +34 -12
- data/docs/codex-appserver.md +0 -175
- data/lib/harnex/adapters/codex.rb +0 -160
- data/lib/harnex/adapters/codex_appserver.rb +0 -407
- data/lib/harnex/codex/app_server/client.rb +0 -347
data/lib/harnex/commands/wait.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require "json"
|
|
2
2
|
require "net/http"
|
|
3
3
|
require "optparse"
|
|
4
|
+
require "stringio"
|
|
4
5
|
require "uri"
|
|
5
6
|
|
|
6
7
|
module Harnex
|
|
@@ -19,7 +20,10 @@ module Harnex
|
|
|
19
20
|
DONE_EXIT_FAILED = 1
|
|
20
21
|
DONE_EXIT_REJECTED_PROOF = 2
|
|
21
22
|
DONE_EXIT_NO_SESSION = 3
|
|
23
|
+
DONE_EXIT_STALLED = 4
|
|
22
24
|
DONE_EXIT_TIMEOUT = 124
|
|
25
|
+
# Stall checks poll the live status API; detection lags by at most this.
|
|
26
|
+
STALL_POLL_MAX_INTERVAL = 15.0
|
|
23
27
|
|
|
24
28
|
# Outcome classes where the process completed but the work was not
|
|
25
29
|
# accepted: proof was missing, invalid, rejected, or nothing happened.
|
|
@@ -45,14 +49,30 @@ module Harnex
|
|
|
45
49
|
"prompt", "busy")
|
|
46
50
|
Without --until, waits for session exit (default).
|
|
47
51
|
--repo PATH Resolve session using PATH's repo root (default: current repo)
|
|
48
|
-
--timeout
|
|
52
|
+
--timeout DUR Hard observer cap (seconds, or suffix s/m/h; default: unlimited)
|
|
53
|
+
--max-wait DUR Alias for --timeout
|
|
54
|
+
--heartbeat DUR Flush observed progress to stderr at this interval (default: off)
|
|
55
|
+
--after-seq N With --until done/task_complete/task_failed: ignore every
|
|
56
|
+
event at or before events-log seq N (the after_seq that
|
|
57
|
+
`harnex send` reports), so the wait covers the turn that
|
|
58
|
+
message started or joined, not an earlier result.
|
|
59
|
+
--since TIME With --until done: ignore events before TIME (ISO 8601),
|
|
60
|
+
e.g. an earlier launch of a lane relaunched under the
|
|
61
|
+
same id (each launch restarts the events-log seq)
|
|
62
|
+
--stall-after DUR
|
|
63
|
+
With --until done: exit 4 when the live session shows no
|
|
64
|
+
progress for DUR (Pi/structured: model, tool or turn
|
|
65
|
+
events during an active turn; PTY: transcript idle).
|
|
66
|
+
Never nudges or stops the worker.
|
|
49
67
|
-h, --help Show this help
|
|
50
68
|
|
|
51
69
|
Common patterns:
|
|
52
|
-
#{program_name} --id
|
|
53
|
-
#{program_name} --id
|
|
54
|
-
#{program_name} --id
|
|
55
|
-
#{program_name} --id
|
|
70
|
+
#{program_name} --id pi-i-42 --until done --timeout 900
|
|
71
|
+
#{program_name} --id pi-i-42 --until done --stall-after 25m --timeout 4h
|
|
72
|
+
#{program_name} --id pi-i-42 --until done --after-seq 812 --timeout 9m
|
|
73
|
+
#{program_name} --id pi-i-42 --until task_complete --timeout 900
|
|
74
|
+
#{program_name} --id pi-i-42 --until prompt --timeout 120
|
|
75
|
+
#{program_name} --id pi-i-42
|
|
56
76
|
|
|
57
77
|
Exit codes for --until done:
|
|
58
78
|
0 completed with accepted work
|
|
@@ -60,10 +80,13 @@ module Harnex
|
|
|
60
80
|
2 completed but proof rejected (completed_no_activity,
|
|
61
81
|
report_missing, report_invalid, report_rejected)
|
|
62
82
|
3 no such session (no live, start, event, or terminal signal)
|
|
63
|
-
|
|
83
|
+
4 stalled: no progress for --stall-after (worker left running)
|
|
84
|
+
124 observer deadline elapsed (does not stop the worker)
|
|
64
85
|
|
|
65
86
|
Gotchas:
|
|
66
87
|
done is the safest work-level fence for monitors.
|
|
88
|
+
A bare wait returns only on process exit; sessions without
|
|
89
|
+
--auto-stop stay alive after their turn (stderr notes say so).
|
|
67
90
|
While the session's pid is alive, --until done blocks (up to
|
|
68
91
|
--timeout); "no signal yet" from a live worker is never terminal.
|
|
69
92
|
task_complete/task_failed are event predicates; prompt/busy are live state polls.
|
|
@@ -74,26 +97,72 @@ module Harnex
|
|
|
74
97
|
TEXT
|
|
75
98
|
end
|
|
76
99
|
|
|
77
|
-
|
|
100
|
+
# Reuse the shared duration grammar while preserving wait's numeric
|
|
101
|
+
# timeout syntax and guarding numeric fallback/forwarded Float values.
|
|
102
|
+
def self.duration(value, option_name:)
|
|
103
|
+
seconds = begin
|
|
104
|
+
Harnex.parse_duration_seconds(value, option_name: option_name)
|
|
105
|
+
rescue OptionParser::InvalidArgument => invalid_duration
|
|
106
|
+
# Preserve wait --timeout's former Float syntax (e.g. .5 or 1e-3),
|
|
107
|
+
# also used by Float#to_s when watch forwards very small intervals.
|
|
108
|
+
begin
|
|
109
|
+
Float(value)
|
|
110
|
+
rescue ArgumentError, TypeError
|
|
111
|
+
raise invalid_duration
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
unless seconds.finite? && seconds.positive?
|
|
115
|
+
raise OptionParser::InvalidArgument, "#{option_name} must be a finite positive duration"
|
|
116
|
+
end
|
|
117
|
+
seconds
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def initialize(argv, out: nil, err: nil)
|
|
78
121
|
@argv = argv.dup
|
|
122
|
+
@out = out
|
|
123
|
+
@err = err
|
|
79
124
|
@options = {
|
|
80
125
|
id: nil,
|
|
81
126
|
until_state: nil,
|
|
82
127
|
repo_path: Dir.pwd,
|
|
83
128
|
timeout: nil,
|
|
129
|
+
heartbeat: nil,
|
|
130
|
+
stall_after: nil,
|
|
131
|
+
after_seq: nil,
|
|
132
|
+
since: nil,
|
|
84
133
|
help: false
|
|
85
134
|
}
|
|
86
135
|
end
|
|
87
136
|
|
|
88
137
|
def run
|
|
138
|
+
@out ||= $stdout
|
|
139
|
+
@err ||= $stderr
|
|
89
140
|
parser.parse!(@argv)
|
|
90
141
|
if @options[:help]
|
|
91
|
-
puts self.class.usage
|
|
142
|
+
@out.puts self.class.usage
|
|
92
143
|
return 0
|
|
93
144
|
end
|
|
94
145
|
|
|
95
146
|
raise "--id is required for harnex wait" unless @options[:id]
|
|
147
|
+
if @options[:stall_after] && @options[:until_state] != "done"
|
|
148
|
+
raise OptionParser::InvalidOption, "--stall-after requires --until done"
|
|
149
|
+
end
|
|
150
|
+
if @options[:after_seq] && !(["done"] + EVENT_PREDICATES).include?(@options[:until_state])
|
|
151
|
+
raise OptionParser::InvalidOption, "--after-seq requires --until done, task_complete, or task_failed"
|
|
152
|
+
end
|
|
96
153
|
|
|
154
|
+
@started_at = monotonic
|
|
155
|
+
@observation = { "state" => "unknown", "last_event" => "unknown", "seq" => nil }
|
|
156
|
+
if @options[:timeout] || @options[:heartbeat]
|
|
157
|
+
observe_in_child
|
|
158
|
+
else
|
|
159
|
+
wait
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
private
|
|
164
|
+
|
|
165
|
+
def wait
|
|
97
166
|
if @options[:until_state]
|
|
98
167
|
case @options[:until_state]
|
|
99
168
|
when "done"
|
|
@@ -108,14 +177,180 @@ module Harnex
|
|
|
108
177
|
end
|
|
109
178
|
end
|
|
110
179
|
|
|
111
|
-
|
|
180
|
+
# Only the read-side observer runs in this child, never the session or any
|
|
181
|
+
# stop/marker action. A process boundary lets the parent enforce its cap
|
|
182
|
+
# during blocking Ruby/HTTP probes, even probes with slow ensure clauses or
|
|
183
|
+
# broad rescue handlers. No asynchronous Ruby exception crosses a probe.
|
|
184
|
+
# The private process group also contains repo-resolution subprocesses.
|
|
185
|
+
def observe_in_child
|
|
186
|
+
reader, writer = IO.pipe
|
|
187
|
+
writer.sync = true
|
|
188
|
+
pid = fork do
|
|
189
|
+
exit_code = 1
|
|
190
|
+
begin
|
|
191
|
+
Process.setpgid(0, 0)
|
|
192
|
+
reader.close
|
|
193
|
+
@observation_writer = writer
|
|
194
|
+
@out = StringIO.new
|
|
195
|
+
@err = DiagnosticStream.new { |text| send_observation_message(["warning", text]) }
|
|
196
|
+
code = wait
|
|
197
|
+
send_observation_message(["result", @out.string, code])
|
|
198
|
+
exit_code = 0
|
|
199
|
+
rescue Exception => error # transport errors too; never run inherited at_exit hooks
|
|
200
|
+
begin
|
|
201
|
+
send_observation_message(["error", error.class.name, error.message, error.backtrace])
|
|
202
|
+
rescue IOError, SystemCallError
|
|
203
|
+
nil # the caller may already have closed the pipe
|
|
204
|
+
end
|
|
205
|
+
ensure
|
|
206
|
+
exit! exit_code
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
writer.close
|
|
210
|
+
deadline = @options[:timeout] && @started_at + @options[:timeout]
|
|
211
|
+
next_heartbeat = @options[:heartbeat] && @started_at + @options[:heartbeat]
|
|
212
|
+
buffer = +""
|
|
213
|
+
|
|
214
|
+
loop do
|
|
215
|
+
now = monotonic
|
|
216
|
+
return emit_timeout if deadline && now >= deadline
|
|
217
|
+
|
|
218
|
+
# Drain complete records without blocking on a partial pipe write.
|
|
219
|
+
if (newline = buffer.index("\n"))
|
|
220
|
+
message = JSON.parse(buffer.slice!(0..newline))
|
|
221
|
+
case message[0]
|
|
222
|
+
when "observation" then @observation = message[1]
|
|
223
|
+
when "warning" then @err.write(message[1])
|
|
224
|
+
when "result"
|
|
225
|
+
return emit_timeout if deadline && monotonic >= deadline
|
|
226
|
+
|
|
227
|
+
@out.write(message[1])
|
|
228
|
+
return message[2]
|
|
229
|
+
when "error"
|
|
230
|
+
raise RuntimeError, "harnex wait observer: #{message[1]}: #{message[2]}", message[3]
|
|
231
|
+
end
|
|
232
|
+
next
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
if next_heartbeat && now >= next_heartbeat
|
|
236
|
+
emit_heartbeat
|
|
237
|
+
# Do not burst missed heartbeats after a scheduler pause.
|
|
238
|
+
next_heartbeat = now + @options[:heartbeat]
|
|
239
|
+
end
|
|
240
|
+
wake_at = [deadline, next_heartbeat].compact.min
|
|
241
|
+
# Clamp even finite enormous durations to an OS-safe select timeout.
|
|
242
|
+
pause = [[wake_at - monotonic, 0].max, POLL_INTERVAL].min
|
|
243
|
+
IO.select([reader], nil, nil, pause)
|
|
244
|
+
chunk = reader.read_nonblock(16_384, exception: false)
|
|
245
|
+
raise "harnex wait: observer exited without a result" if chunk.nil?
|
|
246
|
+
|
|
247
|
+
buffer << chunk unless chunk == :wait_readable
|
|
248
|
+
end
|
|
249
|
+
ensure
|
|
250
|
+
reader&.close unless reader&.closed?
|
|
251
|
+
writer&.close unless writer&.closed?
|
|
252
|
+
reap_observer(pid) if pid
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# SIGKILL bypasses slow Ruby ensure handlers. Reap without an unbounded
|
|
256
|
+
# wait/join. Uninterruptible kernel I/O or a descheduled OS cannot be given
|
|
257
|
+
# a universal preemption guarantee; no helper thread is left waiting on it.
|
|
258
|
+
def reap_observer(pid)
|
|
259
|
+
begin
|
|
260
|
+
Process.kill("KILL", -pid)
|
|
261
|
+
rescue Errno::ESRCH
|
|
262
|
+
nil
|
|
263
|
+
end
|
|
264
|
+
begin
|
|
265
|
+
Process.kill("KILL", pid)
|
|
266
|
+
rescue Errno::ESRCH
|
|
267
|
+
nil
|
|
268
|
+
end
|
|
269
|
+
cleanup_deadline = monotonic + EVENT_POLL_INTERVAL
|
|
270
|
+
loop do
|
|
271
|
+
return if Process.waitpid(pid, Process::WNOHANG)
|
|
272
|
+
break if monotonic >= cleanup_deadline
|
|
273
|
+
|
|
274
|
+
sleep 0.005
|
|
275
|
+
end
|
|
276
|
+
@err.puts("harnex wait: OS has not reaped the cancelled observer yet")
|
|
277
|
+
rescue Errno::ECHILD
|
|
278
|
+
nil
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
class DiagnosticStream
|
|
282
|
+
def initialize(&write)
|
|
283
|
+
@write = write
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def write(text)
|
|
287
|
+
@write.call(text)
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def puts(text)
|
|
291
|
+
write("#{text}\n")
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def send_observation_message(message)
|
|
296
|
+
@observation_writer.write(JSON.generate(message) + "\n")
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def observe(**fields)
|
|
300
|
+
updated = @observation.merge(fields.transform_keys(&:to_s))
|
|
301
|
+
return if updated == @observation
|
|
302
|
+
|
|
303
|
+
@observation = updated
|
|
304
|
+
send_observation_message(["observation", updated]) if @observation_writer
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
def emit_heartbeat
|
|
308
|
+
# These are observations, not model/tool activity or a log-age proxy.
|
|
309
|
+
values = %w[state last_event seq].map do |key|
|
|
310
|
+
value = @observation[key] || "unknown"
|
|
311
|
+
"#{key}=#{value.to_s.gsub(/[[:space:][:cntrl:]]/, '_')[0, 128]}"
|
|
312
|
+
end
|
|
313
|
+
@err.puts("harnex wait: id=#{@options[:id]} waited=#{(monotonic - @started_at).round(1)}s #{values.join(' ')}")
|
|
314
|
+
@err.flush
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
def emit_timeout
|
|
318
|
+
payload = { ok: false, id: @options[:id], status: "timeout",
|
|
319
|
+
waited_seconds: (monotonic - @started_at).round(1) }
|
|
320
|
+
case @options[:until_state]
|
|
321
|
+
when "done"
|
|
322
|
+
payload.merge!(wait_result: "timeout", done: false,
|
|
323
|
+
work_state: @observation["live"] ? "running" : "unknown")
|
|
324
|
+
when *EVENT_PREDICATES
|
|
325
|
+
# Event waits retain their existing timeout payload shape.
|
|
326
|
+
when nil
|
|
327
|
+
payload[:pid] = @observation["pid"] if @observation["pid"]
|
|
328
|
+
else
|
|
329
|
+
payload[:state] = @observation["state"]
|
|
330
|
+
end
|
|
331
|
+
@out.puts JSON.generate(payload)
|
|
332
|
+
DONE_EXIT_TIMEOUT
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# All output goes through per-instance streams, including legacy wait
|
|
336
|
+
# diagnostics. Resolve defaults in run so callers can still capture IO.
|
|
337
|
+
def puts(text)
|
|
338
|
+
@out.puts(text)
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def warn(text)
|
|
342
|
+
@err.puts(text)
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def monotonic
|
|
346
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
347
|
+
end
|
|
112
348
|
|
|
113
349
|
def wait_until_event(predicate)
|
|
114
350
|
repo_root = Harnex.resolve_repo_root(@options[:repo_path])
|
|
115
351
|
events_path = Harnex.events_log_path(repo_root, @options[:id])
|
|
116
352
|
registry = Harnex.read_registry(repo_root, @options[:id])
|
|
117
|
-
start_time =
|
|
118
|
-
deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
|
|
353
|
+
start_time = @started_at
|
|
119
354
|
|
|
120
355
|
unless registry || File.exist?(events_path)
|
|
121
356
|
warn("harnex wait: no session found with id #{@options[:id].inspect}")
|
|
@@ -131,21 +366,17 @@ module Harnex
|
|
|
131
366
|
return status if status
|
|
132
367
|
|
|
133
368
|
target_pid = registry && registry["pid"]
|
|
369
|
+
observe(state: "running", pid: target_pid) if target_pid
|
|
134
370
|
|
|
135
371
|
loop do
|
|
136
372
|
status, offset, task_complete_seen = scan_events(events_path, offset, predicate, task_complete_seen, start_time)
|
|
137
373
|
return status if status
|
|
138
374
|
|
|
139
|
-
if deadline && Time.now >= deadline
|
|
140
|
-
waited = (Time.now - start_time).round(1)
|
|
141
|
-
puts JSON.generate(ok: false, id: @options[:id], status: "timeout", waited_seconds: waited)
|
|
142
|
-
return 124
|
|
143
|
-
end
|
|
144
|
-
|
|
145
375
|
if target_pid && !Harnex.alive_pid?(target_pid)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
376
|
+
observe(state: "exited")
|
|
377
|
+
final_event_deadline ||= monotonic + FINAL_EVENT_GRACE_SECONDS
|
|
378
|
+
if monotonic >= final_event_deadline
|
|
379
|
+
waited = (monotonic - start_time).round(1)
|
|
149
380
|
puts JSON.generate(ok: false, id: @options[:id], state: "exited", waited_seconds: waited)
|
|
150
381
|
return 1
|
|
151
382
|
end
|
|
@@ -168,7 +399,10 @@ module Harnex
|
|
|
168
399
|
event = parse_event(line)
|
|
169
400
|
next unless event
|
|
170
401
|
next if stale_event?(event, min_ts)
|
|
402
|
+
next if before_after_seq?(event)
|
|
171
403
|
|
|
404
|
+
observe(last_event: event_type(event), seq: event["seq"])
|
|
405
|
+
observe(state: event["state"]) if event_type(event) == "agent_state" && event["state"]
|
|
172
406
|
task_complete_seen = true if %w[task_complete task_failed].include?(event_type(event))
|
|
173
407
|
if matches?(event, predicate, task_complete_seen)
|
|
174
408
|
return [emit_event_match(event, start_time, predicate), f.pos, task_complete_seen]
|
|
@@ -180,6 +414,15 @@ module Harnex
|
|
|
180
414
|
[nil, offset, task_complete_seen]
|
|
181
415
|
end
|
|
182
416
|
|
|
417
|
+
# #81: events at or before --after-seq belong to an earlier turn. An event
|
|
418
|
+
# without a seq cannot be placed after it, so it never matches either.
|
|
419
|
+
def before_after_seq?(event)
|
|
420
|
+
return false unless @options[:after_seq]
|
|
421
|
+
|
|
422
|
+
seq = event["seq"]
|
|
423
|
+
!seq.is_a?(Integer) || seq <= @options[:after_seq]
|
|
424
|
+
end
|
|
425
|
+
|
|
183
426
|
def stale_event?(event, min_ts)
|
|
184
427
|
return false unless min_ts
|
|
185
428
|
|
|
@@ -235,7 +478,7 @@ module Harnex
|
|
|
235
478
|
end
|
|
236
479
|
|
|
237
480
|
def emit_event_match(event, start_time, predicate)
|
|
238
|
-
waited = (
|
|
481
|
+
waited = (monotonic - start_time).round(1)
|
|
239
482
|
payload = {
|
|
240
483
|
ok: true,
|
|
241
484
|
id: @options[:id],
|
|
@@ -243,30 +486,48 @@ module Harnex
|
|
|
243
486
|
seq: event["seq"],
|
|
244
487
|
waited_seconds: waited
|
|
245
488
|
}
|
|
489
|
+
payload[:turn] = event["turn"] if event["turn"]
|
|
490
|
+
payload[:follow_ups] = event["follow_ups"] if event["follow_ups"]
|
|
491
|
+
%w[end_sha commits turn_commits model final_message final_message_path final_message_truncated].each do |key|
|
|
492
|
+
payload[key.to_sym] = event[key] if event.key?(key)
|
|
493
|
+
end
|
|
246
494
|
exit_code = 0
|
|
247
495
|
if predicate == "done"
|
|
248
496
|
failed = done_event_failed?(event)
|
|
249
|
-
|
|
497
|
+
outcome_class = event["outcome_class"] || stop_outcome_class(event["stop"])
|
|
498
|
+
exit_code = failed ? done_failure_exit_code(outcome_class) : DONE_EXIT_DONE
|
|
499
|
+
# A session seen gone by now has exited; one never observed stays
|
|
500
|
+
# "running", the historical event-match default.
|
|
501
|
+
exited = @observation["live"] == false
|
|
250
502
|
payload.merge!(
|
|
251
503
|
ok: !failed,
|
|
252
504
|
status: failed ? "failed" : "done",
|
|
253
505
|
wait_result: done_wait_result(exit_code),
|
|
254
|
-
state: "running",
|
|
255
|
-
process_state: "running",
|
|
256
|
-
terminal:
|
|
506
|
+
state: exited ? "exited" : "running",
|
|
507
|
+
process_state: exited ? "exited" : "running",
|
|
508
|
+
terminal: exited,
|
|
257
509
|
task_complete: !failed,
|
|
258
510
|
task_failed: failed,
|
|
259
511
|
done: !failed,
|
|
260
512
|
work_state: failed ? "failed" : "completed",
|
|
261
|
-
outcome_class:
|
|
513
|
+
outcome_class: outcome_class,
|
|
262
514
|
artifact_report_status: event["artifact_report_status"]
|
|
263
515
|
)
|
|
264
516
|
payload[:last_error] = event["message"] || event["error"] if failed
|
|
517
|
+
payload[:stop] = event["stop"] if event["stop"].is_a?(Hash)
|
|
265
518
|
end
|
|
266
519
|
puts JSON.generate(payload)
|
|
267
520
|
exit_code
|
|
268
521
|
end
|
|
269
522
|
|
|
523
|
+
# A turn failed by a caller's stop or the runtime budget carries the
|
|
524
|
+
# stop provenance, not a class; name it as the dispatch row will.
|
|
525
|
+
def stop_outcome_class(stop)
|
|
526
|
+
return nil unless stop.is_a?(Hash)
|
|
527
|
+
|
|
528
|
+
{ "manual" => "stopped_by_caller", "runtime_budget" => "timeout" }[stop["reason"].to_s]
|
|
529
|
+
end
|
|
530
|
+
|
|
270
531
|
def done_failure_exit_code(outcome_class)
|
|
271
532
|
REJECTED_PROOF_CLASSES.include?(outcome_class.to_s) ? DONE_EXIT_REJECTED_PROOF : DONE_EXIT_FAILED
|
|
272
533
|
end
|
|
@@ -288,25 +549,32 @@ module Harnex
|
|
|
288
549
|
repo_root = Harnex.resolve_repo_root(@options[:repo_path])
|
|
289
550
|
events_path = Harnex.events_log_path(repo_root, @options[:id])
|
|
290
551
|
exit_path = Harnex.exit_status_path(repo_root, @options[:id])
|
|
291
|
-
start_time =
|
|
292
|
-
deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
|
|
552
|
+
start_time = @started_at
|
|
293
553
|
|
|
294
554
|
offset = 0
|
|
295
555
|
task_complete_seen = false
|
|
296
556
|
observed_live = nil
|
|
297
557
|
session_started_at = nil
|
|
558
|
+
next_stall_check = nil
|
|
298
559
|
|
|
299
560
|
loop do
|
|
300
561
|
live = live_session(repo_root)
|
|
562
|
+
observe(state: live ? "running" : "unknown", pid: live && live["pid"], live: !!live)
|
|
301
563
|
if live
|
|
302
564
|
observed_live = live
|
|
303
565
|
session_started_at ||= parse_wait_time(live["started_at"])
|
|
304
566
|
end
|
|
305
567
|
|
|
306
568
|
status, offset, task_complete_seen =
|
|
307
|
-
scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: session_started_at)
|
|
569
|
+
scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: done_min_ts(session_started_at))
|
|
308
570
|
return status if status
|
|
309
571
|
|
|
572
|
+
if @options[:stall_after] && live && (next_stall_check.nil? || monotonic >= next_stall_check)
|
|
573
|
+
next_stall_check = monotonic + stall_poll_interval
|
|
574
|
+
stall = Harnex::Stall.stalled(fetch_status_payload(live), @options[:stall_after])
|
|
575
|
+
return emit_stall(stall, start_time) if stall
|
|
576
|
+
end
|
|
577
|
+
|
|
310
578
|
unless live
|
|
311
579
|
if observed_live
|
|
312
580
|
return resolve_after_exit(repo_root, events_path, exit_path, offset,
|
|
@@ -328,18 +596,45 @@ module Harnex
|
|
|
328
596
|
end
|
|
329
597
|
end
|
|
330
598
|
|
|
331
|
-
if deadline && Time.now >= deadline
|
|
332
|
-
waited = (Time.now - start_time).round(1)
|
|
333
|
-
puts JSON.generate(ok: false, id: @options[:id], status: "timeout", wait_result: "timeout",
|
|
334
|
-
waited_seconds: waited, done: false,
|
|
335
|
-
work_state: live ? "running" : "unknown")
|
|
336
|
-
return DONE_EXIT_TIMEOUT
|
|
337
|
-
end
|
|
338
|
-
|
|
339
599
|
sleep EVENT_POLL_INTERVAL
|
|
340
600
|
end
|
|
341
601
|
end
|
|
342
602
|
|
|
603
|
+
def done_min_ts(session_started_at)
|
|
604
|
+
[session_started_at, @options[:since]].compact.max
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
def stall_poll_interval
|
|
608
|
+
[[@options[:stall_after] / 4.0, 0.5].max, STALL_POLL_MAX_INTERVAL].min
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
# Only a registry row has the live API; start-record liveness cannot be
|
|
612
|
+
# probed, so a stall stays unobserved there rather than guessed.
|
|
613
|
+
def fetch_status_payload(live)
|
|
614
|
+
return nil unless live["host"] && live["port"]
|
|
615
|
+
|
|
616
|
+
uri = URI("http://#{live['host']}:#{live['port']}/status")
|
|
617
|
+
request = Net::HTTP::Get.new(uri)
|
|
618
|
+
request["Authorization"] = "Bearer #{live['token']}" if live["token"]
|
|
619
|
+
response = Net::HTTP.start(uri.host, uri.port, open_timeout: 1, read_timeout: 2) { |http| http.request(request) }
|
|
620
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
621
|
+
|
|
622
|
+
JSON.parse(response.body)
|
|
623
|
+
rescue StandardError
|
|
624
|
+
nil
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
# The worker is left running: this is an observer verdict, like a timeout.
|
|
628
|
+
def emit_stall(stall, start_time)
|
|
629
|
+
waited = (monotonic - start_time).round(1)
|
|
630
|
+
warn("harnex wait: #{@options[:id]} shows no progress for #{stall['idle_s'].round}s " \
|
|
631
|
+
"(clock=#{stall['clock']}, threshold #{stall['threshold_s'].round}s); worker left running")
|
|
632
|
+
puts JSON.generate(ok: false, id: @options[:id], status: "stalled", wait_result: "stalled",
|
|
633
|
+
state: "running", process_state: "running", terminal: false, task_complete: false,
|
|
634
|
+
done: false, work_state: "running", stall: stall, waited_seconds: waited)
|
|
635
|
+
DONE_EXIT_STALLED
|
|
636
|
+
end
|
|
637
|
+
|
|
343
638
|
# Registry row first; when it is not visible from this context, an
|
|
344
639
|
# uncompleted dispatch-start row with an alive pid still proves liveness.
|
|
345
640
|
def live_session(repo_root)
|
|
@@ -357,7 +652,7 @@ module Harnex
|
|
|
357
652
|
await_exit_status(exit_path)
|
|
358
653
|
|
|
359
654
|
status, _offset, _seen =
|
|
360
|
-
scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: session_started_at)
|
|
655
|
+
scan_events(events_path, offset, "done", task_complete_seen, start_time, min_ts: done_min_ts(session_started_at))
|
|
361
656
|
return status if status
|
|
362
657
|
|
|
363
658
|
if File.exist?(exit_path) && fresh_exit_status?(exit_path, observed_live, session_started_at)
|
|
@@ -367,7 +662,7 @@ module Harnex
|
|
|
367
662
|
terminal = done_status(repo_root, min_started_at: session_started_at)
|
|
368
663
|
return emit_done_terminal_status(terminal) if terminal
|
|
369
664
|
|
|
370
|
-
waited = (
|
|
665
|
+
waited = (monotonic - start_time).round(1)
|
|
371
666
|
puts JSON.generate(ok: false, id: @options[:id], state: "exited", process_state: "exited",
|
|
372
667
|
terminal: true, task_complete: false, done: false, work_state: "unknown",
|
|
373
668
|
wait_result: "failed", waited_seconds: waited)
|
|
@@ -399,8 +694,7 @@ module Harnex
|
|
|
399
694
|
def wait_until_state
|
|
400
695
|
repo_root = Harnex.resolve_repo_root(@options[:repo_path])
|
|
401
696
|
target_state = @options[:until_state]
|
|
402
|
-
start_time =
|
|
403
|
-
deadline = @options[:timeout] ? start_time + @options[:timeout] : nil
|
|
697
|
+
start_time = @started_at
|
|
404
698
|
|
|
405
699
|
registry = Harnex.read_registry(repo_root, @options[:id])
|
|
406
700
|
unless registry
|
|
@@ -417,31 +711,25 @@ module Harnex
|
|
|
417
711
|
|
|
418
712
|
loop do
|
|
419
713
|
unless Harnex.alive_pid?(target_pid)
|
|
420
|
-
waited = (
|
|
714
|
+
waited = (monotonic - start_time).round(1)
|
|
421
715
|
puts JSON.generate(ok: false, id: @options[:id], state: "exited", waited_seconds: waited)
|
|
422
716
|
return 1
|
|
423
717
|
end
|
|
424
718
|
|
|
425
719
|
state = fetch_agent_state(host, port, token)
|
|
720
|
+
observe(state: state || "unknown", pid: target_pid)
|
|
426
721
|
if state == target_state
|
|
427
|
-
waited = (
|
|
722
|
+
waited = (monotonic - start_time).round(1)
|
|
428
723
|
puts JSON.generate(ok: true, id: @options[:id], state: state, waited_seconds: waited)
|
|
429
724
|
return 0
|
|
430
725
|
end
|
|
431
726
|
|
|
432
|
-
if deadline && Time.now >= deadline
|
|
433
|
-
waited = (Time.now - start_time).round(1)
|
|
434
|
-
puts JSON.generate(ok: false, id: @options[:id], state: state || "unknown", waited_seconds: waited, status: "timeout")
|
|
435
|
-
return 124
|
|
436
|
-
end
|
|
437
|
-
|
|
438
727
|
sleep POLL_INTERVAL
|
|
439
728
|
end
|
|
440
729
|
end
|
|
441
730
|
|
|
442
731
|
def wait_until_exit
|
|
443
732
|
repo_root = Harnex.resolve_repo_root(@options[:repo_path])
|
|
444
|
-
deadline = @options[:timeout] ? Time.now + @options[:timeout] : nil
|
|
445
733
|
exit_path = Harnex.exit_status_path(repo_root, @options[:id])
|
|
446
734
|
|
|
447
735
|
registry = Harnex.read_registry(repo_root, @options[:id])
|
|
@@ -458,7 +746,9 @@ module Harnex
|
|
|
458
746
|
end
|
|
459
747
|
|
|
460
748
|
target_pid = registry["pid"]
|
|
749
|
+
observe(state: "running", pid: target_pid)
|
|
461
750
|
warn("harnex wait: watching session #{@options[:id]} (pid #{target_pid})")
|
|
751
|
+
hint = exit_wait_hint(repo_root, registry)
|
|
462
752
|
|
|
463
753
|
loop do
|
|
464
754
|
unless Harnex.alive_pid?(target_pid)
|
|
@@ -473,22 +763,75 @@ module Harnex
|
|
|
473
763
|
return 1
|
|
474
764
|
end
|
|
475
765
|
|
|
476
|
-
if
|
|
477
|
-
|
|
478
|
-
|
|
766
|
+
if hint && (settled = settled_event_type(hint))
|
|
767
|
+
warn("harnex wait: note: #{@options[:id]} reached #{settled} but is still alive; " \
|
|
768
|
+
"this wait still returns only on process exit. Use --until done for a work-level fence.")
|
|
769
|
+
hint = nil
|
|
479
770
|
end
|
|
480
771
|
|
|
481
772
|
sleep POLL_INTERVAL
|
|
482
773
|
end
|
|
483
774
|
end
|
|
484
775
|
|
|
776
|
+
# A bare wait fences process exit, but an interactive session stays alive
|
|
777
|
+
# at its prompt after a turn. Say so on entry, and once more if the turn
|
|
778
|
+
# settles mid-wait, so a waiter on the wrong predicate learns it before
|
|
779
|
+
# its timeout (#49). --auto-stop sessions exit on their own: no hint.
|
|
780
|
+
# Returns the event-scan cursor for the mid-wait check, or nil when done.
|
|
781
|
+
def exit_wait_hint(repo_root, registry)
|
|
782
|
+
return nil if registry["auto_stop"] == true
|
|
783
|
+
|
|
784
|
+
cursor = {
|
|
785
|
+
path: Harnex.events_log_path(repo_root, @options[:id]),
|
|
786
|
+
offset: 0,
|
|
787
|
+
min_ts: parse_wait_time(registry["started_at"])
|
|
788
|
+
}
|
|
789
|
+
if (settled = settled_event_type(cursor))
|
|
790
|
+
warn("harnex wait: note: #{@options[:id]} already reached #{settled} and stays alive at its prompt; " \
|
|
791
|
+
"this wait returns only on process exit. Use --until done for a work-level fence.")
|
|
792
|
+
return nil
|
|
793
|
+
end
|
|
794
|
+
|
|
795
|
+
reason =
|
|
796
|
+
if registry["auto_stop"] == false
|
|
797
|
+
"was not started with --auto-stop, so it stays alive after its turn"
|
|
798
|
+
else
|
|
799
|
+
"may stay alive at its prompt after its turn unless started with --auto-stop"
|
|
800
|
+
end
|
|
801
|
+
warn("harnex wait: note: #{@options[:id]} #{reason}; this wait returns only on process exit. " \
|
|
802
|
+
"Use --until done for a work-level fence.")
|
|
803
|
+
cursor
|
|
804
|
+
end
|
|
805
|
+
|
|
806
|
+
def settled_event_type(cursor)
|
|
807
|
+
path = cursor[:path]
|
|
808
|
+
return nil unless File.exist?(path) && File.size(path) > cursor[:offset]
|
|
809
|
+
|
|
810
|
+
File.open(path, "r") do |f|
|
|
811
|
+
f.seek(cursor[:offset])
|
|
812
|
+
f.each_line do |line|
|
|
813
|
+
# Only whole lines advance the cursor; a partial tail is re-read.
|
|
814
|
+
break unless line.end_with?("\n")
|
|
815
|
+
|
|
816
|
+
cursor[:offset] = f.pos
|
|
817
|
+
event = parse_event(line)
|
|
818
|
+
next unless event
|
|
819
|
+
next if stale_event?(event, cursor[:min_ts])
|
|
820
|
+
|
|
821
|
+
type = event_type(event)
|
|
822
|
+
return type if %w[task_complete task_failed].include?(type)
|
|
823
|
+
end
|
|
824
|
+
end
|
|
825
|
+
nil
|
|
826
|
+
end
|
|
827
|
+
|
|
485
828
|
# Subprocess death races the parent's DISPATCH-row write; the exit-status
|
|
486
829
|
# file is written *after* the row, so polling it bounds the race.
|
|
487
830
|
def await_exit_status(exit_path)
|
|
488
831
|
return if File.exist?(exit_path)
|
|
489
832
|
|
|
490
|
-
grace_deadline =
|
|
491
|
-
until File.exist?(exit_path) ||
|
|
833
|
+
grace_deadline = monotonic + exit_status_grace_seconds
|
|
834
|
+
until File.exist?(exit_path) || monotonic >= grace_deadline
|
|
492
835
|
sleep EXIT_STATUS_GRACE_POLL_INTERVAL
|
|
493
836
|
end
|
|
494
837
|
end
|
|
@@ -622,6 +965,8 @@ module Harnex
|
|
|
622
965
|
work_state: work_state,
|
|
623
966
|
outcome_class: status["outcome_class"],
|
|
624
967
|
artifact_report_status: status["artifact_report_status"],
|
|
968
|
+
stop: status["stop"],
|
|
969
|
+
runtime_budget: status["runtime_budget"],
|
|
625
970
|
exit: status["exit"],
|
|
626
971
|
exit_code: status["exit_code"],
|
|
627
972
|
ended_at: status["ended_at"],
|
|
@@ -635,7 +980,27 @@ module Harnex
|
|
|
635
980
|
opts.on("--id ID", "Session ID to wait for") { |value| @options[:id] = Harnex.normalize_id(value) }
|
|
636
981
|
opts.on("--until STATE", "Wait until session reaches STATE") { |value| @options[:until_state] = value }
|
|
637
982
|
opts.on("--repo PATH", "Resolve session using PATH's repo root") { |value| @options[:repo_path] = value }
|
|
638
|
-
opts.on("--timeout
|
|
983
|
+
opts.on("--timeout DUR", "Hard observer cap") do |value|
|
|
984
|
+
@options[:timeout] = self.class.duration(value, option_name: "--timeout")
|
|
985
|
+
end
|
|
986
|
+
opts.on("--max-wait DUR", "Alias for --timeout") do |value|
|
|
987
|
+
@options[:timeout] = self.class.duration(value, option_name: "--max-wait")
|
|
988
|
+
end
|
|
989
|
+
opts.on("--heartbeat DUR", "Flush progress to stderr") do |value|
|
|
990
|
+
@options[:heartbeat] = self.class.duration(value, option_name: "--heartbeat")
|
|
991
|
+
end
|
|
992
|
+
opts.on("--after-seq N", Integer, "Ignore events at or before events-log seq N") do |value|
|
|
993
|
+
raise OptionParser::InvalidArgument, "--after-seq must be 0 or greater" if value.negative?
|
|
994
|
+
|
|
995
|
+
@options[:after_seq] = value
|
|
996
|
+
end
|
|
997
|
+
opts.on("--since TIME", "With --until done: ignore events before TIME") do |value|
|
|
998
|
+
@options[:since] = parse_wait_time(value) ||
|
|
999
|
+
raise(OptionParser::InvalidArgument, "--since must be an ISO 8601 time")
|
|
1000
|
+
end
|
|
1001
|
+
opts.on("--stall-after DUR", "With --until done: exit 4 after DUR without progress") do |value|
|
|
1002
|
+
@options[:stall_after] = self.class.duration(value, option_name: "--stall-after")
|
|
1003
|
+
end
|
|
639
1004
|
opts.on("-h", "--help", "Show help") { @options[:help] = true }
|
|
640
1005
|
end
|
|
641
1006
|
end
|