harnex 0.12.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +592 -0
- data/GUIDE.md +222 -68
- data/README.md +78 -48
- data/TECHNICAL.md +235 -98
- data/agents/gpt.md +60 -0
- data/docs/agents.md +119 -0
- data/docs/dispatch-telemetry.md +184 -50
- data/docs/events.md +20 -7
- data/docs/lanes.md +202 -0
- data/docs/mcp.md +102 -0
- data/docs/pi-rpc.md +81 -7
- data/docs/upgrading-to-0.29.md +87 -0
- data/guides/01_dispatch.md +101 -34
- data/guides/04_monitoring.md +223 -25
- data/guides/05_naming.md +0 -1
- data/guides/06_claude.md +96 -0
- data/lib/harnex/adapters/base.rb +4 -8
- data/lib/harnex/adapters/pi.rb +92 -12
- data/lib/harnex/adapters.rb +1 -16
- data/lib/harnex/artifact_report.rb +206 -12
- data/lib/harnex/cli.rb +68 -50
- data/lib/harnex/commands/ask.rb +278 -0
- data/lib/harnex/commands/delegate.rb +202 -0
- data/lib/harnex/commands/doctor.rb +34 -36
- data/lib/harnex/commands/events.rb +3 -3
- data/lib/harnex/commands/history.rb +2 -8
- data/lib/harnex/commands/land.rb +150 -0
- data/lib/harnex/commands/lanes.rb +84 -0
- data/lib/harnex/commands/logs.rb +3 -3
- data/lib/harnex/commands/mcp.rb +208 -0
- data/lib/harnex/commands/orchestration.rb +88 -9
- data/lib/harnex/commands/pane.rb +3 -3
- data/lib/harnex/commands/preflight.rb +566 -0
- data/lib/harnex/commands/reply.rb +125 -0
- data/lib/harnex/commands/result.rb +108 -0
- data/lib/harnex/commands/run.rb +248 -64
- data/lib/harnex/commands/send.rb +121 -16
- data/lib/harnex/commands/status.rb +32 -4
- data/lib/harnex/commands/stop.rb +14 -4
- data/lib/harnex/commands/telemetry.rb +19 -1
- data/lib/harnex/commands/wait.rb +421 -56
- data/lib/harnex/commands/watch.rb +134 -21
- data/lib/harnex/context_file.rb +85 -0
- data/lib/harnex/core.rb +100 -15
- data/lib/harnex/dispatch_history.rb +157 -10
- data/lib/harnex/lanes.rb +513 -0
- data/lib/harnex/mcp.rb +362 -0
- data/lib/harnex/orchestration.rb +128 -5
- data/lib/harnex/owned_process_group.rb +50 -0
- data/lib/harnex/owner_stream.rb +399 -0
- data/lib/harnex/pricing.rb +4 -9
- data/lib/harnex/runtime/activity_tracker.rb +107 -0
- data/lib/harnex/runtime/api_server.rb +11 -1
- data/lib/harnex/runtime/completion_notifier.rb +90 -11
- data/lib/harnex/runtime/inbox.rb +26 -7
- data/lib/harnex/runtime/message.rb +5 -2
- data/lib/harnex/runtime/runtime_budget.rb +96 -0
- data/lib/harnex/runtime/session.rb +684 -294
- data/lib/harnex/runtime/stop_request.rb +39 -0
- data/lib/harnex/stall.rb +46 -0
- data/lib/harnex/telemetry_reconciler.rb +42 -20
- data/lib/harnex/terminal_status.rb +5 -5
- data/lib/harnex/version.rb +2 -2
- data/lib/harnex.rb +16 -0
- data/recipes/01_fire_and_watch.md +14 -14
- data/recipes/02_chain_implement.md +35 -31
- data/recipes/03_buddy.md +1 -1
- metadata +34 -12
- data/docs/codex-appserver.md +0 -175
- data/lib/harnex/adapters/codex.rb +0 -160
- data/lib/harnex/adapters/codex_appserver.rb +0 -407
- data/lib/harnex/codex/app_server/client.rb +0 -347
|
@@ -39,10 +39,28 @@ module Harnex
|
|
|
39
39
|
RELIABILITY_FIELDS = %w[
|
|
40
40
|
adapter_close real_disconnections stream_interruptions stalls force_resumes compactions recovered
|
|
41
41
|
].freeze
|
|
42
|
-
SUCCESSFUL_TURN_STATUSES = %w[completed success succeeded].freeze
|
|
43
42
|
PROOF_REJECTION_CLASSES = %w[
|
|
44
43
|
completed_no_activity report_missing report_invalid report_rejected
|
|
45
44
|
].freeze
|
|
45
|
+
# Closed dispatch-row `outcome.class` vocabulary (#57). Proof classes come
|
|
46
|
+
# from completion assessment; process classes refine an otherwise unknown
|
|
47
|
+
# or failed terminal from signals Harnex already holds (exit path, stop
|
|
48
|
+
# provenance, final input state), never from transcript text.
|
|
49
|
+
PROCESS_OUTCOME_CLASSES = %w[
|
|
50
|
+
boot_failure permission_prompt timeout stopped_by_caller process_failed
|
|
51
|
+
].freeze
|
|
52
|
+
# provider_capacity (#82): the turn failed after the provider adapter's
|
|
53
|
+
# own retries ran out (Pi `auto_retry_end success:false`): capacity or
|
|
54
|
+
# rate limits, not the work. Structured signal only, never text.
|
|
55
|
+
OUTCOME_CLASSES = (
|
|
56
|
+
%w[completed_with_proof completed_with_activity] + PROOF_REJECTION_CLASSES +
|
|
57
|
+
%w[task_failed provider_capacity] + PROCESS_OUTCOME_CLASSES + %w[unknown]
|
|
58
|
+
).freeze
|
|
59
|
+
# Pi tools whose structured results carry an integer exit code.
|
|
60
|
+
PI_COMMAND_TOOLS = %w[bash].freeze
|
|
61
|
+
MAX_PENDING_PI_COMMANDS = 256
|
|
62
|
+
# Inline cap for a turn's final reply; the full text is in its file.
|
|
63
|
+
FINAL_MESSAGE_LIMIT = 4000
|
|
46
64
|
class EventCounters
|
|
47
65
|
def initialize
|
|
48
66
|
@counts = {
|
|
@@ -95,8 +113,15 @@ module Harnex
|
|
|
95
113
|
:output_log_path, :events_log_path, :started_at, :ended_at, :exit_code, :term_signal,
|
|
96
114
|
:require_artifact_report
|
|
97
115
|
|
|
98
|
-
def initialize(adapter:, command:, repo_root:, host:, port: nil, id: DEFAULT_ID, watch: nil, description: nil, meta: nil, artifact_report_path: nil, require_artifact_report: false, inbox_ttl: Inbox::DEFAULT_TTL, auto_stop: false, on_done: nil, launch_cwd: nil, child_cwd: nil, completion_notifier: nil)
|
|
116
|
+
def initialize(adapter:, command:, repo_root:, host:, port: nil, id: DEFAULT_ID, watch: nil, description: nil, meta: nil, artifact_report_path: nil, require_artifact_report: false, inbox_ttl: Inbox::DEFAULT_TTL, auto_stop: false, on_done: nil, launch_cwd: nil, child_cwd: nil, completion_notifier: nil, max_runtime_s: nil)
|
|
99
117
|
@adapter = adapter
|
|
118
|
+
@activity = ActivityTracker.new(transport: adapter.transport)
|
|
119
|
+
@runtime_budget = RuntimeBudget.new(seconds: max_runtime_s) unless max_runtime_s.nil?
|
|
120
|
+
@runtime_child_ready = false
|
|
121
|
+
@runtime_enforcement_started = false
|
|
122
|
+
@runtime_expiry_recorded = false
|
|
123
|
+
@runtime_enforced_at = nil
|
|
124
|
+
@stop_request = nil
|
|
100
125
|
@command = command
|
|
101
126
|
@repo_root = repo_root
|
|
102
127
|
@launch_cwd = File.expand_path(launch_cwd.to_s.empty? ? repo_root : launch_cwd)
|
|
@@ -133,6 +158,7 @@ module Harnex
|
|
|
133
158
|
@events_mutex = Mutex.new
|
|
134
159
|
@stop_mutex = Mutex.new
|
|
135
160
|
@auto_stop_mutex = Mutex.new
|
|
161
|
+
@runtime_mutex = Mutex.new
|
|
136
162
|
@injected_count = 0
|
|
137
163
|
@last_injected_at = nil
|
|
138
164
|
@started_at = Time.now
|
|
@@ -141,12 +167,17 @@ module Harnex
|
|
|
141
167
|
@completion_notification_registered = false
|
|
142
168
|
@completion_notification_sent = false
|
|
143
169
|
@pending_completion_notification = nil
|
|
170
|
+
@completion_turn = 1
|
|
171
|
+
@turn_follow_ups_sent = 0
|
|
172
|
+
@pi_follow_up_pending = 0
|
|
173
|
+
@turn_final_text = nil
|
|
144
174
|
@completion_notifier = completion_notifier || CompletionNotifier.new(
|
|
145
175
|
repo_root: repo_root,
|
|
146
176
|
id: @id,
|
|
147
177
|
session_id: @session_id,
|
|
148
178
|
receipt_path: @artifact_report_path,
|
|
149
179
|
hook_command: on_done,
|
|
180
|
+
owner_id: @meta.is_a?(Hash) ? @meta["orchestration_session_id"] : nil,
|
|
150
181
|
started_at: @started_at,
|
|
151
182
|
event_sink: ->(type, payload) { emit_event(type, **payload) }
|
|
152
183
|
)
|
|
@@ -158,6 +189,13 @@ module Harnex
|
|
|
158
189
|
@event_counters = EventCounters.new
|
|
159
190
|
@observed_commands_mutex = Mutex.new
|
|
160
191
|
@observed_commands = []
|
|
192
|
+
@pi_pending_commands = {}
|
|
193
|
+
@pi_unobserved_commands = 0
|
|
194
|
+
# Final gates and the failure flag cover the whole command history; the
|
|
195
|
+
# retained @observed_commands window is capped at MAX_COMMANDS (#68).
|
|
196
|
+
@final_gates = Harnex::ArtifactReport.normalize_final_gates(meta.is_a?(Hash) ? meta["final_gates"] : nil)
|
|
197
|
+
@final_gate_exits = {}
|
|
198
|
+
@commands_had_failures = false
|
|
161
199
|
@artifact_claims = {}
|
|
162
200
|
@artifact_claims_loaded = false
|
|
163
201
|
@artifact_report_generated = false
|
|
@@ -166,7 +204,6 @@ module Harnex
|
|
|
166
204
|
@git_end = {}
|
|
167
205
|
@usage_summary = {}
|
|
168
206
|
@context_summary = {}
|
|
169
|
-
@rpc_context_telemetry = nil
|
|
170
207
|
@ended_at = nil
|
|
171
208
|
@exit_reason = nil
|
|
172
209
|
@last_error = nil
|
|
@@ -183,6 +220,9 @@ module Harnex
|
|
|
183
220
|
@pi_last_assistant_stop_reason = nil
|
|
184
221
|
@pi_last_assistant_error = nil
|
|
185
222
|
@pi_last_retry_error = nil
|
|
223
|
+
@pi_awaiting_run_start = false
|
|
224
|
+
@pi_idle_stop = false
|
|
225
|
+
@pi_transport_failed = false
|
|
186
226
|
@auto_stop = !!auto_stop
|
|
187
227
|
@auto_stop_fired = false
|
|
188
228
|
@auto_stop_seen_busy = false
|
|
@@ -195,7 +235,6 @@ module Harnex
|
|
|
195
235
|
@output_buffer.force_encoding(Encoding::BINARY)
|
|
196
236
|
@state_machine = SessionState.new(adapter)
|
|
197
237
|
@inbox = Inbox.new(self, @state_machine, ttl: inbox_ttl)
|
|
198
|
-
@rate_limits = nil
|
|
199
238
|
@parent_harnex_id = ENV["HARNEX_ID"].to_s.strip
|
|
200
239
|
@parent_harnex_id = nil if @parent_harnex_id.empty?
|
|
201
240
|
end
|
|
@@ -224,10 +263,13 @@ module Harnex
|
|
|
224
263
|
prune_retained_logs
|
|
225
264
|
prepare_output_log
|
|
226
265
|
prepare_events_log
|
|
266
|
+
start_runtime_budget!
|
|
227
267
|
|
|
228
268
|
return run_structured if structured_transport?
|
|
229
269
|
|
|
230
270
|
run_pty
|
|
271
|
+
ensure
|
|
272
|
+
@runtime_budget&.cancel
|
|
231
273
|
end
|
|
232
274
|
|
|
233
275
|
def run_pty
|
|
@@ -235,6 +277,8 @@ module Harnex
|
|
|
235
277
|
spawn_args << { chdir: child_cwd } if child_cwd
|
|
236
278
|
@reader, @writer, @pid = PTY.spawn(*spawn_args)
|
|
237
279
|
@writer.sync = true
|
|
280
|
+
capture_pty_process_group!
|
|
281
|
+
runtime_child_started!
|
|
238
282
|
arm_auto_stop_after_initial_context
|
|
239
283
|
emit_started_event
|
|
240
284
|
emit_git_start_event
|
|
@@ -243,8 +287,8 @@ module Harnex
|
|
|
243
287
|
sync_window_size
|
|
244
288
|
@server = ApiServer.new(self)
|
|
245
289
|
@server.start
|
|
246
|
-
persist_registry
|
|
247
290
|
register_completion_notifier!
|
|
291
|
+
persist_registry
|
|
248
292
|
append_dispatch_start_record
|
|
249
293
|
|
|
250
294
|
stdin_state = STDIN.tty? ? STDIN.raw! : nil
|
|
@@ -257,10 +301,14 @@ module Harnex
|
|
|
257
301
|
@term_signal = status.signaled? ? status.termsig : nil
|
|
258
302
|
@exit_code = status.exited? ? status.exitstatus : 128 + status.termsig
|
|
259
303
|
@ended_at = Time.now
|
|
304
|
+
@runtime_budget&.cancel
|
|
305
|
+
emit_event("process_exited", code: @exit_code, signal: @term_signal) if @runtime_budget
|
|
306
|
+
stop_for_runtime_budget! if runtime_budget_stop?
|
|
260
307
|
|
|
261
308
|
enforce_required_artifact_report!
|
|
262
309
|
normalize_work_acceptance_exit_code!
|
|
263
310
|
normalize_auto_stop_exit_code!
|
|
311
|
+
normalize_runtime_exit_code!
|
|
264
312
|
drain_auto_stop_threads
|
|
265
313
|
output_thread.join(1)
|
|
266
314
|
finalize_session!
|
|
@@ -298,9 +346,15 @@ module Harnex
|
|
|
298
346
|
output_log_path: output_log_path,
|
|
299
347
|
events_log_path: events_log_path,
|
|
300
348
|
artifact_report_path: artifact_report_path,
|
|
301
|
-
artifact_claims_path: artifact_claims_path
|
|
349
|
+
artifact_claims_path: artifact_claims_path,
|
|
350
|
+
auto_stop: @auto_stop
|
|
302
351
|
}
|
|
303
352
|
payload.merge!(log_activity_snapshot)
|
|
353
|
+
payload[:activity] = @activity.snapshot
|
|
354
|
+
running = pi_running_commands
|
|
355
|
+
payload[:running_commands] = running unless running.empty?
|
|
356
|
+
payload[:stop] = @stop_request&.to_h
|
|
357
|
+
payload[:runtime_budget] = runtime_budget_snapshot
|
|
304
358
|
payload[:description] = description if description
|
|
305
359
|
|
|
306
360
|
if watch
|
|
@@ -374,14 +428,24 @@ module Harnex
|
|
|
374
428
|
inject_sequence([{ text: text, newline: newline }])
|
|
375
429
|
end
|
|
376
430
|
|
|
377
|
-
def inject_stop(turn_id: nil, interrupt: true)
|
|
431
|
+
def inject_stop(turn_id: nil, interrupt: true, reason: "manual", origin: "api")
|
|
432
|
+
StopRequest.validate!(reason: reason, origin: origin)
|
|
378
433
|
unless structured_transport?
|
|
379
434
|
raise "session is not running" unless pid && Harnex.alive_pid?(pid)
|
|
380
435
|
end
|
|
381
436
|
|
|
382
|
-
return { ok: true, signal: "already_requested" } if stop_requested!
|
|
437
|
+
return { ok: true, signal: "already_requested" } if stop_requested!(reason: reason, origin: origin)
|
|
438
|
+
|
|
439
|
+
if runtime_budget_stop?
|
|
440
|
+
@state_machine.force_busy!
|
|
441
|
+
signal_rpc_done! unless @pid
|
|
442
|
+
return { ok: true, signal: "terminate_sent" }
|
|
443
|
+
end
|
|
383
444
|
|
|
384
445
|
if structured_transport?
|
|
446
|
+
# There is nothing to abort after Pi's accepted settlement. In
|
|
447
|
+
# particular, do not race an abort RPC/write against the TERM below.
|
|
448
|
+
interrupt = false if adapter.transport == :stdio_jsonl_rpc && @pi_idle_stop
|
|
385
449
|
if adapter.respond_to?(:terminate_subprocess)
|
|
386
450
|
Thread.new do
|
|
387
451
|
begin
|
|
@@ -416,11 +480,13 @@ module Harnex
|
|
|
416
480
|
{ ok: true, signal: "exit_sequence_sent" }
|
|
417
481
|
end
|
|
418
482
|
|
|
419
|
-
def inject_via_adapter(text:, submit:, enter_only:, force: false)
|
|
483
|
+
def inject_via_adapter(text:, submit:, enter_only:, force: false, follow_up: false)
|
|
420
484
|
if structured_transport?
|
|
421
|
-
return inject_via_structured(text: text, submit: submit, enter_only: enter_only, force: force
|
|
485
|
+
return inject_via_structured(text: text, submit: submit, enter_only: enter_only, force: force,
|
|
486
|
+
follow_up: follow_up)
|
|
422
487
|
end
|
|
423
488
|
|
|
489
|
+
after_seq = current_event_seq
|
|
424
490
|
snapshot = adapter.wait_for_sendable(method(:screen_snapshot), submit: submit, enter_only: enter_only, force: force)
|
|
425
491
|
payload = adapter.build_send_payload(
|
|
426
492
|
text: text,
|
|
@@ -440,38 +506,85 @@ module Harnex
|
|
|
440
506
|
result.merge(
|
|
441
507
|
cli: adapter.key,
|
|
442
508
|
input_state: payload[:input_state],
|
|
443
|
-
force: payload[:force]
|
|
509
|
+
force: payload[:force],
|
|
510
|
+
after_seq: after_seq
|
|
444
511
|
)
|
|
445
512
|
.tap { emit_send_event(text, force: payload[:force]) }
|
|
446
513
|
end
|
|
447
514
|
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
515
|
+
# #79: a structured adapter that can queue a message into the running turn
|
|
516
|
+
# (Pi's followUp) takes it at once instead of waiting in the inbox TTL.
|
|
517
|
+
def accepts_follow_up?
|
|
518
|
+
structured_transport? && adapter.respond_to?(:follow_up_supported?) && adapter.follow_up_supported? &&
|
|
519
|
+
@state_machine.state == :busy && !@stop_requested && !@session_finalized
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
# The newest events-log seq. Terminal events of a turn injected after this
|
|
523
|
+
# point always carry a larger seq (`wait --after-seq`, #81).
|
|
524
|
+
def current_event_seq
|
|
525
|
+
@events_mutex.synchronize { @events_log_seq }
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
def inject_via_structured(text:, submit:, enter_only:, force: false, follow_up: false)
|
|
529
|
+
if @runtime_budget&.expired?
|
|
530
|
+
stop_for_runtime_budget!
|
|
531
|
+
raise ArgumentError, "session runtime budget is exhausted"
|
|
532
|
+
end
|
|
533
|
+
payload_options = { text: text, submit: submit, enter_only: enter_only, screen_text: nil, force: force }
|
|
534
|
+
payload_options[:follow_up] = true if follow_up
|
|
535
|
+
payload = adapter.build_send_payload(**payload_options)
|
|
456
536
|
dispatch = payload.fetch(:dispatch).dup
|
|
457
537
|
dispatch[:model] = meta_hash["model"] if meta_hash["model"] && !dispatch.key?(:model)
|
|
458
538
|
dispatch[:effort] = meta_hash["effort"] if meta_hash["effort"] && !dispatch.key?(:effort)
|
|
539
|
+
queued_into_run = dispatch[:streaming_behavior] == "followUp"
|
|
540
|
+
dispatch_options = dispatch
|
|
459
541
|
|
|
460
542
|
turn_id = nil
|
|
543
|
+
after_seq = nil
|
|
461
544
|
@inject_mutex.synchronize do
|
|
545
|
+
@completion_transition_lock.synchronize do
|
|
546
|
+
raise "session is stopping or disconnected" if @stop_requested || @session_finalized || @pi_transport_failed
|
|
547
|
+
|
|
548
|
+
after_seq = current_event_seq
|
|
549
|
+
# The inbox checked "busy" without this lock; a run that settled
|
|
550
|
+
# since then is done, so this is a new prompt, not a follow-up.
|
|
551
|
+
if queued_into_run && @state_machine.state != :busy
|
|
552
|
+
queued_into_run = false
|
|
553
|
+
dispatch_options = dispatch.reject { |key, _| key == :streaming_behavior }
|
|
554
|
+
end
|
|
555
|
+
# Count the follow-up for the turn it joins before releasing the
|
|
556
|
+
# lock, so a settlement during the RPC cannot misattribute it.
|
|
557
|
+
@turn_follow_ups_sent += 1 if queued_into_run
|
|
558
|
+
# A follow-up joins the running turn: Pi delivers it before
|
|
559
|
+
# agent_settled, so that settlement already covers it and nothing
|
|
560
|
+
# is reset here. If Pi has in fact gone idle it starts a new run,
|
|
561
|
+
# whose agent_start begins the next turn.
|
|
562
|
+
unless queued_into_run
|
|
563
|
+
# Reserve work before the blocking RPC. Steering an active run does
|
|
564
|
+
# not reset its clocks, but a new prompt cannot reuse old proof.
|
|
565
|
+
@activity.start_turn
|
|
566
|
+
begin_pi_run!
|
|
567
|
+
@pi_awaiting_run_start = true
|
|
568
|
+
end
|
|
569
|
+
end
|
|
462
570
|
begin
|
|
463
|
-
turn_id = adapter.dispatch(**
|
|
571
|
+
turn_id = adapter.dispatch(**dispatch_options)
|
|
464
572
|
rescue StandardError => e
|
|
465
|
-
|
|
573
|
+
@completion_transition_lock.synchronize do
|
|
574
|
+
mark_task_failed(status: "dispatch_error", error: e.message) unless @session_finalized || @stop_requested
|
|
575
|
+
end
|
|
466
576
|
raise
|
|
467
577
|
end
|
|
468
|
-
@
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
578
|
+
@completion_transition_lock.synchronize do
|
|
579
|
+
unless @session_finalized
|
|
580
|
+
@injected_count += 1
|
|
581
|
+
@last_injected_at = Time.now
|
|
582
|
+
refresh_registry
|
|
583
|
+
end
|
|
584
|
+
end
|
|
472
585
|
end
|
|
473
586
|
|
|
474
|
-
emit_send_event(
|
|
587
|
+
emit_send_event(dispatch_options.fetch(:prompt, text), force: payload[:force])
|
|
475
588
|
{
|
|
476
589
|
ok: true,
|
|
477
590
|
cli: adapter.key,
|
|
@@ -480,7 +593,9 @@ module Harnex
|
|
|
480
593
|
newline: false,
|
|
481
594
|
input_state: payload[:input_state],
|
|
482
595
|
force: payload[:force],
|
|
483
|
-
turn_id: turn_id
|
|
596
|
+
turn_id: turn_id,
|
|
597
|
+
after_seq: after_seq,
|
|
598
|
+
delivery: queued_into_run ? "follow_up" : "prompt"
|
|
484
599
|
}
|
|
485
600
|
end
|
|
486
601
|
|
|
@@ -508,29 +623,43 @@ module Harnex
|
|
|
508
623
|
end
|
|
509
624
|
|
|
510
625
|
def structured_transport?
|
|
511
|
-
|
|
626
|
+
adapter.transport == :stdio_jsonl_rpc
|
|
512
627
|
end
|
|
513
628
|
|
|
514
629
|
def run_structured
|
|
515
630
|
adapter.on_notification { |msg| handle_structured_notification(msg) }
|
|
516
631
|
adapter.on_disconnect { |err| handle_structured_disconnect(err) }
|
|
632
|
+
if adapter.respond_to?(:on_spawn)
|
|
633
|
+
adapter.on_spawn do |child_pid|
|
|
634
|
+
@pid = child_pid
|
|
635
|
+
runtime_child_started!
|
|
636
|
+
end
|
|
637
|
+
end
|
|
517
638
|
|
|
518
639
|
adapter.start_rpc(env: child_env, cwd: child_cwd || repo_root)
|
|
519
640
|
@pid = adapter.pid
|
|
520
|
-
|
|
641
|
+
runtime_child_started!
|
|
642
|
+
@state_machine.force_prompt! unless @stop_requested
|
|
521
643
|
emit_started_event
|
|
522
644
|
emit_git_start_event
|
|
523
645
|
|
|
524
646
|
install_signal_handlers
|
|
525
647
|
@server = ApiServer.new(self)
|
|
526
648
|
@server.start
|
|
527
|
-
persist_registry
|
|
528
649
|
register_completion_notifier!
|
|
650
|
+
persist_registry
|
|
529
651
|
append_dispatch_start_record
|
|
530
652
|
|
|
531
653
|
watch_thread = start_watch_thread
|
|
532
654
|
@inbox.start
|
|
533
|
-
|
|
655
|
+
begin
|
|
656
|
+
dispatch_initial_prompt
|
|
657
|
+
rescue StandardError
|
|
658
|
+
raise unless @runtime_budget&.expired?
|
|
659
|
+
|
|
660
|
+
stop_for_runtime_budget!
|
|
661
|
+
raise unless runtime_budget_stop?
|
|
662
|
+
end
|
|
534
663
|
|
|
535
664
|
if @pid
|
|
536
665
|
begin
|
|
@@ -556,14 +685,39 @@ module Harnex
|
|
|
556
685
|
@exit_code = 0
|
|
557
686
|
end
|
|
558
687
|
@ended_at = Time.now
|
|
688
|
+
@runtime_budget&.cancel
|
|
559
689
|
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
690
|
+
@completion_transition_lock.synchronize do
|
|
691
|
+
# Keep raw process evidence separate from logical work/cleanup status.
|
|
692
|
+
if adapter.transport == :stdio_jsonl_rpc || @runtime_budget
|
|
693
|
+
emit_event("process_exited", code: @exit_code, signal: @term_signal)
|
|
694
|
+
end
|
|
695
|
+
stop_for_runtime_budget! if runtime_budget_stop?
|
|
696
|
+
normalize_pi_stop_exit_code! if adapter.transport == :stdio_jsonl_rpc
|
|
697
|
+
normalize_runtime_exit_code!
|
|
698
|
+
enforce_required_artifact_report!
|
|
699
|
+
normalize_work_acceptance_exit_code!
|
|
700
|
+
normalize_auto_stop_exit_code!
|
|
701
|
+
normalize_runtime_exit_code!
|
|
702
|
+
end
|
|
563
703
|
drain_auto_stop_threads
|
|
564
704
|
finalize_session!
|
|
565
705
|
watch_thread&.kill
|
|
566
706
|
@exit_code
|
|
707
|
+
rescue StandardError
|
|
708
|
+
raise unless runtime_budget_stop?
|
|
709
|
+
|
|
710
|
+
# The owned child may expire while initialize/handshake is blocked,
|
|
711
|
+
# before normal registration. Preserve a typed terminal record and wake.
|
|
712
|
+
@ended_at ||= Time.now
|
|
713
|
+
@runtime_budget&.cancel
|
|
714
|
+
stop_for_runtime_budget!
|
|
715
|
+
normalize_runtime_exit_code!
|
|
716
|
+
emit_started_event
|
|
717
|
+
append_dispatch_start_record
|
|
718
|
+
register_completion_notifier! unless @completion_notification_registered
|
|
719
|
+
drain_auto_stop_threads
|
|
720
|
+
@exit_code
|
|
567
721
|
ensure
|
|
568
722
|
finalize_session!
|
|
569
723
|
@inbox.stop
|
|
@@ -589,92 +743,7 @@ module Harnex
|
|
|
589
743
|
end
|
|
590
744
|
|
|
591
745
|
def handle_structured_notification(message)
|
|
592
|
-
|
|
593
|
-
when :stdio_jsonrpc
|
|
594
|
-
handle_rpc_notification(message)
|
|
595
|
-
when :stdio_jsonl_rpc
|
|
596
|
-
handle_jsonl_notification(message)
|
|
597
|
-
end
|
|
598
|
-
end
|
|
599
|
-
|
|
600
|
-
def handle_rpc_notification(message)
|
|
601
|
-
method = message["method"]
|
|
602
|
-
params = message["params"] || {}
|
|
603
|
-
|
|
604
|
-
case method
|
|
605
|
-
when "thread/started"
|
|
606
|
-
@rpc_thread_id = params.dig("thread", "id")
|
|
607
|
-
when "turn/started"
|
|
608
|
-
@turn_started_seen = true
|
|
609
|
-
@state_machine.force_busy!
|
|
610
|
-
emit_event("turn_started", turnId: params.dig("turn", "id"))
|
|
611
|
-
when "turn/completed"
|
|
612
|
-
@state_machine.force_prompt!
|
|
613
|
-
turn = params["turn"] || {}
|
|
614
|
-
status = turn["status"]
|
|
615
|
-
turn_id = turn["id"] || params["turnId"]
|
|
616
|
-
payload = { turnId: turn_id }
|
|
617
|
-
payload[:status] = status if status
|
|
618
|
-
payload[:tokenUsage] = params["tokenUsage"] if params["tokenUsage"].is_a?(Hash)
|
|
619
|
-
if successful_turn_status?(status)
|
|
620
|
-
record_successful_completion(payload)
|
|
621
|
-
else
|
|
622
|
-
mark_task_failed(
|
|
623
|
-
turn_id: turn_id,
|
|
624
|
-
status: status,
|
|
625
|
-
error: extract_turn_error_message(turn),
|
|
626
|
-
codex_error_info: extract_turn_error_info(turn)
|
|
627
|
-
)
|
|
628
|
-
end
|
|
629
|
-
schedule_auto_stop("turn_completed", interrupt: false)
|
|
630
|
-
when "item/completed"
|
|
631
|
-
emit_event("item_completed", item: params["item"])
|
|
632
|
-
@event_counters.record_item(params["item"])
|
|
633
|
-
record_observed_command(params["item"])
|
|
634
|
-
text = render_item_text(params["item"])
|
|
635
|
-
record_synthesized(text) if text
|
|
636
|
-
when "thread/compacted"
|
|
637
|
-
emit_event("compaction", **params)
|
|
638
|
-
when "thread/tokenUsage/updated"
|
|
639
|
-
# Schema: ThreadTokenUsageUpdatedNotification carries
|
|
640
|
-
# `tokenUsage: { last, total, modelContextWindow? }` where each
|
|
641
|
-
# breakdown has camelCase {input,output,cachedInput,reasoningOutput,total}Tokens.
|
|
642
|
-
# Snapshot cumulative `total` for usage and aggregate `last` separately
|
|
643
|
-
# as conservative active-context pressure.
|
|
644
|
-
if params["tokenUsage"].is_a?(Hash)
|
|
645
|
-
@token_usage = params["tokenUsage"]
|
|
646
|
-
record_rpc_context_sample(@token_usage)
|
|
647
|
-
end
|
|
648
|
-
when "thread/status/changed"
|
|
649
|
-
# State machine reflects RPC state; no event needed.
|
|
650
|
-
nil
|
|
651
|
-
when "account/rateLimits/updated"
|
|
652
|
-
@rate_limits = params
|
|
653
|
-
when "error"
|
|
654
|
-
message = extract_error_notification_message(params)
|
|
655
|
-
@last_error = message unless message.to_s.empty?
|
|
656
|
-
@state_machine.force_busy!
|
|
657
|
-
emit_event(
|
|
658
|
-
"error",
|
|
659
|
-
source: "error_notification",
|
|
660
|
-
message: message,
|
|
661
|
-
codex_error_info: extract_error_notification_info(params),
|
|
662
|
-
will_retry: params["willRetry"],
|
|
663
|
-
threadId: params["threadId"],
|
|
664
|
-
turnId: params["turnId"]
|
|
665
|
-
)
|
|
666
|
-
emit_event("throttle_429", source: "error_notification", message: message) if throttle_429?(message, params)
|
|
667
|
-
signal_rpc_done! if params["turnId"].to_s.empty?
|
|
668
|
-
end
|
|
669
|
-
rescue StandardError => e
|
|
670
|
-
warn("harnex: rpc notification handler error: #{e.message}")
|
|
671
|
-
end
|
|
672
|
-
|
|
673
|
-
def successful_turn_status?(status)
|
|
674
|
-
text = status.to_s
|
|
675
|
-
return true if text.empty?
|
|
676
|
-
|
|
677
|
-
SUCCESSFUL_TURN_STATUSES.include?(text)
|
|
746
|
+
handle_jsonl_notification(message) if adapter.transport == :stdio_jsonl_rpc
|
|
678
747
|
end
|
|
679
748
|
|
|
680
749
|
def record_successful_completion(payload)
|
|
@@ -682,6 +751,12 @@ module Harnex
|
|
|
682
751
|
end
|
|
683
752
|
|
|
684
753
|
def record_successful_completion_locked(payload)
|
|
754
|
+
return false if @stop_requested || @session_finalized
|
|
755
|
+
if @runtime_budget&.expired?
|
|
756
|
+
stop_for_runtime_budget!
|
|
757
|
+
return false
|
|
758
|
+
end
|
|
759
|
+
@activity.settle
|
|
685
760
|
assessment = completion_gate_required? ? assess_completion_proof : { accepted: true }
|
|
686
761
|
unless assessment[:accepted]
|
|
687
762
|
mark_task_failed(
|
|
@@ -717,16 +792,17 @@ module Harnex
|
|
|
717
792
|
event_payload[:outcome_class] = @completion_outcome_class if @completion_outcome_class
|
|
718
793
|
event_payload[:artifact_report_status] = @completion_report_status if @completion_report_status
|
|
719
794
|
event_payload[:artifact_report_path] = artifact_report_path
|
|
795
|
+
event_payload[:turn] = @completion_turn
|
|
796
|
+
event_payload[:follow_ups] = follow_ups_summary if follow_ups_summary
|
|
797
|
+
event_payload.merge!(final_message_fields).merge!(turn_git_fields)
|
|
720
798
|
emit_event("task_complete", **event_payload)
|
|
721
799
|
publish_completion_notification("task_complete")
|
|
722
800
|
true
|
|
723
801
|
end
|
|
724
802
|
|
|
803
|
+
# Pi turns are accepted as settled unless the caller demands a receipt.
|
|
725
804
|
def completion_gate_required?
|
|
726
|
-
|
|
727
|
-
return false unless adapter.transport == :stdio_jsonrpc
|
|
728
|
-
|
|
729
|
-
@auto_stop || (adapter.respond_to?(:initial_prompt) && !adapter.initial_prompt.to_s.empty?)
|
|
805
|
+
require_artifact_report
|
|
730
806
|
end
|
|
731
807
|
|
|
732
808
|
def assess_completion_proof
|
|
@@ -764,7 +840,7 @@ module Harnex
|
|
|
764
840
|
def git_activity_observed?(snapshot = nil)
|
|
765
841
|
return false if @git_start[:sha].to_s.empty?
|
|
766
842
|
|
|
767
|
-
snapshot ||= Harnex.git_capture_end(
|
|
843
|
+
snapshot ||= Harnex.git_capture_end(git_observation_root, @git_start)
|
|
768
844
|
snapshot[:commits].to_i.positive? || Array(snapshot[:changed_paths]).any? ||
|
|
769
845
|
(!snapshot[:sha].to_s.empty? && snapshot[:sha].to_s != @git_start[:sha].to_s)
|
|
770
846
|
end
|
|
@@ -788,8 +864,9 @@ module Harnex
|
|
|
788
864
|
@completion_transition_lock.synchronize { mark_task_failed_locked(**attributes) }
|
|
789
865
|
end
|
|
790
866
|
|
|
791
|
-
def mark_task_failed_locked(turn_id: nil, status: nil, error: nil,
|
|
867
|
+
def mark_task_failed_locked(turn_id: nil, status: nil, error: nil, outcome_class: nil, artifact_report_status: nil, diagnostics: nil, publish: true)
|
|
792
868
|
@last_completed_at = nil if outcome_class
|
|
869
|
+
@activity.settle
|
|
793
870
|
@last_failed_at = Time.now
|
|
794
871
|
@last_failed_status = status.to_s.empty? ? "failed" : status.to_s
|
|
795
872
|
@last_error = error.to_s unless error.to_s.empty?
|
|
@@ -798,12 +875,15 @@ module Harnex
|
|
|
798
875
|
@completion_diagnostics = Array(diagnostics).first(Harnex::ArtifactReport::MAX_DIAGNOSTICS) if diagnostics
|
|
799
876
|
|
|
800
877
|
payload = { status: @last_failed_status }
|
|
878
|
+
payload[:stop] = @stop_request.to_h if @stop_request
|
|
801
879
|
payload[:turnId] = turn_id if turn_id
|
|
802
880
|
payload[:message] = error unless error.to_s.empty?
|
|
803
|
-
payload[:codex_error_info] = codex_error_info if codex_error_info
|
|
804
881
|
payload[:outcome_class] = outcome_class if outcome_class
|
|
805
882
|
payload[:artifact_report_status] = artifact_report_status if artifact_report_status
|
|
806
883
|
payload[:diagnostics] = @completion_diagnostics unless @completion_diagnostics.empty?
|
|
884
|
+
payload[:turn] = @completion_turn
|
|
885
|
+
payload[:follow_ups] = follow_ups_summary if follow_ups_summary
|
|
886
|
+
payload.merge!(final_message_fields).merge!(turn_git_fields)
|
|
807
887
|
emit_event("task_failed", **payload)
|
|
808
888
|
if publish
|
|
809
889
|
enforce_required_artifact_report!
|
|
@@ -812,50 +892,29 @@ module Harnex
|
|
|
812
892
|
end
|
|
813
893
|
end
|
|
814
894
|
|
|
815
|
-
def
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
params["message"]
|
|
821
|
-
end
|
|
822
|
-
end
|
|
823
|
-
|
|
824
|
-
def extract_error_notification_info(params)
|
|
825
|
-
error = params["error"]
|
|
826
|
-
error.is_a?(Hash) ? error["codexErrorInfo"] : nil
|
|
827
|
-
end
|
|
828
|
-
|
|
829
|
-
def throttle_429?(message, params)
|
|
830
|
-
return true if message.to_s.match?(/\b429\b|rate limit/i)
|
|
831
|
-
|
|
832
|
-
error = params["error"]
|
|
833
|
-
error.is_a?(Hash) && error.values.any? { |value| value.to_s.match?(/\b429\b|rate limit/i) }
|
|
834
|
-
end
|
|
835
|
-
|
|
836
|
-
def extract_turn_error_message(turn)
|
|
837
|
-
error = turn["error"]
|
|
838
|
-
return error["message"] if error.is_a?(Hash)
|
|
839
|
-
return error if error.is_a?(String)
|
|
840
|
-
|
|
841
|
-
nil
|
|
842
|
-
end
|
|
895
|
+
def handle_jsonl_notification(message)
|
|
896
|
+
@completion_transition_lock.synchronize do
|
|
897
|
+
# Stop owns the terminal decision. Late abort/settled/start callbacks
|
|
898
|
+
# cannot resurrect interrupted work or invalidate settled idle cleanup.
|
|
899
|
+
return if @stop_requested || @session_finalized || @pi_transport_failed
|
|
843
900
|
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
error.is_a?(Hash) ? error["codexErrorInfo"] : nil
|
|
901
|
+
handle_jsonl_notification_locked(message)
|
|
902
|
+
end
|
|
847
903
|
end
|
|
848
904
|
|
|
849
|
-
def
|
|
905
|
+
def handle_jsonl_notification_locked(message)
|
|
850
906
|
event_type = message["type"].to_s
|
|
907
|
+
return if event_type == "agent_settled" && @pi_awaiting_run_start
|
|
851
908
|
|
|
909
|
+
@activity.observe_pi(message)
|
|
852
910
|
case event_type
|
|
853
911
|
when "agent_start"
|
|
854
|
-
|
|
855
|
-
@
|
|
912
|
+
begin_pi_run!
|
|
913
|
+
@pi_awaiting_run_start = false
|
|
856
914
|
when "turn_start"
|
|
915
|
+
begin_pi_run!
|
|
916
|
+
@pi_awaiting_run_start = false
|
|
857
917
|
@turn_started_seen = true
|
|
858
|
-
@state_machine.force_busy!
|
|
859
918
|
emit_event("turn_started")
|
|
860
919
|
when "agent_end"
|
|
861
920
|
capture_pi_agent_end(message)
|
|
@@ -863,6 +922,8 @@ module Harnex
|
|
|
863
922
|
emit_event("agent_end", will_retry: message["willRetry"] == true)
|
|
864
923
|
adapter.request_session_stats_async if adapter.respond_to?(:request_session_stats_async)
|
|
865
924
|
when "agent_settled"
|
|
925
|
+
return if @pi_awaiting_run_start
|
|
926
|
+
|
|
866
927
|
@state_machine.force_prompt!
|
|
867
928
|
emit_event("agent_settled", stop_reason: @pi_last_assistant_stop_reason)
|
|
868
929
|
settle_pi_task!
|
|
@@ -891,11 +952,17 @@ module Harnex
|
|
|
891
952
|
record_synthesized(text) if text
|
|
892
953
|
end
|
|
893
954
|
when "tool_execution_start"
|
|
894
|
-
|
|
955
|
+
if pi_command_tool?(message)
|
|
956
|
+
@event_counters.record_item({ "type" => "commandExecution" })
|
|
957
|
+
start_pi_command(message)
|
|
958
|
+
else
|
|
959
|
+
@event_counters.record_item({ "type" => "dynamicToolCall" })
|
|
960
|
+
end
|
|
895
961
|
record_synthesized(
|
|
896
962
|
"tool: #{message["toolName"] || "tool"}#{message["args"] ? " #{summarize(message["args"])}" : ""}"
|
|
897
963
|
)
|
|
898
964
|
when "tool_execution_end"
|
|
965
|
+
finish_pi_command(message) if pi_command_tool?(message)
|
|
899
966
|
tool_name = message["toolName"] || "tool"
|
|
900
967
|
status = message["isError"] ? "error" : "ok"
|
|
901
968
|
record_synthesized("tool-result: #{tool_name} (#{status})")
|
|
@@ -909,7 +976,10 @@ module Harnex
|
|
|
909
976
|
error_message: message["errorMessage"]
|
|
910
977
|
)
|
|
911
978
|
when "queue_update"
|
|
912
|
-
|
|
979
|
+
# Counts only: the queue holds message text. A follow-up still
|
|
980
|
+
# pending when the run settles never reached the model.
|
|
981
|
+
@pi_follow_up_pending = Array(message["followUp"]).length
|
|
982
|
+
emit_event("queue_update", follow_up: @pi_follow_up_pending, steering: Array(message["steering"]).length)
|
|
913
983
|
when "auto_retry_start", "auto_retry_end"
|
|
914
984
|
payload = message.reject { |k, _| k == "type" }
|
|
915
985
|
@pi_last_retry_error = message["finalError"].to_s if event_type == "auto_retry_end" && message["success"] == false
|
|
@@ -950,12 +1020,15 @@ module Harnex
|
|
|
950
1020
|
end
|
|
951
1021
|
|
|
952
1022
|
def handle_structured_disconnect(error)
|
|
953
|
-
handle_rpc_disconnect(error)
|
|
1023
|
+
@completion_transition_lock.synchronize { handle_rpc_disconnect(error) }
|
|
954
1024
|
end
|
|
955
1025
|
|
|
956
1026
|
def handle_rpc_disconnect(error)
|
|
1027
|
+
return if @session_finalized
|
|
1028
|
+
|
|
957
1029
|
msg = error.is_a?(Hash) ? error["message"] : error&.message
|
|
958
|
-
|
|
1030
|
+
expected_cleanup = error.nil? && (@stop_requested || runtime_budget_stop?) && !@pi_transport_failed
|
|
1031
|
+
if expected_cleanup
|
|
959
1032
|
signal_rpc_done!
|
|
960
1033
|
return
|
|
961
1034
|
end
|
|
@@ -965,6 +1038,14 @@ module Harnex
|
|
|
965
1038
|
end
|
|
966
1039
|
@last_error = msg.to_s unless msg.to_s.empty?
|
|
967
1040
|
@state_machine.force_busy!
|
|
1041
|
+
return if @pi_transport_failed
|
|
1042
|
+
|
|
1043
|
+
@pi_transport_failed = true
|
|
1044
|
+
@pi_idle_stop = false
|
|
1045
|
+
mark_task_failed(
|
|
1046
|
+
status: "disconnected", error: msg,
|
|
1047
|
+
outcome_class: "task_failed", artifact_report_status: "rejected"
|
|
1048
|
+
)
|
|
968
1049
|
emit_event("disconnected", source: "transport", message: msg) rescue nil
|
|
969
1050
|
if adapter.respond_to?(:terminate_subprocess)
|
|
970
1051
|
Thread.new do
|
|
@@ -976,11 +1057,12 @@ module Harnex
|
|
|
976
1057
|
signal_rpc_done!
|
|
977
1058
|
end
|
|
978
1059
|
|
|
979
|
-
def normal_auto_stop_disconnect?(message)
|
|
980
|
-
message.to_s.empty? && @auto_stop_fired && (task_complete? || task_failed?)
|
|
981
|
-
end
|
|
982
|
-
|
|
983
1060
|
def dispatch_initial_prompt
|
|
1061
|
+
if @runtime_budget&.expired?
|
|
1062
|
+
stop_for_runtime_budget!
|
|
1063
|
+
return
|
|
1064
|
+
end
|
|
1065
|
+
return if @stop_requested
|
|
984
1066
|
return unless adapter.respond_to?(:initial_prompt)
|
|
985
1067
|
|
|
986
1068
|
prompt = adapter.initial_prompt
|
|
@@ -989,48 +1071,88 @@ module Harnex
|
|
|
989
1071
|
inject_via_structured(text: prompt, submit: true, enter_only: false, force: false)
|
|
990
1072
|
end
|
|
991
1073
|
|
|
992
|
-
|
|
993
|
-
|
|
1074
|
+
# Exact-string gate matching against harness-observed exits; the last
|
|
1075
|
+
# run of a gate decides it. Caller holds @observed_commands_mutex.
|
|
1076
|
+
def note_command_exit_locked(command, exit_code)
|
|
1077
|
+
@commands_had_failures = true unless exit_code.zero?
|
|
1078
|
+
@final_gates.each do |gate|
|
|
1079
|
+
@final_gate_exits[gate] = exit_code if Harnex::ArtifactReport.final_gate_matches?(gate, command)
|
|
1080
|
+
end
|
|
1081
|
+
end
|
|
994
1082
|
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
when "mcpToolCall", "dynamicToolCall"
|
|
999
|
-
name = item["tool"] || "tool"
|
|
1000
|
-
args = item["arguments"]
|
|
1001
|
-
"tool: #{name}#{args ? " #{summarize(args)}" : ""}"
|
|
1002
|
-
when "commandExecution"
|
|
1003
|
-
"command: #{item["command"]}"
|
|
1004
|
-
else
|
|
1005
|
-
item["text"]
|
|
1083
|
+
def final_gates_snapshot
|
|
1084
|
+
@observed_commands_mutex.synchronize do
|
|
1085
|
+
@final_gates.map { |gate| { "cmd" => gate, "exit_code" => @final_gate_exits[gate] } }
|
|
1006
1086
|
end
|
|
1007
1087
|
end
|
|
1008
1088
|
|
|
1009
|
-
def
|
|
1010
|
-
|
|
1089
|
+
def commands_had_failures?
|
|
1090
|
+
@observed_commands_mutex.synchronize { @commands_had_failures }
|
|
1091
|
+
end
|
|
1092
|
+
|
|
1093
|
+
def pi_command_tool?(message)
|
|
1094
|
+
PI_COMMAND_TOOLS.include?(message["toolName"].to_s)
|
|
1095
|
+
end
|
|
1096
|
+
|
|
1097
|
+
# Pi bash calls become proof only from structured evidence: command text
|
|
1098
|
+
# from the start event's args and an integer exit from the end event's
|
|
1099
|
+
# result.structuredContent, correlated by toolCallId. Aborts, timeouts,
|
|
1100
|
+
# extension-replaced results, and older Pi releases carry no structured
|
|
1101
|
+
# exit; those calls are counted as unobserved, never inferred from prose.
|
|
1102
|
+
# Shell commands Pi is running right now (bounded), so a stall report can
|
|
1103
|
+
# say whether the worker waits on a long command or on the model.
|
|
1104
|
+
def pi_running_commands
|
|
1105
|
+
@observed_commands_mutex.synchronize do
|
|
1106
|
+
@pi_pending_commands.values.compact.last(3).map { |command| command[0, 160] }
|
|
1107
|
+
end
|
|
1108
|
+
rescue StandardError
|
|
1109
|
+
[]
|
|
1110
|
+
end
|
|
1011
1111
|
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1112
|
+
def start_pi_command(message)
|
|
1113
|
+
call_id = message["toolCallId"].to_s
|
|
1114
|
+
args = message["args"]
|
|
1115
|
+
command = args["command"] if args.is_a?(Hash)
|
|
1116
|
+
command = command.is_a?(String) && !command.strip.empty? ? Harnex::ArtifactReport.bounded_string(command) : nil
|
|
1117
|
+
@observed_commands_mutex.synchronize do
|
|
1118
|
+
if call_id.empty?
|
|
1119
|
+
@pi_unobserved_commands += 1
|
|
1120
|
+
next
|
|
1121
|
+
end
|
|
1122
|
+
if @pi_pending_commands.key?(call_id)
|
|
1123
|
+
@pi_pending_commands.delete(call_id)
|
|
1124
|
+
@pi_unobserved_commands += 1
|
|
1125
|
+
elsif @pi_pending_commands.length >= MAX_PENDING_PI_COMMANDS
|
|
1126
|
+
@pi_pending_commands.shift
|
|
1127
|
+
@pi_unobserved_commands += 1
|
|
1128
|
+
end
|
|
1129
|
+
@pi_pending_commands[call_id] = command
|
|
1130
|
+
end
|
|
1131
|
+
end
|
|
1019
1132
|
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
}.compact
|
|
1133
|
+
def finish_pi_command(message)
|
|
1134
|
+
call_id = message["toolCallId"].to_s
|
|
1135
|
+
result = message["result"]
|
|
1136
|
+
structured = result["structuredContent"] if result.is_a?(Hash)
|
|
1137
|
+
exit_code = structured["exit_code"] if structured.is_a?(Hash)
|
|
1138
|
+
duration_ms = message["durationMs"]
|
|
1139
|
+
duration = duration_ms / 1000.0 if duration_ms.is_a?(Numeric) && duration_ms.finite? && duration_ms >= 0
|
|
1028
1140
|
@observed_commands_mutex.synchronize do
|
|
1029
|
-
@
|
|
1141
|
+
command = @pi_pending_commands.delete(call_id) unless call_id.empty?
|
|
1142
|
+
unless command && exit_code.is_a?(Integer)
|
|
1143
|
+
@pi_unobserved_commands += 1
|
|
1144
|
+
next
|
|
1145
|
+
end
|
|
1146
|
+
|
|
1147
|
+
@observed_commands << {
|
|
1148
|
+
"cmd" => command,
|
|
1149
|
+
"exit_code" => exit_code,
|
|
1150
|
+
"status" => message["isError"] == true ? "failed" : "completed",
|
|
1151
|
+
"duration_s" => duration
|
|
1152
|
+
}.compact
|
|
1030
1153
|
@observed_commands = @observed_commands.last(Harnex::ArtifactReport::MAX_COMMANDS)
|
|
1154
|
+
note_command_exit_locked(command, exit_code)
|
|
1031
1155
|
end
|
|
1032
|
-
rescue ArgumentError, TypeError
|
|
1033
|
-
nil
|
|
1034
1156
|
end
|
|
1035
1157
|
|
|
1036
1158
|
def summarize(value)
|
|
@@ -1040,6 +1162,24 @@ module Harnex
|
|
|
1040
1162
|
""
|
|
1041
1163
|
end
|
|
1042
1164
|
|
|
1165
|
+
def begin_pi_run!
|
|
1166
|
+
begin_notification_turn!
|
|
1167
|
+
@activity.start_turn
|
|
1168
|
+
reset_work_outcome!
|
|
1169
|
+
reset_pi_run_outcome!
|
|
1170
|
+
@state_machine.force_busy!
|
|
1171
|
+
end
|
|
1172
|
+
|
|
1173
|
+
def reset_work_outcome!
|
|
1174
|
+
@last_completed_at = nil
|
|
1175
|
+
@last_failed_at = nil
|
|
1176
|
+
@last_failed_status = nil
|
|
1177
|
+
@completion_outcome_class = nil
|
|
1178
|
+
@completion_report_status = nil
|
|
1179
|
+
@completion_diagnostics = []
|
|
1180
|
+
@last_error = nil
|
|
1181
|
+
end
|
|
1182
|
+
|
|
1043
1183
|
def reset_pi_run_outcome!
|
|
1044
1184
|
@pi_last_assistant_stop_reason = nil
|
|
1045
1185
|
@pi_last_assistant_error = nil
|
|
@@ -1056,6 +1196,9 @@ module Harnex
|
|
|
1056
1196
|
def capture_pi_assistant_message(message)
|
|
1057
1197
|
return unless message.is_a?(Hash) && message["role"] == "assistant"
|
|
1058
1198
|
|
|
1199
|
+
text = pi_extract_message_text(message).to_s
|
|
1200
|
+
@turn_final_text = text unless text.strip.empty?
|
|
1201
|
+
|
|
1059
1202
|
reason = message["stopReason"].to_s
|
|
1060
1203
|
@pi_last_assistant_stop_reason = reason unless reason.empty?
|
|
1061
1204
|
error = message["errorMessage"].to_s
|
|
@@ -1063,7 +1206,7 @@ module Harnex
|
|
|
1063
1206
|
end
|
|
1064
1207
|
|
|
1065
1208
|
def settle_pi_task!
|
|
1066
|
-
return if @stop_requested
|
|
1209
|
+
return if @stop_requested || @pi_transport_failed
|
|
1067
1210
|
|
|
1068
1211
|
reason = @pi_last_assistant_stop_reason
|
|
1069
1212
|
case reason
|
|
@@ -1072,7 +1215,8 @@ module Harnex
|
|
|
1072
1215
|
when "error", "aborted", "length"
|
|
1073
1216
|
error = @pi_last_assistant_error || @pi_last_retry_error
|
|
1074
1217
|
error = "Pi agent settled with stopReason=#{reason}" if error.to_s.empty?
|
|
1075
|
-
|
|
1218
|
+
capacity = reason == "error" && !@pi_last_retry_error.to_s.empty?
|
|
1219
|
+
mark_task_failed(status: reason, error: error, outcome_class: capacity ? "provider_capacity" : nil)
|
|
1076
1220
|
when nil, ""
|
|
1077
1221
|
mark_task_failed(
|
|
1078
1222
|
status: "missing_final_message",
|
|
@@ -1217,6 +1361,9 @@ module Harnex
|
|
|
1217
1361
|
@completion_notification_registered = true
|
|
1218
1362
|
deliver_completion_notification(@pending_completion_notification) if @pending_completion_notification
|
|
1219
1363
|
end
|
|
1364
|
+
rescue CompletionNotifier::RegistrationError
|
|
1365
|
+
# A named primary must have a durable monitor handoff before any prompt.
|
|
1366
|
+
raise
|
|
1220
1367
|
rescue StandardError => e
|
|
1221
1368
|
warn("harnex: completion marker cleanup failed (#{e.class})")
|
|
1222
1369
|
emit_event("completion_notification_error", component: "cleanup", error_class: e.class.name)
|
|
@@ -1236,6 +1383,23 @@ module Harnex
|
|
|
1236
1383
|
false
|
|
1237
1384
|
end
|
|
1238
1385
|
|
|
1386
|
+
# #81: work that starts after this turn's result was published is the
|
|
1387
|
+
# next turn, so its own result is published (and reaches the owner) too.
|
|
1388
|
+
def begin_notification_turn!
|
|
1389
|
+
turn = @completion_notification_mutex.synchronize do
|
|
1390
|
+
next nil unless @completion_notification_sent
|
|
1391
|
+
|
|
1392
|
+
@completion_notification_sent = false
|
|
1393
|
+
@turn_follow_ups_sent = 0
|
|
1394
|
+
@turn_final_text = nil
|
|
1395
|
+
@completion_turn += 1
|
|
1396
|
+
end
|
|
1397
|
+
return unless turn
|
|
1398
|
+
|
|
1399
|
+
emit_event("work_turn", turn: turn)
|
|
1400
|
+
@completion_notifier.turn_started(turn) if @completion_notifier.respond_to?(:turn_started)
|
|
1401
|
+
end
|
|
1402
|
+
|
|
1239
1403
|
def deliver_completion_notification(snapshot)
|
|
1240
1404
|
@completion_notification_sent = true
|
|
1241
1405
|
@pending_completion_notification = nil
|
|
@@ -1247,10 +1411,59 @@ module Harnex
|
|
|
1247
1411
|
{
|
|
1248
1412
|
outcome: outcome, work_state: outcome == "completed" ? "completed" : "failed",
|
|
1249
1413
|
outcome_class: @completion_outcome_class, artifact_report_status: @completion_report_status,
|
|
1250
|
-
end_sha: summary_string(@git_end[:sha]), terminal_signal: terminal_signal
|
|
1414
|
+
end_sha: summary_string(@git_end[:sha]), terminal_signal: terminal_signal,
|
|
1415
|
+
turn: @completion_turn, follow_ups: follow_ups_summary,
|
|
1416
|
+
final_message: final_message_fields[:final_message],
|
|
1417
|
+
final_message_path: final_message_fields[:final_message_path],
|
|
1418
|
+
turn_details: (@last_turn_git_fields || {}).slice(:turn_commits, :model)
|
|
1251
1419
|
}
|
|
1252
1420
|
end
|
|
1253
1421
|
|
|
1422
|
+
# The lane's head and commit count since dispatch start, refreshed at the
|
|
1423
|
+
# turn's end so a result names the commit it produced.
|
|
1424
|
+
def turn_git_fields
|
|
1425
|
+
snapshot = refresh_git_end_snapshot
|
|
1426
|
+
return {} if snapshot.nil? || snapshot[:sha].to_s.empty?
|
|
1427
|
+
|
|
1428
|
+
commits = snapshot[:commits]
|
|
1429
|
+
turn_commits = commits && (commits - @commits_at_last_result.to_i)
|
|
1430
|
+
@commits_at_last_result = commits if commits
|
|
1431
|
+
fields = { end_sha: snapshot[:sha], commits: commits, turn_commits: turn_commits }.compact
|
|
1432
|
+
model = adapter.respond_to?(:current_model) ? adapter.current_model.to_s : ""
|
|
1433
|
+
fields[:model] = model unless model.empty?
|
|
1434
|
+
@last_turn_git_fields = fields
|
|
1435
|
+
rescue StandardError
|
|
1436
|
+
{}
|
|
1437
|
+
end
|
|
1438
|
+
|
|
1439
|
+
# #85: the worker's last reply this turn, so a supervisor gets the answer
|
|
1440
|
+
# with the result instead of reading the transcript. Written once per
|
|
1441
|
+
# turn; the inline copy is bounded.
|
|
1442
|
+
def final_message_fields
|
|
1443
|
+
text = @turn_final_text.to_s.strip
|
|
1444
|
+
return {} if text.empty?
|
|
1445
|
+
|
|
1446
|
+
@final_message_written ||= {}
|
|
1447
|
+
path = Harnex.final_message_path(repo_root, id, @completion_turn)
|
|
1448
|
+
unless @final_message_written[@completion_turn] == text
|
|
1449
|
+
File.write(path, text + "\n")
|
|
1450
|
+
@final_message_written[@completion_turn] = text
|
|
1451
|
+
end
|
|
1452
|
+
fields = { final_message: text[0, FINAL_MESSAGE_LIMIT], final_message_path: path }
|
|
1453
|
+
fields[:final_message_truncated] = true if text.length > FINAL_MESSAGE_LIMIT
|
|
1454
|
+
fields
|
|
1455
|
+
rescue StandardError
|
|
1456
|
+
{ final_message: text[0, FINAL_MESSAGE_LIMIT] }
|
|
1457
|
+
end
|
|
1458
|
+
|
|
1459
|
+
# Follow-ups sent into this turn and how many Pi still held when it
|
|
1460
|
+
# settled (0 means every one reached the model).
|
|
1461
|
+
def follow_ups_summary
|
|
1462
|
+
return nil unless @turn_follow_ups_sent.positive?
|
|
1463
|
+
|
|
1464
|
+
{ sent: @turn_follow_ups_sent, pending: @pi_follow_up_pending }
|
|
1465
|
+
end
|
|
1466
|
+
|
|
1254
1467
|
def completion_notification_outcome(terminal_signal)
|
|
1255
1468
|
signal = terminal_signal.to_s
|
|
1256
1469
|
return "error" if %w[dispatch_error finalization].include?(signal)
|
|
@@ -1311,7 +1524,9 @@ module Harnex
|
|
|
1311
1524
|
artifact_report_status: @completion_report_status,
|
|
1312
1525
|
started_at: @started_at.iso8601,
|
|
1313
1526
|
exited_at: Time.now.iso8601,
|
|
1314
|
-
injected_count: @injected_count
|
|
1527
|
+
injected_count: @injected_count,
|
|
1528
|
+
stop: @stop_request&.to_h,
|
|
1529
|
+
runtime_budget: runtime_budget_snapshot
|
|
1315
1530
|
}
|
|
1316
1531
|
payload[:signal] = @term_signal if @term_signal
|
|
1317
1532
|
Harnex.write_registry(exit_path, payload)
|
|
@@ -1451,6 +1666,9 @@ module Harnex
|
|
|
1451
1666
|
end
|
|
1452
1667
|
|
|
1453
1668
|
def emit_started_event
|
|
1669
|
+
return if @started_event_emitted
|
|
1670
|
+
|
|
1671
|
+
@started_event_emitted = true
|
|
1454
1672
|
payload = { pid: @pid }
|
|
1455
1673
|
payload[:meta] = meta if meta
|
|
1456
1674
|
emit_event("started", **payload)
|
|
@@ -1462,16 +1680,49 @@ module Harnex
|
|
|
1462
1680
|
|
|
1463
1681
|
def emit_git_start_event
|
|
1464
1682
|
@git_start = Harnex.git_capture_start(
|
|
1465
|
-
|
|
1683
|
+
git_observation_root,
|
|
1466
1684
|
exclude_paths: git_observation_excluded_paths
|
|
1467
1685
|
)
|
|
1468
1686
|
return if @git_start.empty?
|
|
1469
1687
|
|
|
1470
|
-
|
|
1688
|
+
payload = { phase: "start", sha: @git_start[:sha], branch: @git_start[:branch] }
|
|
1689
|
+
payload[:git_root] = git_observation_root if git_lane?
|
|
1690
|
+
emit_event("git", **payload)
|
|
1691
|
+
end
|
|
1692
|
+
|
|
1693
|
+
# #80: a lane launched with --cwd <worktree> --root <checkout> commits in
|
|
1694
|
+
# its own repository. Git is observed there (SHAs, branch, activity,
|
|
1695
|
+
# receipt); --root still decides where telemetry rows land, and its head
|
|
1696
|
+
# is kept as root_commit_sha.
|
|
1697
|
+
def git_observation_root
|
|
1698
|
+
return @git_observation_root if defined?(@git_observation_root)
|
|
1699
|
+
|
|
1700
|
+
root_top = canonical_git_toplevel(repo_root)
|
|
1701
|
+
lane_top = child_cwd && canonical_git_toplevel(child_cwd)
|
|
1702
|
+
@git_observation_root = lane_top && lane_top != root_top ? lane_top : repo_root
|
|
1703
|
+
end
|
|
1704
|
+
|
|
1705
|
+
def git_lane?
|
|
1706
|
+
git_observation_root != repo_root
|
|
1707
|
+
end
|
|
1708
|
+
|
|
1709
|
+
def canonical_git_toplevel(path)
|
|
1710
|
+
top = DispatchHistory.git_toplevel(path.to_s)
|
|
1711
|
+
top && File.realpath(top)
|
|
1712
|
+
rescue StandardError
|
|
1713
|
+
nil
|
|
1714
|
+
end
|
|
1715
|
+
|
|
1716
|
+
def root_commit_sha
|
|
1717
|
+
return nil unless git_lane?
|
|
1718
|
+
|
|
1719
|
+
@root_commit_sha ||= Harnex.git_output(repo_root, "rev-parse", "HEAD")
|
|
1720
|
+
rescue StandardError
|
|
1721
|
+
nil
|
|
1471
1722
|
end
|
|
1472
1723
|
|
|
1473
1724
|
def git_observation_excluded_paths
|
|
1474
|
-
git_root = DispatchHistory.find_git_root(
|
|
1725
|
+
git_root = DispatchHistory.find_git_root(git_observation_root)
|
|
1475
1726
|
return [] unless git_root
|
|
1476
1727
|
|
|
1477
1728
|
root = File.expand_path(git_root)
|
|
@@ -1488,7 +1739,8 @@ module Harnex
|
|
|
1488
1739
|
@context_summary = normalized_context_summary(summary)
|
|
1489
1740
|
emit_event("usage", **@usage_summary)
|
|
1490
1741
|
|
|
1491
|
-
@git_end = Harnex.git_capture_end(
|
|
1742
|
+
@git_end = Harnex.git_capture_end(git_observation_root, @git_start)
|
|
1743
|
+
root_commit_sha
|
|
1492
1744
|
return if @git_end.empty?
|
|
1493
1745
|
|
|
1494
1746
|
emit_event(
|
|
@@ -1531,10 +1783,14 @@ module Harnex
|
|
|
1531
1783
|
end
|
|
1532
1784
|
|
|
1533
1785
|
def finalize_session!
|
|
1534
|
-
|
|
1535
|
-
|
|
1786
|
+
@runtime_budget&.cancel
|
|
1787
|
+
@completion_transition_lock.synchronize do
|
|
1788
|
+
return if @session_finalized
|
|
1789
|
+
return unless @events_log
|
|
1536
1790
|
|
|
1537
|
-
|
|
1791
|
+
@session_finalized = true
|
|
1792
|
+
@activity.settle
|
|
1793
|
+
end
|
|
1538
1794
|
@ended_at ||= Time.now
|
|
1539
1795
|
begin
|
|
1540
1796
|
emit_session_end_telemetry
|
|
@@ -1556,12 +1812,123 @@ module Harnex
|
|
|
1556
1812
|
emit_exit_event
|
|
1557
1813
|
end
|
|
1558
1814
|
|
|
1559
|
-
def stop_requested!
|
|
1560
|
-
@
|
|
1561
|
-
|
|
1815
|
+
def stop_requested!(reason: "manual", origin: "api")
|
|
1816
|
+
@completion_transition_lock.synchronize do
|
|
1817
|
+
@stop_mutex.synchronize do
|
|
1818
|
+
return true if @stop_requested || @session_finalized
|
|
1819
|
+
|
|
1820
|
+
work_state = task_failed? ? "failed" : (task_complete? ? "completed" : "running")
|
|
1821
|
+
@stop_request = StopRequest.new(
|
|
1822
|
+
reason: reason, origin: origin, work_state: work_state,
|
|
1823
|
+
runtime_limit_s: reason == "runtime_budget" ? @runtime_budget&.snapshot&.fetch(:limit_s) : nil
|
|
1824
|
+
)
|
|
1825
|
+
@stop_requested = true
|
|
1826
|
+
emit_event("stop_requested", **@stop_request.to_h)
|
|
1827
|
+
if adapter.transport == :stdio_jsonl_rpc
|
|
1828
|
+
@pi_idle_stop = task_complete? && !@pi_transport_failed &&
|
|
1829
|
+
!@pi_awaiting_run_start && @state_machine.to_s == "prompt" &&
|
|
1830
|
+
(adapter.state == :prompt || runtime_budget_stop?)
|
|
1831
|
+
unless @pi_idle_stop || task_failed?
|
|
1832
|
+
mark_task_failed(
|
|
1833
|
+
status: runtime_budget_stop? ? "runtime_budget" : "interrupted",
|
|
1834
|
+
error: runtime_budget_stop? ? "worker runtime budget expired" : "Pi stopped before accepted settlement"
|
|
1835
|
+
)
|
|
1836
|
+
end
|
|
1837
|
+
elsif runtime_budget_stop? && !task_complete? && !task_failed?
|
|
1838
|
+
mark_task_failed(status: "runtime_budget", error: "worker runtime budget expired")
|
|
1839
|
+
end
|
|
1840
|
+
false
|
|
1841
|
+
end
|
|
1842
|
+
end
|
|
1843
|
+
end
|
|
1844
|
+
|
|
1845
|
+
def start_runtime_budget!
|
|
1846
|
+
@runtime_budget&.start { schedule_runtime_stop }
|
|
1847
|
+
end
|
|
1848
|
+
|
|
1849
|
+
def runtime_child_started!
|
|
1850
|
+
@runtime_child_ready = true
|
|
1851
|
+
schedule_runtime_stop if @runtime_budget&.expired?
|
|
1852
|
+
end
|
|
1853
|
+
|
|
1854
|
+
def schedule_runtime_stop
|
|
1855
|
+
return unless @runtime_child_ready
|
|
1856
|
+
|
|
1857
|
+
# This lock never encloses RPC, receipt, output or event I/O. Physical
|
|
1858
|
+
# enforcement must not wait for the logical completion/stop locks, and
|
|
1859
|
+
# an earlier advisory stop must not disarm the fixed safety deadline.
|
|
1860
|
+
@runtime_mutex.synchronize do
|
|
1861
|
+
return if @session_finalized
|
|
1862
|
+
return if @runtime_enforcement_started && @runtime_enforced_pid == @pid
|
|
1863
|
+
|
|
1864
|
+
@runtime_enforcement_started = true
|
|
1865
|
+
@runtime_enforced_pid = @pid
|
|
1866
|
+
@runtime_enforced_at ||= Time.now.utc.iso8601(3)
|
|
1867
|
+
end
|
|
1868
|
+
terminate_runtime_child_async
|
|
1869
|
+
track_auto_stop_thread(Thread.new { stop_for_runtime_budget! })
|
|
1870
|
+
end
|
|
1871
|
+
|
|
1872
|
+
def stop_for_runtime_budget!
|
|
1873
|
+
@runtime_budget&.check!
|
|
1874
|
+
@completion_transition_lock.synchronize do
|
|
1875
|
+
return if @session_finalized
|
|
1876
|
+
|
|
1877
|
+
stop_requested!(reason: "runtime_budget", origin: "runtime")
|
|
1878
|
+
unless task_complete? || task_failed?
|
|
1879
|
+
mark_task_failed(status: "runtime_budget", error: "worker runtime budget expired")
|
|
1880
|
+
end
|
|
1881
|
+
unless @runtime_expiry_recorded
|
|
1882
|
+
@runtime_expiry_recorded = true
|
|
1883
|
+
emit_event("runtime_budget_expired", **runtime_budget_snapshot)
|
|
1884
|
+
end
|
|
1885
|
+
signal_rpc_done! unless @pid
|
|
1886
|
+
end
|
|
1887
|
+
end
|
|
1888
|
+
|
|
1889
|
+
def runtime_budget_stop?
|
|
1890
|
+
@runtime_enforcement_started
|
|
1891
|
+
end
|
|
1892
|
+
|
|
1893
|
+
def runtime_budget_snapshot
|
|
1894
|
+
@runtime_budget&.snapshot&.merge(enforced: !!@runtime_enforcement_started, enforced_at: @runtime_enforced_at)
|
|
1895
|
+
end
|
|
1896
|
+
|
|
1897
|
+
def capture_pty_process_group!
|
|
1898
|
+
@owned_pty_group = @pid if Process.getpgid(@pid) == @pid
|
|
1899
|
+
rescue Errno::ESRCH
|
|
1900
|
+
nil
|
|
1901
|
+
end
|
|
1902
|
+
|
|
1903
|
+
def terminate_runtime_child_async
|
|
1904
|
+
thread = Thread.new do
|
|
1905
|
+
if adapter.respond_to?(:terminate_subprocess)
|
|
1906
|
+
adapter.terminate_subprocess
|
|
1907
|
+
elsif (child_pid = @pid)
|
|
1908
|
+
target = @owned_pty_group ? -@owned_pty_group : child_pid
|
|
1909
|
+
Process.kill("TERM", target)
|
|
1910
|
+
sleep Adapters::Pi::STOP_TERM_GRACE_SECONDS
|
|
1911
|
+
Process.kill("KILL", target) if Harnex.alive_pid?(target)
|
|
1912
|
+
end
|
|
1913
|
+
rescue Errno::ESRCH
|
|
1914
|
+
nil
|
|
1915
|
+
rescue StandardError => e
|
|
1916
|
+
warn("harnex: runtime termination failed (#{e.class})")
|
|
1917
|
+
end
|
|
1918
|
+
track_auto_stop_thread(thread)
|
|
1919
|
+
end
|
|
1920
|
+
|
|
1921
|
+
def normalize_runtime_exit_code!
|
|
1922
|
+
return unless runtime_budget_stop?
|
|
1562
1923
|
|
|
1563
|
-
|
|
1564
|
-
|
|
1924
|
+
if task_complete? && !task_failed? && @stop_request.to_h[:work_state] == "completed"
|
|
1925
|
+
if @exit_code == 0 || (@exit_code == 143 && [nil, 15].include?(@term_signal))
|
|
1926
|
+
@exit_code = 0
|
|
1927
|
+
@term_signal = nil
|
|
1928
|
+
end
|
|
1929
|
+
else
|
|
1930
|
+
@exit_code = 124
|
|
1931
|
+
@term_signal = nil
|
|
1565
1932
|
end
|
|
1566
1933
|
end
|
|
1567
1934
|
|
|
@@ -1601,7 +1968,7 @@ module Harnex
|
|
|
1601
1968
|
|
|
1602
1969
|
thread = Thread.new do
|
|
1603
1970
|
begin
|
|
1604
|
-
inject_stop(turn_id: turn_id, interrupt: interrupt)
|
|
1971
|
+
inject_stop(turn_id: turn_id, interrupt: interrupt, reason: "completion", origin: "auto_stop")
|
|
1605
1972
|
rescue StandardError => e
|
|
1606
1973
|
warn("harnex: auto-stop failed after #{reason}: #{e.message}")
|
|
1607
1974
|
end
|
|
@@ -1614,8 +1981,6 @@ module Harnex
|
|
|
1614
1981
|
end
|
|
1615
1982
|
|
|
1616
1983
|
def drain_auto_stop_threads
|
|
1617
|
-
return unless @auto_stop
|
|
1618
|
-
|
|
1619
1984
|
threads = @auto_stop_mutex.synchronize { @auto_stop_threads.dup }
|
|
1620
1985
|
return if threads.empty?
|
|
1621
1986
|
|
|
@@ -1633,7 +1998,7 @@ module Harnex
|
|
|
1633
1998
|
timed_out.each(&:kill)
|
|
1634
1999
|
@exit_code = 1 if @exit_code.nil? || @exit_code.zero?
|
|
1635
2000
|
@term_signal = nil if @exit_code == 1
|
|
1636
|
-
emit_event("auto_stop_teardown_timeout", grace_seconds: grace_seconds, threads: timed_out.size)
|
|
2001
|
+
emit_event(@auto_stop ? "auto_stop_teardown_timeout" : "stop_teardown_timeout", grace_seconds: grace_seconds, threads: timed_out.size)
|
|
1637
2002
|
end
|
|
1638
2003
|
|
|
1639
2004
|
def auto_stop_teardown_grace_seconds
|
|
@@ -1697,7 +2062,11 @@ module Harnex
|
|
|
1697
2062
|
turn: receipt_turn_payload(successful),
|
|
1698
2063
|
usage: build_summary_usage,
|
|
1699
2064
|
claims: @artifact_claims,
|
|
1700
|
-
command_observation: command_observation_status
|
|
2065
|
+
command_observation: command_observation_status,
|
|
2066
|
+
commands_unobserved: unobserved_command_count,
|
|
2067
|
+
stop: @stop_request&.to_h,
|
|
2068
|
+
final_gates: final_gates_snapshot,
|
|
2069
|
+
commands_had_failures: commands_had_failures?
|
|
1701
2070
|
)
|
|
1702
2071
|
@artifact_report_generated = true
|
|
1703
2072
|
@artifact_report_write_error = nil
|
|
@@ -1707,7 +2076,7 @@ module Harnex
|
|
|
1707
2076
|
def refresh_git_end_snapshot
|
|
1708
2077
|
return @git_end = {} if @git_start[:sha].to_s.empty?
|
|
1709
2078
|
|
|
1710
|
-
snapshot = Harnex.git_capture_end(
|
|
2079
|
+
snapshot = Harnex.git_capture_end(git_observation_root, @git_start)
|
|
1711
2080
|
@git_end = snapshot unless snapshot.empty?
|
|
1712
2081
|
@git_end
|
|
1713
2082
|
end
|
|
@@ -1737,7 +2106,21 @@ module Harnex
|
|
|
1737
2106
|
end
|
|
1738
2107
|
|
|
1739
2108
|
def command_observation_status
|
|
1740
|
-
adapter.transport
|
|
2109
|
+
case adapter.transport
|
|
2110
|
+
when :stdio_jsonl_rpc
|
|
2111
|
+
if unobserved_command_count.zero?
|
|
2112
|
+
"observed"
|
|
2113
|
+
else
|
|
2114
|
+
observed_commands_snapshot.empty? ? "unsupported" : "partial"
|
|
2115
|
+
end
|
|
2116
|
+
else "unsupported"
|
|
2117
|
+
end
|
|
2118
|
+
end
|
|
2119
|
+
|
|
2120
|
+
# Pi command-tool calls that ended without structured exit evidence, plus
|
|
2121
|
+
# any still in flight when the receipt is written.
|
|
2122
|
+
def unobserved_command_count
|
|
2123
|
+
@observed_commands_mutex.synchronize { @pi_unobserved_commands + @pi_pending_commands.length }
|
|
1741
2124
|
end
|
|
1742
2125
|
|
|
1743
2126
|
def receipt_successful?
|
|
@@ -1821,7 +2204,26 @@ module Harnex
|
|
|
1821
2204
|
@term_signal = nil if @exit_code == 1
|
|
1822
2205
|
end
|
|
1823
2206
|
|
|
2207
|
+
def normalize_pi_stop_exit_code!
|
|
2208
|
+
if @pi_idle_stop && task_complete? && !@pi_transport_failed
|
|
2209
|
+
# Pi may handle TERM itself and call process.exit(143), in which case
|
|
2210
|
+
# wait reports a normal exit with no signal. The idle-stop latch, not
|
|
2211
|
+
# exit 143 alone, is the authority to preserve the settled proof.
|
|
2212
|
+
if @exit_code == 0 || (@exit_code == 143 && [nil, 15].include?(@term_signal))
|
|
2213
|
+
@exit_code = 0
|
|
2214
|
+
@term_signal = nil
|
|
2215
|
+
else
|
|
2216
|
+
mark_task_failed(status: "process_exit", error: "Pi idle cleanup exited unexpectedly")
|
|
2217
|
+
end
|
|
2218
|
+
end
|
|
2219
|
+
if task_failed?
|
|
2220
|
+
@exit_code = 1 if @exit_code.nil? || @exit_code.zero? || @term_signal
|
|
2221
|
+
@term_signal = nil if @exit_code == 1
|
|
2222
|
+
end
|
|
2223
|
+
end
|
|
2224
|
+
|
|
1824
2225
|
def normalize_auto_stop_exit_code!
|
|
2226
|
+
return normalize_pi_stop_exit_code! if adapter.transport == :stdio_jsonl_rpc
|
|
1825
2227
|
return unless @auto_stop
|
|
1826
2228
|
return unless @auto_stop_fired
|
|
1827
2229
|
|
|
@@ -1851,6 +2253,7 @@ module Harnex
|
|
|
1851
2253
|
end
|
|
1852
2254
|
|
|
1853
2255
|
def boot_failure_exit?
|
|
2256
|
+
return false if @stop_requested
|
|
1854
2257
|
return false unless structured_transport?
|
|
1855
2258
|
return false if @turn_started_seen
|
|
1856
2259
|
|
|
@@ -1876,7 +2279,10 @@ module Harnex
|
|
|
1876
2279
|
attribution: attribution,
|
|
1877
2280
|
outcome: outcome,
|
|
1878
2281
|
attempt: build_summary_attempt,
|
|
1879
|
-
reliability: build_summary_reliability
|
|
2282
|
+
reliability: build_summary_reliability,
|
|
2283
|
+
activity: @activity.snapshot,
|
|
2284
|
+
stop: @stop_request&.to_h,
|
|
2285
|
+
runtime_budget: runtime_budget_snapshot
|
|
1880
2286
|
}
|
|
1881
2287
|
queue = build_summary_queue
|
|
1882
2288
|
record[:queue] = queue if queue
|
|
@@ -1912,6 +2318,8 @@ module Harnex
|
|
|
1912
2318
|
issue: passthrough["issue"],
|
|
1913
2319
|
plan: passthrough["plan"],
|
|
1914
2320
|
task_brief: passthrough["task_brief"],
|
|
2321
|
+
context_file: passthrough["context_file"],
|
|
2322
|
+
context_sha256: passthrough["context_sha256"],
|
|
1915
2323
|
repo: repo_root,
|
|
1916
2324
|
branch: @git_start[:branch],
|
|
1917
2325
|
start_sha: @git_start[:sha],
|
|
@@ -2073,8 +2481,12 @@ module Harnex
|
|
|
2073
2481
|
|
|
2074
2482
|
def build_summary_outcome(artifact_payload)
|
|
2075
2483
|
receipt_outcome = artifact_payload&.dig("outcome") || {}
|
|
2076
|
-
|
|
2077
|
-
|
|
2484
|
+
base_class, report_status = summary_proof_classification
|
|
2485
|
+
outcome_class = refine_terminal_outcome_class(base_class)
|
|
2486
|
+
# Status/source derive from the proof verdict, not the refined label.
|
|
2487
|
+
proof_failure = %w[
|
|
2488
|
+
completed_no_activity report_missing report_invalid report_rejected task_failed provider_capacity
|
|
2489
|
+
].include?(base_class)
|
|
2078
2490
|
harness_receipt = artifact_payload&.dig("receipt", "author") == "harnex"
|
|
2079
2491
|
status = receipt_outcome["status"]
|
|
2080
2492
|
status = nil if status == "accepted" && proof_failure
|
|
@@ -2122,6 +2534,36 @@ module Harnex
|
|
|
2122
2534
|
["unknown", @completion_report_status]
|
|
2123
2535
|
end
|
|
2124
2536
|
|
|
2537
|
+
# Only an unknown or failed terminal is refined; completions and proof
|
|
2538
|
+
# verdicts stand. A stop or deadline outranks task_failed only when the
|
|
2539
|
+
# work was still running at the stop: a failure that preceded the stop
|
|
2540
|
+
# is the real cause. Anything unrecognized stays unknown.
|
|
2541
|
+
def refine_terminal_outcome_class(base)
|
|
2542
|
+
base = "unknown" unless OUTCOME_CLASSES.include?(base)
|
|
2543
|
+
# Finalization always writes a receipt, so a process that ended without
|
|
2544
|
+
# any completion signal reaches here as report_rejected. That is
|
|
2545
|
+
# Harnex's own record of an unsuccessful process, not a verdict on
|
|
2546
|
+
# finished work: classify the process instead.
|
|
2547
|
+
base = "unknown" if base == "report_rejected" && receipt_rejection_without_completion?
|
|
2548
|
+
return base unless %w[unknown task_failed].include?(base)
|
|
2549
|
+
|
|
2550
|
+
stop = @stop_request&.to_h || {}
|
|
2551
|
+
stopped_while_running = stop[:work_state] == "running"
|
|
2552
|
+
return "timeout" if stop[:reason] == "runtime_budget" && stopped_while_running
|
|
2553
|
+
return "timeout" if @exit_code == 124 && base == "unknown"
|
|
2554
|
+
return "stopped_by_caller" if stop[:reason] == "manual" && stopped_while_running
|
|
2555
|
+
return base if base == "task_failed"
|
|
2556
|
+
return "boot_failure" if @exit_reason == "boot_failure"
|
|
2557
|
+
return "permission_prompt" if @state_machine&.state == :blocked
|
|
2558
|
+
return "process_failed" if @exit_reason == "failure"
|
|
2559
|
+
|
|
2560
|
+
"unknown"
|
|
2561
|
+
end
|
|
2562
|
+
|
|
2563
|
+
def receipt_rejection_without_completion?
|
|
2564
|
+
@completion_outcome_class.nil? && !task_complete? && !task_failed?
|
|
2565
|
+
end
|
|
2566
|
+
|
|
2125
2567
|
def build_summary_attempt
|
|
2126
2568
|
{
|
|
2127
2569
|
"run_id" => id,
|
|
@@ -2218,8 +2660,7 @@ module Harnex
|
|
|
2218
2660
|
|
|
2219
2661
|
records = {}
|
|
2220
2662
|
duplicates = {}
|
|
2221
|
-
|
|
2222
|
-
record = JSON.parse(line)
|
|
2663
|
+
DispatchHistory.each_record(path) do |record|
|
|
2223
2664
|
next unless DispatchHistory.end_record?(record)
|
|
2224
2665
|
|
|
2225
2666
|
dispatch_id = summary_string(record["id"])
|
|
@@ -2231,8 +2672,6 @@ module Harnex
|
|
|
2231
2672
|
elsif !duplicates.key?(dispatch_id)
|
|
2232
2673
|
records[dispatch_id] = record
|
|
2233
2674
|
end
|
|
2234
|
-
rescue JSON::ParserError
|
|
2235
|
-
next
|
|
2236
2675
|
end
|
|
2237
2676
|
records
|
|
2238
2677
|
rescue StandardError
|
|
@@ -2341,7 +2780,8 @@ module Harnex
|
|
|
2341
2780
|
turn_count: @injected_count,
|
|
2342
2781
|
tool_calls: summary_tool_calls(counters),
|
|
2343
2782
|
commands_executed: counters[:commands_executed],
|
|
2344
|
-
|
|
2783
|
+
# No adapter reports provider rate-limit payloads; kept for the schema.
|
|
2784
|
+
rate_limits: nil,
|
|
2345
2785
|
output_lines: output_measurements[:lines],
|
|
2346
2786
|
output_bytes: output_measurements[:bytes],
|
|
2347
2787
|
event_records: @events_log_seq,
|
|
@@ -2394,7 +2834,6 @@ module Harnex
|
|
|
2394
2834
|
|
|
2395
2835
|
def summary_agent_session_id
|
|
2396
2836
|
@usage_summary[:agent_session_id] ||
|
|
2397
|
-
@rpc_thread_id ||
|
|
2398
2837
|
(adapter.thread_id if adapter.respond_to?(:thread_id))
|
|
2399
2838
|
end
|
|
2400
2839
|
|
|
@@ -2409,28 +2848,7 @@ module Harnex
|
|
|
2409
2848
|
end
|
|
2410
2849
|
|
|
2411
2850
|
def summary_service_tier
|
|
2412
|
-
summary_string(meta_hash["service_tier"])
|
|
2413
|
-
end
|
|
2414
|
-
|
|
2415
|
-
def service_tier_from_command
|
|
2416
|
-
command.each_with_index do |arg, index|
|
|
2417
|
-
text = arg.to_s
|
|
2418
|
-
if text == "-c" || text == "--config"
|
|
2419
|
-
parsed = parse_service_tier_config(command[index + 1])
|
|
2420
|
-
return parsed if parsed
|
|
2421
|
-
end
|
|
2422
|
-
parsed = parse_service_tier_config(text)
|
|
2423
|
-
return parsed if parsed
|
|
2424
|
-
end
|
|
2425
|
-
nil
|
|
2426
|
-
end
|
|
2427
|
-
|
|
2428
|
-
def parse_service_tier_config(value)
|
|
2429
|
-
text = value.to_s
|
|
2430
|
-
match = text.match(/(?:\A|=)service_tier=\\?"?([^"\s]+)\\?"?/)
|
|
2431
|
-
return nil unless match
|
|
2432
|
-
|
|
2433
|
-
summary_string(match[1])
|
|
2851
|
+
summary_string(meta_hash["service_tier"])
|
|
2434
2852
|
end
|
|
2435
2853
|
|
|
2436
2854
|
def legacy_summary_event_counters
|
|
@@ -2493,6 +2911,9 @@ module Harnex
|
|
|
2493
2911
|
end
|
|
2494
2912
|
|
|
2495
2913
|
def append_dispatch_start_record
|
|
2914
|
+
return if @dispatch_start_recorded
|
|
2915
|
+
|
|
2916
|
+
@dispatch_start_recorded = true
|
|
2496
2917
|
DispatchHistory.append(dispatch_history_path, DispatchHistory.build_start_record(self))
|
|
2497
2918
|
rescue StandardError => e
|
|
2498
2919
|
warn("harnex: failed to write dispatch start record: #{e.message}")
|
|
@@ -2530,49 +2951,16 @@ module Harnex
|
|
|
2530
2951
|
end
|
|
2531
2952
|
end
|
|
2532
2953
|
|
|
2533
|
-
#
|
|
2534
|
-
#
|
|
2954
|
+
# Pi RPC reports usage directly (session stats); PTY adapters parse
|
|
2955
|
+
# transcript tails when supported.
|
|
2535
2956
|
def collect_session_summary
|
|
2536
|
-
if adapter.
|
|
2537
|
-
summary_from_token_usage
|
|
2538
|
-
elsif adapter.respond_to?(:collect_session_summary)
|
|
2957
|
+
if adapter.respond_to?(:collect_session_summary)
|
|
2539
2958
|
adapter.collect_session_summary
|
|
2540
2959
|
else
|
|
2541
2960
|
adapter.parse_session_summary(transcript_tail)
|
|
2542
2961
|
end
|
|
2543
2962
|
end
|
|
2544
2963
|
|
|
2545
|
-
def summary_from_token_usage
|
|
2546
|
-
summary = {}
|
|
2547
|
-
if @token_usage.is_a?(Hash) && @token_usage["total"].is_a?(Hash)
|
|
2548
|
-
total = @token_usage["total"]
|
|
2549
|
-
summary.merge!(
|
|
2550
|
-
input_tokens: total["inputTokens"],
|
|
2551
|
-
output_tokens: total["outputTokens"],
|
|
2552
|
-
reasoning_tokens: total["reasoningOutputTokens"],
|
|
2553
|
-
cached_tokens: total["cachedInputTokens"],
|
|
2554
|
-
total_tokens: total["totalTokens"]
|
|
2555
|
-
)
|
|
2556
|
-
end
|
|
2557
|
-
summary[:context] = @rpc_context_telemetry.snapshot if @rpc_context_telemetry
|
|
2558
|
-
summary
|
|
2559
|
-
end
|
|
2560
|
-
|
|
2561
|
-
def record_rpc_context_sample(token_usage)
|
|
2562
|
-
return unless adapter.context_telemetry_supported?
|
|
2563
|
-
|
|
2564
|
-
@rpc_context_telemetry ||= ContextTelemetry.new(
|
|
2565
|
-
status: "estimated",
|
|
2566
|
-
source: adapter.context_telemetry_source
|
|
2567
|
-
)
|
|
2568
|
-
last = token_usage["last"]
|
|
2569
|
-
last = {} unless last.is_a?(Hash)
|
|
2570
|
-
@rpc_context_telemetry.record(
|
|
2571
|
-
tokens: last["totalTokens"],
|
|
2572
|
-
window_tokens: token_usage["modelContextWindow"]
|
|
2573
|
-
)
|
|
2574
|
-
end
|
|
2575
|
-
|
|
2576
2964
|
def transcript_tail
|
|
2577
2965
|
return "" unless File.file?(output_log_path)
|
|
2578
2966
|
|
|
@@ -2595,7 +2983,9 @@ module Harnex
|
|
|
2595
2983
|
event = {
|
|
2596
2984
|
schema_version: 1,
|
|
2597
2985
|
seq: @events_log_seq,
|
|
2598
|
-
|
|
2986
|
+
# Millisecond precision lets teardown races be measured from the
|
|
2987
|
+
# events log (#36/#37); readers parse it with Time.iso8601.
|
|
2988
|
+
ts: Time.now.utc.iso8601(3),
|
|
2599
2989
|
id: id,
|
|
2600
2990
|
type: type
|
|
2601
2991
|
}.merge(payload)
|