harnex 0.12.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +592 -0
- data/GUIDE.md +222 -68
- data/README.md +78 -48
- data/TECHNICAL.md +235 -98
- data/agents/gpt.md +60 -0
- data/docs/agents.md +119 -0
- data/docs/dispatch-telemetry.md +184 -50
- data/docs/events.md +20 -7
- data/docs/lanes.md +202 -0
- data/docs/mcp.md +102 -0
- data/docs/pi-rpc.md +81 -7
- data/docs/upgrading-to-0.29.md +87 -0
- data/guides/01_dispatch.md +101 -34
- data/guides/04_monitoring.md +223 -25
- data/guides/05_naming.md +0 -1
- data/guides/06_claude.md +96 -0
- data/lib/harnex/adapters/base.rb +4 -8
- data/lib/harnex/adapters/pi.rb +92 -12
- data/lib/harnex/adapters.rb +1 -16
- data/lib/harnex/artifact_report.rb +206 -12
- data/lib/harnex/cli.rb +68 -50
- data/lib/harnex/commands/ask.rb +278 -0
- data/lib/harnex/commands/delegate.rb +202 -0
- data/lib/harnex/commands/doctor.rb +34 -36
- data/lib/harnex/commands/events.rb +3 -3
- data/lib/harnex/commands/history.rb +2 -8
- data/lib/harnex/commands/land.rb +150 -0
- data/lib/harnex/commands/lanes.rb +84 -0
- data/lib/harnex/commands/logs.rb +3 -3
- data/lib/harnex/commands/mcp.rb +208 -0
- data/lib/harnex/commands/orchestration.rb +88 -9
- data/lib/harnex/commands/pane.rb +3 -3
- data/lib/harnex/commands/preflight.rb +566 -0
- data/lib/harnex/commands/reply.rb +125 -0
- data/lib/harnex/commands/result.rb +108 -0
- data/lib/harnex/commands/run.rb +248 -64
- data/lib/harnex/commands/send.rb +121 -16
- data/lib/harnex/commands/status.rb +32 -4
- data/lib/harnex/commands/stop.rb +14 -4
- data/lib/harnex/commands/telemetry.rb +19 -1
- data/lib/harnex/commands/wait.rb +421 -56
- data/lib/harnex/commands/watch.rb +134 -21
- data/lib/harnex/context_file.rb +85 -0
- data/lib/harnex/core.rb +100 -15
- data/lib/harnex/dispatch_history.rb +157 -10
- data/lib/harnex/lanes.rb +513 -0
- data/lib/harnex/mcp.rb +362 -0
- data/lib/harnex/orchestration.rb +128 -5
- data/lib/harnex/owned_process_group.rb +50 -0
- data/lib/harnex/owner_stream.rb +399 -0
- data/lib/harnex/pricing.rb +4 -9
- data/lib/harnex/runtime/activity_tracker.rb +107 -0
- data/lib/harnex/runtime/api_server.rb +11 -1
- data/lib/harnex/runtime/completion_notifier.rb +90 -11
- data/lib/harnex/runtime/inbox.rb +26 -7
- data/lib/harnex/runtime/message.rb +5 -2
- data/lib/harnex/runtime/runtime_budget.rb +96 -0
- data/lib/harnex/runtime/session.rb +684 -294
- data/lib/harnex/runtime/stop_request.rb +39 -0
- data/lib/harnex/stall.rb +46 -0
- data/lib/harnex/telemetry_reconciler.rb +42 -20
- data/lib/harnex/terminal_status.rb +5 -5
- data/lib/harnex/version.rb +2 -2
- data/lib/harnex.rb +16 -0
- data/recipes/01_fire_and_watch.md +14 -14
- data/recipes/02_chain_implement.md +35 -31
- data/recipes/03_buddy.md +1 -1
- metadata +34 -12
- data/docs/codex-appserver.md +0 -175
- data/lib/harnex/adapters/codex.rb +0 -160
- data/lib/harnex/adapters/codex_appserver.rb +0 -407
- data/lib/harnex/codex/app_server/client.rb +0 -347
|
@@ -0,0 +1,566 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "net/http"
|
|
3
|
+
require "open3"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "shellwords"
|
|
6
|
+
require "tmpdir"
|
|
7
|
+
require "uri"
|
|
8
|
+
|
|
9
|
+
module Harnex
|
|
10
|
+
# `harnex preflight ADAPTER`: one tiny end-to-end dispatch that proves an
|
|
11
|
+
# adapter can work in a named cwd before an unattended run launches (#56).
|
|
12
|
+
# It drives the real `harnex run` path in a child process and judges the
|
|
13
|
+
# result from harness-observed evidence: the registry, the files the worker
|
|
14
|
+
# had to write, and the dispatch row's receipt fields. Never worker prose.
|
|
15
|
+
class Preflight
|
|
16
|
+
SCHEMA = "harnex.preflight.v1"
|
|
17
|
+
PHASE = "preflight"
|
|
18
|
+
# A worker never outlives its probe by more than this, even if preflight
|
|
19
|
+
# itself is killed: `--max-runtime` is enforced by the session.
|
|
20
|
+
RUNTIME_BACKSTOP_S = 60.0
|
|
21
|
+
DEFAULT_TIMEOUT_S = 180.0
|
|
22
|
+
DEFAULT_REGISTRATION_TIMEOUT_S = 30.0
|
|
23
|
+
PERMISSION_GRACE_S = 15.0
|
|
24
|
+
POLL_INTERVAL_S = 0.5
|
|
25
|
+
ROW_GRACE_S = 10.0
|
|
26
|
+
STATUSES = %w[pass fail unsupported skipped].freeze
|
|
27
|
+
CHECKS = %w[
|
|
28
|
+
registration boot permission task_read workspace_write command_exit
|
|
29
|
+
receipt claims completion stop
|
|
30
|
+
].freeze
|
|
31
|
+
# The first failing check names the verdict's failure class; process
|
|
32
|
+
# classes reuse the #57 outcome vocabulary where one applies.
|
|
33
|
+
FAILURE_CLASSES = {
|
|
34
|
+
"registration" => "registration_timeout",
|
|
35
|
+
"launch" => "launch_failed",
|
|
36
|
+
"boot" => "boot_failure",
|
|
37
|
+
"permission" => "permission_prompt",
|
|
38
|
+
"task_read" => "task_read_denied",
|
|
39
|
+
"workspace_write" => "workspace_write_denied",
|
|
40
|
+
"command_exit" => "command_unobserved",
|
|
41
|
+
"receipt" => "receipt_missing",
|
|
42
|
+
"claims" => "claims_missing",
|
|
43
|
+
"completion" => "completion_missing",
|
|
44
|
+
"stop" => "timeout",
|
|
45
|
+
"commit" => "commit_failed"
|
|
46
|
+
}.freeze
|
|
47
|
+
|
|
48
|
+
def self.usage(program_name = "harnex preflight")
|
|
49
|
+
<<~TEXT
|
|
50
|
+
Usage: #{program_name} ADAPTER [options] [--] [cli-args...]
|
|
51
|
+
|
|
52
|
+
Runs one tiny end-to-end dispatch through `harnex run ADAPTER` and prints
|
|
53
|
+
a JSON verdict. Exits 0 only when every required check passes.
|
|
54
|
+
|
|
55
|
+
Options:
|
|
56
|
+
--cwd DIR Workspace to dispatch in (default: current directory)
|
|
57
|
+
--task-root DIR Where the pinned task file is written (default: #{Dir.tmpdir})
|
|
58
|
+
--commit-check Also prove a commit in a throwaway scratch repo
|
|
59
|
+
(never the caller's repo)
|
|
60
|
+
--timeout DUR Per-dispatch deadline (default: #{DEFAULT_TIMEOUT_S.to_i}s)
|
|
61
|
+
--registration-timeout DUR
|
|
62
|
+
Registration deadline (default: #{DEFAULT_REGISTRATION_TIMEOUT_S.to_i}s)
|
|
63
|
+
--model NAME Passed to `harnex run --model`
|
|
64
|
+
--effort LEVEL Passed to `harnex run --effort`
|
|
65
|
+
--id ID Dispatch id (default: pf-<adapter>-<random>)
|
|
66
|
+
-h, --help Show this help
|
|
67
|
+
|
|
68
|
+
Checks: #{CHECKS.first(5).join(', ')},
|
|
69
|
+
#{CHECKS.drop(5).join(', ')}, plus commit with --commit-check.
|
|
70
|
+
Each is pass, fail, unsupported (the adapter cannot expose that evidence,
|
|
71
|
+
e.g. PTY command exits), or skipped. The verdict also reports usage, cost
|
|
72
|
+
and context telemetry provenance for the adapter.
|
|
73
|
+
|
|
74
|
+
Common patterns:
|
|
75
|
+
#{program_name} pi --cwd ~/repo -- --no-approve
|
|
76
|
+
#{program_name} pi --cwd ~/repo --commit-check --model openai/gpt-5.1 -- --no-approve
|
|
77
|
+
#{program_name} pi --cwd ~/repo && harnex run pi --cwd ~/repo --context "..."
|
|
78
|
+
|
|
79
|
+
Gotchas:
|
|
80
|
+
Preflight spends one real model turn (two with --commit-check).
|
|
81
|
+
Its dispatch row lands in history with phase=preflight and is excluded
|
|
82
|
+
from orchestration report rollups unless --include-preflight is passed.
|
|
83
|
+
Run it for the chosen adapter and one declared fallback before an
|
|
84
|
+
unattended queue; both must sit inside your dispatch-model policy.
|
|
85
|
+
It proves plumbing (boot, files, commands, receipts), not model quality.
|
|
86
|
+
TEXT
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def initialize(argv, out: $stdout, err: $stderr, harnex_command: nil)
|
|
90
|
+
@argv = argv.dup
|
|
91
|
+
@out = out
|
|
92
|
+
@err = err
|
|
93
|
+
@harnex_command = harnex_command || [Gem.ruby, File.expand_path("../../../bin/harnex", __dir__)]
|
|
94
|
+
@options = {
|
|
95
|
+
cwd: nil,
|
|
96
|
+
task_root: nil,
|
|
97
|
+
commit_check: false,
|
|
98
|
+
timeout_s: DEFAULT_TIMEOUT_S,
|
|
99
|
+
registration_timeout_s: DEFAULT_REGISTRATION_TIMEOUT_S,
|
|
100
|
+
model: nil,
|
|
101
|
+
effort: nil,
|
|
102
|
+
id: nil,
|
|
103
|
+
help: false
|
|
104
|
+
}
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def run
|
|
108
|
+
adapter, child_args = parse(@argv)
|
|
109
|
+
if @options[:help]
|
|
110
|
+
@out.puts self.class.usage
|
|
111
|
+
return 0
|
|
112
|
+
end
|
|
113
|
+
raise OptionParser::MissingArgument, "ADAPTER" if adapter.nil?
|
|
114
|
+
|
|
115
|
+
verdict = execute(adapter, child_args)
|
|
116
|
+
@out.puts JSON.generate(verdict)
|
|
117
|
+
verdict["ok"] ? 0 : 1
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
private
|
|
121
|
+
|
|
122
|
+
def execute(adapter, child_args)
|
|
123
|
+
started = monotonic
|
|
124
|
+
cwd = File.expand_path(@options[:cwd] || Dir.pwd)
|
|
125
|
+
raise OptionParser::InvalidArgument, "--cwd #{cwd} is not a directory" unless File.directory?(cwd)
|
|
126
|
+
|
|
127
|
+
task_root = File.expand_path(@options[:task_root] || Dir.tmpdir)
|
|
128
|
+
id = Harnex.normalize_id(@options[:id] || "pf-#{adapter.gsub(/[^a-z0-9]+/i, '-')}-#{SecureRandom.hex(3)}")
|
|
129
|
+
probe = run_probe(adapter, child_args, id: id, cwd: cwd, task_root: task_root, commit: false)
|
|
130
|
+
checks = probe.fetch(:checks)
|
|
131
|
+
if @options[:commit_check]
|
|
132
|
+
checks << if checks.any? { |check| check["status"] == "fail" }
|
|
133
|
+
check("commit", "skipped", "workspace probe failed; commit probe not attempted")
|
|
134
|
+
else
|
|
135
|
+
commit_check(adapter, child_args, id: "#{id}-commit", task_root: task_root)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
failed = checks.find { |item| item["status"] == "fail" }
|
|
140
|
+
failure = failed && FAILURE_CLASSES.fetch(failed["name"], failed["name"])
|
|
141
|
+
failure = "launch_failed" if failed && failed["name"] == "registration" && !probe[:registration_timeout]
|
|
142
|
+
# A failed or missing completion is named by the #57 outcome class.
|
|
143
|
+
failure = probe.dig(:row, "outcome", "class") || failure if failed && failed["name"] == "completion"
|
|
144
|
+
{
|
|
145
|
+
"schema" => SCHEMA,
|
|
146
|
+
"ok" => failed.nil?,
|
|
147
|
+
"failure" => failure,
|
|
148
|
+
"adapter" => adapter,
|
|
149
|
+
"id" => id,
|
|
150
|
+
"cwd" => cwd,
|
|
151
|
+
"task_root" => task_root,
|
|
152
|
+
"checks" => checks,
|
|
153
|
+
"outcome_class" => probe.dig(:row, "outcome", "class"),
|
|
154
|
+
"telemetry" => telemetry_capability(probe[:row]),
|
|
155
|
+
"dispatch_history_path" => Harnex::DispatchHistory.path_for(cwd),
|
|
156
|
+
"duration_s" => (monotonic - started).round(1)
|
|
157
|
+
}
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def commit_check(adapter, child_args, id:, task_root:)
|
|
161
|
+
scratch = Dir.mktmpdir("harnex-preflight-commit")
|
|
162
|
+
git(scratch, "init", "-q")
|
|
163
|
+
git(scratch, "-c", "user.name=harnex", "-c", "user.email=harnex@localhost",
|
|
164
|
+
"commit", "-q", "--allow-empty", "-m", "harnex preflight base")
|
|
165
|
+
base = git(scratch, "rev-parse", "HEAD")
|
|
166
|
+
probe = run_probe(adapter, child_args, id: id, cwd: scratch, task_root: task_root, commit: true)
|
|
167
|
+
head = git(scratch, "rev-parse", "HEAD")
|
|
168
|
+
committed = git(scratch, "show", "#{head}:preflight.txt").strip if head != base
|
|
169
|
+
if head == base
|
|
170
|
+
failed = probe[:checks].find { |item| item["status"] == "fail" }
|
|
171
|
+
check("commit", "fail", "no commit in scratch repo#{failed ? " (#{failed['name']}: #{failed['detail']})" : ''}")
|
|
172
|
+
elsif committed == probe[:token]
|
|
173
|
+
check("commit", "pass", "scratch repo advanced to #{head[0, 12]}")
|
|
174
|
+
else
|
|
175
|
+
check("commit", "fail", "commit #{head[0, 12]} does not carry the task token")
|
|
176
|
+
end
|
|
177
|
+
rescue StandardError => e
|
|
178
|
+
check("commit", "fail", "commit probe error: #{e.class}: #{e.message}")
|
|
179
|
+
ensure
|
|
180
|
+
FileUtils.rm_rf(scratch) if scratch
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def run_probe(adapter, child_args, id:, cwd:, task_root:, commit:)
|
|
184
|
+
token = "harnex-preflight-token-#{SecureRandom.hex(6)}"
|
|
185
|
+
gate = "echo harnex-preflight-#{SecureRandom.hex(4)}"
|
|
186
|
+
task_path = File.join(task_root, "harnex-preflight-#{id}.md")
|
|
187
|
+
out_path = commit ? File.join(cwd, "preflight.txt") : File.join(cwd, ".harnex", "preflight", "#{id}.txt")
|
|
188
|
+
harnex_dir = File.join(cwd, ".harnex")
|
|
189
|
+
harnex_dir_existed = File.directory?(harnex_dir)
|
|
190
|
+
FileUtils.mkdir_p(task_root)
|
|
191
|
+
File.write(task_path, "#{token}\n")
|
|
192
|
+
|
|
193
|
+
launch = launch(adapter, child_args, id: id, cwd: cwd,
|
|
194
|
+
prompt: prompt(id, task_path, out_path, gate, commit: commit), gate: gate)
|
|
195
|
+
watch = if launch[:ok]
|
|
196
|
+
watch_session(id, cwd)
|
|
197
|
+
else
|
|
198
|
+
# A late registration must not leave an orphan worker behind.
|
|
199
|
+
stop_session(id, cwd) if launch[:launched]
|
|
200
|
+
{}
|
|
201
|
+
end
|
|
202
|
+
row = launch[:ok] || launch[:launched] ? wait_for_row(id, cwd) : nil
|
|
203
|
+
checks = evaluate(launch, watch, row, token: token, out_path: out_path)
|
|
204
|
+
{ checks: checks, row: row, token: token, registration_timeout: launch[:launched] }
|
|
205
|
+
ensure
|
|
206
|
+
# Interrupts and errors must not orphan the worker; normal paths have
|
|
207
|
+
# already torn it down, so this is a no-op then.
|
|
208
|
+
stop_session(id, cwd) if id && cwd && Harnex.read_registry(cwd, id)
|
|
209
|
+
FileUtils.rm_f(task_path) if task_path
|
|
210
|
+
unless commit || out_path.nil?
|
|
211
|
+
FileUtils.rm_f(out_path)
|
|
212
|
+
# Leave the caller's workspace as found: drop the probe's directories
|
|
213
|
+
# when they are empty, including a .harnex/ the worker had to create.
|
|
214
|
+
[File.dirname(out_path), (harnex_dir unless harnex_dir_existed)].compact.each do |dir|
|
|
215
|
+
Dir.rmdir(dir) if File.directory?(dir) && Dir.empty?(dir)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def prompt(id, task_path, out_path, gate, commit:)
|
|
221
|
+
claims = %(printf '{"claims":{"summary":"preflight #{id} ok","verdict":"pass"}}' > "$HARNEX_ARTIFACT_CLAIMS_PATH")
|
|
222
|
+
steps = [
|
|
223
|
+
"Read the task file: #{task_path}",
|
|
224
|
+
"Write exactly the token from that file, and nothing else, to: #{out_path}",
|
|
225
|
+
"Run this exact shell command: #{gate}"
|
|
226
|
+
]
|
|
227
|
+
steps << "Run this exact shell command: git add preflight.txt && git commit -q -m harnex-preflight" if commit
|
|
228
|
+
steps << "Run this exact shell command: #{claims}"
|
|
229
|
+
steps << "Reply with the single word DONE."
|
|
230
|
+
lines = ["Harnex preflight check #{id}. This is a plumbing test: follow these steps exactly, in order, and do nothing else."]
|
|
231
|
+
steps.each_with_index { |step, index| lines << "#{index + 1}. #{step}" }
|
|
232
|
+
lines.join("\n")
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def launch(adapter, child_args, id:, cwd:, prompt:, gate:)
|
|
236
|
+
argv = @harnex_command + [
|
|
237
|
+
"run", adapter, "--detach", "--id", id, "--cwd", cwd,
|
|
238
|
+
"--description", "harnex preflight",
|
|
239
|
+
"--context", prompt, "--auto-stop", "--final-gate", gate,
|
|
240
|
+
"--phase", PHASE,
|
|
241
|
+
"--timeout", @options[:registration_timeout_s].to_s,
|
|
242
|
+
"--max-runtime", (@options[:registration_timeout_s] + @options[:timeout_s] + RUNTIME_BACKSTOP_S).ceil.to_s
|
|
243
|
+
]
|
|
244
|
+
argv += ["--model", @options[:model]] if @options[:model]
|
|
245
|
+
argv += ["--effort", @options[:effort]] if @options[:effort]
|
|
246
|
+
argv += ["--"] + child_args unless child_args.empty?
|
|
247
|
+
stdout, stderr, status = Open3.capture3(*argv, chdir: cwd)
|
|
248
|
+
payload = begin
|
|
249
|
+
JSON.parse(stdout.lines.last.to_s)
|
|
250
|
+
rescue JSON::ParserError
|
|
251
|
+
{}
|
|
252
|
+
end
|
|
253
|
+
{
|
|
254
|
+
ok: status.success? && payload["ok"] == true,
|
|
255
|
+
# A registration timeout can still leave a live or finished session;
|
|
256
|
+
# anything else (bad flags, repo policy, missing binary) never launched.
|
|
257
|
+
launched: stderr.include?("did not register"),
|
|
258
|
+
payload: payload,
|
|
259
|
+
stderr: stderr.strip.lines.last(3).join.strip,
|
|
260
|
+
pid: payload["pid"]
|
|
261
|
+
}
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Polls until the session's registry disappears (teardown done) or the
|
|
265
|
+
# deadline passes. A prompt the adapter reports as blocking (trust or
|
|
266
|
+
# confirmation) that persists past PERMISSION_GRACE_S fails fast.
|
|
267
|
+
def watch_session(id, cwd)
|
|
268
|
+
deadline = monotonic + @options[:timeout_s]
|
|
269
|
+
blocked_since = nil
|
|
270
|
+
result = { registered: true }
|
|
271
|
+
loop do
|
|
272
|
+
registry = Harnex.read_registry(cwd, id)
|
|
273
|
+
return result.merge(exited: true) unless registry
|
|
274
|
+
|
|
275
|
+
state = live_agent_state(registry)
|
|
276
|
+
if state == "blocked"
|
|
277
|
+
blocked_since ||= monotonic
|
|
278
|
+
if monotonic - blocked_since >= PERMISSION_GRACE_S
|
|
279
|
+
stop_session(id, cwd)
|
|
280
|
+
return result.merge(blocked: true, exited: wait_registry_gone(id, cwd))
|
|
281
|
+
end
|
|
282
|
+
else
|
|
283
|
+
blocked_since = nil
|
|
284
|
+
end
|
|
285
|
+
if monotonic >= deadline
|
|
286
|
+
stop_session(id, cwd)
|
|
287
|
+
return result.merge(timed_out: true, exited: wait_registry_gone(id, cwd))
|
|
288
|
+
end
|
|
289
|
+
sleep POLL_INTERVAL_S
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
def live_agent_state(registry)
|
|
294
|
+
uri = URI("http://#{registry['host']}:#{registry['port']}/status")
|
|
295
|
+
request = Net::HTTP::Get.new(uri)
|
|
296
|
+
request["Authorization"] = "Bearer #{registry['token']}" if registry["token"]
|
|
297
|
+
response = Net::HTTP.start(uri.host, uri.port, open_timeout: 1, read_timeout: 1) { |http| http.request(request) }
|
|
298
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
299
|
+
|
|
300
|
+
JSON.parse(response.body)["agent_state"]
|
|
301
|
+
rescue StandardError
|
|
302
|
+
nil
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# Straight to the session API: `harnex stop --repo` resolves the Git
|
|
306
|
+
# top-level, but `run --cwd DIR` registers under DIR itself.
|
|
307
|
+
def stop_session(id, cwd)
|
|
308
|
+
registry = Harnex.read_registry(cwd, id)
|
|
309
|
+
return unless registry
|
|
310
|
+
|
|
311
|
+
uri = URI("http://#{registry['host']}:#{registry['port']}/stop")
|
|
312
|
+
request = Net::HTTP::Post.new(uri)
|
|
313
|
+
request["Authorization"] = "Bearer #{registry['token']}" if registry["token"]
|
|
314
|
+
request["Content-Type"] = "application/json"
|
|
315
|
+
request.body = JSON.generate(reason: "manual", origin: "cli")
|
|
316
|
+
Net::HTTP.start(uri.host, uri.port, open_timeout: 2, read_timeout: 5) { |http| http.request(request) }
|
|
317
|
+
rescue StandardError
|
|
318
|
+
nil
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
def wait_registry_gone(id, cwd)
|
|
322
|
+
deadline = monotonic + ROW_GRACE_S
|
|
323
|
+
sleep POLL_INTERVAL_S while Harnex.read_registry(cwd, id) && monotonic < deadline
|
|
324
|
+
Harnex.read_registry(cwd, id).nil?
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
def wait_for_row(id, cwd)
|
|
328
|
+
path = Harnex::DispatchHistory.path_for(cwd)
|
|
329
|
+
deadline = monotonic + ROW_GRACE_S
|
|
330
|
+
loop do
|
|
331
|
+
row = last_end_row(path, id)
|
|
332
|
+
return row if row || monotonic >= deadline
|
|
333
|
+
|
|
334
|
+
sleep POLL_INTERVAL_S
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def last_end_row(path, id)
|
|
339
|
+
return nil unless File.file?(path)
|
|
340
|
+
|
|
341
|
+
File.readlines(path).reverse_each do |line|
|
|
342
|
+
row = JSON.parse(line)
|
|
343
|
+
return row if row.is_a?(Hash) && row["record_type"] == "dispatch_end" && row["id"] == id
|
|
344
|
+
rescue JSON::ParserError
|
|
345
|
+
next
|
|
346
|
+
end
|
|
347
|
+
nil
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
def evaluate(launch, watch, row, token:, out_path:)
|
|
351
|
+
checks = []
|
|
352
|
+
outcome_class = row&.dig("outcome", "class")
|
|
353
|
+
unless launch[:ok]
|
|
354
|
+
boot_failed = outcome_class == "boot_failure"
|
|
355
|
+
checks << check("registration", boot_failed ? "skipped" : "fail",
|
|
356
|
+
launch[:stderr].empty? ? "harnex run failed before registration" : launch[:stderr])
|
|
357
|
+
checks << check("boot", boot_failed ? "fail" : "skipped",
|
|
358
|
+
boot_failed ? "session died before its first turn" : "not reached")
|
|
359
|
+
(CHECKS - %w[registration boot]).each { |name| checks << check(name, "skipped", "not reached") }
|
|
360
|
+
return checks
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
checks << check("registration", "pass", "registered as #{launch.dig(:payload, 'mode')} pid #{launch[:pid]}")
|
|
364
|
+
checks << if outcome_class == "boot_failure"
|
|
365
|
+
check("boot", "fail", "session died before its first turn (#{outcome_class || 'no row'})")
|
|
366
|
+
else
|
|
367
|
+
check("boot", "pass", "session alive after registration")
|
|
368
|
+
end
|
|
369
|
+
checks << if watch[:blocked] || outcome_class == "permission_prompt"
|
|
370
|
+
check("permission", "fail", "adapter stayed on a trust/confirmation prompt")
|
|
371
|
+
else
|
|
372
|
+
check("permission", "pass", "no blocking prompt observed")
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
written = File.file?(out_path) ? File.read(out_path).strip : nil
|
|
376
|
+
if watch[:timed_out] || watch[:blocked]
|
|
377
|
+
reason = watch[:blocked] ? "not reached: blocked on a prompt" : "not reached: the dispatch timed out"
|
|
378
|
+
%w[task_read workspace_write command_exit receipt claims completion].each { |name| checks << check(name, "skipped", reason) }
|
|
379
|
+
checks << stop_check(watch, row)
|
|
380
|
+
return checks
|
|
381
|
+
end
|
|
382
|
+
if turn_failed_before_work?(row, written)
|
|
383
|
+
reason = "not reached: the turn failed before doing any work"
|
|
384
|
+
%w[task_read workspace_write command_exit receipt claims].each { |name| checks << check(name, "skipped", reason) }
|
|
385
|
+
checks << check("completion", "fail", "turn failed: #{redact(row.dig('actual', 'last_error')) || row.dig('outcome', 'class')}")
|
|
386
|
+
checks << stop_check(watch, row, expected: "failure")
|
|
387
|
+
return checks
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
checks << if written.nil?
|
|
391
|
+
check("task_read", "skipped", "no output file to compare")
|
|
392
|
+
elsif written == token
|
|
393
|
+
check("task_read", "pass", "output carries the task token")
|
|
394
|
+
else
|
|
395
|
+
check("task_read", "fail", "output does not carry the task token (task file unreadable?)")
|
|
396
|
+
end
|
|
397
|
+
checks << if written.nil?
|
|
398
|
+
check("workspace_write", "fail", "#{out_path} was not written")
|
|
399
|
+
else
|
|
400
|
+
check("workspace_write", "pass", "wrote #{out_path}")
|
|
401
|
+
end
|
|
402
|
+
checks << command_check(row)
|
|
403
|
+
checks << receipt_check(row)
|
|
404
|
+
checks << claims_check(row)
|
|
405
|
+
checks << completion_check(row)
|
|
406
|
+
checks << stop_check(watch, row)
|
|
407
|
+
checks
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
def command_check(row)
|
|
411
|
+
return check("command_exit", "fail", "no dispatch row") unless row
|
|
412
|
+
|
|
413
|
+
gates = row.dig("validation", "final_gates") || {}
|
|
414
|
+
observation = row.dig("observed", "command_observation")
|
|
415
|
+
case gates["status"]
|
|
416
|
+
when "pass" then check("command_exit", "pass", "gate command exit observed as 0")
|
|
417
|
+
when "fail"
|
|
418
|
+
if gates["observed_fail"].to_i.positive?
|
|
419
|
+
check("command_exit", "fail", "gate command observed with a nonzero exit")
|
|
420
|
+
else
|
|
421
|
+
check("command_exit", "fail", "gate command was never run (command observation was complete)")
|
|
422
|
+
end
|
|
423
|
+
else
|
|
424
|
+
if observation == "unsupported"
|
|
425
|
+
check("command_exit", "unsupported", "adapter does not expose command exits")
|
|
426
|
+
else
|
|
427
|
+
check("command_exit", "fail", "gate command not observed (command_observation=#{observation || 'none'})")
|
|
428
|
+
end
|
|
429
|
+
end
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
def receipt_check(row)
|
|
433
|
+
return check("receipt", "fail", "no dispatch row") unless row
|
|
434
|
+
|
|
435
|
+
report = row["artifact_report"] || {}
|
|
436
|
+
if report["ingest_status"] != "ok" || report["author"] != "harnex"
|
|
437
|
+
check("receipt", "fail", "no harness receipt ingested (#{report['ingest_status'] || 'missing'})")
|
|
438
|
+
elsif %w[accepted no_change].include?(row.dig("outcome", "status"))
|
|
439
|
+
check("receipt", "pass", "harness receipt #{row.dig('outcome', 'status')}")
|
|
440
|
+
else
|
|
441
|
+
check("receipt", "fail", "harness receipt #{row.dig('outcome', 'status') || 'unknown'}")
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
def claims_check(row)
|
|
446
|
+
summary = row&.dig("claims", "summary").to_s
|
|
447
|
+
if summary.include?("preflight")
|
|
448
|
+
check("claims", "pass", "worker claims ingested")
|
|
449
|
+
else
|
|
450
|
+
check("claims", "fail", "worker claims not ingested from HARNEX_ARTIFACT_CLAIMS_PATH")
|
|
451
|
+
end
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
def completion_check(row)
|
|
455
|
+
return check("completion", "fail", "no dispatch row") unless row
|
|
456
|
+
|
|
457
|
+
if row["terminal_event"] == "task_complete"
|
|
458
|
+
check("completion", "pass", "task_complete (#{row.dig('outcome', 'class')})")
|
|
459
|
+
else
|
|
460
|
+
check("completion", "fail", "terminal #{row['terminal_event'] || 'none'} (#{row.dig('outcome', 'class') || 'unknown'})")
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# expected: "failure" when the turn already failed; then a clean exit
|
|
465
|
+
# path only proves teardown, and stop passes on that alone.
|
|
466
|
+
def stop_check(watch, row, expected: "success")
|
|
467
|
+
return check("stop", "fail", "did not finish within #{@options[:timeout_s].round}s") if watch[:timed_out]
|
|
468
|
+
return check("stop", "fail", "session did not stop cleanly") unless watch[:exited] && row
|
|
469
|
+
|
|
470
|
+
exit_reason = row.dig("actual", "exit")
|
|
471
|
+
if exit_reason == expected
|
|
472
|
+
check("stop", "pass", expected == "success" ? "auto-stop exited cleanly" : "session torn down after the failed turn")
|
|
473
|
+
else
|
|
474
|
+
check("stop", "fail", "exit #{exit_reason || 'unknown'}")
|
|
475
|
+
end
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
# A provider/auth/model error fails the turn before any tool runs. Report
|
|
479
|
+
# that root cause instead of blaming the workspace or the receipt.
|
|
480
|
+
def turn_failed_before_work?(row, written)
|
|
481
|
+
return false unless row && row["terminal_event"] == "task_failed"
|
|
482
|
+
|
|
483
|
+
written.nil? && Array(row.dig("observed", "commands")).empty? && row.dig("actual", "tool_calls").to_i.zero?
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
def redact(text)
|
|
487
|
+
return nil if text.nil?
|
|
488
|
+
|
|
489
|
+
text.to_s.gsub(/\b(sk|rk|pk)-[A-Za-z0-9_*\-]{4,}/, "\\1-[redacted]").gsub(/Bearer\s+\S+/, "Bearer [redacted]")[0, 240]
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
# usage/cost/context provenance from the dispatch row; never absent.
|
|
493
|
+
def telemetry_capability(row)
|
|
494
|
+
return { "usage" => "unavailable", "cost" => "unavailable", "context" => "unavailable" } unless row
|
|
495
|
+
|
|
496
|
+
usage = row["usage"] || {}
|
|
497
|
+
cost = if usage["cost_usd"].nil?
|
|
498
|
+
"unsupported"
|
|
499
|
+
elsif usage["cost_source"] == "provider_reported"
|
|
500
|
+
"observed"
|
|
501
|
+
else
|
|
502
|
+
"estimated"
|
|
503
|
+
end
|
|
504
|
+
{
|
|
505
|
+
"usage" => usage["status"] || "missing",
|
|
506
|
+
"cost" => cost,
|
|
507
|
+
"cost_source" => usage["cost_source"],
|
|
508
|
+
"context" => row.dig("context", "status") || "missing"
|
|
509
|
+
}
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
def check(name, status, detail)
|
|
513
|
+
{ "name" => name, "status" => status, "detail" => detail.to_s }
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
def git(dir, *args)
|
|
517
|
+
stdout, stderr, status = Open3.capture3("git", "-C", dir, *args)
|
|
518
|
+
raise "git #{args.first} failed: #{stderr.strip}" unless status.success?
|
|
519
|
+
|
|
520
|
+
stdout.strip
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
def parse(argv)
|
|
524
|
+
adapter = nil
|
|
525
|
+
child_args = []
|
|
526
|
+
index = 0
|
|
527
|
+
while index < argv.length
|
|
528
|
+
arg = argv[index]
|
|
529
|
+
case arg
|
|
530
|
+
when "--"
|
|
531
|
+
child_args = argv[(index + 1)..] || []
|
|
532
|
+
break
|
|
533
|
+
when "-h", "--help" then @options[:help] = true
|
|
534
|
+
when "--cwd" then @options[:cwd] = value(argv, index += 1, arg)
|
|
535
|
+
when "--task-root" then @options[:task_root] = value(argv, index += 1, arg)
|
|
536
|
+
when "--commit-check" then @options[:commit_check] = true
|
|
537
|
+
when "--timeout"
|
|
538
|
+
@options[:timeout_s] = Harnex.parse_duration_seconds(value(argv, index += 1, arg), option_name: arg)
|
|
539
|
+
when "--registration-timeout"
|
|
540
|
+
@options[:registration_timeout_s] = Harnex.parse_duration_seconds(value(argv, index += 1, arg), option_name: arg)
|
|
541
|
+
when "--model" then @options[:model] = value(argv, index += 1, arg)
|
|
542
|
+
when "--effort" then @options[:effort] = value(argv, index += 1, arg)
|
|
543
|
+
when "--id" then @options[:id] = value(argv, index += 1, arg)
|
|
544
|
+
else
|
|
545
|
+
raise OptionParser::InvalidOption, arg if arg.start_with?("-")
|
|
546
|
+
raise OptionParser::NeedlessArgument, arg if adapter
|
|
547
|
+
|
|
548
|
+
adapter = arg
|
|
549
|
+
end
|
|
550
|
+
index += 1
|
|
551
|
+
end
|
|
552
|
+
[adapter, child_args]
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
def value(argv, index, name)
|
|
556
|
+
text = argv[index]
|
|
557
|
+
raise OptionParser::MissingArgument, name if text.nil? || text.start_with?("-")
|
|
558
|
+
|
|
559
|
+
text
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
def monotonic
|
|
563
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
564
|
+
end
|
|
565
|
+
end
|
|
566
|
+
end
|