@jbkjr/relay 0.0.0-stage → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (296) hide show
  1. package/LICENSE +21 -0
  2. package/LICENSES/Apache-2.0.txt +201 -0
  3. package/LICENSES/README.md +19 -0
  4. package/LICENSES/codex-NOTICE.txt +6 -0
  5. package/README.md +393 -2
  6. package/SECURITY.md +145 -0
  7. package/adapters/briefs/claude.md +43 -0
  8. package/adapters/briefs/codex.md +56 -0
  9. package/adapters/briefs/models.json +33 -0
  10. package/adapters/build.mjs +8 -0
  11. package/adapters/claude/.claude-plugin/marketplace.json +13 -0
  12. package/adapters/claude/relay/.claude-plugin/plugin.json +5 -0
  13. package/adapters/claude/relay/agents/codex.md +21 -0
  14. package/adapters/claude/relay/agents/operator.md +21 -0
  15. package/adapters/claude/relay/skills/ask/SKILL.md +20 -0
  16. package/adapters/claude/relay/skills/chat/SKILL.md +13 -0
  17. package/adapters/claude/relay/skills/delegate/SKILL.md +21 -0
  18. package/adapters/claude/relay/skills/discuss/SKILL.md +19 -0
  19. package/adapters/claude/relay/skills/review/SKILL.md +21 -0
  20. package/adapters/claude/relay/skills/tree/SKILL.md +10 -0
  21. package/adapters/claude/relay/skills/write-for-codex/SKILL.md +78 -0
  22. package/adapters/codex/.agents/plugins/marketplace.json +20 -0
  23. package/adapters/codex/AGENTS.snippet.md +5 -0
  24. package/adapters/codex/relay/.codex-plugin/plugin.json +6 -0
  25. package/adapters/codex/relay/skills/relay-ask/SKILL.md +20 -0
  26. package/adapters/codex/relay/skills/relay-chat/SKILL.md +13 -0
  27. package/adapters/codex/relay/skills/relay-delegate/SKILL.md +21 -0
  28. package/adapters/codex/relay/skills/relay-discuss/SKILL.md +19 -0
  29. package/adapters/codex/relay/skills/relay-review/SKILL.md +21 -0
  30. package/adapters/codex/relay/skills/relay-tree/SKILL.md +10 -0
  31. package/adapters/codex/relay/skills/relay-write-for-claude/SKILL.md +66 -0
  32. package/adapters/generate.mjs +124 -0
  33. package/bin/relay +59 -0
  34. package/codex-pin.json +12 -0
  35. package/dist/backends/acp.js +1424 -0
  36. package/dist/backends/claude.js +890 -0
  37. package/dist/backends/cli-version.js +112 -0
  38. package/dist/backends/codex-app.js +1539 -0
  39. package/dist/backends/codex-daybreak.js +75 -0
  40. package/dist/backends/codex-env.js +656 -0
  41. package/dist/backends/codex-network.js +304 -0
  42. package/dist/backends/codex-pin.js +345 -0
  43. package/dist/backends/codex-shell-policy.js +182 -0
  44. package/dist/backends/codex-stage2.js +998 -0
  45. package/dist/backends/codex-subagents.js +47 -0
  46. package/dist/backends/codex.js +641 -0
  47. package/dist/backends/copilot.js +246 -0
  48. package/dist/backends/env.js +147 -0
  49. package/dist/backends/gemini.js +607 -0
  50. package/dist/backends/grok.js +337 -0
  51. package/dist/backends/index.js +8 -0
  52. package/dist/backends/native-temp.js +187 -0
  53. package/dist/backends/never-admission.js +90 -0
  54. package/dist/backends/run.js +67 -0
  55. package/dist/backends/stream.js +230 -0
  56. package/dist/backends/tool-discovery.js +15 -0
  57. package/dist/backends/types.js +264 -0
  58. package/dist/build-info.json +5 -0
  59. package/dist/cli.js +344 -0
  60. package/dist/confine/bridge.js +113 -0
  61. package/dist/confine/broker-core.js +76 -0
  62. package/dist/confine/broker-http.js +650 -0
  63. package/dist/confine/broker-names.js +9 -0
  64. package/dist/confine/broker.js +258 -0
  65. package/dist/confine/canary.js +380 -0
  66. package/dist/confine/diagnostics.js +122 -0
  67. package/dist/confine/gemini-tools-main.js +8 -0
  68. package/dist/confine/gemini-tools.js +181 -0
  69. package/dist/confine/hardlinks.js +58 -0
  70. package/dist/confine/harness-config.js +16 -0
  71. package/dist/confine/index.js +117 -0
  72. package/dist/confine/leases.js +217 -0
  73. package/dist/confine/paths.js +6 -0
  74. package/dist/confine/peer.js +115 -0
  75. package/dist/confine/port-policy.js +8 -0
  76. package/dist/confine/ports.js +365 -0
  77. package/dist/confine/private-root.js +208 -0
  78. package/dist/confine/probe-stage2.js +501 -0
  79. package/dist/confine/probe.js +1109 -0
  80. package/dist/confine/proxy.js +783 -0
  81. package/dist/confine/sbcheck.js +208 -0
  82. package/dist/confine/seatbelt-never.js +41 -0
  83. package/dist/confine/seatbelt-stage2.js +139 -0
  84. package/dist/confine/seatbelt.js +1129 -0
  85. package/dist/confine/sockets.js +36 -0
  86. package/dist/confine/turn-probe.js +19 -0
  87. package/dist/confine/turn.js +86 -0
  88. package/dist/core/adoption.js +124 -0
  89. package/dist/core/api-keys.js +423 -0
  90. package/dist/core/approvals.js +740 -0
  91. package/dist/core/build-identity.js +248 -0
  92. package/dist/core/caller.js +749 -0
  93. package/dist/core/calls.js +102 -0
  94. package/dist/core/claude-settings.js +1045 -0
  95. package/dist/core/config.js +453 -0
  96. package/dist/core/context.js +186 -0
  97. package/dist/core/db.js +753 -0
  98. package/dist/core/dispatch.js +110 -0
  99. package/dist/core/env-broker.js +505 -0
  100. package/dist/core/env-policy.js +182 -0
  101. package/dist/core/escalation-authority.js +76 -0
  102. package/dist/core/escalation-contract.js +13 -0
  103. package/dist/core/escalation-person-path.js +131 -0
  104. package/dist/core/escalation-policy.js +79 -0
  105. package/dist/core/escalation-runtime.js +135 -0
  106. package/dist/core/escalation-state.js +42 -0
  107. package/dist/core/escalations.js +191 -0
  108. package/dist/core/events.js +235 -0
  109. package/dist/core/executables.js +147 -0
  110. package/dist/core/extensions.js +645 -0
  111. package/dist/core/families.js +204 -0
  112. package/dist/core/git.js +657 -0
  113. package/dist/core/identity.js +113 -0
  114. package/dist/core/liveness.js +68 -0
  115. package/dist/core/mediated-network-provider.js +81 -0
  116. package/dist/core/never-checkpoint.js +76 -0
  117. package/dist/core/never-coverage.js +97 -0
  118. package/dist/core/never-network.js +49 -0
  119. package/dist/core/never-policy.js +302 -0
  120. package/dist/core/never-proxy-policy.js +109 -0
  121. package/dist/core/never-schema.js +61 -0
  122. package/dist/core/never-tools.js +56 -0
  123. package/dist/core/notices.js +48 -0
  124. package/dist/core/ops.js +959 -0
  125. package/dist/core/outcome.js +224 -0
  126. package/dist/core/params.js +32 -0
  127. package/dist/core/presence.js +174 -0
  128. package/dist/core/procs.js +546 -0
  129. package/dist/core/prompts.js +363 -0
  130. package/dist/core/protected-config.js +238 -0
  131. package/dist/core/raw-network.js +192 -0
  132. package/dist/core/redact.js +152 -0
  133. package/dist/core/refusal.js +33 -0
  134. package/dist/core/relay-command.js +310 -0
  135. package/dist/core/reported.js +3 -0
  136. package/dist/core/restart-overlay.js +651 -0
  137. package/dist/core/retention.js +1135 -0
  138. package/dist/core/secrets.js +329 -0
  139. package/dist/core/session-token.js +175 -0
  140. package/dist/core/session.js +744 -0
  141. package/dist/core/shell.js +2 -0
  142. package/dist/core/shipped-pin.js +47 -0
  143. package/dist/core/shown.js +42 -0
  144. package/dist/core/slots.js +67 -0
  145. package/dist/core/stage2-executors.js +453 -0
  146. package/dist/core/stage2-switch.js +5 -0
  147. package/dist/core/steering.js +149 -0
  148. package/dist/core/storage.js +358 -0
  149. package/dist/core/toml.js +175 -0
  150. package/dist/core/views.js +324 -0
  151. package/dist/core/wait.js +114 -0
  152. package/dist/core/wake.js +207 -0
  153. package/dist/core/watch.js +247 -0
  154. package/dist/core/worktree.js +213 -0
  155. package/dist/hosts/claude-desktop.js +274 -0
  156. package/dist/hosts/claude.js +249 -0
  157. package/dist/hosts/codex.js +345 -0
  158. package/dist/hosts/desktop-shared.js +20 -0
  159. package/dist/hosts/index.js +11 -0
  160. package/dist/hosts/types.js +4 -0
  161. package/dist/install/basics.js +18 -0
  162. package/dist/install/claude-init.js +133 -0
  163. package/dist/install/common.js +655 -0
  164. package/dist/install/doctor.js +635 -0
  165. package/dist/install/qualify-codex.js +205 -0
  166. package/dist/install.js +524 -0
  167. package/dist/mcp/elicit.js +209 -0
  168. package/dist/mcp/gateway.js +218 -0
  169. package/dist/mcp/host.js +253 -0
  170. package/dist/mcp/server.js +1124 -0
  171. package/dist/mcp/stdio.js +37 -0
  172. package/dist/modes/discussion.js +393 -0
  173. package/dist/modes/review.js +722 -0
  174. package/dist/native/relay-sbcheck +0 -0
  175. package/dist/targets/briefs.js +86 -0
  176. package/dist/targets/claude.js +94 -0
  177. package/dist/targets/codex.js +66 -0
  178. package/dist/targets/copilot.js +93 -0
  179. package/dist/targets/gemini.js +119 -0
  180. package/dist/targets/grok.js +126 -0
  181. package/dist/targets/index.js +192 -0
  182. package/dist/targets/types.js +18 -0
  183. package/dist/targets/verified.js +35 -0
  184. package/dist/worker.js +113 -0
  185. package/docs/UPGRADING.md +1518 -0
  186. package/docs/pinned-codex.md +333 -0
  187. package/experiments/stage2-redteam/lib/oracles.mjs +214 -0
  188. package/experiments/stage2-redteam/lib/report.mjs +35 -0
  189. package/npm-shrinkwrap.json +1632 -0
  190. package/package.json +82 -4
  191. package/scripts/codex-qualify/codex-pin-store.mjs +353 -0
  192. package/scripts/codex-qualify/codex-qualifier-process.mjs +265 -0
  193. package/scripts/codex-qualify/codex-qualifier.example.json +5 -0
  194. package/scripts/codex-qualify/codex-qualifier.mjs +281 -0
  195. package/scripts/codex-qualify/codex-shipped-pin.mjs +158 -0
  196. package/scripts/codex-qualify/codex-stage2-pin.mjs +48 -0
  197. package/scripts/codex-qualify/codex-stage2.mjs +2381 -0
  198. package/scripts/codex-qualify/install-codex-qualifier.mjs +98 -0
  199. package/scripts/codex-qualify/main-module.mjs +14 -0
  200. package/scripts/codex-qualify/relay-codex-qualifier.plist +37 -0
  201. package/scripts/codex-qualify/scratch-env.mjs +31 -0
  202. package/scripts/codex-qualify/session.mjs +96 -0
  203. package/scripts/codex-qualify/stage2-escalation-cases.mjs +125 -0
  204. package/test/codex-permissions.test.ts +875 -0
  205. package/test/fixtures/boundary-probes.mjs +17 -0
  206. package/test/fixtures/denial-stream-host.mjs +7 -0
  207. package/test/fixtures/families.json +246 -0
  208. package/test/fixtures/install-snapshots/claude-desktop-foreign.txt +158 -0
  209. package/test/fixtures/install-snapshots/claude-desktop-fresh.txt +101 -0
  210. package/test/fixtures/install-snapshots/claude-desktop-installed.txt +161 -0
  211. package/test/fixtures/install-snapshots/claude-foreign.txt +133 -0
  212. package/test/fixtures/install-snapshots/claude-fresh.txt +77 -0
  213. package/test/fixtures/install-snapshots/claude-installed.txt +133 -0
  214. package/test/fixtures/install-snapshots/codex-foreign.txt +159 -0
  215. package/test/fixtures/install-snapshots/codex-fresh.txt +107 -0
  216. package/test/fixtures/install-snapshots/codex-installed.txt +161 -0
  217. package/test/fixtures/link-parent.mjs +6 -0
  218. package/test/fixtures/mediated-command.cjs +72 -0
  219. package/test/fixtures/native-skill-isolation.json +280 -0
  220. package/test/fixtures/pinned-codex-app-server.mjs +25 -0
  221. package/test/fixtures/qualified-ext-adapters.mjs +36 -0
  222. package/test/fixtures/refusals/README.md +52 -0
  223. package/test/fixtures/refusals/claude-source-result.json +7 -0
  224. package/test/fixtures/refusals/openai-app-capacity.json +24 -0
  225. package/test/fixtures/refusals/openai-app-cyberPolicy.json +24 -0
  226. package/test/fixtures/refusals/openai-gpt-6-astra-max.json +12 -0
  227. package/test/fixtures/refusals/openai-gpt-6.1-sol-xhigh.json +12 -0
  228. package/test/fixtures/refusals/openai-review-capacity.json +24 -0
  229. package/test/fixtures/relay-0.2.0.sql +57 -0
  230. package/test/fixtures/relay-0.9.1.gen.mjs +212 -0
  231. package/test/fixtures/relay-0.9.1.sql +340 -0
  232. package/test/fixtures/slow-boundary-probes.mjs +37 -0
  233. package/test/fixtures/stable-surfaces/config.json +1122 -0
  234. package/test/fixtures/stable-surfaces/escalation.json +128 -0
  235. package/test/fixtures/stable-surfaces/events.json +37 -0
  236. package/test/fixtures/stable-surfaces/gemini-native-resume.json +4 -0
  237. package/test/fixtures/stable-surfaces/inbox.jsonl +2 -0
  238. package/test/fixtures/stable-surfaces/notice.txt +1 -0
  239. package/test/fixtures/stable-surfaces/refusal-item.json +28 -0
  240. package/test/fixtures/stable-surfaces/refusal-review.json +23 -0
  241. package/test/fixtures/stable-surfaces/registry.json +49 -0
  242. package/test/fixtures/stable-surfaces/tools-desktop.json +575 -0
  243. package/test/fixtures/stable-surfaces/tools.json +716 -0
  244. package/test/fixtures/stable-surfaces/usage.txt +46 -0
  245. package/test/fixtures/stage2-acp-native.mjs +138 -0
  246. package/test/fixtures/stage2-app-server.mjs +23 -0
  247. package/test/fixtures/stage2-catalog.json +91 -0
  248. package/test/fixtures/stage2-hostile-web.mjs +35 -0
  249. package/test/fixtures/stage2-native-network.mjs +62 -0
  250. package/test/fixtures/stage2-worker.ts +87 -0
  251. package/test/fixtures/status-views/expected.json +7825 -0
  252. package/test/fixtures/status-views/registries.json +1 -0
  253. package/test/fixtures/steering.json +209 -0
  254. package/test/helpers.ts +740 -0
  255. package/test/host-env.ts +7 -0
  256. package/test/keychain-decoy.ts +197 -0
  257. package/test/live-scope.ts +44 -0
  258. package/test/mediated-network.test.ts +192 -0
  259. package/test/mediated-proxy.test.ts +264 -0
  260. package/test/native-temp-alias.ts +10 -0
  261. package/test/never-policy.test.ts +629 -0
  262. package/test/never-tools.test.ts +136 -0
  263. package/test/process-home.ts +18 -0
  264. package/test/proxy.test.ts +531 -0
  265. package/test/restart-overlay-live.test.ts +85 -0
  266. package/test/restart-overlay-profile.test.ts +66 -0
  267. package/test/restart-overlay.test.ts +416 -0
  268. package/test/stable-surfaces.test.ts +249 -0
  269. package/test/stage2-escalation-live.test.ts +544 -0
  270. package/test/stage2-escalation-qualification.test.ts +170 -0
  271. package/test/stage2-escalation.test.ts +747 -0
  272. package/test/stage2-executor.test.ts +849 -0
  273. package/test/stage2-grant.test.ts +603 -0
  274. package/test/stage2-native-live.test.ts +647 -0
  275. package/test/stage2-network-decisions.test.ts +70 -0
  276. package/test/stage2-probe.test.ts +845 -0
  277. package/test/stage2-standing-live.test.ts +203 -0
  278. package/test/stubs/broker-host.mjs +59 -0
  279. package/test/stubs/broker-peer.mjs +20 -0
  280. package/test/stubs/confined-harness.mjs +91 -0
  281. package/test/stubs/ext-broken-server.mjs +2 -0
  282. package/test/stubs/ext-odd-server.mjs +21 -0
  283. package/test/stubs/ext-server.mjs +31 -0
  284. package/test/stubs/fake-host-cli.mjs +218 -0
  285. package/test/stubs/procargs.mjs +30 -0
  286. package/test/stubs/stub-acp +4 -0
  287. package/test/stubs/stub-agent.mjs +322 -0
  288. package/test/stubs/stub-claude +4 -0
  289. package/test/stubs/stub-codex +4 -0
  290. package/test/stubs/targets/acp.mjs +307 -0
  291. package/test/stubs/targets/claude.mjs +149 -0
  292. package/test/stubs/targets/codex.mjs +479 -0
  293. package/test/stubs/targets/gemini.mjs +604 -0
  294. package/test/temp-root.ts +11 -0
  295. package/test/wp4-adapters.test.ts +353 -0
  296. package/test/wp4-interlock.test.ts +136 -0
@@ -0,0 +1,2381 @@
1
+ #!/usr/bin/env node
2
+ // Stage 2's qualification of a Codex CLI release (docs/codex-confinement-design.md §7.2): Codex's
3
+ // commands run in `codex exec-server` inside relay's own Seatbelt profile, bound to the app server's
4
+ // thread as its only environment. Model-free: a fake Responses API on loopback drives each turn
5
+ // with scripted tool calls, every Codex and relay home is a scratch directory, and every listener
6
+ // is this script's own, on a random high port. No person's credential is read, and nothing
7
+ // connects to a service of the Mac's.
8
+ //
9
+ // Each item has up to three kinds of part:
10
+ // codex what this Codex release does, run directly against it: an executor this script starts
11
+ // under relay's stage-2 profile (built by relay's own code from the checkout), and an app
12
+ // server with relay's own stage-2 flags (codexFlags with stage2Plan), as the design's
13
+ // probes did (docs/research/codex-confinement-probes/);
14
+ // control the same calls where they must escape (no environment, or no turn binding), so that a
15
+ // refusal in the codex part means something;
16
+ // relay the same through relay's own path: codexBackend.start with stage 2 switched on
17
+ // (openStage2, Stage2Session, startCodexAppTurn). A build without that path (WP-C not
18
+ // merged) reports the part "pending integration", which is never a pass.
19
+ // An item passes only when each of its parts passed. A failed part fails the version.
20
+ //
21
+ // Usage: node codex-stage2.mjs <relay checkout> <scratch dir> <version> <codex bin> [options]
22
+ // --old-bin <path> an older Codex (0.163) that relay must refuse (item 13); without it, not run
23
+ // --source <dir> a codex-rs checkout at this version's tag, for its bundled model catalog;
24
+ // without it the catalog is fetched from GitHub at the tag
25
+ // --items 1,5,10 only these items (the rest are recorded "not run")
26
+ // Writes <scratch dir>/stage2-verdict.json: { ok, state, why, verdictSchema, version, items }.
27
+ import crypto from "node:crypto";
28
+ import fs from "node:fs";
29
+ import http from "node:http";
30
+ import net from "node:net";
31
+ import os from "node:os";
32
+ import path from "node:path";
33
+ import readline from "node:readline";
34
+ import { setTimeout as delay } from "node:timers/promises";
35
+ import { execFileSync, spawn, spawnSync } from "node:child_process";
36
+ import { NO_SESSION_EVIDENCE, noLoginSession } from "./session.mjs";
37
+ import { scratchEnv } from "./scratch-env.mjs";
38
+ import { stageQualificationPin } from "./codex-stage2-pin.mjs";
39
+ import { CURRENT_VERDICT_SCHEMA } from "./codex-pin-store.mjs";
40
+ import { ESCALATION_CASES, runEscalationItem } from "./stage2-escalation-cases.mjs";
41
+ import { isMain } from "./main-module.mjs";
42
+
43
+ // Set before any child starts. Importing pure helpers does not create scratch state.
44
+ let stageScratch;
45
+ const stageChildEnv = (extra = {}) => scratchEnv(stageScratch, extra);
46
+
47
+ export const PENDING = "pending integration";
48
+
49
+ /** The qualification items: §7.2's twelve, and the older-Codex refusal of §2.2 step 2 (W163). */
50
+ export const ITEMS = {
51
+ 1: "commands, apply_patch and code mode's nested tools confined to the roots, and a denied port unreachable; control without the environment (W1, W5, B0)",
52
+ 2: "a restricted Codex profile in the environment fails closed: exit 71 (B3)",
53
+ 3: "a local stdio MCP server starts outside the executor; under relay's stage-2 settings only authenticated relay_tools runs (W2)",
54
+ 4: "an exec-policy allow rule and an approved require_escalated command stay confined (W3)",
55
+ 5: "an unreachable executor: no fallback to local (W4)",
56
+ 6: "the executor token is memory-only: in no argv, environment or file (X7, W7)",
57
+ 7: "cross-turn authentication: no token, a wrong token and a guessed one are refused (X7, W6, W7)",
58
+ 8: "survivors: background, nohup, setsid, a terminal session and a block-port listener are gone after teardown (C5, X3)",
59
+ 9: "executor death mid-turn: the app server is stopped first, the turn fails, a fresh listener on its port sees nothing",
60
+ 10: "the tool list under relay's version-specific stage-2 settings (prediction parent tools pinned off where supported) is within the allowlist for every catalog model; code mode's isolate has no Node, file or network bindings; explicit effort reaches the model on fresh and resumed turns (§2.5, §9.3)",
61
+ 11: "a subagent's command runs in the relay environment; under relay's settings (no local environment) a resumed thread has no environment until bound per turn (turn/start.environments), then keeps commands and subagents in the executor, and unbound runs nothing (control: with local, unbound runs locally); no turn starts by itself on a resume under relay's flags, an active goal's included (control: with goals on it does); with goals on and no local, the goal's own turn, steered into by a bound turn/start, runs nothing (control: with local it runs there); relay refuses a resume with a queued message",
62
+ 12: "plugins off and discovery prefetch off where supported: no curated sync or host-side discovery traffic, with an authenticated discovery positive control",
63
+ 13: "an older Codex (0.163): nothing escapes, and relay's admission refuses it (STAGE2_MIN_CODEX)",
64
+ 14: "HTTP without local: exact entry, listings, direct and code-mode calls, and a 75-second call",
65
+ 15: "credentials/front: missing variable, wrong token, foreign peer, broken checker, required startup and app-server-first mid-turn failure",
66
+ 16: "relay stdio coexists with relay_tools; person-defined relay_tools refuses admission",
67
+ 17: "NO_PROXY protects the bearer from a decoy proxy, with a leaking negative control",
68
+ 18: "authenticated reconnect after broker loss and app-server EOF within one second",
69
+ 19: "404 replays; tombstone and cap JSON-RPC errors never replay or evict",
70
+ 20: "C6: no broker-token name or value in commands (plain, terminal, nested code mode) through the real executor",
71
+ 21: "C1: a self-started goal continuation cannot call the disarmed broker",
72
+ 22: "C8: v1/v2 broker session counts, churn and resume under canonical pins",
73
+ ...Object.fromEntries(Object.entries(ESCALATION_CASES).map(([id, c]) => [id, c.name])),
74
+ 30: "MCP result delivered exactly once beyond the former 3600-second timeout (opt-in hour-long probe)",
75
+ };
76
+
77
+ /**
78
+ * What code mode's isolate may hold beyond ECMAScript's own globals, as Codex 0.164.0 installs them
79
+ * (code-mode-runtime/src/runtime/globals.rs): its helpers and the nested tools. A global outside
80
+ * ISOLATE_GLOBALS fails item 10, so a release that gives the model's JavaScript a new binding (fetch,
81
+ * require, a file API) waits for the maintainer. Recorded from the 0.164.0 run (docs/qualification/codex-stage2-0.164.0.md).
82
+ */
83
+ export const ISOLATE_HELPERS = ["tools", "ALL_TOOLS", "clearTimeout", "setTimeout", "text", "image", "audio", "generatedImage", "store", "load", "notify", "yield_control", "exit",
84
+ // 0.170.0: emits one MCP-shaped content block (or each of an iterable's) as `exec` output, as
85
+ // text, image and audio do (runtime/callbacks.rs content_block_callback, content.js). Text and
86
+ // inline media only; a file reference is refused (runtime/value.rs mcp_output_item).
87
+ "content"];
88
+ /** Names that must not exist in the isolate at all: Node's, the web platform's network and workers. */
89
+ export const ISOLATE_FORBIDDEN = ["require", "process", "fetch", "WebSocket", "XMLHttpRequest", "EventSource", "Worker", "Deno", "Bun", "module", "exports", "Buffer", "console", "importScripts", "navigator"];
90
+ /** Every global name the 0.164.0 isolate has (Object.getOwnPropertyNames(globalThis)): ECMAScript's, V8's, ISOLATE_HELPERS and settled.js's two promise combinators. */
91
+ export const ISOLATE_GLOBALS = [
92
+ "ALL_TOOLS", "AggregateError", "Array", "ArrayBuffer", "AsyncDisposableStack", "BigInt", "BigInt64Array", "BigUint64Array", "Boolean", "DataView", "Date", "DisposableStack", "Error",
93
+ "EvalError", "FinalizationRegistry", "Float16Array", "Float32Array", "Float64Array", "Function", "Infinity", "Int16Array", "Int32Array", "Int8Array", "Intl", "Iterator", "JSON", "Map",
94
+ "Math", "NaN", "Number", "Object", "Promise", "Proxy", "RangeError", "ReferenceError", "Reflect", "RegExp", "Set", "String", "SuppressedError", "Symbol", "SyntaxError", "TypeError",
95
+ "URIError", "Uint16Array", "Uint32Array", "Uint8Array", "Uint8ClampedArray", "WeakMap", "WeakRef", "WeakSet", "as_settled", "audio", "clearTimeout", "content", "decodeURI", "decodeURIComponent",
96
+ "encodeURI", "encodeURIComponent", "escape", "eval", "exit", "generatedImage", "globalThis", "image", "isFinite", "isNaN", "load", "notify", "parseFloat", "parseInt", "setTimeout",
97
+ "store", "stream_settled", "text", "tools", "undefined", "unescape", "yield_control",
98
+ ];
99
+
100
+ // ---------------------------------------------------------------------------------------------
101
+ // Pure helpers (test/release-watch.test.ts).
102
+
103
+ /** `key=value` lines a probe command printed (`echo wt=$?`), the last value of each key. */
104
+ export function markers(text) {
105
+ const out = {};
106
+ for (const m of String(text ?? "").matchAll(/^([A-Za-z_][\w-]*)=(.*)$/gm)) out[m[1]] = m[2].trim();
107
+ return out;
108
+ }
109
+
110
+ /** The text of a tool call's output as the Responses API carries it back: a string, or content parts. */
111
+ export function outputText(output) {
112
+ if (typeof output === "string") return output;
113
+ if (Array.isArray(output)) return output.map((p) => (typeof p === "string" ? p : p?.text ?? "")).join("");
114
+ if (output && typeof output === "object") return String(output.text ?? output.content ?? JSON.stringify(output));
115
+ return "";
116
+ }
117
+
118
+ /** The tools nested in code mode's `exec` tool, as its description declares them (`declare const tools: { name(`). */
119
+ export function nestedTools(description) {
120
+ return [...new Set([...String(description ?? "").matchAll(/declare const tools: \{ ([A-Za-z_$][\w$]*)\(/g)].map((m) => m[1]))];
121
+ }
122
+
123
+ /**
124
+ * Every tool a Responses request offers the model, in the form relay's allowlist takes
125
+ * (codex-stage2.ts stage2ToolsProblem): a function, custom or hosted tool by its name (a hosted one
126
+ * by its type), a namespace by its own name and each of its tools by theirs, and the tools code
127
+ * mode's `exec` declares to its JavaScript. From the request's `tools` and from the
128
+ * `additional_tools` input items through which Codex offers code mode's tools (catalog entries with
129
+ * tool_mode code_mode_only).
130
+ */
131
+ export function offeredTools(body) {
132
+ const lists = [body?.tools ?? [], ...(body?.input ?? []).filter((i) => i?.type === "additional_tools" || i?.type === "tool_search_output").map((i) => i.tools ?? [])];
133
+ const out = [];
134
+ for (const tools of lists) {
135
+ for (const t of tools) {
136
+ if (!t || typeof t !== "object") continue;
137
+ out.push(t.name ?? t.type);
138
+ for (const s of [t, ...(Array.isArray(t.tools) ? t.tools : [])]) {
139
+ if (s !== t) out.push(s?.name ?? s?.type);
140
+ if (s?.name === "exec" && s.description) out.push(...nestedTools(s.description));
141
+ }
142
+ }
143
+ }
144
+ return [...new Set(out.filter(Boolean))];
145
+ }
146
+
147
+ /** 0.166's trusted configuration_update can supersede the request's pinned effort baseline. */
148
+ export function effortEvidence(body) {
149
+ const updates = (body?.input ?? []).filter((i) => i?.type === "configuration_update").map((i) => i.reasoning?.effort ?? null);
150
+ const request = body?.reasoning?.effort ?? null;
151
+ return { request, updates, effective: updates.length ? updates.at(-1) : request };
152
+ }
153
+
154
+ /** Discovery warmed at startup by 0.166's plugin_discovery_prefetch (not installed-plugin sync). */
155
+ export const PLUGIN_DISCOVERY_PATHS = [
156
+ "/backend-api/ps/plugins/list",
157
+ "/backend-api/plugins/featured",
158
+ "/backend-api/ps/plugins/suggested/codex",
159
+ ];
160
+
161
+ /**
162
+ * 0.170.0 removes plugin_discovery_prefetch (features/src/lib.rs, Stage::Removed): startup no
163
+ * longer warms the catalog (`ps/plugins/list`) or the featured ids (core-plugins manager.rs
164
+ * maybe_start_plugin_startup_tasks_for_config). With plugins and remote plugins on it still
165
+ * refreshes the installed plugins and the recommendations at startup, the host-side discovery
166
+ * relay's pins must suppress, so those are the positive control's paths there.
167
+ */
168
+ export const PLUGIN_STARTUP_DISCOVERY_PATHS = [
169
+ "/backend-api/ps/plugins/installed",
170
+ "/backend-api/ps/plugins/suggested/codex",
171
+ ];
172
+
173
+ /** The authenticated discovery the positive control must see on `version` ([major, minor, patch]). */
174
+ export function pluginDiscoveryPaths(version) {
175
+ const [a = 0, b = 0] = version;
176
+ return a > 0 || b >= 170 ? PLUGIN_STARTUP_DISCOVERY_PATHS : PLUGIN_DISCOVERY_PATHS;
177
+ }
178
+
179
+ /**
180
+ * A request to the run's fake ChatGPT service that is plugin discovery: one of the positive
181
+ * control's paths, or any other path with a `plugins` segment (installed, suggested, a later
182
+ * endpoint), so the negatives stay at least as wide as the positive. Codex 0.166.0 also calls
183
+ * the same service with plugins off (account check, user settings, analytics events); those are
184
+ * not plugin discovery and do not count.
185
+ */
186
+ export function pluginDiscoveryRequest(r) {
187
+ const p = String(r?.path ?? "");
188
+ // Also anything of the plugin service (`/backend-api/ps/`): with plugins on, 0.170.0 also POSTs
189
+ // its MCP endpoint (`ps/mcp`), which has no `plugins` segment.
190
+ return PLUGIN_DISCOVERY_PATHS.includes(p) || /\/plugins(\/|$)/.test(p) || /^\/backend-api\/ps(\/|$)/.test(p);
191
+ }
192
+
193
+ /** Item 12's sync and prefetch evidence. Quiet negatives alone can never establish suppression. */
194
+ export function pluginParts({ control, off, hasPrefetch, prefetchOn, pluginsOnly, discoveryPaths = PLUGIN_DISCOVERY_PATHS }) {
195
+ const syncAttempted = (x) => Boolean(x.tmp?.some((f) => f.startsWith("plugins")) || x.proxied.some((s) => /github/.test(s)));
196
+ // Any request to the refusing proxy still counts; at the fake service only plugin discovery does.
197
+ const attempted = (x) => syncAttempted(x) || x.proxied.length > 0 || (x.discovery ?? []).some(pluginDiscoveryRequest);
198
+ const problems = [...(off.readback ? [off.readback] : []), ...(attempted(off) ? [`with relay's plugin pins Codex still started plugin sync or discovery (${JSON.stringify(off)})`] : [])];
199
+ const prefetchProblems = [];
200
+ if (hasPrefetch) {
201
+ if (prefetchOn?.features?.plugins !== true || prefetchOn?.features?.plugin_discovery_prefetch !== true || prefetchOn?.features?.remote_plugin !== true) prefetchProblems.push("the discovery positive-control pins did not read back: the prefetch check cannot tell");
202
+ const seen = new Set((prefetchOn?.discovery ?? []).filter((r) => r.method === "GET" && r.authenticated === true).map((r) => r.path));
203
+ const missing = discoveryPaths.filter((p) => !seen.has(p));
204
+ if (missing.length) prefetchProblems.push(`with plugins and prefetch on, no authenticated discovery request reached the fake for ${missing.join(", ")}: the prefetch check cannot tell`);
205
+ if (!pluginsOnly || pluginsOnly.features?.plugins !== false || pluginsOnly.features?.plugin_discovery_prefetch !== true || attempted(pluginsOnly)) problems.push("plugins=false did not suppress host traffic with discovery prefetch explicitly on, or the control pins did not read back");
206
+ }
207
+ return {
208
+ control: judged(syncAttempted(control) ? [] : ["with plugins on Codex started no plugin sync either: the check cannot tell"], control),
209
+ codex: judged(problems, { ...off, pluginsOnly }),
210
+ prefetch: hasPrefetch ? judged(prefetchProblems, prefetchOn ?? {}) : part("passed", "this version has no discovery prefetch feature", { supported: false }),
211
+ };
212
+ }
213
+
214
+ /**
215
+ * The messages of a request's input, as text: which conversation (parent or subagent) it is. User
216
+ * and developer messages, and the agent messages through which Codex hands a subagent its task
217
+ * (`agent_message`, author /root). Tool calls and their outputs are left out, so a parent whose
218
+ * spawn call carries a subagent's task is not taken for the subagent.
219
+ */
220
+ export function messageList(input) {
221
+ const out = [];
222
+ for (const i of input ?? []) {
223
+ // 0.170.0's untrusted additional context (a goal's objective, untrusted_as_tool_output on by
224
+ // default): a function_call_output with no call_id, in namespace `agent`
225
+ // (context-fragments additional_context.rs). It is what starts a goal's own turn, so it routes
226
+ // like a message; a real tool call's output always has a call_id.
227
+ if (i?.type === "function_call_output" && !i.call_id) {
228
+ out.push(typeof i.output === "string" ? i.output : (Array.isArray(i.output) ? i.output : i.output?.content ?? []).map((c) => c?.text ?? "").join(""));
229
+ continue;
230
+ }
231
+ if (i?.type !== "message" && i?.type !== "agent_message") continue;
232
+ // An agent message's payload is a content part of its own ("encrypted_content", in clear text to a model provider like this one).
233
+ out.push(typeof i.content === "string" ? i.content : (i.content ?? []).map((c) => c?.text ?? c?.encrypted_content ?? "").join(""));
234
+ }
235
+ return out;
236
+ }
237
+ export const messageTexts = (input) => messageList(input).join("\n");
238
+
239
+ /**
240
+ * What the fake model answers to a request: the conversation it belongs to (by the latest message
241
+ * that contains some conversation's `match`, the first such conversation; else the one without
242
+ * `match`: a resumed turn's input holds every earlier turn's messages, so the latest decides) and
243
+ * that conversation's next scripted call,
244
+ * numbered by the outputs its input already holds; or a final message when its calls are spent. A
245
+ * call given as a function is made when it is due (so it can name a port opened during the turn).
246
+ * Returns { conv, step, item }.
247
+ */
248
+ export function nextReply(conversations, body) {
249
+ let conv = -1;
250
+ for (const text of messageList(body.input).reverse()) {
251
+ conv = conversations.findIndex((c) => c.match && text.includes(c.match));
252
+ if (conv >= 0) break;
253
+ }
254
+ if (conv < 0) conv = conversations.findIndex((c) => !c.match);
255
+ if (conv < 0) return { conv: -1, step: 0, item: finalMessage("done") };
256
+ const prefix = `c${conv}-`;
257
+ const outs = (body.input ?? []).filter((i) => /(?:_call_output|tool_search_output)$/.test(i?.type ?? "") && String(i.call_id ?? "").startsWith(prefix));
258
+ const step = outs.length;
259
+ let call = conversations[conv].calls?.[step];
260
+ if (typeof call === "function") call = call(body);
261
+ if (!call) return { conv, step, item: finalMessage(conversations[conv].final ?? "done") };
262
+ return { conv, step, item: { ...call, call_id: `${prefix}${step}`, id: `i${conv}-${step}` } };
263
+ }
264
+
265
+ const finalMessage = (text) => ({ type: "message", role: "assistant", id: `m${crypto.randomUUID().slice(0, 8)}`, content: [{ type: "output_text", text }] });
266
+
267
+ const USAGE = { input_tokens: 0, input_tokens_details: null, output_tokens: 0, output_tokens_details: null, total_tokens: 0 };
268
+ /** A Responses API stream answering with one output item. */
269
+ export function sse(item) {
270
+ const id = `r${crypto.randomUUID().slice(0, 8)}`;
271
+ return [
272
+ { type: "response.created", response: { id } },
273
+ { type: "response.output_item.done", item },
274
+ { type: "response.completed", response: { id, usage: USAGE } },
275
+ ].map((e) => `event: ${e.type}\ndata: ${JSON.stringify(e)}\n\n`).join("");
276
+ }
277
+
278
+ /**
279
+ * Every spelling of this Mac's own addresses a denied port must be refused at (design §5.2, §7.1):
280
+ * loopback in each form, the any-address, their IPv4-mapped IPv6 forms (which Seatbelt's
281
+ * `localhost` does not name), and every interface address with its mapped form; link-local IPv6
282
+ * with its interface. From `os.networkInterfaces()` shaped input.
283
+ */
284
+ export function localSpellings(interfaces = os.networkInterfaces()) {
285
+ const out = ["127.0.0.1", "::1", "localhost", "0.0.0.0", "::", "::ffff:127.0.0.1", "::ffff:0.0.0.0", "0:0:0:0:0:ffff:0:0", "0:0:0:0:0:ffff:7f00:1", "fe80::1%lo0"];
286
+ for (const [name, list] of Object.entries(interfaces ?? {})) {
287
+ for (const a of list ?? []) {
288
+ if (a.family === "IPv4" || a.family === 4) out.push(a.address, `::ffff:${a.address}`);
289
+ else if (/^fe80:/i.test(a.address)) out.push(`${a.address.replace(/%.*$/, "")}%${name}`);
290
+ else out.push(a.address);
291
+ }
292
+ }
293
+ return [...new Set(out)];
294
+ }
295
+
296
+ /**
297
+ * The command that tries TCP connects from inside the sandbox with relay's own Node, printing one
298
+ * `net <kind> <address> <outcome>` line each: the errno (EPERM is Seatbelt's refusal), CONNECTED or
299
+ * TIMEOUT. `deny` addresses at the denied port, `ok` at the allowed one.
300
+ */
301
+ export function netProbe(node, denyPort, deny, okPort, ok) {
302
+ const js = `const net=require("net");const t=(k,h,p)=>new Promise(r=>{const s=net.connect({host:h,port:p});s.setTimeout(2500,()=>{s.destroy();r([k,h,"TIMEOUT"])});s.on("connect",()=>{s.destroy();r([k,h,"CONNECTED"])});s.on("error",e=>r([k,h,e.code||"ERROR"]))});(async()=>{for(const [k,h,p] of ${JSON.stringify([...deny.map((h) => ["deny", h, denyPort]), ...ok.map((h) => ["ok", h, okPort])])}){const x=await t(k,h,p);console.log("net "+x.join(" "))}})()`;
303
+ return `${node} -e '${js}'`;
304
+ }
305
+
306
+ /** The `net <kind> <address> <outcome>` lines of a probe's output: [{ kind, address, outcome }]. */
307
+ export function netLines(text) {
308
+ return [...String(text ?? "").matchAll(/^net (deny|ok) (\S+) (\S+)$/gm)].map((m) => ({ kind: m[1], address: m[2], outcome: m[3] }));
309
+ }
310
+
311
+ /** What a probe's lines show wrong: a denied address not refused with EPERM, an allowed one not connected, or an address with no line. */
312
+ export function netProblems(lines, deny, ok) {
313
+ const p = [];
314
+ for (const a of deny) {
315
+ const l = lines.find((x) => x.kind === "deny" && x.address === a);
316
+ if (!l) p.push(`no result for the denied port at ${a}: the check cannot tell`);
317
+ else if (l.outcome !== "EPERM") p.push(`the denied port at ${a}: ${l.outcome}, not EPERM`);
318
+ }
319
+ for (const a of ok) {
320
+ const l = lines.find((x) => x.kind === "ok" && x.address === a);
321
+ if (l?.outcome !== "CONNECTED") p.push(`a port that is not denied, at ${a}: ${l?.outcome ?? "no result"}: the network check cannot tell`);
322
+ }
323
+ return p;
324
+ }
325
+
326
+ /** The JavaScript the isolate check runs in code mode's `exec`: what exists, what imports do, and every global name, printed as ISOLATE=<json>. */
327
+ export const ISOLATE_PROBE = [
328
+ "const r = { types: {}, imports: {} };",
329
+ // Each name both as a global property and as a bare identifier (typeof of an undeclared name does not throw).
330
+ ...ISOLATE_FORBIDDEN.map((k) => `r.types[${JSON.stringify(k)}] = typeof globalThis[${JSON.stringify(k)}] !== "undefined" ? typeof globalThis[${JSON.stringify(k)}] : typeof ${k};`),
331
+ `for (const s of ["node:fs", "fs", "./x.js", "file:///etc/hosts"]) { try { const m = await import(s); r.imports[s] = "loaded " + Object.keys(m).length; } catch (e) { r.imports[s] = "refused: " + String(e && e.message || e).slice(0, 120); } }`,
332
+ "r.globals = Object.getOwnPropertyNames(globalThis).sort();",
333
+ 'text("ISOLATE=" + JSON.stringify(r));',
334
+ ].join("\n");
335
+
336
+ /**
337
+ * The isolate check's output against what is allowed: every forbidden name undefined, every import
338
+ * refused, and no global outside ISOLATE_GLOBALS. Returns the problems, or one saying the probe
339
+ * printed nothing (which fails: the check could not tell).
340
+ */
341
+ export function isolateProblems(text, allowed = ISOLATE_GLOBALS) {
342
+ const m = /ISOLATE=(\{.*\})/.exec(String(text ?? ""));
343
+ if (!m) return [`the isolate probe printed no result (${JSON.stringify(String(text ?? "").slice(0, 200))}): the check cannot tell`];
344
+ let r;
345
+ try { r = JSON.parse(m[1]); } catch { return ["the isolate probe's result is not JSON: the check cannot tell"]; }
346
+ const p = [];
347
+ for (const [k, t] of Object.entries(r.types ?? {})) if (t !== "undefined") p.push(`code mode's isolate has ${k} (${t})`);
348
+ for (const [s, v] of Object.entries(r.imports ?? {})) if (!String(v).startsWith("refused")) p.push(`code mode's isolate can import ${s} (${v})`);
349
+ const extra = (r.globals ?? []).filter((g) => !allowed.includes(g));
350
+ if (extra.length) p.push(`code mode's isolate has globals relay has not qualified: ${extra.join(", ")}`);
351
+ if (!Array.isArray(r.globals) || !r.globals.length) p.push("the isolate probe listed no globals: the check cannot tell");
352
+ return p;
353
+ }
354
+
355
+ /** One item's state from its parts: failed if any failed, pending if any is pending, not run if any did not run, else passed. */
356
+ export function itemState(parts) {
357
+ const states = Object.values(parts).map((p) => p.state);
358
+ if (!states.length) return "not run";
359
+ if (states.includes("failed")) return "failed";
360
+ if (states.includes(PENDING)) return PENDING;
361
+ if (states.includes("not run")) return "not run";
362
+ return states.every((s) => s === "passed") ? "passed" : "failed";
363
+ }
364
+
365
+ /**
366
+ * The run's verdict: failed if any item failed or is missing (an item the driver lost is never a
367
+ * pass), pending integration if any is pending, not run if any did not run, else passed. `why`
368
+ * names each item that did not pass, with the reasons of its parts.
369
+ */
370
+ export function verdictOf(items) {
371
+ const all = {};
372
+ for (const id of Object.keys(ITEMS)) all[id] = items[id] ?? { state: "failed", why: ["the driver recorded no result for it"], parts: {} };
373
+ const states = Object.values(all).map((i) => i.state);
374
+ const state = states.includes("failed") ? "failed" : states.includes(PENDING) ? PENDING : states.includes("not run") ? "not run" : "passed";
375
+ const why = Object.entries(all).filter(([, i]) => i.state !== "passed").map(([id, i]) => `item ${id} ${i.state}${i.why?.length ? `: ${i.why.join("; ")}` : ""}`);
376
+ return { ok: state === "passed", state, why, verdictSchema: CURRENT_VERDICT_SCHEMA, items: all };
377
+ }
378
+
379
+ /** An item from its parts, with the reasons of every part that did not pass. */
380
+ export function item(parts) {
381
+ const why = [];
382
+ for (const [name, p] of Object.entries(parts)) if (p.state !== "passed") why.push(...(p.why?.length ? p.why : [p.state]).map((w) => `${name}: ${w}`));
383
+ return { state: itemState(parts), why, parts };
384
+ }
385
+
386
+ export const part = (state, why = [], detail = {}) => ({ state, why: [].concat(why).filter(Boolean), ...(Object.keys(detail).length ? { detail } : {}) });
387
+ /** A part judged from a list of problems: passed when there are none. */
388
+ export const judged = (problems, detail = {}) => part(problems.length ? "failed" : "passed", problems, detail);
389
+ /** Shared red-team suite diagnostics become the qualification driver's array-valued parts. */
390
+ export const escalationParts = (parts) => Object.fromEntries(Object.entries(parts).map(([name, { state, why, ...detail }]) => [name, part(state, why, detail)]));
391
+ /** Whether an error is the build's "not implemented" (the skeleton of stage 2 without its package). */
392
+ export const notImplemented = (err) => /not implemented: stage 2/.test(String(err?.message ?? err));
393
+
394
+ /** The command line options: { checkout, dir, version, bin, oldBin, source, only }, or a usage error. */
395
+ export function parseArgs(argv) {
396
+ const [checkout, dir, version, bin, ...rest] = argv;
397
+ if (!bin || !/^\d+\.\d+\.\d+/.test(version ?? "")) throw new Error("usage: codex-stage2.mjs <relay checkout> <scratch dir> <version> <codex bin> [--old-bin <path>] [--source <codex-rs dir>] [--items 1,2]");
398
+ const opt = (k) => {
399
+ const i = rest.indexOf(k);
400
+ if (i < 0) return null;
401
+ if (!rest[i + 1] || rest[i + 1].startsWith("--")) throw new Error(`${k} needs a value`);
402
+ return rest[i + 1];
403
+ };
404
+ const unknown = rest.filter((a, i) => a.startsWith("--") && !["--old-bin", "--source", "--items"].includes(a) && !(i > 0 && ["--old-bin", "--source", "--items"].includes(rest[i - 1])));
405
+ if (unknown.length) throw new Error(`unknown option ${unknown[0]}`);
406
+ const only = (opt("--items") ?? "").split(",").filter(Boolean);
407
+ for (const id of only) if (!(id in ITEMS)) throw new Error(`no item ${id}`);
408
+ return { checkout: path.resolve(checkout), dir: path.resolve(dir), version, bin, oldBin: opt("--old-bin"), source: opt("--source"), only };
409
+ }
410
+
411
+ // ---------------------------------------------------------------------------------------------
412
+ // The live harness: own listeners, the fake model, scratch Codex homes, the executor and the app server.
413
+
414
+ const SYSTEM_PATH = "/usr/bin:/bin:/usr/sbin:/sbin";
415
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
416
+ /** Every process this run started, by pid, each stopped and checked gone at the end. */
417
+ const started = new Map();
418
+ /** Ports of macOS's own services that no check may ever touch, and relay's reserved range, which free ports avoid. */
419
+ const NEVER = new Set([22, 5900, 3283, 3031, 445, 548, 631]);
420
+
421
+ /** A free loopback port: bound on 0 and closed again. Never one of the ports no check may touch. */
422
+ export async function freePort() {
423
+ for (;;) {
424
+ const p = await new Promise((res, rej) => { const s = net.createServer(); s.on("error", rej); s.listen(0, "127.0.0.1", () => { const q = s.address().port; s.close(() => res(q)); }); });
425
+ if (!NEVER.has(p) && !(p >= 47200 && p < 47264)) return p;
426
+ }
427
+ }
428
+
429
+ /** A dual-stack TCP listener of this run's own (on a random high port, or `port`), logging every accept. */
430
+ export async function decoy(name, port = 0) {
431
+ const accepts = [];
432
+ const sockets = new Set();
433
+ const s = net.createServer((c) => { accepts.push(c.remoteAddress); sockets.add(c); c.on("error", () => {}); c.on("close", () => sockets.delete(c)); c.end(`${name}\n`); });
434
+ await new Promise((res, rej) => { s.on("error", rej); s.listen({ port, host: port ? "127.0.0.1" : "::", ipv6Only: false }, res); });
435
+ return { name, port: s.address().port, accepts, close: () => new Promise((r) => { for (const c of sockets) c.destroy(); s.close(() => r()); }) };
436
+ }
437
+
438
+ /**
439
+ * A plain HTTP CONNECT proxy of this run's own that refuses every request and logs it, set as the
440
+ * app server's HTTPS_PROXY so that nothing Codex fetches by itself (the curated plugin sync) leaves
441
+ * the Mac, while its attempts are seen.
442
+ */
443
+ export async function refusingProxy() {
444
+ const seen = [], authorization = [];
445
+ const s = http.createServer((q, r) => { seen.push(`${q.method} ${q.url}`); authorization.push(q.headers.authorization); r.writeHead(403); r.end(); });
446
+ s.on("connect", (q, sock) => { seen.push(`CONNECT ${q.url}`); sock.on("error", () => {}); sock.end("HTTP/1.1 403 Forbidden\r\n\r\n"); });
447
+ await new Promise((res) => s.listen(0, "127.0.0.1", res));
448
+ return { port: s.address().port, seen, hasBearer: (token) => authorization.includes(`Bearer ${token}`), close: () => new Promise((r) => { s.closeAllConnections?.(); s.close(() => r()); }) };
449
+ }
450
+
451
+ const PLUGIN_FAKE_TOKEN = "relay-stage2-plugin-fake-token";
452
+ const PLUGIN_FAKE_ACCOUNT = "relay-stage2-plugin-fake-account";
453
+
454
+ /** Matches app-server/tests/common/auth_fixtures.rs's ChatGptAuthFixture at rust-v0.166.0. */
455
+ function pluginFakeAuth(home) {
456
+ const encode = (x) => Buffer.from(JSON.stringify(x)).toString("base64url");
457
+ const idToken = `${encode({ alg: "none", typ: "JWT" })}.${encode({ "https://api.openai.com/auth": { chatgpt_user_id: "relay-stage2-plugin-fake-user", chatgpt_account_id: PLUGIN_FAKE_ACCOUNT } })}.${Buffer.from("signature").toString("base64url")}`;
458
+ fs.writeFileSync(path.join(home, "auth.json"), JSON.stringify({
459
+ auth_mode: "chatgpt", OPENAI_API_KEY: null,
460
+ tokens: { id_token: idToken, access_token: PLUGIN_FAKE_TOKEN, refresh_token: "relay-stage2-plugin-fake-refresh", account_id: PLUGIN_FAKE_ACCOUNT },
461
+ last_refresh: new Date().toISOString(),
462
+ }), { mode: 0o600 });
463
+ }
464
+
465
+ /** An empty plugin catalog, shared by item 12's positive and negative cases. Never forwards. */
466
+ async function fakePluginService() {
467
+ const seen = [];
468
+ const s = http.createServer((q, r) => {
469
+ const url = new URL(q.url, "http://127.0.0.1");
470
+ const authenticated = q.headers.authorization === `Bearer ${PLUGIN_FAKE_TOKEN}` && q.headers["chatgpt-account-id"] === PLUGIN_FAKE_ACCOUNT;
471
+ seen.push({ method: q.method, path: url.pathname, query: url.search, authenticated });
472
+ let body;
473
+ if (url.pathname === "/backend-api/plugins/featured") body = [];
474
+ else if (url.pathname === "/backend-api/ps/plugins/suggested/codex") body = { enabled: true, plugins: [] };
475
+ else if (["/backend-api/ps/plugins/list", "/backend-api/ps/plugins/installed", "/backend-api/ps/plugins/workspace/shared"].includes(url.pathname)) body = { plugins: [], pagination: { limit: 200, next_page_token: null } };
476
+ r.writeHead(q.method !== "GET" || body === undefined ? 404 : authenticated ? 200 : 401, { "content-type": "application/json" });
477
+ r.end(JSON.stringify(body ?? {}));
478
+ });
479
+ await new Promise((res, rej) => { s.once("error", rej); s.listen(0, "127.0.0.1", res); });
480
+ return { port: s.address().port, seen, close: () => new Promise((r) => { s.closeAllConnections?.(); s.close(() => r()); }) };
481
+ }
482
+
483
+ /**
484
+ * The fake Responses API: answers each request by `nextReply`, and records what each request offered
485
+ * and carried back. `onRequest(reply, body)` runs before each answer (a turn's hooks: kill the
486
+ * executor, open a port).
487
+ */
488
+ export async function fakeModel(conversations, { onRequest } = {}) {
489
+ const requests = [];
490
+ const s = http.createServer((q, r) => {
491
+ let b = "";
492
+ q.on("data", (d) => (b += d));
493
+ q.on("end", async () => {
494
+ if (q.method !== "POST" || !q.url.includes("/responses")) { r.writeHead(404, { "content-type": "application/json" }); r.end("{}"); return; }
495
+ let body;
496
+ try { body = JSON.parse(b); } catch { r.writeHead(400); r.end(); return; }
497
+ const outputs = (body.input ?? []).filter((i) => /(?:_call_output|tool_search_output)$/.test(i?.type ?? "")).map((i) => ({ call_id: i.call_id, text: outputText(i.output) }));
498
+ const peek = { conv: -1, step: 0 };
499
+ try { Object.assign(peek, nextReply(conversations.map((c) => ({ ...c, calls: (c.calls ?? []).map(() => null) })), body)); } catch {}
500
+ if (onRequest) { try { await onRequest(peek, body); } catch (err) { requests.push({ hookError: String(err?.stack ?? err) }); } }
501
+ let reply;
502
+ try { reply = nextReply(conversations, body); }
503
+ catch (err) {
504
+ requests.push({ scriptError: String(err?.message ?? err) });
505
+ reply = { conv: peek.conv, step: peek.step, item: finalMessage("qualification script failed") };
506
+ }
507
+ requests.push({ conv: reply.conv, step: reply.step, tools: offeredTools(body), model: body.model, outputs, sent: reply.item.type === "message" ? "message" : reply.item.name });
508
+ r.writeHead(200, { "content-type": "text/event-stream" });
509
+ r.end(sse(reply.item));
510
+ });
511
+ });
512
+ await new Promise((res) => s.listen(0, "127.0.0.1", res));
513
+ return {
514
+ port: s.address().port, requests,
515
+ /** The output a scripted call got back, by conversation and step. */
516
+ output: (conv, step) => requests.flatMap((x) => x.outputs ?? []).find((o) => o.call_id === `c${conv}-${step}`)?.text ?? null,
517
+ close: () => new Promise((r) => { s.closeAllConnections?.(); s.close(() => r()); }),
518
+ };
519
+ }
520
+
521
+ /**
522
+ * A scratch CODEX_HOME for the app server: OpenAI's own provider, as a person's Codex has it, pointed
523
+ * at the fake model (openai_base_url) with a fake key in the scratch home, the bundled catalog,
524
+ * nothing of the person's. OpenAI's provider, not a custom one: some tools are offered only with it
525
+ * (ext/web-search's web.run), and item 10 must see what production sees.
526
+ */
527
+ function codexHome(home, { port, model, catalog, extra = "" }) {
528
+ fs.mkdirSync(home, { recursive: true, mode: 0o700 });
529
+ fs.writeFileSync(path.join(home, "config.toml"), [
530
+ `model = ${JSON.stringify(model)}`, `openai_base_url = "http://127.0.0.1:${port}/v1"`, `model_catalog_json = ${JSON.stringify(catalog)}`, extra, "",
531
+ ].join("\n"));
532
+ fs.writeFileSync(path.join(home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "sk-relay-stage2-qualification-fake" }), { mode: 0o600 });
533
+ }
534
+
535
+ /** Records a process this run started; the run's sweep stops any still alive. */
536
+ function track(child, what) {
537
+ const exited = new Promise((r) => child.on("exit", (code, signal) => r({ code, signal })));
538
+ const entry = { pid: child.pid, what, exited, child };
539
+ if (child.pid) started.set(child.pid, entry);
540
+ return entry;
541
+ }
542
+ const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === "EPERM"; } };
543
+ async function stopPid(pid, group = false) {
544
+ // Loaded here, not at the top: release-watch's detect job imports this file (for ITEMS, through
545
+ // records.mjs) before anything is built, so a static import of dist/ fails that whole job.
546
+ const { isGroupSignalRefusal, sendGroupSignal } = group ? await import("../../dist/core/procs.js") : {};
547
+ for (const sig of ["SIGTERM", "SIGKILL"]) {
548
+ try { if (group) sendGroupSignal(pid, sig); else process.kill(pid, sig); }
549
+ catch (e) { if (group && isGroupSignalRefusal(e)) { process.stderr.write(`${e.message}\n`); return false; } }
550
+ for (let i = 0; i < 20 && alive(pid); i++) await sleep(50);
551
+ if (!alive(pid)) return true;
552
+ }
553
+ return !alive(pid);
554
+ }
555
+
556
+ /** A JSON-RPC client of `codex app-server` on stdio, answering approval requests as told. */
557
+ function appServer(bin, args, { cwd, env, approve = false }) {
558
+ const child = spawn(bin, ["app-server", ...args], { cwd, env: stageChildEnv(env), stdio: ["pipe", "pipe", "pipe"] });
559
+ const t = track(child, "app server");
560
+ let stderr = "", id = 1, ended = null, stopPromise;
561
+ const waiting = new Map();
562
+ const notes = [];
563
+ const listeners = [];
564
+ child.stderr.on("data", (d) => { stderr = (stderr + d).slice(-20000); });
565
+ child.stdin.on("error", () => {});
566
+ const end = (why) => { ended ??= why; for (const w of waiting.values()) { clearTimeout(w.timer); w.reject(new Error(ended)); } waiting.clear(); };
567
+ child.on("exit", (code, signal) => end(`the app server exited ${code ?? signal}`));
568
+ child.on("error", (err) => end(`the app server could not start: ${err.message}`));
569
+ const send = (m) => { if (!ended) child.stdin.write(`${JSON.stringify(m)}\n`); };
570
+ readline.createInterface({ input: child.stdout }).on("line", (l) => {
571
+ let m; try { m = JSON.parse(l); } catch { return; }
572
+ if (m.id !== undefined && !m.method && waiting.has(m.id)) { const w = waiting.get(m.id); waiting.delete(m.id); clearTimeout(w.timer); m.error ? w.reject(new Error(JSON.stringify(m.error).slice(0, 600))) : w.resolve(m.result); return; }
573
+ if (m.method && m.id !== undefined) {
574
+ notes.push({ request: m.method, params: m.params });
575
+ // Approvals: accept or decline as the case says; anything else is answered with an error.
576
+ if (/requestApproval$/.test(m.method)) send({ id: m.id, result: { decision: approve ? "accept" : "decline" } });
577
+ else send({ id: m.id, error: { code: -32601, message: "not supported by the qualification" } });
578
+ return;
579
+ }
580
+ if (m.method) { notes.push({ method: m.method, params: m.params }); for (const f of listeners) f(m); }
581
+ });
582
+ return {
583
+ pid: child.pid, exited: t.exited, notes,
584
+ request: (method, params) => new Promise((resolve, reject) => {
585
+ if (ended) { reject(new Error(ended)); return; }
586
+ const n = id++;
587
+ const timer = setTimeout(() => { waiting.delete(n); reject(new Error(`${method} timed out after 120s`)); }, 120_000);
588
+ waiting.set(n, { resolve, reject, timer });
589
+ send({ id: n, method, params });
590
+ }),
591
+ notify: (method, params) => send({ method, ...(params ? { params } : {}) }),
592
+ /** Resolves with the first notification `pred` accepts, or null after `ms`. */
593
+ until: (pred, ms) => new Promise((r) => {
594
+ const hit = notes.find((n) => n.method && pred(n));
595
+ if (hit) { r(hit); return; }
596
+ const timer = setTimeout(() => r(null), ms);
597
+ listeners.push((n) => { if (pred(n)) { clearTimeout(timer); r(n); } });
598
+ }),
599
+ stderr: () => stderr,
600
+ eof: () => child.stdin.end(),
601
+ stop: () => stopPromise ??= (async () => {
602
+ end("the app server was stopped"); child.stdin.end();
603
+ return child.exitCode !== null || child.signalCode !== null ? true : stopPid(child.pid);
604
+ })(),
605
+ };
606
+ }
607
+
608
+ async function initialize(rpc) {
609
+ await rpc.request("initialize", { clientInfo: { name: "relay-qualify", title: "relay stage-2 qualification", version: "0.0.1" }, capabilities: { experimentalApi: true } });
610
+ rpc.notify("initialized");
611
+ }
612
+
613
+ /** Whether a loopback TCP port can be bound now. */
614
+ const bindable = (port) => new Promise((res) => { const s = net.createServer(); s.once("error", () => res(false)); s.listen(port, "127.0.0.1", () => s.close(() => res(true))); });
615
+
616
+ /**
617
+ * The ports a case's executor profile gets, as relay leases them (ports.ts): the executor's port in
618
+ * relay's reserved range (whose deny covers it, so a command cannot open a second session on its own
619
+ * executor), a block of random free ports outside it, and the case's denied decoys first, then
620
+ * relay's static list and range (rules only: nothing connects to them).
621
+ */
622
+ async function casePorts(deny) {
623
+ let execPort = 0;
624
+ for (const p of [...Array(64).keys()].map((i) => 47200 + i).sort(() => Math.random() - 0.5)) if (await bindable(p)) { execPort = p; break; }
625
+ if (!execPort) throw new Error("no free port in relay's reserved range 47200-47263 for the case's executor");
626
+ const block = [];
627
+ while (block.length < 2) { const p = await freePort(); if (p !== execPort && !block.includes(p)) block.push(p); }
628
+ return { execPort, block, deny };
629
+ }
630
+
631
+ /** relay's stage-2 profile for a case, by relay's own code (seatbelt.ts with WP-A's network block), around `bin args`. */
632
+ function executorLaunch(ctx, d, ports, bin, args, extraDenied) {
633
+ const { STATIC_DENIED_TCP, STATIC_DENIED_UDP, RELAY_PORT_RANGE } = ctx.dist.ports;
634
+ const range = Array.from({ length: RELAY_PORT_RANGE.count }, (_, i) => RELAY_PORT_RANGE.first + i);
635
+ const network = { mode: "stage2", deniedTcp: [...new Set([...ports.deny, ...STATIC_DENIED_TCP])], deniedUdp: [...STATIC_DENIED_UDP], relayRange: range, execPort: ports.execPort, bindPorts: ports.block, worktreeSockets: false };
636
+ const policy = { worktree: d.wt, worktreeWritable: true, writable: [], stateDirs: [], tmpDir: d.tmp, denied: [...extraDenied], network, state: { home: "temp" } };
637
+ return ctx.dist.seatbelt.seatbeltLaunch(policy, bin, args);
638
+ }
639
+
640
+ /** `codex exec-server` on the case's executor port under relay's profile, as a detached group leader. Its token's hash only is on its argv. */
641
+ async function startExecutor(ctx, d, { hash, ports, extraDenied = [] }) {
642
+ const args = ["exec-server", "--listen", `ws://127.0.0.1:${ports.execPort}`, "--ws-auth", "capability-token", "--ws-token-sha256", hash];
643
+ const launch = executorLaunch(ctx, d, ports, ctx.bin, args, extraDenied);
644
+ const home = path.join(d.tmp, "codex-home");
645
+ fs.mkdirSync(home, { recursive: true, mode: 0o700 });
646
+ const env = { PATH: `${SYSTEM_PATH}:/opt/homebrew/bin`, ...launch.env, HOME: launch.env.HOME ?? d.tmp, CODEX_HOME: home, TMPDIR: launch.env.TMPDIR ?? d.tmp };
647
+ const child = spawn(launch.command, launch.args, { cwd: d.wt, env: stageChildEnv(env), detached: true, stdio: ["ignore", "pipe", "pipe"] });
648
+ const t = track(child, "executor");
649
+ let out = "", err = "";
650
+ child.stdout.on("data", (x) => { out += x; });
651
+ child.stderr.on("data", (x) => { err = (err + x).slice(-8000); });
652
+ for (let i = 0; i < 200 && !/ws:\/\/\S+/.test(out) && alive(child.pid); i++) await sleep(50);
653
+ const url = /ws:\/\/\S+/.exec(out)?.[0] ?? null;
654
+ if (!url) { await stopPid(child.pid, true); throw new Error(`the executor printed no URL (${err.trim().split("\n").at(-1) ?? "no output"})`); }
655
+ return { pid: child.pid, url, port: ports.execPort, ports, exited: t.exited, sandboxed: ctx.dist.seatbelt.sandboxed(child.pid), err: () => err, stop: () => stopPid(child.pid, true) };
656
+ }
657
+
658
+ /** A case's directories: the worktree (a git repository), the executor's TMPDIR, the app server's home and TMPDIR, and an outside directory no profile lets the job write. */
659
+ function caseDirs(ctx, name) {
660
+ const dir = path.join(ctx.dir, name);
661
+ fs.rmSync(dir, { recursive: true, force: true });
662
+ const d = Object.fromEntries(["wt", "tmp", "home", "user", "apptmp", "out"].map((k) => [k, path.join(dir, k)]));
663
+ for (const p of Object.values(d)) fs.mkdirSync(p, { recursive: true, mode: 0o700 });
664
+ execFileSync("git", ["init", "-q", "-b", "main"], { cwd: d.wt, env: stageChildEnv() });
665
+ fs.writeFileSync(path.join(d.wt, "README.md"), "stage 2 qualification\n");
666
+ return { dir: fs.realpathSync(dir), ...Object.fromEntries(Object.entries(d).map(([k, p]) => [k, fs.realpathSync(p)])) };
667
+ }
668
+
669
+ /**
670
+ * One model-free turn: the fake model, a scratch Codex home, the executor (env "relay"), an
671
+ * unreachable URL (env "unreachable") or none (env "local", the control), the app server, the
672
+ * thread and one turn. `o.during(e)` runs once the turn is over and before teardown; `o.keep`
673
+ * leaves everything running for the caller; `o.appEnv` adds to the app server's environment
674
+ * (relayEnv: relay's own CODEX_EXEC_SERVER_URL=none); `o.onRequest(reply, body)` runs before each
675
+ * model answer. Returns the evidence.
676
+ */
677
+ async function runCase(ctx, o) {
678
+ const d = caseDirs(ctx, o.name);
679
+ const model = await fakeModel(o.conversations(d), o.onRequest ? { onRequest: o.onRequest } : {});
680
+ const token = crypto.randomBytes(32).toString("hex");
681
+ const hash = crypto.createHash("sha256").update(token).digest("hex");
682
+ const e = { d, model, token, hash, ex: null, rpc: null, thread: null, done: null, problems: [] };
683
+ try {
684
+ codexHome(d.home, { port: model.port, model: o.model ?? ctx.model, catalog: ctx.catalog, extra: o.extraConfig?.(d) ?? "" });
685
+ o.prepare?.(d);
686
+ if (o.broker) e.broker = await qualificationBroker(ctx, d, o.broker === true ? {} : o.broker);
687
+ if (o.env === "relay") e.ex = await startExecutor(ctx, d, { hash, ports: await casePorts([...(o.deny ?? []), ...(e.broker ? [Number(new URL(e.broker.url).port)] : [])]), extraDenied: [d.home, d.out] });
688
+ const url = e.ex?.url ?? (o.env === "unreachable" ? `ws://127.0.0.1:${await freePort()}` : null);
689
+ const env = { PATH: SYSTEM_PATH, HOME: d.user, CODEX_HOME: d.home, TMPDIR: d.apptmp, SHELL: "/bin/zsh", RUST_LOG: process.env.QUALIFY_RUST_LOG ?? "warn", NO_PROXY: "127.0.0.1,localhost,::1", no_proxy: "127.0.0.1,localhost,::1", ...(e.broker?.env ?? {}), ...(o.appEnv ?? {}) };
690
+ if (o.missingBrokerToken) delete env.RELAY_BROKER_TOKEN;
691
+ e.rpc = appServer(o.bin ?? ctx.bin, o.flags ? o.flags(d, e.broker) : stage2Flags(ctx, d, o.servers ?? [], e.broker), { cwd: d.wt, env, approve: o.approve });
692
+ if (e.broker) {
693
+ const start = ctx.dist.procs.procStart(e.rpc.pid, { fresh: true });
694
+ if (!start) throw new Error("cannot pin qualification app server");
695
+ e.broker.bindAppServer({ pid: e.rpc.pid, start });
696
+ }
697
+ await initialize(e.rpc);
698
+ if (url) e.add = await e.rpc.request("environment/add", { environmentId: "relay", execServerUrl: url, authBearerToken: token });
699
+ await o.beforeThread?.(e);
700
+ if (o.noTurn) return e;
701
+ e.thread = await e.rpc.request("thread/start", {
702
+ cwd: d.wt, runtimeWorkspaceRoots: [d.wt], permissions: o.perms ?? ":danger-full-access", approvalPolicy: o.policy ?? "on-request", approvalsReviewer: "user",
703
+ ...(url ? { environments: [{ environmentId: "relay", cwd: d.wt }] } : {}),
704
+ });
705
+ await o.beforeTurn?.(e);
706
+ await e.rpc.request("turn/start", { threadId: e.thread.thread.id, input: [{ type: "text", text: o.prompt ?? "go", text_elements: [] }] });
707
+ e.broker?.arm();
708
+ e.done = await e.rpc.until((n) => ownTurnCompleted(n, e.thread.thread.id), o.timeoutMs ?? 60_000);
709
+ if (o.during) await o.during(e);
710
+ return e;
711
+ } catch (err) {
712
+ if (o.keep) await teardown(e);
713
+ throw err;
714
+ } finally {
715
+ if (!o.keep) await teardown(e);
716
+ }
717
+ }
718
+
719
+ async function teardown(e) {
720
+ if (e.rpc) await e.rpc.stop();
721
+ if (e.ex) await e.ex.stop();
722
+ await e.broker?.close();
723
+ await e.model?.close();
724
+ }
725
+
726
+ /** The items' commands, by what they print: `key=status` lines. */
727
+ const sh = (lines) => lines.join("; ");
728
+ const exec = (cmd, extra = {}) => ({ type: "function_call", name: "exec_command", arguments: JSON.stringify({ cmd, login: false, timeout_ms: 10_000, ...extra }) });
729
+ const patch = (file, text) => ({ type: "custom_tool_call", name: "apply_patch", input: `*** Begin Patch\n*** Add File: ${file}\n+${text}\n*** End Patch\n` });
730
+ /** Code mode's own tool: JavaScript run in its isolate, which reaches the nested tools through `tools`. */
731
+ const codeMode = (js) => ({ type: "custom_tool_call", name: "exec", input: js });
732
+ /** A code-mode cell that runs a command and a patch through the nested tools, printing what came back. */
733
+ const nestedCalls = (d) => codeMode([
734
+ `const c = await tools.exec_command({ cmd: ${JSON.stringify(sh([`touch ${d.out}/nested_out`, "echo nestedout=$?", `touch ${d.wt}/nested_in`, "echo nestedin=$?"]))}, login: false });`,
735
+ // A structured result carries the command's output with its line breaks escaped: restore them, so that its key=value lines read.
736
+ "text('\\n' + (typeof c === 'string' ? c : typeof c?.output === 'string' ? c.output : JSON.stringify(c)).replace(/\\\\n/g, '\\n') + '\\n');",
737
+ `try { const p = await tools.apply_patch(${JSON.stringify(`*** Begin Patch\n*** Add File: ${d.out}/nested_patch.txt\n+outside\n*** End Patch\n`)}); text("\\npatch=" + JSON.stringify(p).slice(0, 200)); } catch (e) { text("\\npatch=refused " + String(e && e.message || e).slice(0, 200)); }`,
738
+ ].join("\n"));
739
+ const exists = (p) => fs.existsSync(p);
740
+
741
+ /**
742
+ * What relay adds to a stage-2 app server's environment (codex-stage2.ts STAGE2_APP_SERVER_ENV,
743
+ * CODEX_EXEC_SERVER_URL=none): Codex registers no `local` environment. Cases that stand for relay's
744
+ * own settings run with it; controls that must reach `local` run without.
745
+ */
746
+ const relayEnv = (ctx) => ({ ...ctx.dist.stage2.STAGE2_APP_SERVER_ENV });
747
+ const relayCodex = (ctx, home = ctx.pin.home) => ({ ...ctx.pin, home, stage2: true });
748
+
749
+ /**
750
+ * The app server's flags for a stage-2 turn, by relay's own codexFlags with WP-D's stage2Plan:
751
+ * relay-only MCP (the case's `servers` switched off by name), hooks and plugins off, every feature of
752
+ * STAGE2_FEATURES_OFF pinned off, approvals on-request.
753
+ */
754
+ function stage2Flags(ctx, d, servers = [], broker) {
755
+ const spec = {
756
+ config: ctx.dist.pin.freezeCodexConfig(ctx.dist.config.withDefaults({ child_mcp: "relay-only", child_hooks: false, targets: { codex: relayCodex(ctx, d.home) } })),
757
+ target: ctx.dist.targets.targetDef("codex"), access: "write", cwd: d.wt, relay: null, grant: { network: true, raw_network: true },
758
+ };
759
+ const plan = ctx.dist.stage2.stage2Plan(spec, { mode: "stage2", deniedTcp: [], deniedUdp: [], relayRange: [], execPort: 0, bindPorts: [], worktreeSockets: false });
760
+ if (broker) plan.broker = { url: broker.url, toolTimeoutS: broker.toolTimeoutS ?? 180 };
761
+ const f = ctx.dist.codex.codexFlags(spec, servers, { HOME: d.user, CODEX_HOME: d.home, PATH: SYSTEM_PATH }, [], undefined, plan);
762
+ return [...f.off, ...f.c];
763
+ }
764
+
765
+ // ---------------------------------------------------------------------------------------------
766
+ // relay's own path: codexBackend.start with stage 2 switched on, the fake model, scratch homes.
767
+
768
+ /** The executor relay started for a turn in `cwd`: an `exec-server` with a token hash whose working directory is `cwd`. */
769
+ function executorIn(cwd) {
770
+ const r = spawnSync("/bin/ps", ["-axww", "-o", "pid=,command="], { encoding: "utf8", maxBuffer: 32 << 20, env: stageChildEnv() });
771
+ for (const line of (r.stdout ?? "").split("\n")) {
772
+ const m = /^\s*(\d+)\s+(.*)$/.exec(line);
773
+ if (!m || !/ exec-server --listen ws:\/\/127\.0\.0\.1:\d+ .*--ws-token-sha256 [0-9a-f]{64}/.test(m[2])) continue;
774
+ const pid = Number(m[1]);
775
+ const c = spawnSync("/usr/sbin/lsof", ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], { encoding: "utf8", env: stageChildEnv() });
776
+ if ((c.stdout ?? "").split("\n").some((l) => l === `n${cwd}`)) return { pid, port: Number(/--listen ws:\/\/127\.0\.0\.1:(\d+)/.exec(m[2])[1]), command: m[2] };
777
+ }
778
+ return null;
779
+ }
780
+
781
+ /**
782
+ * Whether relay's stage-2 path runs in this build, tried once with a turn of no calls: null when it
783
+ * does, { pending } when the build lacks WP-C ("not implemented"), { noSession } when relay's own
784
+ * sandbox cannot launch here for want of a login session (a launchd daemon with no desktop login: its
785
+ * refusal carries the apple-events evidence and session.mjs confirms the gap), { failed } when relay
786
+ * refused or failed the turn for another reason (admission on this Mac, say), which fails every relay part.
787
+ */
788
+ async function relayPathMissing(ctx) {
789
+ if (ctx.relayMissing !== undefined) return ctx.relayMissing;
790
+ let out = null;
791
+ try {
792
+ const r = await relayTurn(ctx, { name: "relay-ready", conversations: () => [{ calls: [] }] });
793
+ if (notImplemented(r.result.error)) out = { pending: r.result.error };
794
+ else if (r.result.status !== "ok") out = { failed: `${r.result.status}: ${r.result.error ?? ""}` };
795
+ } catch (err) {
796
+ out = notImplemented(err) ? { pending: err.message } : { failed: String(err?.message ?? err) };
797
+ }
798
+ if (out?.failed && NO_SESSION_EVIDENCE.test(out.failed)) {
799
+ const gap = noLoginSession(ctx.checkout);
800
+ if (gap.gap) out = { noSession: gap.why };
801
+ }
802
+ ctx.relayMissing = out;
803
+ return out;
804
+ }
805
+
806
+ /** A relay part, or PENDING where relay's stage-2 path is not in this build. */
807
+ async function relayPart(ctx, run) {
808
+ const missing = await relayPathMissing(ctx);
809
+ if (missing?.pending) return part(PENDING, `relay's stage-2 path is not in this build (${missing.pending}): to run once WP-C is merged`);
810
+ if (missing?.noSession) return part("not run", `no login session (${missing.noSession}): relay's own sandbox cannot launch here, so run the qualification by hand on a logged-in Mac (docs/qualification/codex-stage2-template.md)`);
811
+ if (missing?.failed) return part("failed", `a turn through relay's stage-2 path with no calls did not succeed: ${missing.failed}`);
812
+ try { return await run(); } catch (err) { return notImplemented(err) ? part(PENDING, String(err.message)) : part("failed", `the relay part failed: ${err.stack?.split("\n").slice(0, 3).join(" ") ?? err}`); }
813
+ }
814
+
815
+ /**
816
+ * The job a relay turn belongs to, in the scratch relay home's registry, with the person's consent
817
+ * to its raw network (admission refuses without it, review R2): one run and job per case directory
818
+ * (a resumed turn is the same job), a turn row for each turn, and relay's own consent flow
819
+ * (rawNetworkConsent files a person-only raw_network request) answered as the person answers it in
820
+ * relay's approval dialog (decide, by "user"). Returns { runId, nodeId, turnId }.
821
+ */
822
+ async function registeredJob(ctx, d, name, grant) {
823
+ const { createRun, insertNode, insertTurn, getNode } = ctx.dist.db;
824
+ ctx.jobs ??= new Map();
825
+ let job = ctx.jobs.get(d.dir);
826
+ if (!job) {
827
+ const runId = createRun(d.wt, ctx.dist.config.withDefaults({ targets: { codex: relayCodex(ctx, d.home) } }), `qualify-${crypto.randomUUID().slice(0, 8)}`);
828
+ const nodeId = `j_q${name.replace(/\W/g, "")}${crypto.randomUUID().slice(0, 6)}`;
829
+ insertNode({ id: nodeId, run_id: runId, kind: "job", target: "codex", task: "stage-2 qualification", cwd: d.wt, mode: "write", depth: 1, status: "running", params_json: JSON.stringify({ grant }) });
830
+ job = { runId, nodeId };
831
+ ctx.jobs.set(d.dir, job);
832
+ }
833
+ const turnId = insertTurn(job.nodeId, "caller", "go", "write", "running");
834
+ let stop = false;
835
+ const answer = (async () => {
836
+ for (let i = 0; i < 200 && !stop; i++) {
837
+ const r = ctx.dist.approvals.pendingRequests(job.runId).find((x) => x.node_id === job.nodeId && x.kind === "raw_network");
838
+ if (r) return ctx.dist.approvals.decide(r.id, true, "user");
839
+ await sleep(25);
840
+ }
841
+ return null;
842
+ })();
843
+ try {
844
+ const got = await ctx.dist.rawNetwork.rawNetworkConsent(getNode(job.nodeId), turnId, "write", () => false);
845
+ if (got) throw new Error(`relay's raw-network consent for the qualification's job: ${JSON.stringify(got)}`);
846
+ } finally { stop = true; await answer; }
847
+ return { ...job, turnId };
848
+ }
849
+
850
+ /**
851
+ * One turn through relay's own path: codexBackend.start with stage 2 switched on in relay's config
852
+ * (targets.codex.stage2), grant.network and grant.raw_network, a scratch CODEX_HOME whose model is
853
+ * the fake one, inside withTurnSecrets (so redaction knows the executor token). `o.dirs` reuses an
854
+ * earlier turn's directories (a resume); `o.sessionId` resumes that thread. `o.onRequest(reply, h)`
855
+ * runs before each model answer, with `h.executor()` (relay's executor for this turn) and
856
+ * `h.capture(what, text)`. Returns { d, model, result, approvals, appPid, captured, tokenIn }.
857
+ */
858
+ async function relayTurn(ctx, o) {
859
+ const d = o.dirs ?? caseDirs(ctx, o.name);
860
+ const captured = [];
861
+ let ex = null;
862
+ const h = { executor: () => (ex ??= executorIn(d.wt)), capture: (what, text) => captured.push({ what, text: String(text ?? "") }), d };
863
+ const model = o.model$ ?? await fakeModel(o.conversations(d), { onRequest: async (reply, body) => { h.executor(); await o.onRequest?.(reply, h, body); } });
864
+ h.model = model;
865
+ const { withTurnSecrets, redact } = ctx.dist.secrets;
866
+ const approvals = [];
867
+ let result, appPid = 0, startRefused = false;
868
+ h.appPid = () => appPid || appServerFor(d.home);
869
+ const tokenIn = [], brokerTokenIn = [];
870
+ try {
871
+ const slug = o.model ?? ctx.model;
872
+ fs.mkdirSync(d.home, { recursive: true, mode: 0o700 });
873
+ fs.writeFileSync(path.join(d.home, "config.toml"), [
874
+ ctx.dist.pin.RELAY_CODEX_CONFIG,
875
+ `model = ${JSON.stringify(slug)}`, `openai_base_url = "http://127.0.0.1:${model.port}/v1"`, `model_catalog_json = ${JSON.stringify(ctx.catalog)}`,
876
+ o.extraConfig?.(d) ?? "", "",
877
+ ].join("\n"));
878
+ // A fake key for the fake model, in the scratch home only.
879
+ fs.writeFileSync(path.join(d.home, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "sk-relay-stage2-qualification-fake" }), { mode: 0o600 });
880
+ o.prepare?.(d);
881
+ // PC generates the owned home's config on every preparation. Preserve this case's fake
882
+ // endpoint/catalog and negative-control settings through the code-only fixture seam.
883
+ const fixtureConfig = fs.readFileSync(path.join(d.home, "config.toml"), "utf8");
884
+ ctx.dist.pin.setCodexHomeConfigForTests((home) => home === d.home ? fixtureConfig : undefined);
885
+ const grant = { network: true, raw_network: true };
886
+ const job = await registeredJob(ctx, d, o.name, grant);
887
+ h.nodeId = job.nodeId;
888
+ const spec = {
889
+ target: ctx.dist.targets.targetDef("codex"), prompt: o.prompt ?? "go", cwd: d.wt, dir: d.dir, logPath: path.join(d.dir, `agent-${o.sessionId ? "resume" : "start"}.log`), access: "write",
890
+ relay: { runId: job.runId, nodeId: job.nodeId, depth: 1, turnId: job.turnId, mayDelegate: o.mayDelegate ?? true, mayAsk: false },
891
+ config: ctx.dist.config.withDefaults({ codex_backend: "app-server", child_auth: "any", codex_endpoint_override: "accept", child_mcp: "relay-only", child_hooks: false, cancel_grace_ms: 2000, targets: { codex: relayCodex(ctx, d.home) } }),
892
+ environment: scratchEnv(ctx.dir, { HOME: d.user, CODEX_HOME: d.home, TMPDIR: d.apptmp, SHELL: "/bin/zsh" }),
893
+ markers: { runId: job.runId, nodeId: job.nodeId, depth: 1 },
894
+ turnId: job.turnId,
895
+ grant,
896
+ ...(o.effort ? { effort: o.effort } : {}),
897
+ escalate: Boolean(o.approve),
898
+ ...(o.sessionId ? { sessionId: o.sessionId } : {}),
899
+ onApproval: async (ask) => { approvals.push(ask); return { decision: o.approve ? "allow" : "deny" }; },
900
+ };
901
+ await withTurnSecrets(async () => {
902
+ const started = await qualificationStart(() => ctx.dist.codex.codexBackend.start(spec), o.expectedStartRefusal);
903
+ startRefused = started.startRefused;
904
+ if (startRefused) { result = started.result; return; }
905
+ const handle = started.handle;
906
+ appPid = handle.pid ?? 0;
907
+ result = await handle.done;
908
+ // The token, wherever it showed: only redaction knows it, so only redaction can tell.
909
+ const label = "[redacted: executor token]";
910
+ for (const c of captured) {
911
+ if (redact(c.text).includes(label)) tokenIn.push(c.what);
912
+ if (redact(c.text).includes("[redacted: broker token]")) brokerTokenIn.push(c.what);
913
+ }
914
+ const walk = (dir) => {
915
+ // Immutable candidate bytes predate the turn and cannot contain its generated tokens.
916
+ if (dir === ctx.pin.pin_store) return;
917
+ let es; try { es = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
918
+ for (const e of es) {
919
+ const p = path.join(dir, e.name);
920
+ if (e.isDirectory()) walk(p);
921
+ else if (e.isFile()) { try { if (redact(fs.readFileSync(p, "latin1")).includes(label)) tokenIn.push(p); } catch {} }
922
+ }
923
+ };
924
+ for (const root of [d.dir, process.env.RELAY_HOME, process.env.HOME]) walk(root);
925
+ });
926
+ await o.after?.({ d, model, result, h, appPid });
927
+ } finally {
928
+ ctx.dist.pin.setCodexHomeConfigForTests(null);
929
+ if (!o.keepModel) await model.close();
930
+ }
931
+ return { d, model, result, startRefused, approvals, appPid, nodeId: h.nodeId, captured, tokenIn, brokerTokenIn, executor: ex };
932
+ }
933
+
934
+ /** The turn's status, with its error, for a part's detail. */
935
+ const turnSaid = (r) => ({ status: r.result?.status ?? null, error: String(r.result?.error ?? "").slice(0, 400) || undefined, warnings: r.result?.warnings?.length ? r.result.warnings : undefined, survivors: r.result?.survivors?.length ? r.result.survivors : undefined });
936
+
937
+ // ---------------------------------------------------------------------------------------------
938
+ // The items. Each returns its parts.
939
+
940
+ /** The checks of item 1's calls, from what they printed and left: the same for the codex and the relay part. */
941
+ function confinedProblems(o, d, m, nested, denyAccepts, okAccepts, net) {
942
+ const p = [];
943
+ if (m.wt !== "0") p.push(`a command could not write the worktree (wt=${m.wt})`);
944
+ if (m.out === "0" || exists(`${d.out}/proof`)) p.push("a command wrote outside the roots");
945
+ p.push(...netProblems(net, DENY_AT, OK_AT));
946
+ if (denyAccepts) p.push(`the denied decoy accepted ${denyAccepts} connection(s)`);
947
+ if (!okAccepts) p.push("the allowed decoy saw no connection: the network check cannot tell");
948
+ if (m.readhome === "0") p.push("a command read the app server's CODEX_HOME");
949
+ if (m.writehome === "0") p.push("a command wrote the app server's CODEX_HOME");
950
+ if (exists(`${d.out}/patched.txt`)) p.push("apply_patch wrote outside the roots");
951
+ if (!exists(`${d.wt}/patched_in.txt`)) p.push("apply_patch could not write inside the worktree");
952
+ if (o.codeMode) {
953
+ if (nested.nestedin !== "0") p.push(`code mode's nested exec_command did not run in the worktree (nestedin=${nested.nestedin}): the check cannot tell`);
954
+ if (nested.nestedout === "0" || exists(`${d.out}/nested_out`)) p.push("code mode's nested exec_command wrote outside the roots");
955
+ if (exists(`${d.out}/nested_patch.txt`)) p.push("code mode's nested apply_patch wrote outside the roots");
956
+ }
957
+ return p;
958
+ }
959
+
960
+ /** Where item 1 tries the denied port (every spelling of this Mac's addresses) and the allowed one. */
961
+ const DENY_AT = localSpellings();
962
+ const OK_AT = ["127.0.0.1"];
963
+
964
+ /** Item 1's calls: a command writing in and out of the roots, probing a denied and an allowed port and the app server's home; patches out and in; a code-mode cell through the nested tools. */
965
+ const item1Calls = (d, deny, ok, codeModeToo) => [
966
+ () => exec(sh([`touch ${d.wt}/in_wt`, "echo wt=$?", `touch ${d.out}/proof`, "echo out=$?",
967
+ netProbe(process.execPath, deny.port, DENY_AT, ok().port, OK_AT),
968
+ `cat ${d.home}/config.toml >/dev/null 2>&1`, "echo readhome=$?", `echo x >> ${d.home}/config.toml`, "echo writehome=$?"]), { timeout_ms: 120_000 }),
969
+ patch(`${d.out}/patched.txt`, "outside"),
970
+ patch("patched_in.txt", "inside"),
971
+ ...(codeModeToo ? [nestedCalls(d)] : []),
972
+ ];
973
+
974
+ /** Item 1 (W1, W5, B0), with a denied decoy port: commands, patches and code mode's nested tools inside the roots only, the decoy unreachable, the app server's home unreadable. */
975
+ async function item1(ctx) {
976
+ const codeModeToo = ctx.codeMode;
977
+ const deny = await decoy("deny"), ok = await decoy("ok");
978
+ try {
979
+ const w = await runCase(ctx, { name: "i1-relay", env: "relay", deny: [deny.port], conversations: (d) => [{ calls: item1Calls(d, deny, () => ok, codeModeToo) }] });
980
+ const m = markers(w.model.output(0, 0)), nested = markers(w.model.output(0, 3)), net1 = netLines(w.model.output(0, 0));
981
+ const p = confinedProblems({ codeMode: codeModeToo }, w.d, m, nested, deny.accepts.length, ok.accepts.length, net1);
982
+ if (!w.done) p.push("the turn did not complete");
983
+ if (!w.ex.sandboxed) p.push("the executor's pid is not sandboxed");
984
+ const codex = judged(p, { markers: m, nested, net: net1, model: ctx.model });
985
+ // B0: the same calls without the environment and without relay's profile must escape, or the check proves nothing.
986
+ const b = await runCase(ctx, { name: "i1-control", env: "local", conversations: (d) => [{ calls: item1Calls(d, deny, () => ok, codeModeToo) }] });
987
+ const cm = markers(b.model.output(0, 0)), cn = markers(b.model.output(0, 3)), cnet = netLines(b.model.output(0, 0));
988
+ const cp = [];
989
+ if (!(cm.out === "0" && exists(`${b.d.out}/patched.txt`))) cp.push(`without the environment the same calls did not write outside (out=${cm.out}): the check cannot tell`);
990
+ // Without relay's profile the denied port answers at loopback, so a refusal inside means the profile refused it.
991
+ if (cnet.find((x) => x.kind === "deny" && x.address === "127.0.0.1")?.outcome !== "CONNECTED") cp.push("without relay's profile the denied decoy was not reachable at 127.0.0.1: the network check cannot tell");
992
+ if (codeModeToo && !exists(`${b.d.out}/nested_out`)) cp.push(`without the environment code mode's nested exec_command did not write outside (nestedout=${cn.nestedout}): the check cannot tell`);
993
+ const control = judged(cp, { markers: cm, nested: cn, net: cnet });
994
+ const relay = await relayPart(ctx, async () => {
995
+ // The denied decoy listens before the turn starts, so relay's listening snapshot denies it; the allowed one opens once the turn runs.
996
+ const deny2 = await decoy("deny-relay");
997
+ let ok2 = null;
998
+ try {
999
+ const r = await relayTurn(ctx, { name: "i1-relaypath", conversations: (d) => [{ calls: item1Calls(d, deny2, () => ok2, codeModeToo) }], onRequest: async (reply) => { if (reply.conv === 0 && reply.step === 0) ok2 ??= await decoy("ok-relay"); } });
1000
+ const rm = markers(r.model.output(0, 0)), rn = markers(r.model.output(0, 3)), rnet = netLines(r.model.output(0, 0));
1001
+ const rp = confinedProblems({ codeMode: codeModeToo }, r.d, rm, rn, deny2.accepts.length, ok2?.accepts.length ?? 0, rnet);
1002
+ if (r.result.status !== "ok") rp.push(`the turn did not succeed (${r.result.status}: ${r.result.error ?? ""})`);
1003
+ if (!r.executor) rp.push("relay's executor for the turn was not seen: the check cannot tell");
1004
+ return judged(rp, { markers: rm, nested: rn, net: rnet, turn: turnSaid(r) });
1005
+ } finally { await deny2.close(); await ok2?.close(); }
1006
+ });
1007
+ return { codex, control, relay };
1008
+ } finally {
1009
+ await deny.close(); await ok.close();
1010
+ }
1011
+ }
1012
+
1013
+ /** Item 2 (B3): a restricted Codex profile nests Codex's own sandbox-exec inside relay's, which fails (exit 71); apply_patch fails too. */
1014
+ async function item2(ctx) {
1015
+ const w = await runCase(ctx, { name: "i2", env: "relay", perms: ":workspace", conversations: () => [{ calls: [exec("echo ran=yes"), patch("patched_in.txt", "inside")] }] });
1016
+ const out = w.model.output(0, 0) ?? "";
1017
+ const done = w.rpc.notes.filter((n) => n.method === "item/completed" && n.params?.item?.type === "commandExecution").map((n) => n.params.item.exitCode);
1018
+ const p = [];
1019
+ if (markers(out).ran === "yes") p.push("a command ran under a restricted Codex profile inside relay's");
1020
+ if (!done.includes(71) && !/sandbox_apply|exit(ed)? (code )?71/i.test(out)) p.push(`the command did not fail with exit 71 (exit codes ${JSON.stringify(done)}; output ${JSON.stringify(out.slice(0, 200))})`);
1021
+ if (exists(`${w.d.wt}/patched_in.txt`)) p.push("apply_patch succeeded under a restricted Codex profile inside relay's");
1022
+ return { codex: judged(p, { exitCodes: done, output: out.slice(0, 300) }) };
1023
+ }
1024
+
1025
+ /** A stand-in stdio MCP server: touches a marker outside every root (so it runs unconfined, outside the executor), then idles. Returns its script. */
1026
+ function mcpScript(d, name) {
1027
+ const script = path.join(d.dir, `mcp-${name}.sh`);
1028
+ fs.writeFileSync(script, `#!/bin/sh\ntouch ${JSON.stringify(path.join(d.out, `mcp-${name}`))}\nexec cat >/dev/null\n`, { mode: 0o700 });
1029
+ return script;
1030
+ }
1031
+ const mcpServer = (d, name) => `[mcp_servers.${name}]\ncommand = "/bin/sh"\nargs = [${JSON.stringify(mcpScript(d, name))}]\nstartup_timeout_sec = 10\n`;
1032
+ const waitFor = async (f, ms) => { for (let i = 0; i < ms / 100 && !exists(f); i++) await sleep(100); return exists(f); };
1033
+ /** An app server with nothing of stage 2's settings: the control cases. */
1034
+ const BARE = () => ["-c", 'default_permissions=":danger-full-access"', "-c", 'approvals_reviewer="user"'];
1035
+
1036
+ /** A stage-2 app server of relay's, running now, whose CODEX_HOME is `home`: its pid, or 0. */
1037
+ const appServerFor = (home) => spawnSync("/usr/bin/pgrep", ["-f", "app-server"], { encoding: "utf8", env: stageChildEnv() }).stdout.split("\n").filter(Boolean).map(Number).find((pid) => psEnv(pid).includes(`CODEX_HOME=${home}`)) ?? 0;
1038
+
1039
+ /**
1040
+ * Item 3 (W2): a stdio MCP server starts outside the executor, in Codex's `local` environment. Under
1041
+ * relay's stage-2 settings, which give the app server no `local` (CODEX_EXEC_SERVER_URL=none), none
1042
+ * runs: the person's are switched off, relay's own entry is switched off, and a stdio server left on
1043
+ * cannot start ("requires a local environment"). So a stage-2 job has no relay tools (design §8).
1044
+ */
1045
+ async function item3(ctx) {
1046
+ // Outside: a server of the person's, with settings that let it run, starts in the local environment.
1047
+ const a = await runCase(ctx, { name: "i3-outside", env: "relay", flags: BARE, extraConfig: (d) => mcpServer(d, "person"), conversations: () => [{ calls: [] }],
1048
+ during: async (e) => { e.ran = await waitFor(`${e.d.out}/mcp-person`, 8000); } });
1049
+ const outside = judged(a.ran ? [] : ["a stdio MCP server did not start outside the executor (it wrote no marker outside the roots)"]);
1050
+ // Relay's settings: its stage-2 flags (the person's server switched off by name, relay's entry off) and its environment (no `local`).
1051
+ // A stand-in at relay's key, switched back on, shows that without `local` no stdio server starts at all.
1052
+ const look = async (name, extraFlags) => runCase(ctx, { name, env: "relay", broker: true, appEnv: relayEnv(ctx), extraConfig: (d) => mcpServer(d, "person"),
1053
+ flags: (d, broker) => [...stage2Flags(ctx, d, ["person"], broker), ...extraFlags(d)],
1054
+ conversations: () => [{ calls: [] }],
1055
+ during: async (e) => {
1056
+ e.relayRan = await waitFor(`${e.d.out}/mcp-relay`, 4000);
1057
+ e.local = await e.rpc.request("environment/status", { environmentId: "local" }).catch((err) => ({ error: err.message }));
1058
+ e.config = (await e.rpc.request("config/read", { cwd: e.d.wt }).catch(() => null))?.config?.mcp_servers ?? null;
1059
+ e.status = await e.rpc.request("mcpServerStatus/list", {}).catch((err) => ({ error: err.message }));
1060
+ },
1061
+ });
1062
+ // Listed is not running: Codex lists every configured server; one it started has its server's info, or a startup error.
1063
+ const startedOf = (b) => (b.status?.data ?? []).filter((x) => x.serverInfo || x.runtimeStatus || (x.toolsError && !/requires a local environment/.test(x.toolsError))).map((x) => x.name);
1064
+ const b = await look("i3-relay-settings", () => []);
1065
+ const p = [];
1066
+ if (b.local?.status !== "unknown") p.push(`under relay's settings Codex reports its local environment as ${JSON.stringify(b.local)}, not unknown`);
1067
+ if (exists(`${b.d.out}/mcp-person`)) p.push("the person's MCP server ran under relay's settings");
1068
+ if (exists(`${b.d.out}/mcp-relay`)) p.push("an MCP server ran at relay's key under relay's settings");
1069
+ const on = Object.entries(b.config ?? {}).filter(([, v]) => v?.enabled !== false).map(([k]) => k);
1070
+ if (!b.config) p.push("config/read did not show the MCP servers: the check cannot tell");
1071
+ else if (on.join(",") !== "relay_tools") p.push(`config/read must show exactly relay_tools on: ${on.join(", ")}`);
1072
+ if (b.status?.error) p.push(`mcpServerStatus/list failed: ${b.status.error}`);
1073
+ if (!startedOf(b).includes("relay_tools") || startedOf(b).some((n) => n !== "relay_tools")) p.push(`Codex must start exactly relay_tools, saw ${startedOf(b).join(", ")}`);
1074
+ const c = await look("i3-relay-key-on", (d) => ["-c", 'mcp_servers.relay.command="/bin/sh"', "-c", `mcp_servers.relay.args=[${JSON.stringify(mcpScript(d, "relay"))}]`, "-c", "mcp_servers.relay.enabled=true"]);
1075
+ const relayListed = (c.status?.data ?? []).find((x) => x.name === "relay");
1076
+ if (c.relayRan || exists(`${c.d.out}/mcp-relay`)) p.push("without a local environment, a stdio MCP server at relay's key still started");
1077
+ if (!/requires a local environment/.test(relayListed?.toolsError ?? "")) p.push(`Codex did not report the stdio server at relay's key as needing a local environment (${JSON.stringify(relayListed?.toolsError ?? relayListed ?? null)}): the check cannot tell why it did not run`);
1078
+ const settings = judged(p, { local: b.local, on, started: startedOf(b), listed: (b.status?.data ?? []).map((x) => x.name), relayKeyOn: { ran: Boolean(c.relayRan), toolsError: relayListed?.toolsError ?? null } });
1079
+ // Through relay: the person's server is in the app server's CODEX_HOME; relay's readback admits the turn, its app server
1080
+ // runs with CODEX_EXEC_SERVER_URL=none, and no MCP server starts, the person's or relay's.
1081
+ const relay = await relayPart(ctx, async () => {
1082
+ let appPs = "";
1083
+ const r = await relayTurn(ctx, { name: "i3-relaypath", extraConfig: (d) => mcpServer(d, "person"), conversations: () => [{ calls: [exec("echo ran=1"), brokerSearch(), brokerCall()] }],
1084
+ onRequest: async (_reply, h) => { if (!appPs) { const pid = appServerFor(h.d.home); appPs = pid ? psEnv(pid) : ""; } } });
1085
+ await sleep(500);
1086
+ const rp = [];
1087
+ if (r.result.status !== "ok") rp.push(`the turn did not succeed (${r.result.status}: ${r.result.error ?? ""})`);
1088
+ if (exists(`${r.d.out}/mcp-person`)) rp.push("the person's MCP server ran in a stage-2 turn through relay");
1089
+ if (!appPs) rp.push("relay's app server was not seen during the turn: the check cannot tell");
1090
+ else if (!/\bCODEX_EXEC_SERVER_URL=none\b/.test(appPs)) rp.push("relay's app server for the stage-2 turn does not run with CODEX_EXEC_SERVER_URL=none");
1091
+ if (/\bRELAY_INJECTED=1\b/.test(appPs) || /mcp_servers\.relay\.enabled=true/.test(appPs)) rp.push("relay's app server for the stage-2 turn was given relay's MCP server");
1092
+ if (!r.model.output(0, 2)?.includes(r.nodeId)) rp.push("relay_tools status did not succeed through the real broker");
1093
+ return judged(rp, { turn: turnSaid(r), appServerNone: /\bCODEX_EXEC_SERVER_URL=none\b/.test(appPs) });
1094
+ });
1095
+ return { outside, settings, relay };
1096
+ }
1097
+
1098
+ /** Item 4's calls: a command an exec-policy rule allows, then one asking to run escalated. */
1099
+ const item4Calls = (d) => [
1100
+ exec(`touch ${d.out}/rule; echo rule=$?`),
1101
+ exec(`touch ${d.out}/esc; echo esc=$?`, { sandbox_permissions: "require_escalated", justification: "qualification" }),
1102
+ ];
1103
+ const rulesFile = (home) => { fs.mkdirSync(path.join(home, "rules"), { recursive: true }); fs.writeFileSync(path.join(home, "rules", "default.rules"), 'prefix_rule(pattern=["touch"], decision="allow")\n'); };
1104
+ function item4Problems(d, r, e, asked) {
1105
+ const p = [];
1106
+ if (exists(`${d.out}/rule`) || r.rule === "0") p.push("a command an exec-policy rule allows wrote outside the roots");
1107
+ if (exists(`${d.out}/esc`) || e.esc === "0") p.push("an approved escalated command wrote outside the roots");
1108
+ if (r.rule === undefined) p.push("the rule's command did not run: the check cannot tell");
1109
+ if (e.esc === undefined) p.push(`the escalated command did not run (approvals asked: ${asked.join(", ") || "none"}): the check cannot tell`);
1110
+ return p;
1111
+ }
1112
+
1113
+ /** Item 4 (W3): a command an exec-policy rule allows, and one approved to run escalated, both still run in the executor. */
1114
+ async function item4(ctx) {
1115
+ const w = await runCase(ctx, { name: "i4", env: "relay", approve: true, prepare: (d) => rulesFile(d.home), conversations: (d) => [{ calls: item4Calls(d) }] });
1116
+ const asked = w.rpc.notes.filter((n) => /requestApproval$/.test(n.request ?? "")).map((n) => n.request);
1117
+ const r = markers(w.model.output(0, 0)), e = markers(w.model.output(0, 1));
1118
+ const codex = judged(item4Problems(w.d, r, e, asked), { approvals: asked, rule: r.rule, esc: e.esc });
1119
+ const relay = await relayPart(ctx, async () => {
1120
+ const t = await relayTurn(ctx, { name: "i4-relaypath", approve: true, prepare: (d) => rulesFile(d.home), conversations: (d) => [{ calls: item4Calls(d) }] });
1121
+ const tr = markers(t.model.output(0, 0)), te = markers(t.model.output(0, 1));
1122
+ const p = item4Problems(t.d, tr, te, t.approvals.map((a) => a.kind ?? "ask"));
1123
+ if (t.result.status !== "ok") p.push(`the turn did not succeed (${t.result.status}: ${t.result.error ?? ""})`);
1124
+ return judged(p, { approvals: t.approvals.map((a) => ({ kind: a.kind, detail: a.detail })), rule: tr.rule, esc: te.esc, turn: turnSaid(t) });
1125
+ });
1126
+ return { codex, relay };
1127
+ }
1128
+
1129
+ /** Item 5 (W4): with the executor's URL unreachable, nothing runs, locally or anywhere. */
1130
+ async function item5(ctx) {
1131
+ const w = await runCase(ctx, { name: "i5", env: "unreachable", conversations: (d) => [{ calls: [exec(`touch ${d.wt}/ran_in; touch ${d.out}/ran_out; echo ran=$?`), patch("patched_in.txt", "inside")] }], timeoutMs: 90_000 });
1132
+ const out = w.model.output(0, 0);
1133
+ const p = [];
1134
+ if (exists(`${w.d.wt}/ran_in`) || exists(`${w.d.out}/ran_out`)) p.push("a command ran without its executor (local fallback)");
1135
+ if (exists(`${w.d.wt}/patched_in.txt`)) p.push("apply_patch ran without its executor (local fallback)");
1136
+ if (markers(out).ran !== undefined) p.push("the command reported running");
1137
+ return { codex: judged(p, { output: String(out ?? "").slice(0, 300) }) };
1138
+ }
1139
+
1140
+ /** Every file under `roots` that holds `needle`, as paths. */
1141
+ function filesHolding(roots, needle) {
1142
+ const hits = [];
1143
+ const walkDir = (d) => {
1144
+ let es; try { es = fs.readdirSync(d, { withFileTypes: true }); } catch { return; }
1145
+ for (const e of es) {
1146
+ const p = path.join(d, e.name);
1147
+ if (e.isDirectory()) walkDir(p);
1148
+ else if (e.isFile()) { try { if (fs.readFileSync(p).includes(needle)) hits.push(p); } catch {} }
1149
+ }
1150
+ };
1151
+ for (const r of roots) walkDir(r);
1152
+ return hits;
1153
+ }
1154
+ const psEnv = (pid) => { try { return execFileSync("/bin/ps", ["eww", "-o", "command=", "-p", String(pid)], { encoding: "utf8", env: stageChildEnv() }); } catch { return ""; } };
1155
+
1156
+ /** Item 6 (X7, W7): the token, made in memory, given to the app server over stdin: in no argv, environment or file, with the command's environment dump in the worktree as the control that the files held something to look at. */
1157
+ async function item6(ctx) {
1158
+ let seen;
1159
+ const w = await runCase(ctx, { name: "i6", env: "relay", conversations: () => [{ calls: [exec("env > env.txt; ls -la $TMPDIR > tmp-ls.txt; echo ok=1")] }],
1160
+ during: async (e) => { seen = { exPs: psEnv(e.ex.pid), appPs: psEnv(e.rpc.pid), stderr: e.rpc.stderr() + e.ex.err() }; } });
1161
+ const codex = item6CodexPart(w, seen);
1162
+ const relay = await relayPart(ctx, async () => {
1163
+ let hashSeen = false, tmpSeen = null, appSeen = 0;
1164
+ const r = await relayTurn(ctx, { name: "i6-relaypath", conversations: () => [{ calls: [exec("env > env.txt; ls -la $TMPDIR > tmp-ls.txt; echo ok=1"), exec("echo two=1")] }],
1165
+ onRequest: async (reply, h) => {
1166
+ if (!(reply.conv === 0 && reply.step === 1)) return;
1167
+ const ex = h.executor();
1168
+ if (!ex) return;
1169
+ const exPs = psEnv(ex.pid);
1170
+ hashSeen = /--ws-token-sha256 [0-9a-f]{64}/.test(exPs);
1171
+ h.capture("the executor's argv and environment", exPs);
1172
+ // The app server: the executor's parent is not it (a detached group); find it by its CODEX_HOME.
1173
+ const apps = appServerSnapshots(spawnSync("/usr/bin/pgrep", ["-f", "app-server"], { encoding: "utf8", env: stageChildEnv() }).stdout.split("\n").filter(Boolean).map(Number), h.d.home);
1174
+ for (const a of apps) h.capture(`the app server's argv and environment (${a.pid})`, a.env);
1175
+ appSeen = apps.length;
1176
+ // The turn's temp dir (the executor's TMPDIR), read now: it is gone after the turn.
1177
+ tmpSeen = /\bTMPDIR=(\S+)/.exec(exPs)?.[1] ?? null;
1178
+ const walkDir = (dir) => { let es; try { es = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } for (const e of es) { const q = path.join(dir, e.name); if (e.isDirectory()) walkDir(q); else if (e.isFile()) { try { h.capture(q, fs.readFileSync(q, "latin1")); } catch {} } } };
1179
+ if (tmpSeen) walkDir(tmpSeen);
1180
+ } });
1181
+ return item6RelayPart(r, { hashSeen, tmpSeen, appSeen });
1182
+ });
1183
+ return { codex, relay };
1184
+ }
1185
+
1186
+ /**
1187
+ * The argv and environment of each of `pids` that is an app server with CODEX_HOME `home`, as
1188
+ * [{ pid, env }] (`read` is psEnv). Each pid is read once, and only a non-empty snapshot that names
1189
+ * that CODEX_HOME is kept: the bytes the token scan reads are the ones that matched, and item 6
1190
+ * counts only those.
1191
+ */
1192
+ export function appServerSnapshots(pids, home, read = psEnv) {
1193
+ const out = [];
1194
+ for (const pid of pids) {
1195
+ const env = read(pid);
1196
+ if (env && env.includes(`CODEX_HOME=${home}`)) out.push({ pid, env });
1197
+ }
1198
+ return out;
1199
+ }
1200
+
1201
+ /** Item 6's relay part, from relay's turn `r` and what was seen during it: the token's hash on the executor's argv, its temp dir, and how many app servers matched the case's CODEX_HOME. */
1202
+ export function item6RelayPart(r, { hashSeen, tmpSeen, appSeen }) {
1203
+ const rp = envDumpProblems(r.model.output(0, 0), r.d.wt);
1204
+ if (r.result.status !== "ok") rp.push(`the turn did not succeed (${r.result.status}: ${r.result.error ?? ""})`);
1205
+ if (!hashSeen) rp.push("the token's hash was not seen in relay's executor's argv: the check looked at the wrong process");
1206
+ if (!tmpSeen) rp.push("the turn's temp dir was not found: the check cannot tell");
1207
+ if (!appSeen) rp.push("no app server with the case's CODEX_HOME was seen, so its argv and environment were not read: the check cannot tell");
1208
+ if (r.tokenIn.length) rp.push(`the executor token is in ${r.tokenIn.join(", ")}`);
1209
+ return judged(rp, { captured: r.captured.length, tmpDir: tmpSeen, appServers: appSeen, turn: turnSaid(r) });
1210
+ }
1211
+
1212
+ /**
1213
+ * Item 6's command, as it must have run for the token check to have looked at anything: it printed
1214
+ * ok=1 and left its environment dump, env.txt, in the worktree `wt` (where the file scan reads it).
1215
+ */
1216
+ export function envDumpProblems(out, wt) {
1217
+ const p = [];
1218
+ if (markers(out).ok !== "1") p.push("the command did not report running (no ok=1): the check cannot tell");
1219
+ let dump = "";
1220
+ try { dump = fs.readFileSync(path.join(wt, "env.txt"), "utf8"); } catch { /* none */ }
1221
+ if (!/^[A-Za-z_][A-Za-z0-9_]*=/m.test(dump)) p.push("the command left no environment dump (env.txt) in the worktree: the check cannot tell");
1222
+ return p;
1223
+ }
1224
+
1225
+ /** Item 6's codex part, from the case `w` (its token, hash, directories and model) and what ps and the logs showed during it (`seen`). */
1226
+ export function item6CodexPart(w, seen) {
1227
+ const p = envDumpProblems(w.model.output(0, 0), w.d.wt);
1228
+ // psEnv gives "" when ps fails: the app server's argv and environment must have been read, and be its.
1229
+ if (!seen.appPs || !(/\bapp-server\b/.test(seen.appPs) || (w.d.home && seen.appPs.includes(`CODEX_HOME=${w.d.home}`)))) p.push("the app server's argv and environment were not read (ps gave nothing, or another process): the check cannot tell");
1230
+ if (seen.exPs.includes(w.token)) p.push("the token is in the executor's argv or environment");
1231
+ if (!seen.exPs.includes(w.hash)) p.push("the token's hash is not in the executor's argv: the check looked at the wrong process");
1232
+ if (seen.appPs.includes(w.token)) p.push("the token is in the app server's argv or environment");
1233
+ if (seen.stderr.includes(w.token)) p.push("the token is in the app server's or executor's output");
1234
+ const files = filesHolding([w.d.dir], w.token);
1235
+ if (files.length) p.push(`the token is in ${files.map((f) => path.relative(w.d.dir, f)).join(", ")}`);
1236
+ return judged(p);
1237
+ }
1238
+
1239
+ /** A probe of the executor's websocket with each of `tokens` ([label, token | null]), printing `label=answered|refused|timeout`. */
1240
+ const wsProbe = (url, tokens) => `const tokens=${JSON.stringify(tokens)};for(const [k,t] of tokens){const r=await new Promise(res=>{const ws=new WebSocket(${JSON.stringify(url)},{headers:t?{Authorization:"Bearer "+t}:{}});const tm=setTimeout(()=>res("timeout"),4000);ws.onopen=()=>ws.send(JSON.stringify({id:1,method:"initialize",params:{clientName:"probe"}}));ws.onmessage=m=>{clearTimeout(tm);res("answered");ws.close()};ws.onerror=e=>{clearTimeout(tm);res("refused")}});console.log(k+"="+r)}process.exit(0)`;
1241
+ const WRONG = () => [["none", null], ["wrong", "0".repeat(64)], ["guessed", crypto.randomBytes(32).toString("hex")]];
1242
+
1243
+ /** Runs the websocket probe under another turn's stage-2 profile (relay's own, its own decoy ports). */
1244
+ async function probeFromOtherTurn(ctx, name, url) {
1245
+ const other = caseDirs(ctx, name);
1246
+ const script = path.join(other.tmp, "probe.mjs");
1247
+ fs.writeFileSync(script, wsProbe(url, WRONG()));
1248
+ const launch = executorLaunch(ctx, other, await casePorts([]), process.execPath, [script], []);
1249
+ const r = spawnSync(launch.command, launch.args, { cwd: other.wt, env: stageChildEnv({ ...launch.env }), encoding: "utf8", timeout: 30_000 });
1250
+ return { inside: markers(r.stdout), status: r.status, stderr: (r.stderr || "").slice(-300) };
1251
+ }
1252
+ const otherTurnProblems = (got) => ["none", "wrong", "guessed"].filter((k) => got.inside[k] !== "refused").map((k) => `another turn's process with ${k === "none" ? "no" : `a ${k}`} token: ${got.inside[k] ?? `no result (exit ${got.status}: ${got.stderr})`}`);
1253
+
1254
+ /** What an unsandboxed, token-less HTTP upgrade to `port` gets back: its status line (a 401 shows the executor listens and checks tokens). */
1255
+ function upgradeStatus(port) {
1256
+ return new Promise((res) => {
1257
+ const c = net.connect(port, "127.0.0.1");
1258
+ let got = "";
1259
+ const done = (v) => { c.destroy(); res(v); };
1260
+ c.setTimeout(3000, () => done(got.split("\r\n")[0] || "timeout"));
1261
+ c.on("error", (e) => done(`error ${e.code}`));
1262
+ c.on("connect", () => c.write(`GET / HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ${crypto.randomBytes(16).toString("base64")}\r\n\r\n`));
1263
+ c.on("data", (d) => { got += d; if (got.includes("\r\n")) done(got.split("\r\n")[0]); });
1264
+ });
1265
+ }
1266
+
1267
+ /** Item 7 (X7, W6, W7): a confined process under another turn's profile cannot use the executor without its token. */
1268
+ async function item7(ctx) {
1269
+ let got;
1270
+ await runCase(ctx, { name: "i7", env: "relay", conversations: () => [{ calls: [] }],
1271
+ during: async (e) => {
1272
+ got = await probeFromOtherTurn(ctx, "i7-other", e.ex.url);
1273
+ // The control, outside every sandbox: the right token is answered, so the refusals above mean something.
1274
+ const ctl = path.join(ctx.dir, "i7-other", "out", "control.mjs");
1275
+ fs.writeFileSync(ctl, wsProbe(e.ex.url, [["right", e.token]]));
1276
+ got.control = markers(spawnSync(process.execPath, [ctl], { encoding: "utf8", timeout: 30_000, env: stageChildEnv() }).stdout);
1277
+ } });
1278
+ const p = otherTurnProblems(got);
1279
+ if (got.control.right !== "answered") p.push(`the right token was not answered from outside (${got.control.right}): the check cannot tell`);
1280
+ const relay = await relayPart(ctx, async () => {
1281
+ let r7 = null;
1282
+ const r = await relayTurn(ctx, { name: "i7-relaypath", conversations: () => [{ calls: [exec("echo one=1")] }],
1283
+ onRequest: async (reply, h) => {
1284
+ if (reply.conv !== 0 || reply.step !== 1 || r7) return;
1285
+ const ex = h.executor();
1286
+ if (!ex) { r7 = { missing: true }; return; }
1287
+ r7 = await probeFromOtherTurn(ctx, "i7-relay-other", `ws://127.0.0.1:${ex.port}`);
1288
+ r7.outside = await upgradeStatus(ex.port);
1289
+ } });
1290
+ const rp = [];
1291
+ if (!r7 || r7.missing) rp.push("relay's executor for the turn was not seen: the check cannot tell");
1292
+ else {
1293
+ rp.push(...otherTurnProblems(r7));
1294
+ if (!/^HTTP\/1\.1 401/.test(r7.outside)) rp.push(`a token-less upgrade from outside got ${JSON.stringify(r7.outside)}, not 401: the check cannot tell`);
1295
+ }
1296
+ if (r.result.status !== "ok") rp.push(`the turn did not succeed (${r.result.status}: ${r.result.error ?? ""})`);
1297
+ return judged(rp, { probe: r7, turn: turnSaid(r) });
1298
+ });
1299
+ return { codex: judged(p, got), relay };
1300
+ }
1301
+
1302
+ /**
1303
+ * Item 8 (C5, X3): relay's teardown. A command leaves a background loop, a nohup'd loop, a setsid'd
1304
+ * loop, a terminal session's loop and a listener on the job's block port (PORT), each writing a
1305
+ * heartbeat into the worktree. After the turn each must be gone, no heartbeat may grow, and the port
1306
+ * must refuse.
1307
+ */
1308
+ async function item8(ctx) {
1309
+ const loop = (wt, n) => `while :; do echo x >> ${wt}/hb-${n}; sleep 0.2; done`;
1310
+ const calls = (d) => [
1311
+ exec(sh([
1312
+ `(${loop(d.wt, "bg")}) >/dev/null 2>&1 & echo $! > ${d.wt}/pid-bg`,
1313
+ // macOS nohup refuses inside relay's sandbox ("can't detach from console"); a loop ignoring SIGHUP is the same survivor.
1314
+ `nohup /bin/sh -c '${loop(d.wt, "nohup")}' </dev/null >/dev/null 2>${d.wt}/nohup.err & echo $! > ${d.wt}/pid-nohup`,
1315
+ `/bin/sh -c 'trap "" HUP; ${loop(d.wt, "hup")}' </dev/null >/dev/null 2>&1 & echo $! > ${d.wt}/pid-hup`,
1316
+ `/usr/bin/perl -MPOSIX -e 'if (fork() == 0) { POSIX::setsid(); open(my $f, ">", "${d.wt}/pid-setsid"); print $f "$$\\n"; close $f; exec("/bin/sh", "-c", "${loop(d.wt, "setsid")}"); }' >/dev/null 2>&1`,
1317
+ `/usr/bin/perl -MIO::Socket::INET -e 'my $s = IO::Socket::INET->new(LocalAddr => "127.0.0.1", LocalPort => $ENV{PORT}, Listen => 5, ReuseAddr => 1) or die "listen: $!"; open(my $f, ">", "${d.wt}/pid-listen"); print $f "$$\\n"; close $f; while (my $c = $s->accept) { print $c "survivor\\n"; close $c; }' >/dev/null 2>&1 &`,
1318
+ "sleep 1", "echo port=$PORT", `echo listening=$(test -s ${d.wt}/pid-listen && echo 1 || echo 0)`, "echo started=1",
1319
+ ])),
1320
+ exec(`/bin/sh -c 'echo $$ > ${d.wt}/pid-pty; ${loop(d.wt, "pty")}'`, { tty: true, yield_time_ms: 1500 }),
1321
+ ];
1322
+ const relay = await relayPart(ctx, async () => {
1323
+ const r = await relayTurn(ctx, { name: "i8-relaypath", conversations: (d) => [{ calls: calls(d) }] });
1324
+ const m = markers(r.model.output(0, 0));
1325
+ const p = [];
1326
+ if (m.started !== "1") p.push(`the survivors' command did not run (${JSON.stringify(String(r.model.output(0, 0) ?? "").slice(0, 200))}): the check cannot tell`);
1327
+ // nohup counts as a survivor only where it starts; where it refuses (macOS, inside a sandbox) that is recorded.
1328
+ const nohupErr = (() => { try { return fs.readFileSync(`${r.d.wt}/nohup.err`, "utf8").trim(); } catch { return ""; } })();
1329
+ const nohupRefused = /can't detach from console/.test(nohupErr) && !exists(`${r.d.wt}/hb-nohup`);
1330
+ const names = ["bg", ...(nohupRefused ? [] : ["nohup"]), "hup", "setsid", "listen", "pty"];
1331
+ const pids = Object.fromEntries(names.map((n) => [n, Number(fs.existsSync(`${r.d.wt}/pid-${n}`) ? fs.readFileSync(`${r.d.wt}/pid-${n}`, "utf8").trim() : 0)]));
1332
+ for (const n of names) if (!(pids[n] > 1)) p.push(`the ${n} survivor recorded no pid: the check cannot tell`);
1333
+ await sleep(1000);
1334
+ const size = (n) => { try { return fs.statSync(`${r.d.wt}/hb-${n}`).size; } catch { return -1; } };
1335
+ const before = Object.fromEntries(names.filter((n) => n !== "listen").map((n) => [n, size(n)]));
1336
+ await sleep(2000);
1337
+ for (const [n, s] of Object.entries(before)) {
1338
+ if (s < 0) p.push(`the ${n} survivor wrote no heartbeat: the check cannot tell`);
1339
+ else if (size(n) !== s) p.push(`the ${n} survivor still writes after teardown`);
1340
+ }
1341
+ const left = names.filter((n) => pids[n] > 1 && alive(pids[n]));
1342
+ if (left.length) p.push(`still running after teardown: ${left.map((n) => `${n} ${pids[n]}`).join(", ")}`);
1343
+ const port = Number(m.port);
1344
+ if (!(port > 0)) p.push("the job's PORT was not seen: the listener check cannot tell");
1345
+ else if (m.listening !== "1") p.push("the block-port listener did not start: the check cannot tell");
1346
+ else {
1347
+ const said = await new Promise((res) => { const c = net.connect(port, "127.0.0.1"); c.setTimeout(2000, () => { c.destroy(); res("timeout"); }); c.on("error", (e) => res(e.code)); c.on("data", (x) => { c.destroy(); res(`answered ${String(x).trim()}`); }); });
1348
+ if (said !== "ECONNREFUSED") p.push(`the job's block port ${port} after teardown: ${said}`);
1349
+ }
1350
+ // Stop whatever this check left, so that a failure does not outlive the run.
1351
+ for (const n of left) await stopPid(pids[n]);
1352
+ return judged(p, { pids, port, nohup: nohupRefused ? `refused to start: ${nohupErr.slice(0, 120)}` : "started", turn: turnSaid(r) });
1353
+ });
1354
+ return { relay };
1355
+ }
1356
+
1357
+ /** Item 9: relay's executor killed mid-turn. The app server must be gone when the turn ends, the turn failed, and a fresh listener on the executor's port must see no connection. */
1358
+ async function item9(ctx) {
1359
+ const relay = await relayPart(ctx, async () => {
1360
+ let killed = null, squat = null;
1361
+ const r = await relayTurn(ctx, {
1362
+ name: "i9-relaypath",
1363
+ conversations: (d) => [{ calls: [exec("echo first=1"), exec(`touch ${d.wt}/after_kill; touch ${d.out}/after_kill; echo after=$?`)] }],
1364
+ onRequest: async (reply, h) => {
1365
+ if (reply.conv !== 0 || reply.step !== 1 || killed) return;
1366
+ const ex = h.executor();
1367
+ if (!ex) { killed = { missing: true }; return; }
1368
+ killed = { pid: ex.pid, port: ex.port };
1369
+ try { process.kill(ex.pid, "SIGKILL"); } catch {}
1370
+ for (let i = 0; i < 40 && alive(ex.pid); i++) await sleep(25);
1371
+ // A listener of this run's own on the executor's port (relay's reserved range), as a squatter would be.
1372
+ for (let i = 0; i < 20 && !squat; i++) { try { squat = await decoy("squat", ex.port); } catch { await sleep(50); } }
1373
+ await sleep(1000);
1374
+ },
1375
+ // When relay reports the turn over, its app server must already be gone (stopped before the turn failed).
1376
+ after: async ({ appPid }) => { if (killed && !killed.missing) killed.appAliveAtEnd = appPid > 1 && alive(appPid); },
1377
+ });
1378
+ await sleep(3000);
1379
+ const p = [];
1380
+ if (!killed || killed.missing) p.push("relay's executor for the turn was not seen: the check cannot tell");
1381
+ if (r.result.status === "ok") p.push("the turn succeeded though its executor was killed");
1382
+ if (killed?.appAliveAtEnd) p.push(`the app server ${r.appPid} still ran when relay reported the turn over`);
1383
+ if (r.appPid > 1 && alive(r.appPid)) { p.push(`the app server ${r.appPid} still runs after the turn`); await stopPid(r.appPid); }
1384
+ if (!squat) p.push("no listener could be opened on the executor's port after it died: the check cannot tell");
1385
+ else if (squat.accepts.length) p.push(`a listener on the dead executor's port got ${squat.accepts.length} connection(s)`);
1386
+ if (exists(`${r.d.wt}/after_kill`) || exists(`${r.d.out}/after_kill`)) p.push("a command ran after the executor was killed");
1387
+ await squat?.close();
1388
+ return judged(p, { killed, accepts: squat?.accepts.length ?? null, turn: turnSaid(r) });
1389
+ });
1390
+ return { relay };
1391
+ }
1392
+
1393
+ /** Every model of the catalog: relay can run any of them (a model named on a job, or hidden ones like Daybreak's). */
1394
+ const catalogModels = (catalog) => JSON.parse(fs.readFileSync(catalog, "utf8")).models;
1395
+
1396
+ /**
1397
+ * Item 10 (§2.5, §9.3): for every catalog model, the tools a stage-2 session offers under relay's
1398
+ * settings (its flags, and no `local` environment: relayEnv) are within relay's allowlist
1399
+ * (stage2ToolsProblem, which admits only relay_tools among MCP servers), with at least
1400
+ * one direct-mode and one code-mode model seen; and code mode's isolate has no Node, file or network
1401
+ * bindings (ISOLATE_GLOBALS).
1402
+ */
1403
+ async function item10(ctx) {
1404
+ const problem = ctx.dist.stage2.stage2ToolsProblem;
1405
+ const per = {};
1406
+ const p = [];
1407
+ for (const m of catalogModels(ctx.catalog)) {
1408
+ const w = await runCase(ctx, { name: `i10-${m.slug}`, env: "relay", broker: true, appEnv: relayEnv(ctx), model: m.slug, conversations: () => [{ calls: [brokerSearch(), brokerCall()] }],
1409
+ beforeThread: async (e) => { e.brokerToolNames = (await e.broker.listTools()).map((t) => t.name); await brokerReadback(ctx, e); } });
1410
+ if (w.brokerReadback) p.push(`${m.slug}: ${w.brokerReadback}`);
1411
+ const names = w.model.requests.some((x) => x.tools) ? [...new Set(w.model.requests.flatMap((x) => x.tools ?? []))] : null;
1412
+ if (!names) { p.push(`${m.slug}: the model was never asked (${w.rpc.stderr().trim().split("\n").at(-1) ?? ""})`); continue; }
1413
+ const extra = problem(names, w.brokerToolNames);
1414
+ per[m.slug] = { toolMode: m.tool_mode ?? "direct", visibility: m.visibility, featuresOff: ctx.dist.stage2.stage2FeaturesOff(ctx.version.split(".").map(Number)), readback: w.brokerReadback, tools: names, ...(extra ? { problem: extra } : {}) };
1415
+ if (extra) p.push(`${m.slug}: ${extra}`);
1416
+ if (!brokerOK(w.model.output(0, 1))) p.push(`${m.slug}: relay_tools was not discovered and called`);
1417
+ }
1418
+ const modes = new Set(Object.values(per).map((x) => x.toolMode));
1419
+ if (!modes.has("direct")) p.push("no direct-mode model (gpt-5.5) was checked: the allowlist's classic half is unqualified");
1420
+ if (!modes.has("code_mode_only")) p.push("no code-mode model was checked");
1421
+ const tools = judged(p, { allowed: ctx.dist.stage2.STAGE2_TOOLS, models: per });
1422
+ // The isolate: a code-mode model's exec cell reports what its JavaScript can reach.
1423
+ const cm = catalogModels(ctx.catalog).find((m) => m.tool_mode === "code_mode_only" && m.visibility === "list");
1424
+ let isolate;
1425
+ if (!cm) isolate = part("failed", "no listed code-mode model in the catalog: the isolate check cannot run");
1426
+ else {
1427
+ const w = await runCase(ctx, { name: "i10-isolate", env: "relay", appEnv: relayEnv(ctx), model: cm.slug, conversations: () => [{ calls: [codeMode(ISOLATE_PROBE)] }] });
1428
+ const out = w.model.output(0, 0);
1429
+ const r = /ISOLATE=(\{.*\})/.exec(String(out ?? ""))?.[1];
1430
+ isolate = judged(isolateProblems(out), { model: cm.slug, result: r ? JSON.parse(r) : String(out ?? "").slice(0, 300) });
1431
+ }
1432
+ const relay = await relayPart(ctx, async () => {
1433
+ const rper = {}, rp = [];
1434
+ for (const m of catalogModels(ctx.catalog)) {
1435
+ const r = await relayTurn(ctx, { name: `i10-relay-${m.slug}`, model: m.slug, conversations: () => [{ calls: [brokerSearch(), brokerCall()] }] });
1436
+ const names = r.model.requests.some((x) => x.tools) ? [...new Set(r.model.requests.flatMap((x) => x.tools ?? []))] : null;
1437
+ if (r.result.status !== "ok" || !r.model.output(0, 1)?.includes(r.nodeId)) rp.push(`${m.slug}: real broker tool did not succeed`);
1438
+ rper[m.slug] = { status: r.result.status, ...(names ? { tools: names } : { error: String(r.result.error ?? "").slice(0, 300) }) };
1439
+ if (names) { const extra = problem(names); if (extra) rp.push(`${m.slug}: ${extra}`); }
1440
+ }
1441
+ const ran = Object.entries(rper).filter(([, x]) => x.tools);
1442
+ if (!ran.some(([s]) => catalogModels(ctx.catalog).find((m) => m.slug === s)?.tool_mode !== "code_mode_only")) rp.push("relay ran no direct-mode model: the check cannot tell");
1443
+ if (!ran.some(([s]) => catalogModels(ctx.catalog).find((m) => m.slug === s)?.tool_mode === "code_mode_only")) rp.push("relay ran no code-mode model: the check cannot tell");
1444
+ return judged(rp, { models: rper });
1445
+ });
1446
+ const effort = await relayPart(ctx, async () => {
1447
+ const m = catalogModels(ctx.catalog).find((m) => m.supported_reasoning_levels?.some((r) => r.effort === "high") && m.supported_reasoning_levels?.some((r) => r.effort === "medium") && m.supports_reasoning_effort_updates);
1448
+ if (!m) return part("failed", "no model supporting effort updates, high and medium: the effort check cannot tell");
1449
+ const evidence = [], ep = [];
1450
+ const turn = async (selected, dirs, sessionId) => {
1451
+ const seen = [];
1452
+ const r = await relayTurn(ctx, { name: "i10-effort", model: m.slug, effort: selected, dirs, sessionId, conversations: () => [{ calls: [] }],
1453
+ onRequest: (_reply, _h, body) => { seen.push(effortEvidence(body)); } });
1454
+ if (r.result.status !== "ok") ep.push(`${selected}: turn failed (${r.result.error ?? r.result.status})`);
1455
+ if (!seen.length || seen.some((s) => s.effective !== selected)) ep.push(`${selected}: explicit effort did not reach every model request`);
1456
+ evidence.push({ selected, resumed: Boolean(sessionId), requests: seen, turn: turnSaid(r) });
1457
+ return r;
1458
+ };
1459
+ const first = await turn("high");
1460
+ if (!first.result.sessionId) ep.push("no thread to resume: the changed-effort check cannot tell");
1461
+ else await turn("medium", first.d, first.result.sessionId);
1462
+ return judged(ep, { model: m.slug, turns: evidence });
1463
+ });
1464
+ return { tools, isolate, relay, effort };
1465
+ }
1466
+
1467
+ /** relay's readback of a thread reply (stage2ThreadProblem; a resumed thread under relay's settings has no environment, never `local`): the refusal, or null. */
1468
+ const threadProblem = (ctx, reply, cwd, resumed = false) => ctx.dist.stage2.stage2ThreadProblem(reply, cwd, "relay", { resumed });
1469
+
1470
+ /**
1471
+ * Item 11: a subagent's command runs in the relay environment (it inherits the parent's
1472
+ * environments). Codex 0.164's thread/resume takes no environments. Under relay's settings, whose
1473
+ * app server has no `local` environment (relayEnv, CODEX_EXEC_SERVER_URL=none), a resumed thread has
1474
+ * none at all until relay binds each turn with turn/start.environments. Checked: a resumed thread in
1475
+ * a new app server replies with no environment (and never `local`), and bound per turn runs its
1476
+ * commands and a subagent's in the executor; the same resume without the binding runs nothing
1477
+ * ("unbound"); the control, that resume with Codex's `local`, runs a command locally (if it did
1478
+ * not, the bound case could not tell whether the binding works). A release that drops or ignores
1479
+ * turn-level binding fails the first; one that changes resume or `none` fails the unbound part or
1480
+ * the control, for the maintainer.
1481
+ *
1482
+ * Turns Codex starts by itself on resume (WP-C's findings on 0.164): an active goal's continuation
1483
+ * and a message queued with thread/queue/add. With `local` they ran there, and a bound turn/start
1484
+ * relay sends then steers into such a turn without rebinding it (review r_0c11d860 C1+X2). relay's
1485
+ * app server has no `local`, so such a turn has nowhere to run a command; relay also pins goals off
1486
+ * and reads it back, refuses a resume while the queue is not empty, and fails a turn on any
1487
+ * turn/started before its own turn/start. Checked here: with relay's flags, a resume with an active
1488
+ * goal starts nothing (control: with goals on, it does); every resume under relay's flags is watched
1489
+ * for a turn relay did not start, and one from any source but the queue fails the item. With goals
1490
+ * on and no `local`, the goal's own turn, steered into by a bound turn/start, runs nothing, and so
1491
+ * does the queue's own turn ("noLocal"); with `local` (the goal control) the goal's turn runs there.
1492
+ */
1493
+ async function item11(ctx) {
1494
+ const v2 = catalogModels(ctx.catalog).find((m) => m.visibility === "list" && m.multi_agent_version === "v2");
1495
+ const model = v2?.slug ?? ctx.model;
1496
+ // Code-mode models offer the multi-agent tools in the "collaboration" namespace; direct models by plain name.
1497
+ const ns = v2?.tool_mode === "code_mode_only" ? { namespace: "collaboration" } : {};
1498
+ // Each subagent its own task name: a thread refuses a second agent under a name it already has.
1499
+ const spawnCall = (task, name) => ({ type: "function_call", name: "spawn_agent", ...ns, arguments: JSON.stringify({ task_name: name, message: task, fork_turns: "none" }) });
1500
+ const waitCall = { type: "function_call", name: "wait_agent", ...ns, arguments: JSON.stringify({ timeout_ms: 20_000 }) };
1501
+ const T = { SUB2: "SUBAGENT-TWO: run the resumed turn's command", SUB: "SUBAGENT-ONE: run the qualification's command", GOAL: "GOAL-OBJECTIVE-FROM-OUTSIDE", GOAL_NL: "GOAL-OBJECTIVE-WITHOUT-LOCAL", QUEUED: "QUEUED-BY-ANOTHER-CLIENT",
1502
+ BARE_RES: "RESUME-WITHOUT-ENVIRONMENTS", UNBOUND: "RESUME-UNBOUND-WITHOUT-LOCAL", RES_GOAL: "RESUME-AFTER-GOAL", RES_Q: "RESUME-AFTER-QUEUE", RES: "RESUME-CHECK" };
1503
+ // Steered into the goal's own turn; it matches no conversation, so the request after it still goes to the goal's.
1504
+ const STEER = "STEERED-INTO-CODEX-OWN-TURN";
1505
+ // Conversation order: the most specific match first (a resumed turn's input holds the earlier turns' messages).
1506
+ const order = ["SUB2", "SUB", "GOAL", "GOAL_NL", "QUEUED", "BARE_RES", "UNBOUND", "RES_GOAL", "RES_Q", "RES", "FIRST"];
1507
+ const C = Object.fromEntries(order.map((k, i) => [k, i]));
1508
+ const conversations = (d) => order.map((k) => ({
1509
+ SUB2: { match: T.SUB2, calls: [exec(`touch ${d.wt}/sub2_in; echo sub2wt=$?; touch ${d.out}/sub2_out; echo sub2out=$?`), brokerSearch(), brokerCall()] },
1510
+ SUB: { match: T.SUB, calls: [exec(`touch ${d.wt}/sub_in; echo subwt=$?; touch ${d.out}/sub_out; echo subout=$?`), brokerSearch(), brokerCall()] },
1511
+ GOAL: { match: T.GOAL, calls: [exec(`touch ${d.out}/goal_out; echo goalout=$?`)] },
1512
+ GOAL_NL: { match: T.GOAL_NL, calls: [exec(`touch ${d.out}/goal_nl_out; echo goalnlout=$?`)] },
1513
+ QUEUED: { match: T.QUEUED, calls: [exec(`touch ${d.out}/queued_out; echo queuedout=$?`)] },
1514
+ BARE_RES: { match: T.BARE_RES, calls: [exec(`touch ${d.out}/bare_out; echo bareout=$?`)] },
1515
+ UNBOUND: { match: T.UNBOUND, calls: [exec(`touch ${d.out}/unbound_out; echo unboundout=$?; touch ${d.wt}/unbound_in; echo unboundin=$?`)] },
1516
+ RES_GOAL: { match: T.RES_GOAL, calls: [exec(`touch ${d.wt}/rg_in; echo rgwt=$?; touch ${d.out}/rg_out; echo rgout=$?`)] },
1517
+ RES_Q: { match: T.RES_Q, calls: [exec(`touch ${d.out}/rq_out; echo rqout=$?`)] },
1518
+ RES: { match: T.RES, calls: [exec(`touch ${d.wt}/res_in; echo reswt=$?; touch ${d.out}/res_out; echo resout=$?`), spawnCall(T.SUB2, "qualify_resumed"), waitCall] },
1519
+ FIRST: { calls: [spawnCall(T.SUB, "qualify_first"), waitCall] },
1520
+ })[k]);
1521
+ const asked = (m, k) => m.requests.filter((x) => x.conv === C[k]).length;
1522
+ const subProblems = (who, m, wtKey, outKey, d, file) => {
1523
+ const p = [];
1524
+ if (m[wtKey] === undefined) p.push(`${who}'s command did not run: the check cannot tell`);
1525
+ else {
1526
+ if (m[wtKey] !== "0") p.push(`${who}'s command could not write the worktree`);
1527
+ if (m[outKey] === "0" || exists(`${d.out}/${file}`)) p.push(`${who}'s command wrote outside the roots: it did not run in the relay environment`);
1528
+ }
1529
+ return p;
1530
+ };
1531
+ // A hook of the case's fake model, set for the steer below: runs before each answer.
1532
+ let modelHook = null;
1533
+ const w = await runCase(ctx, { name: "i11", env: "relay", broker: true, appEnv: relayEnv(ctx), model, conversations, keep: true, timeoutMs: 90_000, onRequest: (reply, body) => modelHook?.(reply, body) });
1534
+ const detail = { model, started: w.thread?.thread?.environments ?? null };
1535
+ const sub = [], resume = [], unbound = [], control = [], selfStart = [], noLocal = [], goalControl = [];
1536
+ const env = { PATH: SYSTEM_PATH, HOME: w.d.user, CODEX_HOME: w.d.home, TMPDIR: w.d.apptmp, NO_PROXY: "127.0.0.1,localhost,::1" };
1537
+ const threadId = w.thread?.thread?.id;
1538
+ /** Another client of the person's, with Codex's defaults (goals on), acting on the stored thread. */
1539
+ const otherClient = async (f) => {
1540
+ const r = appServer(ctx.bin, BARE(), { cwd: w.d.wt, env });
1541
+ try { await initialize(r); return await f(r); } finally { await r.stop(); }
1542
+ };
1543
+ /**
1544
+ * A new app server resuming the thread: relay's stage-2 flags unless `flags`, relay's environment
1545
+ * (no `local`) unless `local`, the executor added, thread/resume, a pause in which Codex may start
1546
+ * a turn by itself, then (with `text`) a turn of ours with `environments`. With `steer`, the first
1547
+ * time the model is asked for `steer` (a conversation), a turn/start bound to the executor is sent,
1548
+ * as relay sends its own, and steers into the turn under way. Returns the reply, our turn's id, the
1549
+ * turns we did not start, `local`'s status and the steer's reply.
1550
+ */
1551
+ const resumed = async ({ text = null, environments = null, flags = null, settleMs = 2500, local = false, steer = null }) => {
1552
+ const broker = await qualificationBroker(ctx, w.d);
1553
+ const base = flags ?? stage2Flags(ctx, w.d);
1554
+ const brokerFlags = stage2Flags(ctx, w.d, [], broker).filter((v, i, a) => v.startsWith("mcp_servers.relay_tools.") || (v === "-c" && a[i + 1]?.startsWith("mcp_servers.relay_tools.")));
1555
+ const r = appServer(ctx.bin, [...base, ...brokerFlags], { cwd: w.d.wt, env: { ...(local ? env : { ...env, ...relayEnv(ctx) }), ...broker.env } });
1556
+ broker.bindAppServer({ pid: r.pid, start: ctx.dist.procs.procStart(r.pid, { fresh: true }) });
1557
+ let steered = null;
1558
+ if (steer) {
1559
+ modelHook = async (reply) => {
1560
+ if (reply.conv !== C[steer] || steered) return;
1561
+ steered = await r.request("turn/start", { threadId, input: [{ type: "text", text: STEER, text_elements: [] }], environments: [{ environmentId: "relay", cwd: w.d.wt }] }).catch((e) => ({ error: e.message.slice(0, 300) }));
1562
+ };
1563
+ }
1564
+ try {
1565
+ await initialize(r);
1566
+ await r.request("environment/add", { environmentId: "relay", execServerUrl: w.ex.url, authBearerToken: w.token });
1567
+ const localStatus = await r.request("environment/status", { environmentId: "local" }).catch((e) => ({ error: e.message.slice(0, 200) }));
1568
+ const reply = await r.request("thread/resume", { threadId, cwd: w.d.wt, runtimeWorkspaceRoots: [w.d.wt], permissions: ":danger-full-access", approvalPolicy: "on-request", approvalsReviewer: "user" });
1569
+ await sleep(settleMs);
1570
+ let own = null;
1571
+ if (text) {
1572
+ const t = await r.request("turn/start", { threadId, input: [{ type: "text", text, text_elements: [] }], ...(environments ? { environments } : {}) });
1573
+ own = t?.turn?.id ?? null;
1574
+ broker.arm();
1575
+ await r.until((n) => n.method === "turn/completed" && (!own || n.params?.turn?.id === own), 90_000);
1576
+ if (text === T.RES) await boundedUntil(() => w.model.output(C.SUB2, 2) !== null, 20_000);
1577
+ } else if (r.notes.some((n) => n.method === "turn/started" && (n.params?.threadId ?? threadId) === threadId)) await r.until((n) => n.method === "turn/completed" && (n.params?.threadId ?? threadId) === threadId, 60_000);
1578
+ // This thread's turns only: a subagent's turns are on threads of their own.
1579
+ const turns = r.notes.filter((n) => n.method === "turn/started" && (n.params?.threadId ?? threadId) === threadId).map((n) => n.params?.turn?.id ?? "?");
1580
+ return { reply, own, selfStarted: turns.filter((id) => id !== own), localStatus, steered };
1581
+ } finally { modelHook = null; await r.stop(); await broker.close(); }
1582
+ };
1583
+ const envIds = (reply) => (reply?.thread?.environments ?? []).map((e) => e.environmentId);
1584
+ try {
1585
+ for (let i = 0; i < 100 && w.model.output(C.SUB, 2) === null; i++) await sleep(200);
1586
+ detail.sub = markers(w.model.output(C.SUB, 0));
1587
+ if (!brokerOK(w.model.output(C.SUB, 2))) sub.push("the first subagent did not call relay_tools");
1588
+ sub.push(...subProblems("the subagent", detail.sub, "subwt", "subout", w.d, "sub_out"));
1589
+ await w.rpc.stop();
1590
+ // Bound as relay must: each resumed turn names the executor as its environment. Under relay's settings
1591
+ // the app server has no `local`, so the resume reply holds no environment, and relay's readback accepts it.
1592
+ const a = await resumed({ text: T.RES, environments: [{ environmentId: "relay", cwd: w.d.wt }] });
1593
+ detail.resumedEnvironments = a.reply?.thread?.environments ?? null;
1594
+ detail.resumedLocal = a.localStatus;
1595
+ if (a.localStatus?.status !== "unknown") resume.push(`under relay's settings Codex reports its local environment as ${JSON.stringify(a.localStatus)}, not unknown`);
1596
+ if (envIds(a.reply).length) resume.push(`under relay's settings the resume reply holds the environments ${JSON.stringify(envIds(a.reply))}, where Codex 0.164 gave none`);
1597
+ detail.resumed = markers(w.model.output(C.RES, 0));
1598
+ if (detail.resumed.reswt !== "0") resume.push(`after thread/resume and turn/start with environments a command could not write the worktree (reswt=${detail.resumed.reswt})`);
1599
+ if (detail.resumed.resout === "0" || exists(`${w.d.out}/res_out`)) resume.push("after thread/resume and turn/start with environments a command wrote outside the roots");
1600
+ for (let i = 0; i < 100 && w.model.output(C.SUB2, 2) === null; i++) await sleep(200);
1601
+ detail.sub2 = markers(w.model.output(C.SUB2, 0));
1602
+ if (!brokerOK(w.model.output(C.SUB2, 2))) resume.push("the resumed subagent did not call relay_tools");
1603
+ detail.spawned2 = String(w.model.output(C.RES, 1) ?? "").slice(0, 300);
1604
+ resume.push(...subProblems("a subagent spawned in the resumed, bound turn", detail.sub2, "sub2wt", "sub2out", w.d, "sub2_out"));
1605
+ detail.readbackOfResumeReply = threadProblem(ctx, a.reply, w.d.wt, true);
1606
+ if (detail.readbackOfResumeReply) resume.push(`relay's readback refuses the resume reply: ${detail.readbackOfResumeReply}`);
1607
+ if (a.selfStarted.length) selfStart.push(`a plain resume under relay's flags started ${a.selfStarted.length} turn(s) relay did not start`);
1608
+ // Unbound, under relay's settings: the same resume without the turn binding runs its command nowhere.
1609
+ const u = await resumed({ text: T.UNBOUND });
1610
+ detail.unbound = { environments: envIds(u.reply), markers: markers(w.model.output(C.UNBOUND, 0)), output: String(w.model.output(C.UNBOUND, 0) ?? "").slice(0, 200) };
1611
+ if (w.model.output(C.UNBOUND, 0) === null) unbound.push("the unbound turn's command was never tried: the check cannot tell");
1612
+ if (exists(`${w.d.out}/unbound_out`) || detail.unbound.markers.unboundout === "0") unbound.push("without the turn binding and without local, a command wrote outside the roots");
1613
+ if (exists(`${w.d.wt}/unbound_in`) || detail.unbound.markers.unboundin === "0") unbound.push("without the turn binding and without local, a command ran (in the worktree): it had an environment relay did not bind");
1614
+ if (u.selfStarted.length) selfStart.push(`a resume without the turn binding, under relay's settings, started ${u.selfStarted.length} turn(s) relay did not start`);
1615
+ // The control: the same resume without the turn binding, with Codex's `local`, must run its command locally.
1616
+ const b = await resumed({ text: T.BARE_RES, local: true });
1617
+ detail.bareResumedEnvironments = b.reply?.thread?.environments ?? null;
1618
+ detail.bareEscaped = exists(`${w.d.out}/bare_out`) || markers(w.model.output(C.BARE_RES, 0)).bareout === "0";
1619
+ if (!detail.bareEscaped) control.push(`a resumed thread without the turn binding, with local, did not run its command locally (environments ${JSON.stringify((detail.bareResumedEnvironments ?? []).map((e) => e.environmentId))}): Codex's resume changed, and the bound case cannot tell whether turn binding works`);
1620
+ if (b.selfStarted.length) selfStart.push(`a resume without the turn binding, under relay's flags with local, started ${b.selfStarted.length} turn(s) relay did not start`);
1621
+ // An active goal set by another client: under relay's flags (goals pinned off) a resume starts nothing, and relay's bound turn alone runs.
1622
+ detail.goalSet = await otherClient(async (r) => { await r.request("thread/goal/set", { threadId, objective: T.GOAL }); return (await r.request("thread/goal/get", { threadId }))?.goal?.status ?? null; }).catch((e) => `error ${e.message.slice(0, 200)}`);
1623
+ const g = await resumed({ text: T.RES_GOAL, environments: [{ environmentId: "relay", cwd: w.d.wt }] });
1624
+ detail.goal = { selfStarted: g.selfStarted.length, goalAsked: asked(w.model, "GOAL"), resumed: markers(w.model.output(C.RES_GOAL, 0)) };
1625
+ if (detail.goalSet !== "active") selfStart.push(`the goal set from another client is ${JSON.stringify(detail.goalSet)}, not active: the goal check cannot tell`);
1626
+ if (g.selfStarted.length || detail.goal.goalAsked) selfStart.push(`with an active goal, a resume under relay's flags started a turn by itself (${g.selfStarted.length} turn(s), the goal's conversation asked ${detail.goal.goalAsked} time(s))`);
1627
+ if (exists(`${w.d.out}/goal_out`)) selfStart.push("the goal's command wrote outside the roots under relay's flags");
1628
+ if (detail.goal.resumed.rgwt !== "0" || detail.goal.resumed.rgout === "0" || exists(`${w.d.out}/rg_out`)) selfStart.push(`relay's bound turn after the goal did not run confined (${JSON.stringify(detail.goal.resumed)})`);
1629
+ // Goals on, no `local` (relay's settings but the goals pin): the goal's own turn starts by itself; a bound
1630
+ // turn/start steers into it (as relay's would); its command, issued after the steer, runs nowhere.
1631
+ // A goal of its own (its own conversation, so the control's call below is still the goal's first).
1632
+ const setGoal = (objective) => otherClient(async (r) => { await r.request("thread/goal/clear", { threadId }).catch(() => {}); await r.request("thread/goal/set", { threadId, objective }); return (await r.request("thread/goal/get", { threadId }))?.goal?.status ?? null; }).catch((e) => `error ${e.message.slice(0, 200)}`);
1633
+ const goalsOn = (d) => stage2Flags(ctx, d).map((a) => (a === "features.goals=false" ? "features.goals=true" : a));
1634
+ const nlSet = await setGoal(T.GOAL_NL);
1635
+ const ng = await resumed({ flags: goalsOn(w.d), settleMs: 6000, steer: "GOAL_NL" });
1636
+ const ngOut = w.model.output(C.GOAL_NL, 0);
1637
+ detail.noLocalGoal = { set: nlSet, localStatus: ng.localStatus, selfStarted: ng.selfStarted.length, steeredInto: ng.steered?.turn?.id ?? ng.steered ?? null, goalAsked: asked(w.model, "GOAL_NL"), output: String(ngOut ?? "").slice(0, 200), escaped: exists(`${w.d.out}/goal_nl_out`) };
1638
+ if (nlSet !== "active") noLocal.push(`the goal set from another client is ${JSON.stringify(nlSet)}, not active: the check cannot tell`);
1639
+ if (ng.localStatus?.status !== "unknown") noLocal.push(`with goals on under relay's environment, Codex reports its local environment as ${JSON.stringify(ng.localStatus)}`);
1640
+ if (!ng.selfStarted.length || !detail.noLocalGoal.goalAsked) noLocal.push(`with goals on and no local, the goal started no turn of its own (${JSON.stringify(detail.noLocalGoal)}): the check cannot tell`);
1641
+ else if (ng.steered?.turn?.id !== ng.selfStarted[0]) noLocal.push(`the bound turn/start did not steer into the goal's own turn (${JSON.stringify(ng.steered)}): the check cannot tell`);
1642
+ if (ngOut === null) noLocal.push("the goal's own turn never tried its command: the check cannot tell");
1643
+ if (detail.noLocalGoal.escaped || markers(ngOut).goalnlout === "0") noLocal.push("with goals on and no local, the goal's own turn, steered into by a bound turn/start, ran its command outside the roots");
1644
+ // The control: with Codex's defaults (goals on, `local`), the same resume starts the goal's turn by itself, and it runs locally.
1645
+ const gcSet = await setGoal(T.GOAL);
1646
+ if (gcSet !== "active") goalControl.push(`the control's goal is ${JSON.stringify(gcSet)}, not active: the check cannot tell`);
1647
+ const gc = await resumed({ flags: BARE(), settleMs: 4000, local: true });
1648
+ detail.goalControl = { selfStarted: gc.selfStarted.length, goalAsked: asked(w.model, "GOAL"), goalEscaped: exists(`${w.d.out}/goal_out`) };
1649
+ if (!gc.selfStarted.length) goalControl.push("with goals on, a resume with an active goal started no turn by itself: Codex's goal continuation changed, and the goal check cannot tell");
1650
+ if (!detail.goalControl.goalEscaped) goalControl.push("with goals on and local, the goal's own turn did not run its command locally: the no-local goal check cannot tell");
1651
+ await otherClient((r) => r.request("thread/goal/clear", { threadId })).catch(() => {});
1652
+ // A message queued by another client: Codex starts it by itself on resume (relay refuses such a resume, and
1653
+ // a message queued after its check is the race). Under relay's settings, without `local`, it runs nothing.
1654
+ detail.queued = await otherClient((r) => r.request("thread/queue/add", { threadId, input: [{ type: "text", text: T.QUEUED, text_elements: [] }], clientUserMessageId: "q1" })).then(() => "queued").catch((e) => `error ${e.message.slice(0, 200)}`);
1655
+ const q = await resumed({ settleMs: 4000 });
1656
+ const qOut = w.model.output(C.QUEUED, 0);
1657
+ detail.queue = { selfStarted: q.selfStarted.length, queuedAsked: asked(w.model, "QUEUED"), output: String(qOut ?? "").slice(0, 200), queuedEscaped: exists(`${w.d.out}/queued_out`) };
1658
+ if (detail.queue.queuedEscaped || markers(qOut).queuedout === "0") noLocal.push("under relay's settings, the queued message's own turn ran its command outside the roots");
1659
+ if (detail.queued === "queued" && !q.selfStarted.length) detail.queue.note = "the queued message started no turn by itself on this resume";
1660
+ } finally {
1661
+ await teardown(w);
1662
+ }
1663
+ // Through relay: a first turn spawning a subagent, relay's own resume of that thread (spec.sessionId) running a command and a subagent, both confined;
1664
+ // then an active goal (relay's resume starts no goal turn) and a queued message (relay refuses the resume, nothing runs).
1665
+ const relay = await relayPart(ctx, async () => {
1666
+ const d = caseDirs(ctx, "i11-relaypath");
1667
+ const m = await fakeModel(conversations(d));
1668
+ const direct = async (f) => {
1669
+ const r = appServer(ctx.bin, BARE(), { cwd: d.wt, env: stageChildEnv({ HOME: d.user, CODEX_HOME: d.home, TMPDIR: d.apptmp }) });
1670
+ try { await initialize(r); return await f(r); } finally { await r.stop(); }
1671
+ };
1672
+ try {
1673
+ const turn = (o) => relayTurn(ctx, { name: "i11-relaypath", dirs: d, model, model$: m, keepModel: true, conversations, ...o });
1674
+ const first = await turn({});
1675
+ const sid = first.result.sessionId;
1676
+ const rp = [];
1677
+ if (first.result.status !== "ok") rp.push(`the first turn did not succeed (${first.result.status}: ${first.result.error ?? ""})`);
1678
+ if (!sid) return judged([...rp, "the first turn reported no thread to resume: the check cannot tell"], { first: turnSaid(first) });
1679
+ const second = await turn({ sessionId: sid, prompt: T.RES });
1680
+ if (second.result.status !== "ok") rp.push(`relay's resumed turn did not succeed (${second.result.status}: ${second.result.error ?? ""}): relay must bind a resumed turn (turn/start.environments), not refuse it`);
1681
+ rp.push(...subProblems("the first turn's subagent", markers(m.output(C.SUB, 0)), "subwt", "subout", d, "sub_out"));
1682
+ const rr = markers(m.output(C.RES, 0));
1683
+ if (rr.reswt === undefined) rp.push("relay's resumed turn ran no command: the check cannot tell");
1684
+ else {
1685
+ if (rr.reswt !== "0") rp.push("relay's resumed turn could not write the worktree");
1686
+ if (rr.resout === "0" || exists(`${d.out}/res_out`)) rp.push("relay's resumed turn wrote outside the roots: its command ran locally");
1687
+ }
1688
+ rp.push(...subProblems("a subagent in relay's resumed turn", markers(m.output(C.SUB2, 0)), "sub2wt", "sub2out", d, "sub2_out"));
1689
+ for (const key of ["SUB", "SUB2"]) if (!m.output(C[key], 2)?.includes(first.nodeId)) rp.push(`${key}: real broker tool did not succeed`);
1690
+ // relay archives a thread after each turn and unarchives it to resume; another client does the same.
1691
+ const goalSet = await direct(async (r) => { await r.request("thread/unarchive", { threadId: sid }); await r.request("thread/goal/set", { threadId: sid, objective: T.GOAL }); return (await r.request("thread/goal/get", { threadId: sid }))?.goal?.status ?? null; }).catch((e) => `error ${e.message.slice(0, 200)}`);
1692
+ const third = await turn({ sessionId: sid, prompt: T.RES_GOAL });
1693
+ const rg = markers(m.output(C.RES_GOAL, 0));
1694
+ if (goalSet !== "active") rp.push(`the goal set from another client is ${JSON.stringify(goalSet)}, not active: the goal check cannot tell`);
1695
+ if (third.result.status !== "ok") rp.push(`relay's resumed turn with an active goal did not succeed (${third.result.status}: ${third.result.error ?? ""})`);
1696
+ if (asked(m, "GOAL") || exists(`${d.out}/goal_out`)) rp.push("relay's resume with an active goal let Codex start the goal's turn");
1697
+ if (rg.rgwt !== "0" || rg.rgout === "0" || exists(`${d.out}/rg_out`)) rp.push(`relay's resumed turn after the goal did not run confined (${JSON.stringify(rg)})`);
1698
+ await direct(async (r) => { await r.request("thread/unarchive", { threadId: sid }).catch(() => {}); await r.request("thread/goal/clear", { threadId: sid }); }).catch(() => {});
1699
+ const queued = await direct(async (r) => { await r.request("thread/unarchive", { threadId: sid }).catch(() => {}); await r.request("thread/queue/add", { threadId: sid, input: [{ type: "text", text: T.QUEUED, text_elements: [] }], clientUserMessageId: "q1" }); return "queued"; }).catch((e) => `error ${e.message.slice(0, 200)}`);
1700
+ const fourth = await turn({ sessionId: sid, prompt: T.RES_Q });
1701
+ if (queued !== "queued") rp.push(`a message could not be queued from another client (${queued}): the queue check cannot tell`);
1702
+ if (fourth.result.status !== "failed" || !/queued/i.test(fourth.result.error ?? "")) rp.push(`relay's resume with a queued message was not refused for it (${fourth.result.status}: ${fourth.result.error ?? ""})`);
1703
+ if (asked(m, "QUEUED") || asked(m, "RES_Q") || exists(`${d.out}/queued_out`) || exists(`${d.out}/rq_out`)) rp.push("with a queued message, a turn ran on relay's resume");
1704
+ return judged(rp, { first: turnSaid(first), resumed: turnSaid(second), goal: { set: goalSet, turn: turnSaid(third) }, queue: { queued, turn: turnSaid(fourth) } });
1705
+ } finally { await m.close(); }
1706
+ });
1707
+ return {
1708
+ subagent: judged(sub, { sub: detail.sub, model }),
1709
+ resume: judged(resume, { started: detail.started, resumed: detail.resumedEnvironments, local: detail.resumedLocal, sub2: detail.sub2, spawned2: detail.spawned2, readbackOfResumeReply: detail.readbackOfResumeReply }),
1710
+ unbound: judged(unbound, detail.unbound ?? {}),
1711
+ control: judged(control, { bareResumed: detail.bareResumedEnvironments, bareEscaped: detail.bareEscaped }),
1712
+ selfStart: judged(selfStart, { goalSet: detail.goalSet, goal: detail.goal, queued: detail.queued, queue: detail.queue }),
1713
+ noLocal: judged(noLocal, { goal: detail.noLocalGoal, queue: detail.queue }),
1714
+ goalControl: judged(goalControl, detail.goalControl ?? {}),
1715
+ relay,
1716
+ };
1717
+ }
1718
+
1719
+ /** Item 12: relay's version-specific plugin pins suppress sync and host-side discovery. The
1720
+ * plugins-on control must sync. Where prefetch exists, authenticated discovery must work against
1721
+ * the run's fake service before its quiet negatives mean anything. The fake model and plugin
1722
+ * service bypass the refusing proxy on loopback; all other host requests go to that proxy. */
1723
+ async function item12(ctx) {
1724
+ const hasPrefetch = ctx.dist.stage2.stage2FeaturesOff(ctx.version.split(".").map(Number)).includes("plugin_discovery_prefetch");
1725
+ const look = async (name, flags, service = null) => {
1726
+ const proxy = await refusingProxy();
1727
+ const url = `http://127.0.0.1:${proxy.port}`;
1728
+ const start = service?.seen.length ?? 0;
1729
+ try {
1730
+ const w = await runCase(ctx, { name, env: "relay", flags, conversations: () => [{ calls: [] }],
1731
+ extraConfig: () => service ? `cli_auth_credentials_store = "file"\nchatgpt_base_url = "http://127.0.0.1:${service.port}/backend-api/"` : "",
1732
+ prepare: (d) => { if (service) pluginFakeAuth(d.home); },
1733
+ appEnv: { ...relayEnv(ctx), HTTPS_PROXY: url, https_proxy: url, HTTP_PROXY: url, http_proxy: url, ALL_PROXY: url, all_proxy: url },
1734
+ beforeThread: async (e) => {
1735
+ const eff = (await e.rpc.request("config/read", { cwd: e.d.wt })).config;
1736
+ e.features = eff?.features;
1737
+ if (name === "i12-plugins-off") e.readback = await ctx.dist.stage2.stage2Readback(e.rpc, eff, e.d.wt, undefined, ctx.dist.stage2.stage2FeaturesOff(ctx.version.split(".").map(Number)));
1738
+ },
1739
+ during: async (e) => {
1740
+ // The same observation window for discovery's positive and negatives; .tmp creation or
1741
+ // installed-plugin sync must not end it before asynchronous discovery gets a chance.
1742
+ if (service) await sleep(9500);
1743
+ else { for (let i = 0; i < 80 && !exists(path.join(e.d.home, ".tmp")) && !proxy.seen.length; i++) await sleep(100); await sleep(1500); }
1744
+ } });
1745
+ return { tmp: exists(path.join(w.d.home, ".tmp")) ? fs.readdirSync(path.join(w.d.home, ".tmp")) : null, proxied: [...proxy.seen],
1746
+ ...(service ? { discovery: service.seen.slice(start), discoveryBaseUrl: `http://127.0.0.1:${service.port}/backend-api/`, observationMs: 9500 } : {}),
1747
+ features: { plugins: w.features?.plugins, plugin_discovery_prefetch: w.features?.plugin_discovery_prefetch, remote_plugin: w.features?.remote_plugin }, readback: w.readback };
1748
+ } finally { await proxy.close(); }
1749
+ };
1750
+ const control = await look("i12-plugins-on", BARE);
1751
+ const service = hasPrefetch ? await fakePluginService() : null;
1752
+ try {
1753
+ const prefetchOn = service ? await look("i12-prefetch-on", () => [...BARE(), "-c", "features.plugins=true", "-c", "features.remote_plugin=true", "-c", "features.plugin_discovery_prefetch=true"], service) : null;
1754
+ const off = await look("i12-plugins-off", (d) => stage2Flags(ctx, d), service);
1755
+ const pluginsOnly = service ? await look("i12-plugins-only", (d) => [...stage2Flags(ctx, d), "-c", "features.plugin_discovery_prefetch=true"], service) : null;
1756
+ return pluginParts({ control, off, hasPrefetch, prefetchOn, pluginsOnly, discoveryPaths: pluginDiscoveryPaths(ctx.version.split(".").map(Number)) });
1757
+ } finally { await service?.close(); }
1758
+ }
1759
+
1760
+ /** Item 13 (W163): on an older Codex given the executor, its command runs and nothing escapes (a case that cannot run shows nothing), and relay's admission refuses it on its version (STAGE2_MIN_CODEX) while admitting this one's. */
1761
+ async function item13(ctx) {
1762
+ if (!ctx.oldBin) return { codex: part("not run", "no older Codex given (--old-bin)") };
1763
+ const said = tryText(ctx.oldBin, ["--version"]);
1764
+ let err = null, w = null;
1765
+ try {
1766
+ w = await runCase(ctx, { name: "i13", bin: ctx.oldBin, env: "relay", flags: BARE, conversations: (d) => [{ calls: [exec(`touch ${d.out}/old_out; echo oldout=$?; touch ${d.wt}/old_in; echo oldin=$?`)] }] });
1767
+ } catch (e) { err = e.message; }
1768
+ const codex = oldCodexPart({ said, err, m: w ? markers(w.model.output(0, 0)) : {}, wroteOut: w ? exists(`${w.d.out}/old_out`) : false });
1769
+ const oldScratch = path.join(ctx.dir, "i13-pin");
1770
+ fs.mkdirSync(oldScratch, { mode: 0o700 });
1771
+ const oldPin = stageQualificationPin({ scratch: oldScratch, bin: ctx.oldBin, version: "0.163.0" });
1772
+ const admission = (pin) => {
1773
+ const config = ctx.dist.pin.freezeCodexConfig(ctx.dist.config.withDefaults({ child_mcp: "relay-only", child_hooks: false, targets: { codex: { ...pin, stage2: true } } }));
1774
+ const spec = { config, target: ctx.dist.targets.targetDef("codex"), access: "write", cwd: ctx.dir, relay: null, grant: { network: true, raw_network: true } };
1775
+ // Every probe but the version's stood in (stage2Admission's own test parameter): this item is about the version.
1776
+ return ctx.dist.stage2.stage2Admission(spec, stageChildEnv({ CODEX_HOME: pin.home }), { interlock: () => null, aliases: () => null, consent: () => null });
1777
+ };
1778
+ const min = ctx.dist.stage2.STAGE2_MIN_CODEX.join(".");
1779
+ const old = admission(oldPin), cur = admission(ctx.pin);
1780
+ const p = [];
1781
+ if (min !== "0.164.0") p.push(`STAGE2_MIN_CODEX is ${min}, not 0.164.0`);
1782
+ if (!old || !old.includes(`needs ${min} or later`)) p.push(`relay's admission ${old ? `refused ${said} for another reason: ${old}` : `admitted ${said}`}`);
1783
+ if (cur) p.push(`relay's admission refused this Codex (${ctx.version}): ${cur}`);
1784
+ return { codex, relay: judged(p, { old, current: cur, min }) };
1785
+ }
1786
+ /**
1787
+ * Item 13's codex part: `said`, the older Codex's --version; `err`, why its case threw (null when it
1788
+ * ran); `m`, its command's markers; `wroteOut`, whether old_out is outside the roots. It passes only
1789
+ * when the command ran (it wrote the worktree) and wrote nothing outside: a case that threw, or a
1790
+ * command that never ran, shows nothing about escaping, so the check cannot tell.
1791
+ */
1792
+ export function oldCodexPart({ said, err = null, m = {}, wroteOut = false }) {
1793
+ const ran = m.oldin === "0", escaped = wroteOut || m.oldout === "0";
1794
+ const p = [];
1795
+ if (escaped) p.push(`${said}: a command wrote outside the roots`);
1796
+ if (err) p.push(`${said}: the case did not run (${String(err).slice(0, 300)}): the check cannot tell`);
1797
+ else if (!ran) p.push(`${said}: the command did not write the worktree (oldin=${m.oldin ?? "none"}): the check cannot tell`);
1798
+ return judged(p, { version: said, refused: err?.slice(0, 300) ?? null, ran, escaped });
1799
+ }
1800
+ const tryText = (bin, args) => { try { return execFileSync(bin, args, { encoding: "utf8", env: stageChildEnv() }).trim(); } catch (e) { return `(${e.message.slice(0, 80)})`; } };
1801
+
1802
+ // Broker qualification (§7.2 items 14-22). These use the production front/core, with a tool
1803
+ // handler only where a deterministic/75-second answer is needed. Relay parts use the real handler.
1804
+ const brokerSearch = () => ({ type: "tool_search_call", execution: "client", arguments: { query: "relay status delegate", limit: 10 } });
1805
+ const brokerCall = (name = "status", args = {}) => ({ type: "function_call", namespace: "mcp__relay_tools", name, arguments: JSON.stringify(args) });
1806
+ export function brokerOK(out) {
1807
+ if (out == null) return false;
1808
+ if (typeof out === "object") {
1809
+ if (out.isError || out.error) return false;
1810
+ if (out.fixture === "BROKER_OK") return true;
1811
+ if (out.result) return brokerOK(out.result);
1812
+ return Array.isArray(out.content) && out.content.some((c) => c.type === "text" && brokerOK(c.text));
1813
+ }
1814
+ const text = String(out);
1815
+ if (/Unexpected token|not valid JSON|Error calling tool|"isError"\s*:\s*true/.test(text)) return false;
1816
+ try { return brokerOK(JSON.parse(text)); } catch {}
1817
+ // Codex can wrap the JSON text in command-style timing/output content parts.
1818
+ return /"fixture"\s*:\s*"BROKER_OK"/.test(text);
1819
+ }
1820
+ export const ownTurnCompleted = (note, threadId) => note.method === "turn/completed" && note.params?.threadId === threadId;
1821
+
1822
+ /** Expected admission refusal throws before a handle exists; unrelated launch errors still fail. */
1823
+ export async function qualificationStart(start, expectedRefusal) {
1824
+ try { return { startRefused: false, handle: await start() }; }
1825
+ catch (err) {
1826
+ if (!expectedRefusal?.(err)) throw err;
1827
+ return { startRefused: true, result: { status: "failed", error: String(err.message ?? err) } };
1828
+ }
1829
+ }
1830
+ const threadParams = (d) => ({ cwd: d.wt, runtimeWorkspaceRoots: [d.wt], permissions: ":danger-full-access", approvalPolicy: "on-request", approvalsReviewer: "user" });
1831
+ const rpcTool = (rpc, threadId) => rpc.request("mcpServer/tool/call", { threadId, server: "relay_tools", tool: "status", arguments: {} });
1832
+ async function boundedUntil(pred, ms = 10_000) {
1833
+ const until = Date.now() + ms;
1834
+ while (Date.now() < until) { if (pred()) return true; await sleep(25); }
1835
+ return false;
1836
+ }
1837
+
1838
+ /** Only safe metadata is retained: never the Authorization header or request body. */
1839
+ export function brokerObservation(head) {
1840
+ const sid = /^mcp-session-id:\s*([^\r\n]+)/im.exec(head)?.[1];
1841
+ return { method: /^(GET|POST|DELETE) /.exec(head)?.[1] ?? null, sid,
1842
+ issued: /^mcp-session-id:\s*([^\r\n]+)/im.exec(head)?.[1], status: /^HTTP\/1\.[01] (\d+)/.exec(head)?.[1] };
1843
+ }
1844
+
1845
+ async function qualificationBroker(ctx, d, o = {}) {
1846
+ const job = await registeredJob(ctx, d, path.basename(d.dir), { network: true, raw_network: true });
1847
+ const diag = { calls: 0, durations: [], peers: 0, refusals: [], fatal: [], events: [], live: new Set(), peak: 0 };
1848
+ const sockets = new Set();
1849
+ const delayStop = new AbortController();
1850
+ let opening = true, peerMode = o.peerMode ?? "ok";
1851
+ const b = await ctx.dist.broker.openHttpBroker({ nodeId: job.nodeId, turnId: job.turnId, mayDelegate: true, mayAsk: false }, await freePort(), {
1852
+ onRefusal: (why) => diag.refusals.push(why),
1853
+ onSession: (event) => {
1854
+ if (event.state === "opened") diag.live.add(event.id);
1855
+ else diag.live.delete(event.id);
1856
+ diag.peak = Math.max(diag.peak, event.live);
1857
+ },
1858
+ peer: (p, socket) => {
1859
+ if (!opening) diag.peers++;
1860
+ return !opening && peerMode === "foreign" ? Promise.resolve("foreign") : !opening && peerMode === "error" ? Promise.resolve({ error: "qualification checker failure" }) : ctx.dist.peer.tcpPeerLookup(p, socket);
1861
+ },
1862
+ toolHandler: async () => {
1863
+ diag.calls++; const at = performance.now();
1864
+ while (performance.now() - at < (o.delayMs ?? 0)) await delay(Math.max(1, o.delayMs - (performance.now() - at)), undefined, { signal: delayStop.signal });
1865
+ diag.durations.push(performance.now() - at);
1866
+ // relayServer decorates status by parsing its handler's text as JSON.
1867
+ return { content: [{ type: "text", text: JSON.stringify({ fixture: "BROKER_OK" }) }] };
1868
+ },
1869
+ network: { connect: net.createConnection, createServer(options, accept) {
1870
+ return net.createServer(options, (socket) => {
1871
+ sockets.add(socket); socket.once("close", () => sockets.delete(socket));
1872
+ // Request metadata is separate from the broker's authoritative lifecycle callback:
1873
+ // session teardown can happen on GET/SSE close without a DELETE request.
1874
+ let incoming = "";
1875
+ socket.on("data", (chunk) => {
1876
+ incoming = (incoming + chunk.toString("latin1")).slice(-16384);
1877
+ const end = incoming.indexOf("\r\n\r\n");
1878
+ if (end < 0) return;
1879
+ const ev = brokerObservation(incoming.slice(0, end)); incoming = "";
1880
+ diag.events.push({ method: ev.method, session: Boolean(ev.sid) });
1881
+ });
1882
+ accept(socket);
1883
+ });
1884
+ } },
1885
+ });
1886
+ opening = false;
1887
+ b.onFatal((why) => diag.fatal.push(why));
1888
+ return { ...b, close: async () => { delayStop.abort(); await b.close(); }, toolTimeoutS: o.toolTimeoutS, diag, peerMode: (mode) => { peerMode = mode; }, breakConnections: () => { for (const s of sockets) s.destroy(); } };
1889
+ }
1890
+
1891
+ async function brokerReadback(ctx, e) {
1892
+ const eff = (await e.rpc.request("config/read", { cwd: e.d.wt })).config;
1893
+ e.brokerReadback = await ctx.dist.stage2.stage2Readback(e.rpc, eff, e.d.wt, { ...e.broker, toolTimeoutS: 180 }, ctx.dist.stage2.stage2FeaturesOff(ctx.version.split(".").map(Number)));
1894
+ e.effective = eff;
1895
+ e.brokerCheckpoint = e.broker.sessionsServed();
1896
+ }
1897
+
1898
+ async function item14(ctx) {
1899
+ const parts = {};
1900
+ for (const [mode, m] of ["direct", "code_mode_only"].map((mode) => [mode, catalogModels(ctx.catalog).find((m) => (m.tool_mode ?? "direct") === mode)])) {
1901
+ if (!m) { parts[mode] = part("failed", `no ${mode} model in catalog`); continue; }
1902
+ const w = await runCase(ctx, { name: `i14-${mode}`, env: "relay", broker: true, appEnv: relayEnv(ctx), model: m.slug,
1903
+ conversations: () => [{ calls: [brokerSearch(), brokerCall()] }], beforeThread: (e) => brokerReadback(ctx, e),
1904
+ beforeTurn: async (e) => {
1905
+ e.threadReadback = await ctx.dist.stage2.stage2ThreadMcpProblem(e.rpc, e.effective, e.thread.thread.id, { ...e.broker, toolTimeoutS: 180 }, e.brokerCheckpoint);
1906
+ e.threadListing = await e.rpc.request("mcpServerStatus/list", { threadId: e.thread.thread.id, detail: "toolsAndAuthOnly" });
1907
+ e.broker.arm();
1908
+ e.explicit = await rpcTool(e.rpc, e.thread.thread.id);
1909
+ } });
1910
+ const p = [];
1911
+ if (w.brokerReadback) p.push(w.brokerReadback);
1912
+ if (w.threadReadback) p.push(w.threadReadback);
1913
+ if (!w.threadListing?.data?.some((x) => x.name === "relay_tools" && x.runtimeStatus === "connected")) p.push("no connected per-thread broker listing");
1914
+ if (!brokerOK(w.explicit)) p.push("mcpServer/tool/call did not return the fixture answer");
1915
+ if (!brokerOK(w.model.output(0, 1)) || !w.done) p.push("model-driven call did not finish through the broker");
1916
+ parts[mode] = judged(p, { model: m.slug, sessions: w.broker.sessionsServed() });
1917
+ }
1918
+ const long = await runCase(ctx, { name: "i14-long", env: "relay", broker: { delayMs: 75_000 }, appEnv: relayEnv(ctx), timeoutMs: 100_000,
1919
+ conversations: () => [{ calls: [brokerSearch(), brokerCall()] }] });
1920
+ parts.long = judged(long.done && brokerOK(long.model.output(0, 1)) && long.broker.diag.calls === 1 && long.broker.diag.durations[0] >= 75_000 ? [] : ["75-second call did not finish exactly once"], { calls: long.broker.diag.calls, durationMs: long.broker.diag.durations[0] });
1921
+ // §14 obligation 1: no manual arm; the first tool call goes through production turn ordering.
1922
+ parts.relay = await relayPart(ctx, async () => {
1923
+ let readbackWaitSawCall = false;
1924
+ ctx.dist.codexEnv.setStage2DepsForTests({ turnReadback: async (...args) => {
1925
+ readbackWaitSawCall = await boundedUntil(() => Boolean(observed?.output(0, 1)), 10_000);
1926
+ return ctx.dist.stage2.stage2TurnReadback(...args);
1927
+ } });
1928
+ let observed;
1929
+ try {
1930
+ const r = await relayTurn(ctx, { name: "i14-production", conversations: () => [{ calls: [brokerSearch(), brokerCall()] }],
1931
+ onRequest: (_reply, h) => { observed = h.model; } });
1932
+ return judged(r.result.status === "ok" && r.model.output(0, 1)?.includes(r.nodeId) && readbackWaitSawCall ? [] : ["first real tool call was not served during production turn readback"], { turn: turnSaid(r), readbackWaitSawCall });
1933
+ } finally { ctx.dist.codexEnv.setStage2DepsForTests(null); }
1934
+ });
1935
+ return parts;
1936
+ }
1937
+
1938
+ async function item15(ctx) {
1939
+ const startup = [], detail = {};
1940
+ for (const mode of ["missing", "wrong", "foreign", "error"]) {
1941
+ const w = await runCase(ctx, { name: `i15-${mode}`, env: "relay", broker: { peerMode: mode }, appEnv: { ...relayEnv(ctx), ...(mode === "wrong" ? { RELAY_BROKER_TOKEN: "wrong-qualification-token" } : {}) }, missingBrokerToken: mode === "missing", noTurn: true,
1942
+ conversations: () => [{ calls: [] }], beforeThread: async (e) => {
1943
+ e.refused = await e.rpc.request("thread/start", threadParams(e.d)).then(() => false, () => true);
1944
+ } });
1945
+ const g = w.broker.diag;
1946
+ detail[mode] = { refused: w.refused, peers: g.peers, fatal: g.fatal.length, refusals: g.refusals };
1947
+ if (!w.refused) startup.push(`${mode}: required broker did not refuse thread/start`);
1948
+ if (["missing", "wrong"].includes(mode) && g.peers !== 0) startup.push(`${mode}: spawned a peer lookup before authentication`);
1949
+ if (mode === "wrong" && !g.refusals.includes("HTTP 401")) startup.push("wrong token did not get 401");
1950
+ if (mode === "foreign" && (!g.refusals.includes("HTTP 403") || g.fatal.length)) startup.push("foreign peer was not a nonfatal 403");
1951
+ if (mode === "error" && !g.fatal.length) startup.push("broken checker did not report fatal");
1952
+ }
1953
+ const credentials = await runCase(ctx, { name: "i15-files", env: "relay", broker: true, appEnv: relayEnv(ctx), conversations: () => [{ calls: [brokerSearch(), brokerCall()] }], during: async (e) => {
1954
+ const token = e.broker.env.RELAY_BROKER_TOKEN;
1955
+ e.leaks = filesHolding([e.d.dir], token);
1956
+ e.argvLeak = tryText("/bin/ps", ["-p", String(e.rpc.pid), "-ww", "-o", "command="]).includes(token);
1957
+ e.stderrLeak = e.rpc.stderr().includes(token);
1958
+ // Unlike the forced startup-refusal case, this is a real foreign kernel peer, holding
1959
+ // the right credential, connecting from the qualifier process rather than the pinned app.
1960
+ e.foreign = await wireBroker(e.broker, "POST", { jsonrpc: "2.0", id: 1, method: "initialize", params: {} });
1961
+ } });
1962
+ const midTurn = await relayPart(ctx, async () => {
1963
+ const order = [], sockets = new Set();
1964
+ let broken = false, appPid = 0, toolServed = false;
1965
+ const kill = process.kill;
1966
+ process.kill = function (pid, signal) {
1967
+ if (pid === -appPid && signal === "SIGKILL") order.push("app-kill");
1968
+ return Reflect.apply(kill, process, [pid, signal]);
1969
+ };
1970
+ ctx.dist.codexEnv.setStage2DepsForTests({ openBroker: async (turn, port) => {
1971
+ const b = await ctx.dist.broker.openHttpBroker(turn, port, {
1972
+ peer: (p, s) => broken ? Promise.resolve({ error: "qualification mid-turn checker failure" }) : ctx.dist.peer.tcpPeerLookup(p, s),
1973
+ network: { connect: net.createConnection, createServer: (o, accept) => net.createServer(o, (s) => { sockets.add(s); s.once("close", () => sockets.delete(s)); accept(s); }) },
1974
+ });
1975
+ return { ...b, onFatal(fail) { b.onFatal((why) => {
1976
+ order.push("fatal"); fail(why);
1977
+ order.push("failure-callback-returned");
1978
+ }); }, async close() {
1979
+ order.push(alive(appPid) ? "close-before-app-exit" : "app-reaped");
1980
+ order.push("broker-close"); await b.close();
1981
+ } };
1982
+ } });
1983
+ try {
1984
+ const r = await relayTurn(ctx, { name: "i15-midturn", conversations: () => [{ calls: [brokerSearch(), brokerCall()] }], onRequest: async (reply, h, body) => {
1985
+ appPid = h.appPid();
1986
+ if (reply.step !== 2) return;
1987
+ toolServed = (body.input ?? []).some((x) => x.call_id === "c0-1" && /_call_output$/.test(x.type) && outputText(x.output).includes(h.nodeId));
1988
+ broken = true;
1989
+ for (const s of sockets) s.destroy();
1990
+ await boundedUntil(() => order.includes("fatal"));
1991
+ } });
1992
+ order.push("turn-result");
1993
+ const p = [];
1994
+ if (!toolServed) p.push("no successful tool call before checker injection");
1995
+ if (r.result.status !== "failed" || !/TCP peer|checker/.test(r.result.error ?? "")) p.push("checker failure did not fail the turn");
1996
+ if (alive(appPid) || !order.includes("app-reaped") || order.includes("close-before-app-exit")) p.push("app server not reaped before broker close/turn result");
1997
+ if (!order.includes("fatal") || order.indexOf("fatal") > order.indexOf("app-reaped")) p.push("missing fatal-before-teardown ordering");
1998
+ if (!order.includes("app-kill") || order.indexOf("app-kill") < order.indexOf("fatal") || order.indexOf("app-kill") > order.indexOf("failure-callback-returned")) p.push("failure callback did not kill the app server first");
1999
+ return judged(p, { order, turn: turnSaid(r) });
2000
+ } finally { process.kill = kill; ctx.dist.codexEnv.setStage2DepsForTests(null); }
2001
+ });
2002
+ const credentialProblems = credentials.leaks.length || credentials.argvLeak || credentials.stderrLeak ? ["broker token leaked to argv, scratch file or stderr"] : [];
2003
+ if (credentials.foreign.status !== 403 || credentials.broker.diag.fatal.length) credentialProblems.push("a real foreign kernel peer with the correct bearer was not a nonfatal 403");
2004
+ return { startup: judged(startup, detail), credentials: judged(credentialProblems, { fileHits: credentials.leaks, argvLeak: credentials.argvLeak, stderrLeak: credentials.stderrLeak, foreignStatus: credentials.foreign.status }), midTurn };
2005
+ }
2006
+
2007
+ async function item16(ctx) {
2008
+ const w = await runCase(ctx, { name: "i16-stdio-relay", env: "relay", broker: true, appEnv: relayEnv(ctx), servers: ["relay", "relay_tools"], extraConfig: (d) => mcpServer(d, "relay"), conversations: () => [{ calls: [brokerSearch(), brokerCall()] }] });
2009
+ const coexist = judged(!exists(`${w.d.out}/mcp-relay`) && brokerOK(w.model.output(0, 1)) ? [] : ["stdio relay collided with relay_tools or ran outside the executor"]);
2010
+ const admission = await relayPart(ctx, async () => {
2011
+ const r = await relayTurn(ctx, { name: "i16-person-entry", extraConfig: () => '[mcp_servers.relay_tools]\ncommand = "/usr/bin/false"', conversations: () => [{ calls: [] }],
2012
+ expectedStartRefusal: (err) => /the person's Codex config defines relay_tools;/.test(err.message ?? "") });
2013
+ const configPath = path.join(r.d.home, "config.toml");
2014
+ const ownEntry = fs.readFileSync(configPath, "utf8").includes('[mcp_servers.relay_tools]\ncommand = "/usr/bin/false"');
2015
+ return judged(ownEntry && r.startRefused && r.result.status === "failed" && r.appPid === 0 && !r.model.requests.length ? [] : ["fixture's scratch relay_tools entry did not refuse before app/model startup"], { turn: turnSaid(r), configPath, fixtureWroteEntry: ownEntry, refusedBeforeStart: r.startRefused });
2016
+ });
2017
+ return { coexist, admission };
2018
+ }
2019
+
2020
+ async function item17(ctx) {
2021
+ const parts = {};
2022
+ for (const bypass of [true, false]) {
2023
+ const proxy = await refusingProxy();
2024
+ try {
2025
+ const url = `http://127.0.0.1:${proxy.port}`;
2026
+ const w = await runCase(ctx, { name: `i17-${bypass}`, env: "relay", broker: true, noTurn: true, appEnv: { ...relayEnv(ctx), HTTP_PROXY: url, HTTPS_PROXY: url, http_proxy: url, https_proxy: url, ALL_PROXY: url, all_proxy: url, NO_PROXY: bypass ? "127.0.0.1" : "", no_proxy: bypass ? "127.0.0.1" : "" }, conversations: () => [{ calls: [] }], beforeThread: async (e) => {
2027
+ e.listed = await e.rpc.request("mcpServerStatus/list", { detail: "toolsAndAuthOnly" }).catch(() => null);
2028
+ } });
2029
+ const token = w.broker.env.RELAY_BROKER_TOKEN;
2030
+ const leaked = proxy.hasBearer(token);
2031
+ parts[bypass ? "codex" : "control"] = judged(bypass ? (leaked || proxy.seen.length || !w.broker.sessionsServed() ? ["bypass did not keep a working broker off the proxy"] : []) : (!leaked ? ["negative control did not deliver bearer to decoy proxy"] : []), { proxyRequests: proxy.seen.length, bearerSeen: leaked });
2032
+ } finally { await proxy.close(); }
2033
+ }
2034
+ return parts;
2035
+ }
2036
+
2037
+ async function item18(ctx) {
2038
+ const e = await runCase(ctx, { name: "i18", env: "relay", broker: true, appEnv: relayEnv(ctx), keep: true, conversations: () => [{ calls: [brokerSearch(), brokerCall()] }] });
2039
+ const hits = [];
2040
+ let squatter;
2041
+ try {
2042
+ const token = e.broker.env.RELAY_BROKER_TOKEN, port = Number(new URL(e.broker.url).port);
2043
+ await e.broker.close();
2044
+ squatter = http.createServer((q, r) => { hits.push({ method: q.method, token: q.headers.authorization === `Bearer ${token}` }); r.writeHead(500); r.end(); });
2045
+ await new Promise((resolve, reject) => { squatter.once("error", reject); squatter.listen(port, "127.0.0.1", resolve); });
2046
+ await boundedUntil(() => hits.length > 0, 6500);
2047
+ const at = Date.now(); e.rpc.eof();
2048
+ const exited = await Promise.race([e.rpc.exited.then(() => true), sleep(1000).then(() => false)]);
2049
+ return { reconnect: judged(hits.some((x) => x.token) ? [] : ["no authenticated reconnect after broker closure (Codex behavior changed)"], { hits }), eof: judged(exited ? [] : ["app server did not exit within 1s of stdin EOF"], { ms: Date.now() - at }) };
2050
+ } finally { squatter?.closeAllConnections(); if (squatter) await new Promise((r) => squatter.close(r)); await teardown(e); }
2051
+ }
2052
+
2053
+ /** A wire response parser used by the fixture and unit tests; malformed/missing responses fail. */
2054
+ export function brokerRpc(text) {
2055
+ try { return JSON.parse(text); } catch {}
2056
+ for (const line of String(text).split("\n")) if (line.startsWith("data:")) { try { return JSON.parse(line.slice(5)); } catch {} }
2057
+ return null;
2058
+ }
2059
+ async function wireBroker(b, method, body, sid) {
2060
+ const r = await fetch(b.url, { method, headers: { Authorization: `Bearer ${b.env.RELAY_BROKER_TOKEN}`, Accept: "application/json, text/event-stream", "Content-Type": "application/json", ...(sid ? { "Mcp-Session-Id": sid, "MCP-Protocol-Version": "2025-03-26" } : {}) }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(5000) });
2061
+ return { status: r.status, sid: r.headers.get("mcp-session-id"), rpc: brokerRpc(await r.text()) };
2062
+ }
2063
+
2064
+ async function item19(ctx) {
2065
+ // Production tombstones and cap, driven by a pinned scratch client (this process).
2066
+ const d = caseDirs(ctx, "i19-sessions"), b = await qualificationBroker(ctx, d);
2067
+ b.bindAppServer({ pid: process.pid, start: ctx.dist.procs.procStart(process.pid, { fresh: true }) }); b.arm();
2068
+ const init = { jsonrpc: "2.0", id: 1, method: "initialize", params: { protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "qualify", version: "1" } } };
2069
+ const call = { jsonrpc: "2.0", id: 2, method: "tools/call", params: { name: "status", arguments: {} } };
2070
+ const sessions = [], p = [];
2071
+ try {
2072
+ for (let i = 0; i < 8; i++) { const r = await wireBroker(b, "POST", init); if (!r.sid || r.status !== 200) p.push(`session ${i} not initialized`); sessions.push(r.sid); }
2073
+ const cap = await wireBroker(b, "POST", init);
2074
+ const first = await wireBroker(b, "POST", call, sessions[0]);
2075
+ await wireBroker(b, "DELETE", null, sessions[0]);
2076
+ const tombstone = await wireBroker(b, "POST", call, sessions[0]);
2077
+ const unknown = await wireBroker(b, "POST", call, "never-issued");
2078
+ if (![cap, tombstone, unknown].every((r) => r.status === 200 && r.rpc?.error)) p.push("cap, tombstone or unknown session was not HTTP 200 JSON-RPC error");
2079
+ if (!brokerOK(first.rpc)) p.push("cap evicted the first session");
2080
+ } finally { await b.close(); }
2081
+ // Actual Codex retry semantics against a deliberately faulty HTTP fixture (404 control).
2082
+ const retry = {};
2083
+ for (const status of [200, 404]) {
2084
+ let calls = 0;
2085
+ const s = http.createServer((q, r) => {
2086
+ let body = ""; q.on("data", (c) => body += c); q.on("end", () => {
2087
+ const msg = brokerRpc(body);
2088
+ if (q.method === "GET") { r.writeHead(405); r.end(); return; }
2089
+ if (q.method === "DELETE" || msg?.id === undefined || msg?.id === null) { r.writeHead(202); r.end(); return; }
2090
+ let result = {};
2091
+ if (msg.method === "initialize") { r.setHeader("mcp-session-id", crypto.randomUUID()); result = { protocolVersion: "2025-03-26", capabilities: { tools: {} }, serverInfo: { name: "retry-control", version: "1" } }; }
2092
+ else if (msg.method === "tools/list") result = { tools: [{ name: "status", description: "retry fixture", inputSchema: { type: "object", properties: {} } }] };
2093
+ else if (msg.method === "tools/call") { calls++; if (calls === 1) { r.writeHead(status, { "content-type": "application/json" }); r.end(JSON.stringify({ jsonrpc: "2.0", id: msg.id, error: { code: -32001, message: "closed session / cap" } })); return; } result = { content: [{ type: "text", text: "BROKER_OK" }] }; }
2094
+ r.writeHead(200, { "content-type": "application/json" }); r.end(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result }));
2095
+ });
2096
+ });
2097
+ await new Promise((r) => s.listen(0, "127.0.0.1", r));
2098
+ try {
2099
+ const endpoint = { url: `http://127.0.0.1:${s.address().port}/mcp` };
2100
+ await runCase(ctx, { name: `i19-retry-${status}`, env: "relay", appEnv: { ...relayEnv(ctx), RELAY_BROKER_TOKEN: "scratch-retry-control" }, flags: (d) => stage2Flags(ctx, d, [], endpoint), conversations: () => [{ calls: [brokerSearch(), brokerCall()] }] });
2101
+ retry[status] = calls;
2102
+ } finally { s.closeAllConnections(); await new Promise((r) => s.close(r)); }
2103
+ }
2104
+ return { sessions: judged(p), retry: judged(retry[200] === 1 && retry[404] === 2 ? [] : ["Codex retry behavior changed: expected 1 POST on 200 error, 2 on 404"], retry) };
2105
+ }
2106
+
2107
+ export function brokerEnvProblems(outputs, tokenSeen = false) {
2108
+ const p = [];
2109
+ for (const [i, out] of outputs.entries()) {
2110
+ if (!String(out ?? "").includes("BROKER_ENV_END")) p.push(`command ${i + 1} printed no environment marker`);
2111
+ if (/RELAY_BROKER_TOKEN/.test(String(out ?? ""))) p.push(`command ${i + 1} exposed the token variable name`);
2112
+ }
2113
+ if (outputs.length !== 3) p.push("expected exactly three command environments");
2114
+ if (tokenSeen) p.push("a command exposed the broker token value");
2115
+ return p;
2116
+ }
2117
+ async function item20(ctx) {
2118
+ const m = catalogModels(ctx.catalog).find((m) => m.tool_mode === "code_mode_only");
2119
+ if (!m) return { relay: part("failed", "no code-mode model for C6") };
2120
+ return { relay: await relayPart(ctx, async () => {
2121
+ const cmd = "/usr/bin/env; echo BROKER_ENV_END";
2122
+ const r = await relayTurn(ctx, { name: "i20", model: m.slug, conversations: () => [{ calls: [exec(cmd), exec(cmd, { tty: true }), codeMode(`const c = await tools.exec_command({ cmd: ${JSON.stringify(cmd)}, login: false }); text(typeof c === 'string' ? c : JSON.stringify(c));`)] }],
2123
+ onRequest: (_reply, h, body) => { for (const x of body.input ?? []) if (/_call_output$/.test(x.type ?? "")) h.capture("command environment", outputText(x.output)); } });
2124
+ const out = [0, 1, 2].map((i) => r.model.output(0, i));
2125
+ const p = brokerEnvProblems(out, r.brokerTokenIn.length > 0);
2126
+ if (r.result.status !== "ok") p.push("C6 turn failed");
2127
+ return judged(p, { turn: turnSaid(r), commands: out.map((x) => ({ ran: Boolean(x?.includes("BROKER_ENV_END")), tokenName: Boolean(x?.includes("RELAY_BROKER_TOKEN")) })), tokenValue: r.brokerTokenIn.length > 0 });
2128
+ }) };
2129
+ }
2130
+
2131
+ async function item21(ctx) {
2132
+ const w = await runCase(ctx, { name: "i21", env: "relay", appEnv: relayEnv(ctx), keep: true, conversations: () => [{ calls: [] }] });
2133
+ let b, r;
2134
+ try {
2135
+ const threadId = w.thread.thread.id;
2136
+ await w.rpc.stop();
2137
+ // A scratch client sets the active goal; the next app server resumes with goals on.
2138
+ r = appServer(ctx.bin, [...BARE(), "--disable", "plugins"], { cwd: w.d.wt, env: { HOME: w.d.user, CODEX_HOME: w.d.home, TMPDIR: w.d.apptmp, PATH: SYSTEM_PATH } });
2139
+ await initialize(r);
2140
+ await r.request("thread/goal/set", { threadId, objective: "BROKER-SELF-START" });
2141
+ await r.stop();
2142
+ b = await qualificationBroker(ctx, w.d);
2143
+ const model = await fakeModel([{ calls: [brokerSearch(), brokerCall("delegate", { target: "claude", mode: "read", task: "must be refused before dispatch" })] }]);
2144
+ await w.model.close(); w.model = model;
2145
+ codexHome(w.d.home, { port: model.port, model: ctx.model, catalog: ctx.catalog });
2146
+ const flags = stage2Flags(ctx, w.d, [], b).map((x) => x === "features.goals=false" ? "features.goals=true" : x);
2147
+ r = appServer(ctx.bin, flags, { cwd: w.d.wt, env: { HOME: w.d.user, CODEX_HOME: w.d.home, TMPDIR: w.d.apptmp, PATH: SYSTEM_PATH, ...relayEnv(ctx), ...b.env, NO_PROXY: "127.0.0.1", no_proxy: "127.0.0.1" } });
2148
+ b.bindAppServer({ pid: r.pid, start: ctx.dist.procs.procStart(r.pid, { fresh: true }) });
2149
+ await initialize(r);
2150
+ await r.request("thread/resume", { threadId, ...threadParams(w.d) });
2151
+ const done = await r.until((n) => n.method === "turn/completed" && n.params?.threadId === threadId, 60_000);
2152
+ return { disarmed: judged(done && /turn has not started/.test(model.output(0, 1) ?? "") && b.diag.calls === 0 && r.notes.some((n) => n.method === "turn/started") ? [] : ["goal did not self-start and get a disarmed refusal before dispatch"], { callsDispatched: b.diag.calls, refused: b.diag.refusals.some((x) => x.includes("turn has not started")) }) };
2153
+ } finally { await r?.stop(); await b?.close(); await teardown(w); }
2154
+ }
2155
+
2156
+ /** Read the current request, including a spawn output not yet stored in fakeModel's history. */
2157
+ export function brokerAgentId(body, conv, step) {
2158
+ const output = (body.input ?? []).find((x) => x.type === "function_call_output" && x.call_id === `c${conv}-${step}`);
2159
+ const text = outputText(output?.output);
2160
+ let id;
2161
+ try { id = JSON.parse(text).agent_id; } catch {}
2162
+ if (typeof id !== "string" || !/^[0-9a-f-]{36}$/.test(id)) throw new Error(`spawn c${conv}-${step} supplied no valid agent_id: ${text.slice(0, 200)}`);
2163
+ return id;
2164
+ }
2165
+
2166
+ function brokerNamespace(body, name) {
2167
+ let found;
2168
+ const walk = (tools, ns) => { for (const t of tools ?? []) { if (t.type === "namespace") walk(t.tools, t.name); else if (t.name === name) found ??= ns; } };
2169
+ walk(body.tools);
2170
+ for (const x of body.input ?? []) if (x.tools) walk(x.tools);
2171
+ return found;
2172
+ }
2173
+
2174
+ export function brokerSessionProblems({ opened, peak, calls, resumed, actors, fatal }, version) {
2175
+ const expected = resumed ? 3 : 8; // readback + parent + (one resumed child or six first-turn children)
2176
+ const p = [];
2177
+ // Named first: everything after a fatal broker fails with 503 ("Transport send error" in Codex).
2178
+ if (fatal?.length) p.push(`${version}: the broker went fatal: ${fatal.join("; ")}`);
2179
+ if (opened !== expected) p.push(`${version}: opened ${opened} sessions, expected ${expected}`);
2180
+ if (calls !== (resumed ? 2 : 7)) p.push(`${version}: parent/child tool calls missing or replayed (${calls})`);
2181
+ // B5's stand-in observed eager DELETE of the threadless readback. The production transport
2182
+ // may overlap that session with the turn; §3.3 deliberately reserves the eighth slot for it.
2183
+ const max = resumed || version === "v1" ? 3 : 8;
2184
+ if (!Number.isSafeInteger(peak) || peak < 2 || peak > max) p.push(`${version}: peak live sessions ${peak} outside 2..${max} (including readback overlap)`);
2185
+ if (!Array.isArray(actors) || actors.length !== (resumed ? 2 : 7)) p.push(`${version}: missing per-parent/child reply evidence`);
2186
+ const expectedActors = ["parent", ...Array.from({ length: resumed ? 1 : 6 }, (_, i) => `child ${resumed ? 6 : i}`)].sort();
2187
+ if (JSON.stringify((actors ?? []).map((a) => a.actor).sort()) !== JSON.stringify(expectedActors)) p.push(`${version}: parent/child reply identities do not match the scripted actors`);
2188
+ for (const actor of actors ?? []) if (!actor.ok) p.push(`${version}: ${actor.actor} did not receive a successful broker reply (${actor.output})`);
2189
+ return p;
2190
+ }
2191
+
2192
+ async function item22(ctx) {
2193
+ const parts = {};
2194
+ for (const version of ["v1", "v2"]) {
2195
+ // B5 shape: force the two runtime paths in a scratch copy of the bundled catalog, even
2196
+ // when this release exposes only v2 models by default. Item 10 checks the unmodified catalog.
2197
+ const catalog = JSON.parse(fs.readFileSync(ctx.catalog, "utf8"));
2198
+ const m = catalog.models.find((m) => m.multi_agent_version === version && m.visibility === "list") ?? catalog.models.find((m) => m.visibility === "list");
2199
+ m.multi_agent_version = version; m.tool_mode = version === "v1" ? "direct" : "code_mode_only";
2200
+ const variant = path.join(ctx.dir, `i22-${version}-catalog.json`);
2201
+ fs.writeFileSync(variant, JSON.stringify(catalog));
2202
+ const variantCtx = { ...ctx, catalog: variant };
2203
+ const fc = (body, name, args) => ({ type: "function_call", name, ...(brokerNamespace(body, name) ? { namespace: brokerNamespace(body, name) } : {}), arguments: JSON.stringify(args) });
2204
+ const parent = (n, offset, conv) => [brokerSearch(), brokerCall(), ...Array.from({ length: n }, (_, i) => {
2205
+ const id = (body) => brokerAgentId(body, conv, 2 + i * (version === "v1" ? 3 : 2));
2206
+ return [
2207
+ (body) => fc(body, "spawn_agent", version === "v1" ? { message: `BROKER-SUB-${offset + i}` } : { task_name: `broker_${offset + i}`, message: `BROKER-SUB-${offset + i}`, fork_turns: "none" }),
2208
+ (body) => fc(body, "wait_agent", version === "v1" ? { targets: [id(body)], timeout_ms: 20_000 } : { timeout_ms: 20_000 }),
2209
+ ...(version === "v1" ? [(body) => fc(body, "close_agent", { target: id(body) })] : []),
2210
+ ];
2211
+ }).flat()];
2212
+ const conversations = () => [
2213
+ { match: "BROKER-PARENT-RESUME", calls: parent(1, 6, 0) },
2214
+ ...Array.from({ length: 7 }, (_, i) => ({ match: `BROKER-SUB-${i}`, calls: [brokerSearch(), brokerCall()] })),
2215
+ { calls: parent(6, 0, 8) },
2216
+ ];
2217
+ // Multi-agent v1 defers its tools too; search them in the same discovery call.
2218
+ const originalSearch = conversations();
2219
+ for (const c of originalSearch) c.calls[0].arguments.query += " spawn agent wait close agent";
2220
+ const w = await runCase(variantCtx, { name: `i22-${version}`, env: "relay", broker: true, appEnv: relayEnv(ctx), model: m.slug, prompt: "BROKER-PARENT-FIRST", keep: true, timeoutMs: 120_000,
2221
+ conversations: () => originalSearch, beforeThread: (e) => brokerReadback(ctx, e) });
2222
+ const counts = (resumed) => {
2223
+ const convs = resumed ? [0, 7] : [8, 1, 2, 3, 4, 5, 6];
2224
+ // The broker's fatal reasons (a TCP peer check that failed, say): a broker gone fatal answers 503 to everything after.
2225
+ return { opened: w.broker.sessionsServed(), peak: w.broker.diag.peak, calls: w.broker.diag.calls, resumed, fatal: [...w.broker.diag.fatal],
2226
+ actors: convs.map((conv, i) => {
2227
+ const output = w.model.output(conv, 1);
2228
+ return { actor: i === 0 ? "parent" : `child ${resumed ? 6 : i - 1}`, ok: brokerOK(output), output: String(output ?? "no output").slice(0, 400) };
2229
+ }),
2230
+ };
2231
+ };
2232
+ let resumedRpc;
2233
+ try {
2234
+ await boundedUntil(() => w.broker.diag.calls >= 7);
2235
+ const first = counts(false);
2236
+ const p = brokerSessionProblems(first, version);
2237
+ if (w.done?.params?.turn?.status !== "completed" || w.brokerReadback) p.push("first parent turn or readback did not succeed");
2238
+ const threadId = w.thread.thread.id;
2239
+ await w.rpc.stop(); await w.broker.close();
2240
+ w.broker = await qualificationBroker(ctx, w.d);
2241
+ resumedRpc = appServer(ctx.bin, stage2Flags(ctx, w.d, [], w.broker), { cwd: w.d.wt, env: { HOME: w.d.user, CODEX_HOME: w.d.home, TMPDIR: w.d.apptmp, PATH: SYSTEM_PATH, ...relayEnv(ctx), ...w.broker.env, NO_PROXY: "127.0.0.1", no_proxy: "127.0.0.1" } });
2242
+ w.broker.bindAppServer({ pid: resumedRpc.pid, start: ctx.dist.procs.procStart(resumedRpc.pid, { fresh: true }) });
2243
+ await initialize(resumedRpc);
2244
+ await resumedRpc.request("environment/add", { environmentId: "relay", execServerUrl: w.ex.url, authBearerToken: w.token });
2245
+ const resumedCase = { rpc: resumedRpc, d: w.d, broker: w.broker };
2246
+ await brokerReadback(ctx, resumedCase);
2247
+ await resumedRpc.request("thread/resume", { threadId, ...threadParams(w.d) });
2248
+ await resumedRpc.request("turn/start", { threadId, input: [{ type: "text", text: "BROKER-PARENT-RESUME", text_elements: [] }], environments: [{ environmentId: "relay", cwd: w.d.wt }] });
2249
+ w.broker.arm();
2250
+ const done = await resumedRpc.until((n) => ownTurnCompleted(n, threadId), 90_000);
2251
+ await boundedUntil(() => w.broker.diag.calls >= 2);
2252
+ const resumed = counts(true);
2253
+ p.push(...brokerSessionProblems(resumed, version));
2254
+ if (done?.params?.turn?.status !== "completed" || resumedCase.brokerReadback) p.push("resumed parent turn or readback did not succeed");
2255
+ for (const r of w.model.requests) if (r.scriptError || r.hookError) p.push(r.scriptError ?? r.hookError);
2256
+ parts[version] = judged(p, { model: m.slug, first, resumed });
2257
+ } finally { await resumedRpc?.stop(); await teardown(w); }
2258
+ }
2259
+ return parts;
2260
+ }
2261
+
2262
+ /** For a run by hand of one case (debugging). */
2263
+ export const internal = { runCase, relayTurn, stage2Flags, BARE };
2264
+
2265
+ // RELAY_TEST_ESCALATION_LONG=readback, for release gates: the readback alone, with the hour-long
2266
+ // call recorded as not run, so the verdict can never promote a pin. The full call runs at pin
2267
+ // qualification and at a gate whose change touches the broker or the tool timeout.
2268
+ async function item30(ctx) {
2269
+ const mode = process.env.RELAY_TEST_ESCALATION_LONG;
2270
+ if (mode !== "1" && mode !== "readback") return { long: part("not run", "opt in with RELAY_TEST_ESCALATION_LONG=1 (takes over one hour), or =readback for the timeout readback alone") };
2271
+ const delayMs = 3_601_000;
2272
+ const toolTimeoutS = (await import(path.join(ctx.checkout, "dist/targets/codex.js"))).codexToolTimeoutS(1);
2273
+ const readback = async e => {
2274
+ const eff = (await e.rpc.request("config/read", { cwd: e.d.wt })).config;
2275
+ const problem = await ctx.dist.stage2.stage2Readback(e.rpc, eff, e.d.wt, { ...e.broker, toolTimeoutS }, ctx.dist.stage2.stage2FeaturesOff(ctx.version.split(".").map(Number)));
2276
+ if (problem) throw new Error(problem);
2277
+ };
2278
+ if (mode === "readback") {
2279
+ await runCase(ctx, { name: "i30-readback", env: "relay", broker: { delayMs, toolTimeoutS }, appEnv: relayEnv(ctx), noTurn: true, beforeThread: readback, conversations: () => [] });
2280
+ return { readback: part("passed", [], { toolTimeoutS }), long: part("not run", "readback only (RELAY_TEST_ESCALATION_LONG=readback): the hour-long call was not made") };
2281
+ }
2282
+ const w = await runCase(ctx, { name: "i30-long", env: "relay", broker: { delayMs, toolTimeoutS }, appEnv: relayEnv(ctx), timeoutMs: delayMs + 120_000,
2283
+ beforeThread: readback, conversations: () => [{ calls: [brokerSearch(), brokerCall()] }] });
2284
+ return { long: judged(w.done && brokerOK(w.model.output(0, 1)) && w.broker.diag.calls === 1 && w.broker.diag.durations[0] >= delayMs ? [] : ["result beyond old timeout was not delivered exactly once"], { toolTimeoutS, calls: w.broker.diag.calls, durationMs: w.broker.diag.durations[0] }) };
2285
+ }
2286
+
2287
+ const RUN = { 1: item1, 2: item2, 3: item3, 4: item4, 5: item5, 6: item6, 7: item7, 8: item8, 9: item9, 10: item10, 11: item11, 12: item12, 13: item13, 14: item14, 15: item15, 16: item16, 17: item17, 18: item18, 19: item19, 20: item20, 21: item21, 22: item22, ...Object.fromEntries(Object.keys(ESCALATION_CASES).map(id => [id, async ctx => escalationParts(await runEscalationItem(ctx, id))])), 30: item30 };
2288
+
2289
+ // ---------------------------------------------------------------------------------------------
2290
+ // The run.
2291
+
2292
+ /** The bundled model catalog of this version (codex-rs/models-manager/models.json): from a local source tree, or from GitHub at the tag. */
2293
+ async function catalogFor(version, source, dir) {
2294
+ const out = path.join(dir, "catalog.json");
2295
+ let text;
2296
+ if (source) text = fs.readFileSync(path.join(source, "models-manager", "models.json"), "utf8");
2297
+ else {
2298
+ const r = await fetch(`https://raw.githubusercontent.com/openai/codex/rust-v${version}/codex-rs/models-manager/models.json`, { headers: { "User-Agent": "relay-release-watch" } });
2299
+ if (!r.ok) throw new Error(`the bundled model catalog of rust-v${version}: HTTP ${r.status}`);
2300
+ text = await r.text();
2301
+ }
2302
+ const models = JSON.parse(text).models;
2303
+ if (!Array.isArray(models) || !models.some((m) => m.visibility === "list")) throw new Error("the bundled model catalog lists no model");
2304
+ fs.writeFileSync(out, text);
2305
+ return out;
2306
+ }
2307
+
2308
+ /** Stops everything this run started that still runs, and anything else whose command names the scratch directory; returns what was left running. */
2309
+ async function sweep(dir) {
2310
+ const left = [];
2311
+ for (const [pid, t] of started) if (alive(pid)) { left.push(`${t.what} ${pid}`); await stopPid(pid, t.what === "executor"); }
2312
+ const r = spawnSync("/usr/bin/pgrep", ["-f", dir], { encoding: "utf8", env: stageChildEnv() });
2313
+ for (const pid of (r.stdout || "").split("\n").filter(Boolean).map(Number)) {
2314
+ if (pid === process.pid) continue;
2315
+ left.push(`${psEnv(pid).slice(0, 120).trim() || "process"} ${pid}`);
2316
+ await stopPid(pid);
2317
+ }
2318
+ // relay's own executors (the relay parts): any exec-server whose working directory is under the scratch dir.
2319
+ const ps = spawnSync("/bin/ps", ["-axww", "-o", "pid=,command="], { encoding: "utf8", maxBuffer: 32 << 20, env: stageChildEnv() });
2320
+ const execs = [];
2321
+ for (const pid of (ps.stdout ?? "").split("\n").map((l) => /^\s*(\d+)\s+(.*)$/.exec(l)).filter((m) => m && / exec-server --listen ws:\/\//.test(m[2])).map((m) => Number(m[1]))) {
2322
+ const c = spawnSync("/usr/sbin/lsof", ["-a", "-p", String(pid), "-d", "cwd", "-Fn"], { encoding: "utf8", env: stageChildEnv() });
2323
+ if (!(c.stdout ?? "").split("\n").some((l) => l.startsWith(`n${dir}/`))) continue;
2324
+ execs.push(pid);
2325
+ left.push(`relay's executor ${pid}`);
2326
+ await stopPid(pid, true);
2327
+ }
2328
+ const still = [...started.keys(), ...execs, ...(r.stdout || "").split("\n").filter(Boolean).map(Number)].filter((p) => p !== process.pid && alive(p));
2329
+ return { left, still };
2330
+ }
2331
+
2332
+ export async function qualify({ checkout, dir, version, bin, oldBin = null, source = null, only = [] }) {
2333
+ fs.rmSync(dir, { recursive: true, force: true });
2334
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
2335
+ dir = fs.realpathSync(dir);
2336
+ stageScratch = dir;
2337
+ // relay's code runs here with a scratch home, relay home and config: nothing of the person's.
2338
+ process.env.HOME = path.join(dir, "home");
2339
+ process.env.RELAY_HOME = path.join(dir, "relay-home");
2340
+ process.env.RELAY_CONFIG = path.join(dir, "relay-config.json");
2341
+ for (const k of Object.keys(process.env)) if (k.startsWith("CODEX_") || k === "RELAY_EXPERIMENTAL_CODEX_STAGE2") delete process.env[k];
2342
+ const childEnv = stageChildEnv();
2343
+ process.env.CODEX_HOME = childEnv.CODEX_HOME;
2344
+ process.env.TMPDIR = childEnv.TMPDIR;
2345
+ fs.mkdirSync(process.env.HOME, { recursive: true, mode: 0o700 });
2346
+ fs.mkdirSync(process.env.RELAY_HOME, { recursive: true, mode: 0o700 });
2347
+ const pin = stageQualificationPin({ scratch: dir, bin, version });
2348
+ fs.writeFileSync(process.env.RELAY_CONFIG, JSON.stringify({ targets: { codex: { ...pin, stage2: true } } }) + "\n");
2349
+ const imp = (p) => import(path.join(checkout, "dist", p));
2350
+ const dist = {
2351
+ seatbelt: await imp("confine/seatbelt.js"), ports: await imp("confine/ports.js"), codex: await imp("backends/codex.js"), secrets: await imp("core/secrets.js"),
2352
+ stage2: await imp("backends/codex-stage2.js"), config: await imp("core/config.js"), targets: await imp("targets/index.js"),
2353
+ db: await imp("core/db.js"), approvals: await imp("core/approvals.js"), rawNetwork: await imp("core/raw-network.js"),
2354
+ broker: await imp("confine/broker-http.js"), peer: await imp("confine/peer.js"), procs: await imp("core/procs.js"), codexEnv: await imp("backends/codex-env.js"),
2355
+ pin: await imp("backends/codex-pin.js"),
2356
+ };
2357
+ const catalog = await catalogFor(version, source, dir);
2358
+ const first = catalogModels(catalog).find((m) => m.visibility === "list");
2359
+ const ctx = { dist, dir, checkout, bin, pin, version, oldBin, catalog, model: first.slug, codeMode: first.tool_mode === "code_mode_only" };
2360
+ const items = {};
2361
+ for (const id of Object.keys(ITEMS)) {
2362
+ if (only.length && !only.includes(id)) { items[id] = item({ run: part("not run", "not selected (--items)") }); continue; }
2363
+ try { items[id] = item(await dist.secrets.withTurnSecrets(() => RUN[id](ctx))); }
2364
+ catch (err) { items[id] = item({ driver: part("failed", `the driver failed: ${err.stack?.split("\n").slice(0, 3).join(" ") ?? err}`) }); }
2365
+ }
2366
+ const { left, still } = await sweep(dir);
2367
+ const v = verdictOf(items);
2368
+ if (left.length) v.why.push(`left running after the items, stopped by the sweep: ${left.join(", ")}`);
2369
+ if (still.length) { v.state = "failed"; v.ok = false; v.why.push(`still running after the sweep: ${still.join(", ")}`); }
2370
+ const verdict = { ...v, version, bin, oldBin, model: ctx.model, relayPath: ctx.relayMissing === undefined ? "not tried" : ctx.relayMissing === null ? "present" : ctx.relayMissing.pending ? `pending: ${ctx.relayMissing.pending}` : ctx.relayMissing.noSession ? `not run: no login session (${ctx.relayMissing.noSession})` : `failed: ${ctx.relayMissing.failed}`, items: v.items };
2371
+ fs.writeFileSync(path.join(dir, "stage2-verdict.json"), `${JSON.stringify(verdict, null, 2)}\n`);
2372
+ return verdict;
2373
+ }
2374
+
2375
+ if (isMain(import.meta.url)) {
2376
+ let o;
2377
+ try { o = parseArgs(process.argv.slice(2)); } catch (err) { console.error(err.message); process.exit(2); }
2378
+ const v = await qualify(o);
2379
+ console.log(JSON.stringify({ state: v.state, relayPath: v.relayPath, items: Object.fromEntries(Object.entries(v.items).map(([k, i]) => [k, i.state])), why: v.why }, null, 2));
2380
+ process.exit(0);
2381
+ }