@tangle-network/agent-bench 0.3.6 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/HARNESS.md +43 -0
  3. package/dist/adapters.js +24 -24
  4. package/dist/benchmarks/_harness.d.ts +1 -1
  5. package/dist/benchmarks/_harness.js +1 -1
  6. package/dist/benchmarks/aec-bench.js +2 -2
  7. package/dist/benchmarks/agentbench.js +2 -2
  8. package/dist/benchmarks/appworld.js +2 -2
  9. package/dist/benchmarks/bfcl.js +2 -2
  10. package/dist/benchmarks/commit0.js +2 -2
  11. package/dist/benchmarks/crag.js +2 -2
  12. package/dist/benchmarks/dabstep.js +2 -2
  13. package/dist/benchmarks/enterpriseops-gym.js +2 -2
  14. package/dist/benchmarks/finresearchbench.js +2 -2
  15. package/dist/benchmarks/humaneval.d.ts +10 -1
  16. package/dist/benchmarks/humaneval.js +5 -3
  17. package/dist/benchmarks/nomiracl.js +2 -2
  18. package/dist/benchmarks/open-rag-bench.js +2 -2
  19. package/dist/benchmarks/programbench.js +2 -2
  20. package/dist/benchmarks/ragbench.js +2 -2
  21. package/dist/benchmarks/swe-bench.js +2 -2
  22. package/dist/benchmarks/t2-ragbench.js +2 -2
  23. package/dist/benchmarks/tau-bench-shared.js +2 -2
  24. package/dist/benchmarks/tau2-bench.js +3 -3
  25. package/dist/benchmarks/tau3-banking.js +3 -3
  26. package/dist/benchmarks/terminal-bench.js +2 -2
  27. package/dist/benchmarks/toollm.js +2 -2
  28. package/dist/benchmarks/webarena-verified.js +2 -2
  29. package/dist/{chunk-CKUVRZ2T.js → chunk-3U5TXJZS.js} +2 -2
  30. package/dist/{chunk-PPYSEKFM.js → chunk-5H5XV76F.js} +73 -15
  31. package/dist/chunk-5H5XV76F.js.map +1 -0
  32. package/dist/{chunk-YCGY7UIZ.js → chunk-7GRVHU22.js} +2 -2
  33. package/dist/{chunk-Z7ML6L77.js → chunk-HWST3SED.js} +2 -2
  34. package/dist/{chunk-SYDW647C.js → chunk-IA2FBTWC.js} +2 -2
  35. package/dist/{chunk-R67DFVLO.js → chunk-IFVINJ4B.js} +2 -2
  36. package/dist/{chunk-R36V2VP7.js → chunk-IZ5M6OAC.js} +2 -2
  37. package/dist/{chunk-ODT47UAY.js → chunk-K3BQGZCT.js} +2 -2
  38. package/dist/{chunk-IFAV6KEM.js → chunk-KP5KD6EN.js} +2 -2
  39. package/dist/{chunk-ZEWMTR5M.js → chunk-MQMRLGOG.js} +2 -2
  40. package/dist/{chunk-TSWPNOYM.js → chunk-NQG5XDSB.js} +2 -2
  41. package/dist/{chunk-7WSD27QQ.js → chunk-PB64GYIG.js} +2 -2
  42. package/dist/{chunk-HBSWHQNJ.js → chunk-RCYQEFNX.js} +3 -3
  43. package/dist/{chunk-J3KDJNX2.js → chunk-RH5F53JT.js} +2 -2
  44. package/dist/{chunk-UAIOHCUK.js → chunk-SFLA7OH3.js} +3 -3
  45. package/dist/{chunk-Y6O2OCUO.js → chunk-SHM6MRRF.js} +2 -2
  46. package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js} +2 -2
  47. package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js} +2 -2
  48. package/dist/{chunk-5SBJCB6W.js → chunk-V7AEBY6U.js} +22 -22
  49. package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js} +18 -2
  50. package/dist/chunk-WSKWVEQB.js.map +1 -0
  51. package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js} +2 -2
  52. package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js} +2 -2
  53. package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js} +2 -2
  54. package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js} +2 -2
  55. package/dist/index.js +24 -24
  56. package/package.json +6 -5
  57. package/scripts/run-package-tests.mjs +30 -8
  58. package/scripts/verify-packed-consumer.mjs +1 -1
  59. package/scripts/verify-pier-agent.mts +1 -0
  60. package/src/benchmarks/_harness.ts +20 -2
  61. package/src/benchmarks/humaneval.test.mts +122 -0
  62. package/src/benchmarks/humaneval.ts +100 -27
  63. package/src/david-attribution.mts +78 -0
  64. package/src/david-goliath.mts +149 -0
  65. package/src/hev-improve.mts +25 -6
  66. package/src/humaneval-object-ablation.mts +201 -0
  67. package/src/live-improve-campaign-mbpp.mts +641 -0
  68. package/src/live-improve-campaign.mts +500 -0
  69. package/src/mbpp-structural.mts +12 -7
  70. package/src/quant-arena/README.md +144 -0
  71. package/src/quant-arena/backtest.test.mts +135 -0
  72. package/src/quant-arena/backtest.ts +218 -0
  73. package/src/quant-arena/data.test.mts +44 -0
  74. package/src/quant-arena/data.ts +141 -0
  75. package/src/quant-arena/driver.test.mts +253 -0
  76. package/src/quant-arena/driver.ts +219 -0
  77. package/src/quant-arena/fixtures/data/PROVENANCE.md +26 -0
  78. package/src/quant-arena/fixtures/data/holdout/IDX.csv +523 -0
  79. package/src/quant-arena/fixtures/data/holdout/S01.csv +523 -0
  80. package/src/quant-arena/fixtures/data/holdout/S02.csv +523 -0
  81. package/src/quant-arena/fixtures/data/holdout/S03.csv +523 -0
  82. package/src/quant-arena/fixtures/data/holdout/S04.csv +523 -0
  83. package/src/quant-arena/fixtures/data/holdout/S05.csv +523 -0
  84. package/src/quant-arena/fixtures/data/holdout/S06.csv +523 -0
  85. package/src/quant-arena/fixtures/data/holdout/S07.csv +523 -0
  86. package/src/quant-arena/fixtures/data/holdout/S08.csv +523 -0
  87. package/src/quant-arena/fixtures/data/holdout/S09.csv +523 -0
  88. package/src/quant-arena/fixtures/data/holdout/S10.csv +523 -0
  89. package/src/quant-arena/fixtures/data/insample/IDX.csv +2087 -0
  90. package/src/quant-arena/fixtures/data/insample/S01.csv +2087 -0
  91. package/src/quant-arena/fixtures/data/insample/S02.csv +2087 -0
  92. package/src/quant-arena/fixtures/data/insample/S03.csv +2087 -0
  93. package/src/quant-arena/fixtures/data/insample/S04.csv +2087 -0
  94. package/src/quant-arena/fixtures/data/insample/S05.csv +2087 -0
  95. package/src/quant-arena/fixtures/data/insample/S06.csv +2087 -0
  96. package/src/quant-arena/fixtures/data/insample/S07.csv +2087 -0
  97. package/src/quant-arena/fixtures/data/insample/S08.csv +2087 -0
  98. package/src/quant-arena/fixtures/data/insample/S09.csv +2087 -0
  99. package/src/quant-arena/fixtures/data/insample/S10.csv +2087 -0
  100. package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +16 -0
  101. package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +5 -0
  102. package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +171 -0
  103. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +119 -0
  104. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +119 -0
  105. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +105 -0
  106. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +102 -0
  107. package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +4 -0
  108. package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +2 -0
  109. package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +84 -0
  110. package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +117 -0
  111. package/src/quant-arena/holdout-certify.mts +206 -0
  112. package/src/quant-arena/holdout-certify.test.mts +82 -0
  113. package/src/quant-arena/leak-audit.test.mts +79 -0
  114. package/src/quant-arena/leak-audit.ts +95 -0
  115. package/src/quant-arena/make-fixtures.mts +161 -0
  116. package/src/quant-arena/multiplicity.test.mts +68 -0
  117. package/src/quant-arena/multiplicity.ts +87 -0
  118. package/src/quant-arena/nautilus-certify.ts +31 -0
  119. package/src/quant-arena/oms.ts +90 -0
  120. package/src/quant-arena/profiles/quant-researcher.profile.json +7 -0
  121. package/src/quant-arena/python/pyproject.toml +8 -0
  122. package/src/quant-arena/python/uv.lock +1297 -0
  123. package/src/quant-arena/python/vbt-worker.py +192 -0
  124. package/src/quant-arena/quant-loop.mts +813 -0
  125. package/src/quant-arena/quant-loop.test.mts +75 -0
  126. package/src/quant-arena/strategies/buy-hold-index/strategy.ts +11 -0
  127. package/src/quant-arena/strategies/equal-weight/strategy.ts +20 -0
  128. package/src/quant-arena/strategies/sma-crossover/strategy.ts +42 -0
  129. package/src/quant-arena/types.ts +133 -0
  130. package/src/quant-arena/vbt-client.ts +321 -0
  131. package/src/quant-arena/vbt-parity.test.mts +183 -0
  132. package/src/quant-arena/windows.test.mts +45 -0
  133. package/src/quant-arena/windows.ts +54 -0
  134. package/src/rollout-ledger/backfill-swe-arena.mts +606 -0
  135. package/src/rollout-ledger/backfill-swe-arena.test.mts +338 -0
  136. package/src/rollout-ledger/settle-capture.mts +442 -0
  137. package/src/rollout-ledger/settle-capture.test.mts +270 -0
  138. package/src/stream-observe.py +45 -0
  139. package/src/stream-observe.tpl.html +247 -0
  140. package/src/supervisor-arena.mts +816 -0
  141. package/src/swe-arena/activation.mts +228 -0
  142. package/src/swe-arena/activation.test.mts +303 -0
  143. package/src/swe-arena/analyze.ts +211 -0
  144. package/src/swe-arena/arms.ts +804 -0
  145. package/src/swe-arena/bootstrap-meta.mts +188 -0
  146. package/src/swe-arena/bootstrap-meta.test.mts +51 -0
  147. package/src/swe-arena/briefing.mts +217 -0
  148. package/src/swe-arena/briefing.test.mts +178 -0
  149. package/src/swe-arena/calibrate.ts +217 -0
  150. package/src/swe-arena/capabilities.mts +76 -0
  151. package/src/swe-arena/capabilities.test.mts +57 -0
  152. package/src/swe-arena/capacity.ts +194 -0
  153. package/src/swe-arena/cell-evidence.mts +437 -0
  154. package/src/swe-arena/cell-evidence.test.mts +248 -0
  155. package/src/swe-arena/diagnosis-ensemble.test.mts +210 -0
  156. package/src/swe-arena/diagnosis-ensemble.ts +520 -0
  157. package/src/swe-arena/execution.test.mts +1170 -0
  158. package/src/swe-arena/factory-command-container.ts +284 -0
  159. package/src/swe-arena/factory-judge-child.mts +228 -0
  160. package/src/swe-arena/factory.test.mts +643 -0
  161. package/src/swe-arena/fixtures/analyze.py +80 -0
  162. package/src/swe-arena/fixtures/excludes.txt +8 -0
  163. package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +51 -0
  164. package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +29 -0
  165. package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +64 -0
  166. package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +48 -0
  167. package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +29 -0
  168. package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +48 -0
  169. package/src/swe-arena/fixtures/factory/loops-28/calibration.md +47 -0
  170. package/src/swe-arena/fixtures/factory/loops-28/manifest.json +30 -0
  171. package/src/swe-arena/fixtures/factory/loops-28/spec.md +50 -0
  172. package/src/swe-arena/fixtures/gen1-salvage/README.md +45 -0
  173. package/src/swe-arena/fixtures/gen1-salvage/cand0-e6d7361.diff +116 -0
  174. package/src/swe-arena/fixtures/gen1-salvage/cand1-76a8590.diff +293 -0
  175. package/src/swe-arena/fixtures/holdout-preregister.log +12 -0
  176. package/src/swe-arena/fixtures/holdout.json +44 -0
  177. package/src/swe-arena/fixtures/instances.json +146 -0
  178. package/src/swe-arena/fixtures/ledger.jsonl +12 -0
  179. package/src/swe-arena/fixtures/patches/pallets__flask-5014.solo.patch +36 -0
  180. package/src/swe-arena/fixtures/patches/pydata__xarray-4687.sup.patch +33 -0
  181. package/src/swe-arena/fixtures/rejudge.jsonl +15 -0
  182. package/src/swe-arena/fixtures/rematch.jsonl +3 -0
  183. package/src/swe-arena/fixtures/rematch2.jsonl +3 -0
  184. package/src/swe-arena/fixtures/rematch3.jsonl +3 -0
  185. package/src/swe-arena/fixtures/run-report/README.md +43 -0
  186. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +173 -0
  187. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +100 -0
  188. package/src/swe-arena/fixtures/run-report/gen3-rollup.json +551 -0
  189. package/src/swe-arena/fixtures/run-report/gen3-rollup.md +64 -0
  190. package/src/swe-arena/fixtures/sup-journal-true.json +19 -0
  191. package/src/swe-arena/fixtures/verify/astropy__astropy-13033.sh +48 -0
  192. package/src/swe-arena/fixtures/verify/django__django-11532.sh +50 -0
  193. package/src/swe-arena/fixtures/verify/matplotlib__matplotlib-20826.sh +76 -0
  194. package/src/swe-arena/fixtures/verify/pydata__xarray-4687.sh +44 -0
  195. package/src/swe-arena/fixtures/verify/pytest-dev__pytest-6197.sh +32 -0
  196. package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh +51 -0
  197. package/src/swe-arena/fixtures/worker-tokens.json +42 -0
  198. package/src/swe-arena/fixtures.ts +237 -0
  199. package/src/swe-arena/gepa-seat.mts +583 -0
  200. package/src/swe-arena/gepa-seat.test.mts +635 -0
  201. package/src/swe-arena/holdout-certify.mts +408 -0
  202. package/src/swe-arena/holdout-certify.test.mts +160 -0
  203. package/src/swe-arena/judge-child.mts +37 -0
  204. package/src/swe-arena/ledger-orphans.mts +77 -0
  205. package/src/swe-arena/ledger-orphans.test.mts +147 -0
  206. package/src/swe-arena/lineage-record.mts +164 -0
  207. package/src/swe-arena/lineage-record.test.mts +115 -0
  208. package/src/swe-arena/manifest.mts +293 -0
  209. package/src/swe-arena/manifest.test.mts +169 -0
  210. package/src/swe-arena/materialize.ts +142 -0
  211. package/src/swe-arena/outer-loop.mts +2795 -0
  212. package/src/swe-arena/outer-loop.test.mts +696 -0
  213. package/src/swe-arena/parity.test.mts +87 -0
  214. package/src/swe-arena/premeasured-from-cells.mts +281 -0
  215. package/src/swe-arena/premeasured-from-cells.test.mts +180 -0
  216. package/src/swe-arena/proc.test.mts +174 -0
  217. package/src/swe-arena/proc.ts +260 -0
  218. package/src/swe-arena/profiles/default-author.profile.json +4 -0
  219. package/src/swe-arena/proposer-fanout.mts +770 -0
  220. package/src/swe-arena/proposer-fanout.test.mts +619 -0
  221. package/src/swe-arena/proposer-provenance.mts +177 -0
  222. package/src/swe-arena/proposer-provenance.test.mts +106 -0
  223. package/src/swe-arena/reconcile.ts +0 -0
  224. package/src/swe-arena/replay.mts +183 -0
  225. package/src/swe-arena/replay.test.mts +300 -0
  226. package/src/swe-arena/run-experiment.mts +727 -0
  227. package/src/swe-arena/run-report.mts +75 -0
  228. package/src/swe-arena/run-supervisor.mjs +297 -0
  229. package/src/swe-arena/run-supervisor.test.mts +500 -0
  230. package/src/swe-arena/score-split.mts +140 -0
  231. package/src/swe-arena/score-split.test.mts +123 -0
  232. package/src/swe-arena/serialized-judge.ts +414 -0
  233. package/src/swe-arena/types.ts +218 -0
  234. package/src/swe-code-improve.mts +328 -0
  235. package/src/swe-emit-patch.mts +104 -0
  236. package/src/swe-improve.mts +232 -0
  237. package/src/swe-jail.ts +2 -2
  238. package/src/swe-local-proof.mts +169 -0
  239. package/src/swe-repro-calibrate.mts +446 -0
  240. package/src/swe-stream.mts +1497 -0
  241. package/src/swe-structural.mts +245 -837
  242. package/dist/chunk-LRRD7NAG.js.map +0 -1
  243. package/dist/chunk-PPYSEKFM.js.map +0 -1
  244. /package/dist/{chunk-CKUVRZ2T.js.map → chunk-3U5TXJZS.js.map} +0 -0
  245. /package/dist/{chunk-YCGY7UIZ.js.map → chunk-7GRVHU22.js.map} +0 -0
  246. /package/dist/{chunk-Z7ML6L77.js.map → chunk-HWST3SED.js.map} +0 -0
  247. /package/dist/{chunk-SYDW647C.js.map → chunk-IA2FBTWC.js.map} +0 -0
  248. /package/dist/{chunk-R67DFVLO.js.map → chunk-IFVINJ4B.js.map} +0 -0
  249. /package/dist/{chunk-R36V2VP7.js.map → chunk-IZ5M6OAC.js.map} +0 -0
  250. /package/dist/{chunk-ODT47UAY.js.map → chunk-K3BQGZCT.js.map} +0 -0
  251. /package/dist/{chunk-IFAV6KEM.js.map → chunk-KP5KD6EN.js.map} +0 -0
  252. /package/dist/{chunk-ZEWMTR5M.js.map → chunk-MQMRLGOG.js.map} +0 -0
  253. /package/dist/{chunk-TSWPNOYM.js.map → chunk-NQG5XDSB.js.map} +0 -0
  254. /package/dist/{chunk-7WSD27QQ.js.map → chunk-PB64GYIG.js.map} +0 -0
  255. /package/dist/{chunk-HBSWHQNJ.js.map → chunk-RCYQEFNX.js.map} +0 -0
  256. /package/dist/{chunk-J3KDJNX2.js.map → chunk-RH5F53JT.js.map} +0 -0
  257. /package/dist/{chunk-UAIOHCUK.js.map → chunk-SFLA7OH3.js.map} +0 -0
  258. /package/dist/{chunk-Y6O2OCUO.js.map → chunk-SHM6MRRF.js.map} +0 -0
  259. /package/dist/{chunk-KDIKRJGB.js.map → chunk-SHYIRB7I.js.map} +0 -0
  260. /package/dist/{chunk-HHXFIHXC.js.map → chunk-SVR2LKYI.js.map} +0 -0
  261. /package/dist/{chunk-5SBJCB6W.js.map → chunk-V7AEBY6U.js.map} +0 -0
  262. /package/dist/{chunk-2PVVP7GN.js.map → chunk-XKEFIFIC.js.map} +0 -0
  263. /package/dist/{chunk-JRWWGMK7.js.map → chunk-XYA4XSNU.js.map} +0 -0
  264. /package/dist/{chunk-X5YKXC6V.js.map → chunk-YSMEKBTD.js.map} +0 -0
  265. /package/dist/{chunk-2XU6OGEN.js.map → chunk-Z4TZ76N7.js.map} +0 -0
@@ -0,0 +1,2795 @@
1
+ /**
2
+ * Round-4 outer loop — agent-runtime's `improve()` in the OPTIMIZER SEAT,
3
+ * proposing code changes to the loops pi supervisor, evaluated by this typed
4
+ * swe-arena harness. Replaces the human/Claude-driven rounds 1-3 recorded in
5
+ * supervisor-lab `.evolve/state.json`.
6
+ *
7
+ * tsx src/swe-arena/outer-loop.mts <config.json> # SPENDS: fires arms + judges
8
+ * tsx src/swe-arena/outer-loop.mts --write-config <path> # emit the default round-4 config
9
+ * tsx src/swe-arena/outer-loop.mts --calibration-smoke [supRunDir] [--analysts N] [--model M]
10
+ *
11
+ * One `runRound()` = one `improve()` call with `surface: 'code'`:
12
+ *
13
+ * (a) DIAGNOSE — the `analyzeGeneration` seam runs the blind diagnosis
14
+ * ensemble (diagnosis-ensemble.ts) over the PREVIOUS round's failure
15
+ * artifacts (round-3 SUP4 run dirs seeded via config) plus every fresh
16
+ * arm run this round produced, and UNIONS the fused findings with
17
+ * `rawTraceDistiller` path-context so the coding agent also greps the raw
18
+ * traces itself (`rawTraceContext: true` names the mechanism; an explicit
19
+ * `analyzeGeneration` wins, so the distiller is composed in directly).
20
+ * (b) PROPOSE — `improvementDriver` + a change-space-constrained
21
+ * `agenticGenerator` edit an isolated git worktree of loops. The DECLARED
22
+ * CHANGE-SPACE is enforced twice: in the generator's verifier (feedback →
23
+ * next shot) and fail-closed in the dispatch below (an out-of-space
24
+ * candidate never reaches a model token).
25
+ * (c) EVALUATE — each candidate surface is a loops commit; the dispatch adds
26
+ * a detached eval worktree at that commit, points the supervisor arm's
27
+ * extension path at it (armProvenance records the commit), runs the
28
+ * 3-instance improvement set through arms.ts + the serialized official
29
+ * judge. Score = resolved count; cost guard = wall ratio vs baseline.
30
+ * (d) ACCEPT/REJECT — keep-if-better per protocol_v2. The loop NEVER ships:
31
+ * `budget.holdout: 'deferred'` makes the lib dispatch zero holdout
32
+ * cells, force `hold`, and omit `lift` — the pre-registered 6-instance
33
+ * holdout costs real money and runs only in a separate, operator-
34
+ * approved run. The would-be-KEEP operator brief is computed post-run
35
+ * from campaign cells; every candidate + verdict persists as staircase
36
+ * rows in `<roundsDir>/gen-<N>.jsonl`.
37
+ *
38
+ * BASELINE: the gate's only denominator is the stored premeasured baseline
39
+ * artifact ({surfaceHash, campaign}) that the lib validates (surface hash,
40
+ * seed, reps, split digest, coverage) before skipping the baseline campaign.
41
+ * A missing artifact = the bootstrap run: the baseline is measured
42
+ * (cache-resumable) and the artifact written for every later run.
43
+ * capabilities.mts fails loud on a stale substrate install that would
44
+ * silently drop the passthrough.
45
+ *
46
+ * SCORING SOURCE: operator-brief evidence + staircase rows derive from the
47
+ * LIB's campaign cells (`improve()` result campaigns in memory; the per-cell
48
+ * `cached-result.json` caches on disk survive resume) — see cell-evidence.mts.
49
+ * The in-process RoundRecorder is dispatch-time only: fail-closed
50
+ * change-space enforcement + candidate diff writing. It is NOT a scoring
51
+ * source — that recorder role mislabeled a resumed run's baseline
52
+ * (r4-mroh3rkt) because cached cells replay without dispatching.
53
+ *
54
+ * Immutable per protocol_v2 (enforced, not advisory): judge + verify scripts,
55
+ * task prompts, model ids, budgets. `assertFrozenArm` pins the arm to the
56
+ * round-3 values; the serialized judge enforces its own 1800s floor; the
57
+ * change space keeps candidates inside extensions/pi/** and the three named
58
+ * src files (plus the `.improve/` raw-trace diagnosis artifact the agentic
59
+ * generator's evidence gate requires).
60
+ */
61
+
62
+ import { appendFile, mkdir, readdir, readFile, rm, symlink, unlink, writeFile } from 'node:fs/promises'
63
+ import { existsSync } from 'node:fs'
64
+ import process from 'node:process'
65
+ import { join } from 'node:path'
66
+ import { fileURLToPath, pathToFileURL } from 'node:url'
67
+ import {
68
+ agenticGenerator,
69
+ improve,
70
+ rawTraceDistiller,
71
+ type CandidateGenerator,
72
+ type Verifier,
73
+ } from '@tangle-network/agent-runtime'
74
+ import { runLocalHarness } from '@tangle-network/agent-runtime/mcp'
75
+ import { makeFinding } from '@tangle-network/agent-eval'
76
+ import {
77
+ FsLabeledScenarioStore,
78
+ surfaceHash,
79
+ type CampaignResult,
80
+ type CodeSurface,
81
+ type DispatchContext,
82
+ type JudgeConfig,
83
+ type MutableSurface,
84
+ type PremeasuredOptimizationBaseline,
85
+ type Scenario,
86
+ } from '@tangle-network/agent-eval/campaign'
87
+ import type { CostLedgerHandle } from '@tangle-network/agent-eval'
88
+ import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
89
+ import {
90
+ baselineDriftWarnings,
91
+ cellsFromCampaign,
92
+ gateEvidenceFromCells,
93
+ instanceVerdictsFromCells,
94
+ loadCampaignCells,
95
+ perInstanceFromCells,
96
+ replicateCoverageComplete,
97
+ replicateRunsFromCells,
98
+ resolvedInstanceCount,
99
+ sumWallSFromCells,
100
+ decideVerdict,
101
+ type R4Artifact,
102
+ type StaircasePerInstance,
103
+ type StaircaseVerdict,
104
+ } from './cell-evidence.mts'
105
+ import { assertSubstratePassthroughs } from './capabilities.mts'
106
+ import {
107
+ loadExcludes,
108
+ runSupervisorArm,
109
+ type SecretsEnv,
110
+ type SupervisorArmSpec,
111
+ type SupervisorArmResult,
112
+ } from './arms.ts'
113
+ import { gatesForArmKind, waitForCapacity, ZAI_CODING_ENDPOINT } from './capacity.ts'
114
+ import {
115
+ defaultAnalysts,
116
+ fusedToAnalystFindings,
117
+ runDiagnosisEnsemble,
118
+ surfacesPlacementRegex,
119
+ type AnalystSpec,
120
+ type SupRunArtifacts,
121
+ } from './diagnosis-ensemble.ts'
122
+ import {
123
+ defaultProposers,
124
+ fanOutLoopsGenerator,
125
+ materializeParetoParents,
126
+ proposerShotHooks,
127
+ type ParetoParentContext,
128
+ type ParetoParentSeed,
129
+ type PrefilterConfig,
130
+ type PrefilterKill,
131
+ type ProposerSpec,
132
+ type SmokeRunner,
133
+ type SmokeVerdict,
134
+ } from './proposer-fanout.mts'
135
+ import { captureProposerProvenance } from './proposer-provenance.mts'
136
+ import { CRASH_ORPHAN_REASON, reconcileCrashOrphansOnDisk } from './ledger-orphans.mts'
137
+ import {
138
+ AUTHOR_BRIEFING_VERSION,
139
+ resolveAuthorBriefing,
140
+ writeEvidenceIndex,
141
+ type BriefingContext,
142
+ } from './briefing.mts'
143
+ import {
144
+ readCommittedPredicate,
145
+ runActivationPredicate,
146
+ ACTIVATION_PREDICATE_RELPATH,
147
+ type ActivationRecord,
148
+ } from './activation.mts'
149
+ import {
150
+ loadOrCreateScoreSplit,
151
+ subScores,
152
+ type ScoreSplit,
153
+ type ScoreSplitConfig,
154
+ } from './score-split.mts'
155
+ import { recordLineageGeneration, type LineageCandidateInput } from './lineage-record.mts'
156
+ import {
157
+ reportSupervisorRound,
158
+ writeSupervisorRunReportSafe,
159
+ } from '@tangle-network/agent-eval/supervisor-run'
160
+ import {
161
+ campaignCoordsFromCellPath,
162
+ createSettleCapture,
163
+ type SettleCapture,
164
+ } from '../rollout-ledger/settle-capture.mts'
165
+ import { findSupervisorRunDir } from './arms.ts'
166
+ import { installProcessSignalAbort, run, runOk } from './proc.ts'
167
+ import { loadInstanceImages } from './run-experiment.mts'
168
+ import {
169
+ createSerializedJudge,
170
+ JUDGE_TIMEOUT_FLOOR_MS,
171
+ type SerializedJudge,
172
+ } from './serialized-judge.ts'
173
+
174
+ // ---------------------------------------------------------------------------
175
+ // The DECLARED CHANGE-SPACE (protocol_v2). Pure + unit-tested.
176
+ // ---------------------------------------------------------------------------
177
+
178
+ export interface ChangeSpace {
179
+ /** Directory prefixes (repo-relative, trailing '/') where edits are allowed. */
180
+ prefixes: string[]
181
+ /** Exact repo-relative files where edits are allowed. */
182
+ files: string[]
183
+ /** Non-code artifact prefixes allowed to change (the agentic generator's
184
+ * raw-trace evidence gate REQUIRES `.improve/raw-trace-diagnosis.md`, which
185
+ * finalize commits — evidence metadata, not supervisor code). */
186
+ metadataPrefixes: string[]
187
+ }
188
+
189
+ export const LOOPS_CHANGE_SPACE: ChangeSpace = {
190
+ prefixes: ['extensions/pi/'],
191
+ files: ['src/worker-evidence.ts', 'src/best-effort.ts', 'src/worker-clone.ts'],
192
+ metadataPrefixes: ['.improve/'],
193
+ }
194
+
195
+ /** Normalize a repo-relative path; `null` = un-normalizable (always a violation). */
196
+ export function normalizeRepoPath(p: string): string | null {
197
+ let s = p.trim().replace(/\\/g, '/')
198
+ if (s.startsWith('"') && s.endsWith('"') && s.length >= 2) {
199
+ // git quotes paths containing spaces/specials; minimal unquote.
200
+ s = s.slice(1, -1).replace(/\\"/g, '"')
201
+ }
202
+ while (s.startsWith('./')) s = s.slice(2)
203
+ if (s.length === 0) return null
204
+ if (s.startsWith('/')) return null // absolute — never a repo-relative candidate path
205
+ const segments = s.split('/')
206
+ if (segments.some((seg) => seg === '..' || seg === '')) return null // traversal / '//' — fail closed
207
+ return s
208
+ }
209
+
210
+ /** Paths that fall OUTSIDE the declared change-space (empty ⇒ compliant). */
211
+ export function changeSpaceViolations(paths: string[], space: ChangeSpace = LOOPS_CHANGE_SPACE): string[] {
212
+ const violations: string[] = []
213
+ for (const raw of paths) {
214
+ const p = normalizeRepoPath(raw)
215
+ if (p === null) {
216
+ violations.push(raw)
217
+ continue
218
+ }
219
+ const allowed =
220
+ space.files.includes(p) ||
221
+ space.prefixes.some((pre) => p.startsWith(pre)) ||
222
+ space.metadataPrefixes.some((pre) => p.startsWith(pre))
223
+ if (!allowed) violations.push(p)
224
+ }
225
+ return violations
226
+ }
227
+
228
+ /** Changed paths from `git status --porcelain=v1 --untracked-files=all`.
229
+ * Renames contribute BOTH sides (removing an out-of-space file is a change). */
230
+ export function porcelainChangedPaths(stdout: string): string[] {
231
+ const paths: string[] = []
232
+ for (const line of stdout.split('\n')) {
233
+ if (line.trim().length === 0) continue
234
+ const entry = line.slice(3)
235
+ const arrow = entry.indexOf(' -> ')
236
+ if (arrow !== -1) {
237
+ paths.push(entry.slice(0, arrow).trim(), entry.slice(arrow + 4).trim())
238
+ } else {
239
+ paths.push(entry.trim())
240
+ }
241
+ }
242
+ return paths.filter((p) => p.length > 0)
243
+ }
244
+
245
+ // ---------------------------------------------------------------------------
246
+ // Dispatch clocks. The campaign's dispatchTimeoutMs races the ENTIRE dispatch
247
+ // — including the endpoint capacity-gate wait — so a legitimate multi-hour
248
+ // capacity hold was billed to the cell's work budget (measured: a 58-min gate
249
+ // hold pushed the astropy baseline cell over the 7200s clock and the whole
250
+ // candidate became 'rejected-incomplete'). Fix: the cell's REAL work clock
251
+ // (`runWithPostGateClock`) starts only after the gates clear, and the campaign
252
+ // clock is widened to cover worst-case gate holds so it can never fire during
253
+ // a legitimate wait. Both clocks still fail loud — a hung arm is bounded by
254
+ // dispatchTimeoutMs post-gate, and the widened campaign clock is the backstop.
255
+ // ---------------------------------------------------------------------------
256
+
257
+ /** Supervisor arms gate on BOTH endpoints (worker z.ai path + brain router path). */
258
+ export const SUPERVISOR_GATE_COUNT = 2
259
+
260
+ /** capacity.ts's default waitCeilingMs (orchestrate.sh: 300 min/gate). */
261
+ export const DEFAULT_GATE_WAIT_CEILING_MS = 300 * 60_000
262
+
263
+ /** Extra time for process/worktree cleanup after the post-gate clock aborts.
264
+ * Judge time is budgeted separately because one verdict may require two full
265
+ * attempts. The campaign must not abandon either attempt or cleanup. */
266
+ export const DISPATCH_CLEANUP_GRACE_MS = 5 * 60_000
267
+
268
+ /** The widened ceiling handed to the campaign: per-cell work budget PLUS the
269
+ * worst-case capacity-gate holds (gates run sequentially, each with its own
270
+ * ceiling). The campaign clock starts at dispatch entry — before the gates —
271
+ * so it must cover them; `waitForCapacity` itself fails the cell at each
272
+ * gate's own ceiling, so total cell time stays bounded. */
273
+ export function campaignDispatchCeilingMs(
274
+ config: Pick<OuterLoopConfig, 'dispatchTimeoutMs' | 'gateWaitCeilingMs' | 'judgeTimeoutMs'>,
275
+ gateCount = SUPERVISOR_GATE_COUNT,
276
+ ): number {
277
+ const judgeSettlementMs = config.judgeTimeoutMs ?? JUDGE_TIMEOUT_FLOOR_MS
278
+ return (
279
+ config.dispatchTimeoutMs +
280
+ gateCount * (config.gateWaitCeilingMs ?? DEFAULT_GATE_WAIT_CEILING_MS) +
281
+ 2 * judgeSettlementMs +
282
+ DISPATCH_CLEANUP_GRACE_MS
283
+ )
284
+ }
285
+
286
+ /** Run `work` under `timeoutMs`, with the clock started AFTER `awaitGates`
287
+ * resolves — a capacity hold is never billed to the cell's work budget.
288
+ * Gate failures (no capacity within a gate's own ceiling) still reject. */
289
+ export async function runWithPostGateClock<T>(opts: {
290
+ awaitGates: (signal?: AbortSignal) => Promise<void>
291
+ work: (signal: AbortSignal) => Promise<T>
292
+ timeoutMs: number
293
+ label?: string
294
+ /** Caller cancellation remains active during both capacity waiting and work. */
295
+ signal?: AbortSignal
296
+ }): Promise<T> {
297
+ opts.signal?.throwIfAborted()
298
+ await opts.awaitGates(opts.signal)
299
+ opts.signal?.throwIfAborted()
300
+ const abort = new AbortController()
301
+ const linked = linkAbortSignals([abort.signal, ...(opts.signal ? [opts.signal] : [])])
302
+ let timer: NodeJS.Timeout | undefined
303
+ let timedOut = false
304
+ const timeoutError = new Error(
305
+ `post-gate dispatch exceeded ${opts.timeoutMs}ms${opts.label ? ` (${opts.label})` : ''} — failed loud, gate wait unbilled`,
306
+ )
307
+ try {
308
+ if (opts.timeoutMs > 0) {
309
+ timer = setTimeout(() => {
310
+ timedOut = true
311
+ abort.abort(timeoutError)
312
+ }, opts.timeoutMs)
313
+ timer.unref?.()
314
+ }
315
+ const result = await opts.work(linked.signal)
316
+ if (timedOut) throw timeoutError
317
+ opts.signal?.throwIfAborted()
318
+ return result
319
+ } catch (err) {
320
+ if (timedOut && err !== timeoutError) {
321
+ const cleanupFailure = err instanceof Error ? err.message : String(err)
322
+ throw new Error(`${timeoutError.message}; cleanup failed: ${cleanupFailure}`, { cause: err })
323
+ }
324
+ if (timedOut) throw timeoutError
325
+ if (opts.signal?.aborted) {
326
+ if (err !== opts.signal.reason) {
327
+ const cleanupFailure = err instanceof Error ? err.message : String(err)
328
+ const interrupted = opts.signal.reason instanceof Error
329
+ ? opts.signal.reason.message
330
+ : String(opts.signal.reason ?? 'caller aborted')
331
+ throw new Error(`${interrupted}; cleanup failed: ${cleanupFailure}`, { cause: err })
332
+ }
333
+ throw opts.signal.reason
334
+ }
335
+ throw err
336
+ } finally {
337
+ if (timer) clearTimeout(timer)
338
+ linked.dispose()
339
+ }
340
+ }
341
+
342
+ function linkAbortSignals(signals: AbortSignal[]): { signal: AbortSignal; dispose: () => void } {
343
+ const controller = new AbortController()
344
+ const listeners = new Map<AbortSignal, () => void>()
345
+ for (const signal of signals) {
346
+ const onAbort = () => controller.abort(signal.reason)
347
+ listeners.set(signal, onAbort)
348
+ if (signal.aborted) {
349
+ onAbort()
350
+ break
351
+ }
352
+ signal.addEventListener('abort', onAbort, { once: true })
353
+ }
354
+ return {
355
+ signal: controller.signal,
356
+ dispose: () => {
357
+ for (const [signal, listener] of listeners) signal.removeEventListener('abort', listener)
358
+ },
359
+ }
360
+ }
361
+
362
+ function withParentCancellation<T extends CandidateGenerator>(generator: T, signal?: AbortSignal): T {
363
+ if (!signal) return generator
364
+ return {
365
+ ...generator,
366
+ async generate(args) {
367
+ signal.throwIfAborted()
368
+ const linked = linkAbortSignals([args.signal, signal])
369
+ try {
370
+ const result = await generator.generate({ ...args, signal: linked.signal })
371
+ signal.throwIfAborted()
372
+ return result
373
+ } finally {
374
+ linked.dispose()
375
+ }
376
+ },
377
+ } as T
378
+ }
379
+
380
+ // ---------------------------------------------------------------------------
381
+ // Scoring primitives — replicate semantics, the pinned baseline, and the
382
+ // protocol_v2 verdict — live in cell-evidence.mts (pure over lib campaign
383
+ // cells). Re-exported here so existing consumers/tests keep one import home.
384
+ // ---------------------------------------------------------------------------
385
+
386
+ export {
387
+ baselineDriftWarnings,
388
+ cellsFromCampaign,
389
+ decideVerdict,
390
+ gateEvidenceFromCells,
391
+ instanceVerdictsFromCells,
392
+ loadCampaignCells,
393
+ loadCandidateCellGroups,
394
+ perInstanceFromCells,
395
+ replicateCoverageComplete,
396
+ replicateRunsFromCells,
397
+ resolvedInstanceCount,
398
+ sumWallSFromCells,
399
+ type EvidenceCell,
400
+ type R4Artifact,
401
+ type ReplicateRun,
402
+ type StaircasePerInstance,
403
+ type StaircaseVerdict,
404
+ } from './cell-evidence.mts'
405
+
406
+ // ---------------------------------------------------------------------------
407
+ // Launch guards. (a) The arms + judge + proposer all die confusingly hours in
408
+ // when the two API keys are absent (the launcher forgot dotenvx) — refuse at
409
+ // t=0 instead. (b) Two outer-loops sharing an outDir corrupt the campaign
410
+ // runDir and the arm-run caches — a pid-file lock with a staleness check makes
411
+ // the race impossible.
412
+ // ---------------------------------------------------------------------------
413
+
414
+ export const REQUIRED_LAUNCH_ENV = ['TANGLE_API_KEY', 'ZAI_API_KEY'] as const
415
+
416
+ export function assertLaunchEnv(env: Record<string, string | undefined> = process.env): void {
417
+ const missing = REQUIRED_LAUNCH_ENV.filter((k) => !env[k] || env[k]!.trim().length === 0)
418
+ if (missing.length > 0) {
419
+ throw new Error(
420
+ `outer-loop: ${missing.join(' + ')} absent from env — launch through dotenvx (dotenvx run -f agent-state.env -f tangle-router.env -- ...)`,
421
+ )
422
+ }
423
+ }
424
+
425
+ /** True when `pid` is a live process (EPERM = alive but not ours — still live). */
426
+ export function isPidAlive(pid: number): boolean {
427
+ try {
428
+ process.kill(pid, 0)
429
+ return true
430
+ } catch (err) {
431
+ return (err as NodeJS.ErrnoException).code === 'EPERM'
432
+ }
433
+ }
434
+
435
+ export const INSTANCE_LOCK_FILENAME = 'outer-loop.pid'
436
+
437
+ export interface InstanceLock {
438
+ path: string
439
+ release: () => Promise<void>
440
+ }
441
+
442
+ /** Single-instance pid-file lock in `outDir`. `wx` creation is the atomic
443
+ * claim; an existing file is honored only while its pid is alive (a crashed
444
+ * loop's stale lock — dead pid or garbage — is reclaimed). Pid reuse can in
445
+ * principle false-positive a stale lock as live; that fails SAFE (refuses to
446
+ * start) and clears on the next reboot cycle. */
447
+ export async function acquireInstanceLock(outDir: string, pid: number = process.pid): Promise<InstanceLock> {
448
+ await mkdir(outDir, { recursive: true })
449
+ const lockPath = join(outDir, INSTANCE_LOCK_FILENAME)
450
+ for (let attempt = 0; attempt < 2; attempt++) {
451
+ try {
452
+ await writeFile(lockPath, `${pid}\n`, { flag: 'wx' })
453
+ return {
454
+ path: lockPath,
455
+ release: async () => {
456
+ const raw = (await readFile(lockPath, 'utf8').catch(() => '')).trim()
457
+ if (raw === String(pid)) await unlink(lockPath).catch(() => {})
458
+ },
459
+ }
460
+ } catch (err) {
461
+ if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err
462
+ const raw = (await readFile(lockPath, 'utf8').catch(() => '')).trim()
463
+ const holder = Number.parseInt(raw, 10)
464
+ if (Number.isInteger(holder) && holder > 0 && holder !== pid && isPidAlive(holder)) {
465
+ throw new Error(
466
+ `outer-loop: another outer-loop (pid ${holder}) holds ${lockPath} — single-instance lock, refusing to race`,
467
+ )
468
+ }
469
+ await unlink(lockPath).catch(() => {}) // stale: dead pid or garbage content
470
+ }
471
+ }
472
+ throw new Error(`outer-loop: could not acquire ${lockPath} after clearing a stale lock`)
473
+ }
474
+
475
+ // ---------------------------------------------------------------------------
476
+ // Staircase rows — accepted successors + rejected dots, one JSONL row each.
477
+ // ---------------------------------------------------------------------------
478
+
479
+ export const STAIRCASE_SCHEMA = 'swe-arena.staircase.v1'
480
+
481
+ export interface StaircaseRow {
482
+ schema: typeof STAIRCASE_SCHEMA
483
+ round: number
484
+ generation: number
485
+ runId: string
486
+ at: string
487
+ /** Candidate surface hash (agent-eval surface identity). */
488
+ candidate: string
489
+ candidateCommit: string | null
490
+ /** Incumbent surface hash the candidate mutated. */
491
+ parent: string
492
+ parentResolvedCount: number
493
+ label?: string
494
+ rationale?: string
495
+ changedFiles: string[]
496
+ changeSpaceViolations: string[]
497
+ perInstance: StaircasePerInstance[]
498
+ resolvedCount: number
499
+ coverageComplete: boolean
500
+ wallS: number
501
+ baselineWallS: number
502
+ costRatio: number | null
503
+ costGuardRatio: number
504
+ /** Whether runOptimization's internal keep-if-better advanced the incumbent
505
+ * to this candidate (composite-only rule; may diverge from `verdict` when
506
+ * the protocol cost guard rejects a gaining candidate — divergence is the
507
+ * signal, so both are recorded). */
508
+ internallyPromoted: boolean
509
+ verdict: StaircaseVerdict
510
+ /** Present only on `rejected-prefilter` dots: which pre-filter stage killed
511
+ * the candidate and why (e.g. `smoke: pallets__flask-5014 unresolved`). */
512
+ killReason?: string
513
+ holdout: 'operator-approval-required' | 'not-run'
514
+ armProvenance: { repo: string; commit: string } | null
515
+ diffPath: string | null
516
+ diffSha256: string | null
517
+ /** GEN-5 public/private sub-scores (selection stays on the combined count;
518
+ * the private sub-score is never surfaced to proposers). */
519
+ split?: {
520
+ publicInstances: string[]
521
+ privateInstances: string[]
522
+ publicResolvedCount: number
523
+ privateResolvedCount: number
524
+ }
525
+ /** GEN-5 activation-gate outcome for this candidate. */
526
+ activation?: ActivationRecord
527
+ }
528
+
529
+ const STAIRCASE_VERDICTS: ReadonlySet<string> = new Set([
530
+ 'accepted',
531
+ 'rejected-no-gain',
532
+ 'rejected-cost',
533
+ 'rejected-out-of-space',
534
+ 'rejected-incomplete',
535
+ 'rejected-prefilter',
536
+ 'quarantined-inactive',
537
+ ])
538
+
539
+ /** Parse + validate one staircase JSONL row. Throws on schema drift. */
540
+ export function parseStaircaseRow(line: string): StaircaseRow {
541
+ const row = JSON.parse(line) as StaircaseRow
542
+ if (row.schema !== STAIRCASE_SCHEMA) throw new Error(`staircase row: unknown schema ${JSON.stringify(row.schema)}`)
543
+ for (const field of ['round', 'generation', 'resolvedCount', 'parentResolvedCount', 'wallS', 'baselineWallS', 'costGuardRatio'] as const) {
544
+ if (typeof row[field] !== 'number') throw new Error(`staircase row: ${field} must be a number`)
545
+ }
546
+ for (const field of ['runId', 'at', 'candidate', 'parent'] as const) {
547
+ if (typeof row[field] !== 'string' || row[field].length === 0) throw new Error(`staircase row: ${field} must be a non-empty string`)
548
+ }
549
+ if (!Array.isArray(row.perInstance)) throw new Error('staircase row: perInstance must be an array')
550
+ if (!Array.isArray(row.changedFiles) || !Array.isArray(row.changeSpaceViolations)) {
551
+ throw new Error('staircase row: changedFiles/changeSpaceViolations must be arrays')
552
+ }
553
+ if (!STAIRCASE_VERDICTS.has(row.verdict)) throw new Error(`staircase row: unknown verdict ${JSON.stringify(row.verdict)}`)
554
+ if (typeof row.coverageComplete !== 'boolean' || typeof row.internallyPromoted !== 'boolean') {
555
+ throw new Error('staircase row: coverageComplete/internallyPromoted must be booleans')
556
+ }
557
+ if (row.costRatio !== null && typeof row.costRatio !== 'number') throw new Error('staircase row: costRatio must be number|null')
558
+ return row
559
+ }
560
+
561
+ // ---------------------------------------------------------------------------
562
+ // Config.
563
+ // ---------------------------------------------------------------------------
564
+
565
+ /** Round 1-3 artifact home (this session's scratchpad). Config-overridable —
566
+ * a future round supplies its own artifact roots. */
567
+ export const DEFAULT_HH_SCRATCHPAD =
568
+ '/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh'
569
+
570
+ export interface SeedArtifactRun {
571
+ iid: string
572
+ arm: string
573
+ dir: string
574
+ patchPath?: string
575
+ /** Official-judge outcome for the seed run (round-3 values pinned in config). */
576
+ resolved: boolean | null
577
+ }
578
+
579
+ export interface FrozenArmParams {
580
+ workerModel: string
581
+ driverModel: string
582
+ budget: number
583
+ maxSandboxes: number
584
+ maxUsd: number
585
+ maxDepth: number
586
+ timeoutMs: number
587
+ envKnobs?: Record<string, string>
588
+ }
589
+
590
+ /** The round-3 (SUP4) arm — protocol_v2 immutables. */
591
+ export const FROZEN_ARM: FrozenArmParams = {
592
+ workerModel: 'zai-coding-plan/glm-5.2',
593
+ driverModel: 'glm-5.2',
594
+ budget: 40,
595
+ maxSandboxes: 4,
596
+ maxUsd: 8,
597
+ maxDepth: 3,
598
+ timeoutMs: 2_800_000,
599
+ }
600
+
601
+ export interface OuterLoopConfig {
602
+ round: number
603
+ /** Improvement set — the arena `improve()` trains on. */
604
+ instances: string[]
605
+ /** Pre-registered holdout. RECORDED here so the flag + operator instruction
606
+ * are self-contained; this driver NEVER runs them. */
607
+ holdoutInstances: string[]
608
+ loopsRepo: string
609
+ loopsBaseRef: string
610
+ armName: string
611
+ arm: FrozenArmParams
612
+ verifyDir: string
613
+ outDir: string
614
+ /** Staircase home, e.g. /home/drew/code/supervisor-lab/.evolve/rounds. */
615
+ roundsDir: string
616
+ secretsDir: string
617
+ envFiles: string[]
618
+ instanceImagesPath?: string
619
+ judgeTimeoutMs?: number
620
+ gateWaitCeilingMs?: number
621
+ capacityModel?: string
622
+ generations: number
623
+ populationSize: number
624
+ /** Replicate cells per (candidate × instance). Default 1. Instances count as
625
+ * resolved only when ALL replicates resolve (see resolvedInstanceCount) —
626
+ * single-rep scoring flips instance outcomes run-to-run. */
627
+ repsPerInstance?: number
628
+ /** Stored `PremeasuredOptimizationBaseline` JSON ({surfaceHash, campaign})
629
+ * from a prior run's baseline campaign — REQUIRED, the gate's only
630
+ * denominator. The LIB validates the artifact (surface hash, seed, reps,
631
+ * split digest, coverage) before skipping the baseline campaign, so a
632
+ * wrong artifact fails loud at t≈0. BOOTSTRAP: when the file does not
633
+ * exist yet, this run MEASURES the baseline (cache-resumable) and WRITES
634
+ * the artifact here for every later run to consume. */
635
+ premeasuredBaselinePath: string
636
+ /** DEPTH for the agentic generator — forwarded as
637
+ * budget.maxImprovementShots; the LIB owns the dial (capabilities.mts
638
+ * fails loud on a substrate that would drop it). */
639
+ maxShots: number
640
+ proposerHarness: 'claude' | 'codex' | 'opencode'
641
+ proposerTimeoutMs: number
642
+ /** GEN-3 proposer fan-out: N proposers author candidates CONCURRENTLY, each
643
+ * an AgentProfile-pinned harness invocation (see proposer-fanout.mts).
644
+ * When set, `populationSize` MUST equal `proposers.length` (one candidate
645
+ * slot per proposer — enforced at launch). Unset = the legacy
646
+ * single-author generator (`proposerHarness` + bare invocation).
647
+ * GEN-6: a spec with `engine` set is a GEPA seat (gepa-seat.mts) — the
648
+ * agent-eval external-GEPA adapter optimizes ONE change-space file as a
649
+ * string against the pre-filter smoke cell; requires `prefilter.enabled`. */
650
+ proposers?: ProposerSpec[]
651
+ /** GEN-3 cheap pre-filter: per candidate, change-space + tsc (the authoring
652
+ * verifier) plus ONE smoke arm cell before any full-evaluation spend.
653
+ * Killed candidates become `rejected-prefilter` staircase dots. */
654
+ prefilter?: PrefilterConfig
655
+ /** GEN-4 Pareto parents: prior-run frontier candidates (loops commits +
656
+ * measured per-instance results) seeded into every author's prompt and
657
+ * the merge seat's explicit input. Seeded at OUR buildPrompt seam, not the
658
+ * lib's `ctx.paretoParents` — the lib frontier is within-run only and a
659
+ * prior campaign cannot be injected without its runDir + ledger receipts
660
+ * (see proposer-fanout.mts). */
661
+ paretoParents?: ParetoParentSeed[]
662
+ /** Replicates per holdout instance in the operator-approved certification
663
+ * run (holdout-certify.mts). Default 2 — the gen-2 winner failed 3/6 vs
664
+ * 4/6 on a 1-rep holdout with exactly one discordant cell, a known
665
+ * single-rep noise class. */
666
+ holdoutRepsPerInstance?: number
667
+ /** SAME-PROTOCOL parent measurement the certification bar compares against:
668
+ * an explicit {iid -> AND-verdict} map measured under the identical
669
+ * reps/fail-closed protocol, or 'measure' — the incumbent runs the same
670
+ * 2-rep holdout first in the certification run. */
671
+ holdoutBaseline?: Record<string, boolean> | 'measure'
672
+ /** GEN-5 public/private score split (score-split.mts): proposers + the
673
+ * pre-filter see only PUBLIC instances' scores/evidence; selection stays
674
+ * on the combined set. Unset = everything public (pre-gen-5 behavior). */
675
+ scoreSplit?: ScoreSplitConfig
676
+ /** GEN-5 MAP+TOOLBOX briefing (briefing.mts): write the per-run evidence
677
+ * index and append the toolbox/permission briefing (change-space
678
+ * overridable) to every author prompt. */
679
+ briefing?: typeof AUTHOR_BRIEFING_VERSION
680
+ /** GEN-5 activation gate (activation.mts): require a machine-checkable
681
+ * activation predicate per candidate (prefilter-enforced) and quarantine
682
+ * candidates whose mechanism never fired in their own campaign traces. */
683
+ activationGate?: boolean
684
+ /** GEN-5 settle-time rollout-ledger capture (rollout-ledger/settle-capture.mts):
685
+ * emit tangle.rollout.v1 lines live after each cell judges, with label-v2
686
+ * rewards. Default path: <outDir>/rollout-ledger.jsonl. */
687
+ rolloutLedger?: { enabled: boolean; path?: string; opencodeDb?: string }
688
+ /** GEN-5 lineage DAG (lineage-record.mts): record every candidate as a
689
+ * LineageNode at <outDir>/.evolve/lineage.jsonl and put the governor's
690
+ * continuation decision in the round summary. */
691
+ lineage?: boolean
692
+ /** GEN-5 evidence map: prior run outDirs whose arm-runs/judge/candidate
693
+ * evidence the authors may mine (rendered into the evidence index). */
694
+ priorEvidenceDirs?: string[]
695
+ /** Router model ids for the blind diagnosis ensemble (config, never a
696
+ * hardcoded unrouted model). */
697
+ analystModels: string[]
698
+ /** Previous round's failure artifacts, diagnosed before generation 0. */
699
+ seedArtifactRuns: SeedArtifactRun[]
700
+ costGuardRatio: number
701
+ dispatchTimeoutMs: number
702
+ }
703
+
704
+ export function assertFrozenArm(arm: FrozenArmParams): void {
705
+ const drift: string[] = []
706
+ for (const key of ['workerModel', 'driverModel', 'budget', 'maxSandboxes', 'maxUsd', 'maxDepth'] as const) {
707
+ if (arm[key] !== FROZEN_ARM[key]) drift.push(`${key}: ${JSON.stringify(arm[key])} != ${JSON.stringify(FROZEN_ARM[key])}`)
708
+ }
709
+ if (drift.length > 0) {
710
+ throw new Error(
711
+ `protocol_v2 violation: arm params are immutable (round-3 frozen values) — ${drift.join('; ')}`,
712
+ )
713
+ }
714
+ }
715
+
716
+ /** Committed per-instance verify scripts (fixtures/verify/<iid>.sh) — the
717
+ * durable home; the experiment's scratchpad copy did not survive a reboot. */
718
+ export const FIXTURES_VERIFY_DIR = fileURLToPath(new URL('./fixtures/verify', import.meta.url))
719
+
720
+ export function defaultRound4Config(
721
+ hh = DEFAULT_HH_SCRATCHPAD,
722
+ opts: { outDirName?: string } = {},
723
+ ): OuterLoopConfig {
724
+ const round3 = [
725
+ { iid: 'astropy__astropy-13033', resolved: false },
726
+ { iid: 'django__django-11532', resolved: false },
727
+ { iid: 'matplotlib__matplotlib-20826', resolved: true },
728
+ ]
729
+ return {
730
+ round: 4,
731
+ instances: round3.map((r) => r.iid),
732
+ holdoutInstances: [
733
+ 'astropy__astropy-14182',
734
+ 'django__django-12774',
735
+ 'django__django-14140',
736
+ 'scikit-learn__scikit-learn-14894',
737
+ 'sympy__sympy-20438',
738
+ 'pytest-dev__pytest-7236',
739
+ ],
740
+ loopsRepo: '/home/drew/code/loops',
741
+ loopsBaseRef: 'feat/supervisor-evidence-flow',
742
+ armName: 'R4',
743
+ arm: { ...FROZEN_ARM },
744
+ verifyDir: FIXTURES_VERIFY_DIR,
745
+ outDir: join(hh, opts.outDirName ?? 'r4'),
746
+ roundsDir: '/home/drew/code/supervisor-lab/.evolve/rounds',
747
+ secretsDir: '/home/drew/company/devops/secrets',
748
+ envFiles: ['agent-state.env', 'tangle-router.env'],
749
+ generations: 1,
750
+ populationSize: 2,
751
+ repsPerInstance: 2,
752
+ // The reps-confirmed baseline artifact (gen-1 measured: astropy F/F,
753
+ // django T/F → F fail-closed, matplotlib T/T = 1/3) lives here once the
754
+ // bootstrap run writes it; the lib validates it on every consumption.
755
+ premeasuredBaselinePath: join(hh, 'r4', 'premeasured-baseline.json'),
756
+ maxShots: 3,
757
+ proposerHarness: 'claude',
758
+ // Per author SHOT (agenticGenerator timeoutMs). 20 min timed out 3× under
759
+ // degraded capacity in gen-1 ("author shot timed out") — doubled to 40 min.
760
+ proposerTimeoutMs: 2_400_000,
761
+ analystModels: ['glm-5.2', 'glm-5.2', 'glm-5.2'],
762
+ seedArtifactRuns: round3.map((r) => ({
763
+ iid: r.iid,
764
+ arm: 'SUP4',
765
+ dir: join(hh, 'runs', r.iid, 'SUP4'),
766
+ patchPath: join(hh, 'patches', `${r.iid}.sup4.patch`),
767
+ resolved: r.resolved,
768
+ })),
769
+ costGuardRatio: 1.2,
770
+ dispatchTimeoutMs: 7_200_000,
771
+ }
772
+ }
773
+
774
+ // ---------------------------------------------------------------------------
775
+ // GEN-3 configuration — proposer fan-out + pre-filter + the widened
776
+ // improvement set + the 2-rep holdout protocol.
777
+ // ---------------------------------------------------------------------------
778
+
779
+ /** The gen-3 improvement set: the round-3 trio plus the three BOTH-FAIL
780
+ * instances from the original head-to-head (solo glm-5.2 ALSO failed them —
781
+ * any resolution beats solo, not just the parent). All six carry committed,
782
+ * dual-calibrated verify fixtures (repro base-fail/gold-pass + gold
783
+ * official-resolved). */
784
+ export const GEN3_IMPROVEMENT_SET = [
785
+ 'astropy__astropy-13033',
786
+ 'django__django-11532',
787
+ 'matplotlib__matplotlib-20826',
788
+ 'pydata__xarray-4687',
789
+ 'pytest-dev__pytest-6197',
790
+ 'sphinx-doc__sphinx-9658',
791
+ ] as const
792
+
793
+ /** Never-registered spare pool, pre-named in case a gen-3 instance has to be
794
+ * replaced (calibration regression, image loss). */
795
+ export const GEN3_SPARE_POOL = [
796
+ 'sympy__sympy-17318',
797
+ 'scikit-learn__scikit-learn-14087',
798
+ 'astropy__astropy-14508',
799
+ ] as const
800
+
801
+ /** Resolve the pre-filter smoke instance. 'cheapest-of-set' picks the
802
+ * improvement-set instance with the smallest summed baseline wall seconds
803
+ * (from the premeasured artifact's cells); with no baseline measurement yet
804
+ * it falls back to the first instance. An explicit iid passes through. */
805
+ export function resolveSmokeInstance(
806
+ smokeInstance: string,
807
+ instances: readonly string[],
808
+ baselineCells: import('./cell-evidence.mts').EvidenceCell[] | null,
809
+ ): string {
810
+ if (smokeInstance !== 'cheapest-of-set') return smokeInstance
811
+ if (instances.length === 0) throw new Error('resolveSmokeInstance: empty improvement set')
812
+ if (baselineCells === null || baselineCells.length === 0) return instances[0]!
813
+ const wall = new Map<string, number>()
814
+ for (const cell of baselineCells) {
815
+ if (cell.artifact === null || cell.artifact.kind !== 'swe-arm') continue
816
+ wall.set(cell.scenarioId, (wall.get(cell.scenarioId) ?? 0) + cell.artifact.wallS)
817
+ }
818
+ let best: string | null = null
819
+ let bestWall = Number.POSITIVE_INFINITY
820
+ for (const iid of instances) {
821
+ const w = wall.get(iid)
822
+ if (w !== undefined && w < bestWall) {
823
+ best = iid
824
+ bestWall = w
825
+ }
826
+ }
827
+ return best ?? instances[0]!
828
+ }
829
+
830
+ /**
831
+ * The gen-3 config: protocol round 4 continues (frozen arm, same holdout
832
+ * registry, same roundsDir staircase) with the gen-3 machinery on:
833
+ *
834
+ * - THREE parallel proposers (all claude, bare default-author profile) that
835
+ * differ by diagnosis slice/lens — fan-out diversity without unproven
836
+ * harness seats; `populationSize` = `proposers.length`.
837
+ * - Pre-filter enabled at the mechanism bar on the cheapest-of-set smoke
838
+ * instance ('pallets__flask-5014' becomes the designated smoke once its
839
+ * verify fixture is authored + calibrated; it has none committed yet).
840
+ * - The 6-instance improvement set. The premeasured-baseline artifact path
841
+ * is NEW (gen3/): the lib validates a premeasured campaign against the
842
+ * FULL scenario split digest, so the 3-instance round-4 artifact cannot
843
+ * seed a 6-instance split — the first gen-3 run is the bootstrap that
844
+ * measures all six (cache-resumable) and writes the artifact; the three
845
+ * new instances are thereby measured on the first round.
846
+ * - Holdout protocol pinned at 2 reps, parent measured under the SAME
847
+ * protocol ('measure'), operator valve unchanged (holdout: 'deferred').
848
+ */
849
+ export function defaultGen3Config(
850
+ hh = DEFAULT_HH_SCRATCHPAD,
851
+ opts: { outDirName?: string } = {},
852
+ ): OuterLoopConfig {
853
+ const base = defaultRound4Config(hh, opts)
854
+ const outDirName = opts.outDirName ?? 'gen3'
855
+ const proposers: ProposerSpec[] = [
856
+ { name: 'default-author', profile: 'default-author.profile.json', harness: 'claude' },
857
+ {
858
+ name: 'mechanics-author',
859
+ profile: 'default-author.profile.json',
860
+ harness: 'claude',
861
+ diagnosisSlice: 'mechanics',
862
+ lens: 'Focus on MECHANICS: worker lifecycle, sandbox/clone contracts, settlement and delivery paths. Prefer code-path fixes over prompt wording.',
863
+ },
864
+ {
865
+ name: 'prompts-author',
866
+ profile: 'default-author.profile.json',
867
+ harness: 'claude',
868
+ diagnosisSlice: 'prompts',
869
+ lens: 'Focus on PROMPTS: worker/brain instruction wording, placement guidance, self-check discipline. Prefer prompt/instruction changes over code-path rewrites.',
870
+ },
871
+ ]
872
+ return {
873
+ ...base,
874
+ instances: [...GEN3_IMPROVEMENT_SET],
875
+ outDir: join(hh, outDirName),
876
+ premeasuredBaselinePath: join(hh, outDirName, 'premeasured-baseline.json'),
877
+ populationSize: proposers.length,
878
+ proposers,
879
+ prefilter: { enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false },
880
+ holdoutRepsPerInstance: 2,
881
+ holdoutBaseline: 'measure',
882
+ }
883
+ }
884
+
885
+ // ---------------------------------------------------------------------------
886
+ // GEN-4 configuration — pinned per-proposer models (recorded in provenance),
887
+ // Pareto-parent seeding from the gen-3 frontier, and a dedicated merge seat.
888
+ // ---------------------------------------------------------------------------
889
+
890
+ /** The gen-3 frontier (run r4-mrwc0awe): winner + runner-up, both 2/6 vs the
891
+ * 1/6 baseline on DIFFERENT instances — complementary lessons, the merge
892
+ * seat's input. Per-instance verdicts are the fail-closed all-reps values
893
+ * from `.evolve/rounds/gen-0.jsonl`. */
894
+ export const GEN3_PARETO_PARENTS: ParetoParentSeed[] = [
895
+ {
896
+ commit: 'cc0d95584c7ea14324cd57c21fe946c7c0f53827',
897
+ label: 'default-author',
898
+ resolvedInstances: ['pydata__xarray-4687', 'sphinx-doc__sphinx-9658'],
899
+ note:
900
+ 'mechanical patch-risk scan (patchRiskWarnings in src/worker-evidence.ts, threaded through ' +
901
+ 'extensions/pi/loops.ts) + never-reword / test-seam / run-the-neighbors worker rules + 3-check ' +
902
+ 'reviewer; pytest-dev__pytest-6197 split 0/1 across reps (near-miss)',
903
+ },
904
+ {
905
+ commit: 'a7a2a982e51551de3a8e796ee2efc448ed405e6a',
906
+ label: 'prompts-author',
907
+ resolvedInstances: ['django__django-11532', 'pydata__xarray-4687'],
908
+ note:
909
+ 'prompt-only: hidden-suite bullet in the supervisor GOAL-authoring section (the django seam), ' +
910
+ 'frozen-behavior worker section + run-the-repo-tests discipline, 2-check reviewer; ' +
911
+ 'sphinx-doc__sphinx-9658 split 0/1 across reps (near-miss)',
912
+ },
913
+ ]
914
+
915
+ /**
916
+ * The gen-4 config: protocol round 4 continues (frozen arm, same holdout
917
+ * registry, same roundsDir staircase) with three changes as a unit:
918
+ *
919
+ * 1. PINNED PER-PROPOSER MODELS — four seats: claude-author (claude CLI on
920
+ * its own login; the resolved model + CLI version are captured into
921
+ * `<outDir>/proposer-provenance.json` at t=0), glm-author (opencode
922
+ * pinned to zai-coding-plan/glm-5.2 via `-m`), codex-author (codex CLI on
923
+ * its ChatGPT login, auth provenance-gated at launch; drop the seat via
924
+ * `includeCodex: false` when the CLI is absent), and merge-author (claude,
925
+ * merge seat).
926
+ * 2. PARETO PARENTS — the gen-3 winner + runner-up diffs and their measured
927
+ * per-instance results seed every author's prompt; the merge seat's task
928
+ * is their coherent union. Seeded at the buildPrompt seam (our seam): the
929
+ * lib's `ctx.paretoParents` frontier is within-run only, and a prior
930
+ * campaign cannot cross runs without its runDir + ledger receipts.
931
+ * 3. PINNED BASELINE — the premeasured artifact at `hh/gen4/` is BUILT from
932
+ * gen-3's measured baseline cells (premeasured-from-cells.mts; gen-3
933
+ * measured astropy F, django F, matplotlib F, xarray F, pytest F,
934
+ * sphinx T — matplotlib/django false under current weather), so gen-4
935
+ * spends nothing re-measuring and fails loud if the loops tip moved.
936
+ */
937
+ export function defaultGen4Config(
938
+ hh = DEFAULT_HH_SCRATCHPAD,
939
+ opts: { outDirName?: string; includeCodex?: boolean } = {},
940
+ ): OuterLoopConfig {
941
+ const base = defaultGen3Config(hh, { outDirName: opts.outDirName ?? 'gen4' })
942
+ const proposers: ProposerSpec[] = [
943
+ { name: 'claude-author', profile: 'default-author.profile.json', harness: 'claude' },
944
+ { name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' },
945
+ ...(opts.includeCodex === false ? [] : [{ name: 'codex-author', harness: 'codex' } satisfies ProposerSpec]),
946
+ { name: 'merge-author', profile: 'default-author.profile.json', harness: 'claude', merge: true },
947
+ ]
948
+ return {
949
+ ...base,
950
+ populationSize: proposers.length,
951
+ proposers,
952
+ paretoParents: [...GEN3_PARETO_PARENTS],
953
+ }
954
+ }
955
+
956
+ // ---------------------------------------------------------------------------
957
+ // GEN-5 configuration — gen-4's shape (4 proposers incl. the merge seat,
958
+ // Pareto parents, premeasured baseline carried forward per the same
959
+ // cell-derivation, 2 reps, deferred holdout) PLUS the gen-5 integration
960
+ // bundle as a unit:
961
+ //
962
+ // 1. MAP+TOOLBOX briefing — per-run evidence index + toolbox/permission
963
+ // briefing (change-space overridable at extensions/pi/author-briefing.md);
964
+ // the 3-analyst diagnosis stays as ONE input among the named tools.
965
+ // 2. PUBLIC/PRIVATE SPLIT — 4 public / 2 private of the 6 instances,
966
+ // deterministically seeded by runId and persisted per outDir; proposers +
967
+ // prefilter see public only, selection stays combined. Small-n caveat
968
+ // documented in score-split.mts.
969
+ // 3. ACTIVATION GATE — required machine-checkable predicate per candidate;
970
+ // never-fired mechanisms are quarantined even on an improved score.
971
+ // 4. SETTLE-TIME ROLLOUT LEDGER — tangle.rollout.v1 lines live per cell,
972
+ // label v2 (contribution-aware workers, baseline-relative proposers).
973
+ // 5. LINEAGE DAG — agent-eval Lineage at <outDir>/.evolve/lineage.jsonl +
974
+ // governor continuation decision in the round summary; staircase rows
975
+ // unchanged (observatory contract).
976
+ // ---------------------------------------------------------------------------
977
+
978
+ export function defaultGen5Config(
979
+ hh = DEFAULT_HH_SCRATCHPAD,
980
+ opts: { outDirName?: string; includeCodex?: boolean } = {},
981
+ ): OuterLoopConfig {
982
+ const base = defaultGen4Config(hh, {
983
+ outDirName: opts.outDirName ?? 'gen5',
984
+ ...(opts.includeCodex !== undefined ? { includeCodex: opts.includeCodex } : {}),
985
+ })
986
+ return {
987
+ ...base,
988
+ scoreSplit: { publicCount: 4 },
989
+ briefing: AUTHOR_BRIEFING_VERSION,
990
+ activationGate: true,
991
+ rolloutLedger: { enabled: true },
992
+ lineage: true,
993
+ priorEvidenceDirs: [join(hh, 'gen4'), join(hh, 'gen3')],
994
+ }
995
+ }
996
+
997
+ // ---------------------------------------------------------------------------
998
+ // Round recorder — dispatch-time change-space fail-closed + diff writing ONLY.
999
+ // NOT a scoring source: scoring reads the lib's campaign cells
1000
+ // (cell-evidence.mts). The prior recorder role — accumulating per-instance
1001
+ // results keyed by dispatch order — mislabeled a resumed run's baseline
1002
+ // (r4-mroh3rkt: cached cells replay without dispatching, so "first dispatched
1003
+ // surface" was a CANDIDATE and the summary published its cells as
1004
+ // "baseline 0/3" while the measured baseline was 1/3).
1005
+ // ---------------------------------------------------------------------------
1006
+
1007
+ interface CandidateRecord {
1008
+ surfaceKey: string
1009
+ commit: string
1010
+ baseCommit: string
1011
+ tag: string
1012
+ changedFiles: string[]
1013
+ violations: string[]
1014
+ diffPath: string | null
1015
+ diffSha256: string | null
1016
+ /** Dispatch-time forensics: which loops checkout ran the arm. Null for a
1017
+ * candidate whose cells were all replayed from cache (never dispatched
1018
+ * in this process). */
1019
+ armProvenance: { repo: string; commit: string } | null
1020
+ }
1021
+
1022
+ class RoundRecorder {
1023
+ readonly byKey = new Map<string, CandidateRecord>()
1024
+ constructor(
1025
+ private readonly loopsRepo: string,
1026
+ private readonly candidatesDir: string,
1027
+ ) {}
1028
+
1029
+ byCommit(commit: string): CandidateRecord | undefined {
1030
+ for (const rec of this.byKey.values()) if (rec.commit === commit) return rec
1031
+ return undefined
1032
+ }
1033
+
1034
+ /** Describe a candidate surface: changed files, change-space violations, and
1035
+ * the written diff. Idempotent and callable POST-RUN too (candidate commits
1036
+ * survive in the loops object store after worktree cleanup), so resumed
1037
+ * candidates that never dispatched here still get full staircase rows. */
1038
+ async ensure(surface: CodeSurface): Promise<CandidateRecord> {
1039
+ const key = surfaceHash(surface)
1040
+ const existing = this.byKey.get(key)
1041
+ if (existing) return existing
1042
+ const names = await runOk('git', [
1043
+ '-C', this.loopsRepo,
1044
+ 'diff', '--name-only', surface.baseCommit, surface.candidateCommit,
1045
+ ])
1046
+ const changedFiles = names.stdout.split('\n').map((s) => s.trim()).filter(Boolean)
1047
+ const violations = changeSpaceViolations(changedFiles)
1048
+ const tag = surface.candidateCommit.slice(0, 10)
1049
+ let diffPath: string | null = null
1050
+ if (surface.candidateCommit !== surface.baseCommit) {
1051
+ const diff = await runOk('git', ['-C', this.loopsRepo, 'diff', surface.baseCommit, surface.candidateCommit])
1052
+ await mkdir(this.candidatesDir, { recursive: true })
1053
+ diffPath = join(this.candidatesDir, `${tag}.patch`)
1054
+ await writeFile(diffPath, diff.stdout)
1055
+ }
1056
+ const rec: CandidateRecord = {
1057
+ surfaceKey: key,
1058
+ commit: surface.candidateCommit,
1059
+ baseCommit: surface.baseCommit,
1060
+ tag,
1061
+ changedFiles,
1062
+ violations,
1063
+ diffPath,
1064
+ diffSha256: surface.patch.sha256,
1065
+ armProvenance: null,
1066
+ }
1067
+ this.byKey.set(key, rec)
1068
+ return rec
1069
+ }
1070
+ }
1071
+
1072
+ // ---------------------------------------------------------------------------
1073
+ // Eval worktrees — a candidate commit gets its own loops checkout so the
1074
+ // candidate worktree managed by the improvement driver stays PRISTINE (its
1075
+ // finalize-time verification rejects any extra file, node_modules included).
1076
+ // ---------------------------------------------------------------------------
1077
+
1078
+ export async function addEvalWorktree(
1079
+ loopsRepo: string,
1080
+ commit: string,
1081
+ dest: string,
1082
+ signal?: AbortSignal,
1083
+ ): Promise<void> {
1084
+ signal?.throwIfAborted()
1085
+ await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest], { timeoutMs: 60_000, signal })
1086
+ signal?.throwIfAborted()
1087
+ await rm(dest, { recursive: true, force: true })
1088
+ signal?.throwIfAborted()
1089
+ await run('git', ['-C', loopsRepo, 'worktree', 'prune'], { timeoutMs: 60_000, signal })
1090
+ signal?.throwIfAborted()
1091
+ await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', dest, commit], { timeoutMs: 60_000, signal })
1092
+ signal?.throwIfAborted()
1093
+ // The loops driver needs deps; a worktree has none. Shared install is safe:
1094
+ // arms never write into the loops checkout (state goes to ws/.loops + runDir).
1095
+ await symlink(join(loopsRepo, 'node_modules'), join(dest, 'node_modules'), 'dir')
1096
+ }
1097
+
1098
+ export async function removeEvalWorktree(loopsRepo: string, dest: string): Promise<void> {
1099
+ await unlink(join(dest, 'node_modules')).catch(() => {})
1100
+ const res = await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest], { timeoutMs: 60_000 })
1101
+ if (res.code !== 0) {
1102
+ await rm(dest, { recursive: true, force: true })
1103
+ await run('git', ['-C', loopsRepo, 'worktree', 'prune'], { timeoutMs: 60_000 })
1104
+ }
1105
+ }
1106
+
1107
+ // ---------------------------------------------------------------------------
1108
+ // The constrained proposer: agenticGenerator + change-space verifier + the
1109
+ // round-4 task prompt. `improve(surface:'code')` requires the generator via
1110
+ // `code.generator` so the runtime owns candidate-worktree cleanup.
1111
+ // ---------------------------------------------------------------------------
1112
+
1113
+ /** Mirrors the agentic generator's raw-trace evidence contract — the exact
1114
+ * artifact path its gate checks for. */
1115
+ export const RAW_TRACE_DIAGNOSIS_PATH = '.improve/raw-trace-diagnosis.md'
1116
+
1117
+ export function changeSpaceInstruction(space: ChangeSpace = LOOPS_CHANGE_SPACE): string {
1118
+ return [
1119
+ 'DECLARED CHANGE-SPACE (hard constraint, enforced by an automated gate):',
1120
+ `- You may ONLY edit files under: ${space.prefixes.map((p) => `${p}**`).join(', ')}`,
1121
+ `- and these exact files: ${space.files.join(', ')}`,
1122
+ `- plus the diagnosis artifact ${RAW_TRACE_DIAGNOSIS_PATH}.`,
1123
+ '- Everything else is IMMUTABLE for this experiment: the official judge, the per-instance verify scripts,',
1124
+ ' task prompts, model ids, and budgets live outside your reach and candidates whose diff touches any',
1125
+ ' other path are REJECTED before they are ever evaluated.',
1126
+ ].join('\n')
1127
+ }
1128
+
1129
+ export function round4BuildPrompt(args: { report: unknown; findings: Array<Record<string, unknown>> }): string {
1130
+ const lines: string[] = [
1131
+ 'You are the optimizer of the "loops" pi SUPERVISOR — an agent that plans, spawns sandboxed coding',
1132
+ 'workers, and settles a delivered patch for SWE-bench Verified instances (glm-5.2 in both seats, frozen).',
1133
+ 'Round-3 state: the supervisor resolves 1/3 of its improvement set (matplotlib resolved; astropy + django',
1134
+ 'deliver self-verify-passing patches the OFFICIAL maintainer test suite still rejects).',
1135
+ '',
1136
+ 'GOAL: raise the official resolved count on the improvement set WITHOUT raising cost/arm by more than 20%.',
1137
+ 'Make the smallest coherent change to the supervisor implementation that addresses the diagnosis below,',
1138
+ 'then stop. Do not commit — leave changes in the working tree.',
1139
+ '',
1140
+ changeSpaceInstruction(),
1141
+ '',
1142
+ 'Diagnosis findings (blind multi-analyst ensemble + raw-trace context):',
1143
+ ]
1144
+ for (const f of args.findings) {
1145
+ const severity = typeof f.severity === 'string' ? f.severity : 'info'
1146
+ const subject = typeof f.subject === 'string' ? ` [${f.subject}]` : ''
1147
+ const claim = typeof f.claim === 'string' ? f.claim : JSON.stringify(f)
1148
+ lines.push(`- (${severity})${subject} ${claim}`)
1149
+ if (typeof f.recommended_action === 'string') lines.push(` → ${f.recommended_action}`)
1150
+ }
1151
+ const hasRawTrace = args.findings.some(
1152
+ (f) => f.analyst_id === 'raw-trace-distiller' || f.area === 'raw-trace-context',
1153
+ )
1154
+ if (hasRawTrace) {
1155
+ lines.push(
1156
+ '',
1157
+ 'Raw trace evidence requirement:',
1158
+ '- Inspect at least one raw trace path named above before editing.',
1159
+ `- Write ${RAW_TRACE_DIAGNOSIS_PATH} in this worktree.`,
1160
+ '- Include the exact trace path(s) inspected, the failure mechanism, and the code change made.',
1161
+ '- A candidate without this file, or with only this file changed, is discarded.',
1162
+ )
1163
+ }
1164
+ return lines.join('\n')
1165
+ }
1166
+
1167
+ /** Purge gitignored artifacts from a candidate worktree with `git clean -Xdff`.
1168
+ *
1169
+ * The proposer agent may run a dependency install inside its worktree to
1170
+ * verify its own change (measured: round-4 gen-0 cand-1 left a real pnpm
1171
+ * `node_modules/` — 38k paths — after editing loops.ts). Ignored paths are
1172
+ * invisible to the change-space check (`git status` honors .gitignore), but
1173
+ * the improvement driver's finalize-time surface verification rejects ANY
1174
+ * extra path, ignored included (`ls-files --others --ignored`), killing the
1175
+ * whole run. `-X` deletes only ignored paths, so tracked edits and untracked
1176
+ * non-ignored deliverables (e.g. .improve/raw-trace-diagnosis.md) survive;
1177
+ * the doubled `-f` clears nested git dirs some packages ship. */
1178
+ export async function purgeIgnoredArtifacts(
1179
+ worktreePath: string,
1180
+ signal?: AbortSignal,
1181
+ ): Promise<void> {
1182
+ await runOk('git', ['-C', worktreePath, 'clean', '-Xdff'], {
1183
+ ...(signal ? { signal } : {}),
1184
+ })
1185
+ }
1186
+
1187
+ /** Verifier run after each generator shot: ignored-dirt purge first (the
1188
+ * finalize precondition), then change-space compliance (cheap,
1189
+ * feedback-rich), then `tsc --noEmit` with the main repo's
1190
+ * node_modules linked in TEMPORARILY (the link must not survive — the
1191
+ * driver's finalize-time surface verification rejects any extra path). */
1192
+ export function loopsCandidateVerifier(loopsRepo: string): Verifier {
1193
+ return async (worktreePath: string, signal?: AbortSignal) => {
1194
+ signal?.throwIfAborted()
1195
+ await purgeIgnoredArtifacts(worktreePath, signal)
1196
+ signal?.throwIfAborted()
1197
+ const status = await runOk(
1198
+ 'git',
1199
+ ['-C', worktreePath, 'status', '--porcelain=v1', '--untracked-files=all'],
1200
+ { ...(signal ? { signal } : {}) },
1201
+ )
1202
+ signal?.throwIfAborted()
1203
+ const violations = changeSpaceViolations(porcelainChangedPaths(status.stdout))
1204
+ if (violations.length > 0) {
1205
+ return {
1206
+ ok: false,
1207
+ feedback:
1208
+ `CHANGE-SPACE VIOLATION — these paths are outside the declared change-space:\n` +
1209
+ violations.map((v) => ` - ${v}`).join('\n') +
1210
+ `\n${changeSpaceInstruction()}\nRevert or relocate those edits (git checkout -- <path> / rm for untracked).`,
1211
+ }
1212
+ }
1213
+ const nm = join(worktreePath, 'node_modules')
1214
+ let linked = false
1215
+ signal?.throwIfAborted()
1216
+ if (!existsSync(nm)) {
1217
+ await symlink(join(loopsRepo, 'node_modules'), nm, 'dir')
1218
+ linked = true
1219
+ }
1220
+ try {
1221
+ const tsc = join(loopsRepo, 'node_modules', '.bin', 'tsc')
1222
+ const res = await run(tsc, ['--noEmit'], {
1223
+ cwd: worktreePath,
1224
+ timeoutMs: 300_000,
1225
+ ...(signal ? { signal } : {}),
1226
+ })
1227
+ signal?.throwIfAborted()
1228
+ if (res.code !== 0) {
1229
+ return {
1230
+ ok: false,
1231
+ feedback: `tsc --noEmit failed (rc=${res.code}${res.timedOut ? ', timeout' : ''}):\n${(res.stdout + res.stderr).slice(0, 4000)}`,
1232
+ }
1233
+ }
1234
+ return { ok: true }
1235
+ } finally {
1236
+ if (linked) await unlink(nm).catch(() => {})
1237
+ }
1238
+ }
1239
+ }
1240
+
1241
+ /** Ambient auth vars that hijack the claude CLI away from its claude.ai login.
1242
+ * The run is launched under dotenvx, and agent-state.env injects an
1243
+ * ANTHROPIC_API_KEY meant for other tooling; the claude CLI prefers env-key
1244
+ * auth over the logged-in account and exits 1 immediately when that key's org
1245
+ * is over its usage cap (reproduced 2026-07-20: `claude -p` under the run env
1246
+ * → rc=1, "API Error: 400 You have reached your specified API usage limits";
1247
+ * same command with these vars unset → rc=0). The author shot must run on the
1248
+ * CLI's own login, so the leaked auth is stripped for the shot subprocess
1249
+ * only — the rest of the run keeps its env untouched. */
1250
+ const CLAUDE_AMBIENT_AUTH_VARS = ['ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_BASE_URL'] as const
1251
+
1252
+ /** Same failure class for the gen-4 codex seat: agent-state.env injects an
1253
+ * OPENAI_API_KEY meant for other tooling, and the codex CLI prefers env-key
1254
+ * auth over its ChatGPT login. The codex author shot must run on the CLI's
1255
+ * own login (`codex login status` is provenance-gated at launch), so the
1256
+ * leaked auth is stripped for the shot subprocess only. */
1257
+ const CODEX_AMBIENT_AUTH_VARS = ['OPENAI_API_KEY', 'OPENAI_BASE_URL'] as const
1258
+
1259
+ export function proposerShotEnv(harness: OuterLoopConfig['proposerHarness']): NodeJS.ProcessEnv {
1260
+ const env: NodeJS.ProcessEnv = { ...process.env }
1261
+ if (harness === 'claude') {
1262
+ for (const name of CLAUDE_AMBIENT_AUTH_VARS) delete env[name]
1263
+ }
1264
+ if (harness === 'codex') {
1265
+ for (const name of CODEX_AMBIENT_AUTH_VARS) delete env[name]
1266
+ }
1267
+ return env
1268
+ }
1269
+
1270
+ export function constrainedLoopsGenerator(config: OuterLoopConfig): CandidateGenerator {
1271
+ const shotDir = join(config.outDir, 'proposer-shots')
1272
+ // The run-wide CostLedger the current generate() call rides — captured so
1273
+ // onShotCompleted can settle each shot's spend into it. maxConcurrency is 1
1274
+ // and shots run inside generate(), so a single slot cannot interleave.
1275
+ let activeLedger: CostLedgerHandle | undefined
1276
+ let activePhase: string | undefined
1277
+ const inner = agenticGenerator({
1278
+ harness: config.proposerHarness,
1279
+ timeoutMs: config.proposerTimeoutMs,
1280
+ buildPrompt: (args) =>
1281
+ round4BuildPrompt(args as unknown as { report: unknown; findings: Array<Record<string, unknown>> }),
1282
+ verify: loopsCandidateVerifier(config.loopsRepo),
1283
+ runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(config.proposerHarness) }),
1284
+ // Three runs died as "author shot exited with code 1" with the shot's
1285
+ // stderr lost (nothing wires receipt persistence by default). Persist every
1286
+ // attempted shot — receipt plus bounded stream tails — so the NEXT failure
1287
+ // names its cause from disk. Shared implementation with the gen-3 fan-out
1288
+ // authors (proposer-fanout.mts): receipt persistence + spend settlement
1289
+ // into the run ledger for the claude/opencode paths whose shots would
1290
+ // otherwise read $0.
1291
+ onShotCompleted: proposerShotHooks({
1292
+ shotDir,
1293
+ harness: config.proposerHarness,
1294
+ ledger: () => activeLedger,
1295
+ phase: () => activePhase,
1296
+ }),
1297
+ })
1298
+ return {
1299
+ kind: `round4-constrained:${inner.kind}`,
1300
+ proposesWithoutFindings: true,
1301
+ generate: (args) => {
1302
+ activeLedger = args.costLedger
1303
+ activePhase = args.costPhase
1304
+ // args.maxShots is the LIB's dial (budget.maxImprovementShots → the
1305
+ // improvement driver); capabilities.mts guarantees it is threaded.
1306
+ return inner.generate(args)
1307
+ },
1308
+ }
1309
+ }
1310
+
1311
+ // ---------------------------------------------------------------------------
1312
+ // runRound. (The evaluated R4Artifact type lives in cell-evidence.mts with
1313
+ // the scoring that consumes it.)
1314
+ // ---------------------------------------------------------------------------
1315
+
1316
+ /** Ledger model id for the dockerized official judge's $0 receipts. */
1317
+ export const OFFICIAL_JUDGE_MODEL = 'swe-bench-official-judge'
1318
+
1319
+ function asCodeSurface(surface: MutableSurface): CodeSurface {
1320
+ if (typeof surface !== 'object' || surface === null || surface.kind !== 'code') {
1321
+ throw new Error('outer-loop: expected a CodeSurface (improve surface:"code" contract)')
1322
+ }
1323
+ return surface
1324
+ }
1325
+
1326
+ const log = (msg: string): void => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`)
1327
+
1328
+ export async function runRound(config: OuterLoopConfig, signal?: AbortSignal): Promise<void> {
1329
+ signal?.throwIfAborted()
1330
+ assertFrozenArm(config.arm)
1331
+ if (config.instances.length === 0) throw new Error('outer-loop: empty improvement set')
1332
+ const overlap = config.instances.filter((i) => config.holdoutInstances.includes(i))
1333
+ if (overlap.length > 0) {
1334
+ throw new Error(`outer-loop: improvement set leaks into the pre-registered holdout: ${overlap.join(', ')}`)
1335
+ }
1336
+ const reps = config.repsPerInstance ?? 1
1337
+ if (!Number.isInteger(reps) || reps < 1) {
1338
+ throw new Error(`outer-loop: repsPerInstance must be a positive integer, got ${JSON.stringify(config.repsPerInstance)}`)
1339
+ }
1340
+ if (config.proposers !== undefined) {
1341
+ if (config.proposers.length === 0) throw new Error('outer-loop: config.proposers must not be empty when set')
1342
+ if (config.proposers.length !== config.populationSize) {
1343
+ throw new Error(
1344
+ `outer-loop: populationSize ${config.populationSize} != proposers.length ${config.proposers.length} — ` +
1345
+ 'the fan-out assigns exactly one candidate slot per proposer',
1346
+ )
1347
+ }
1348
+ }
1349
+ // Stale-install guard: the resolved substrate must thread the passthroughs
1350
+ // this run depends on. Fails loud — a silent drop would re-spend the
1351
+ // premeasured baseline and pin the depth dial (see capabilities.mts).
1352
+ assertSubstratePassthroughs(log)
1353
+
1354
+ // GEN-4 model-identity provenance at t=0: harness CLI versions, the claude
1355
+ // seat's resolved settings model, codex auth, and every explicit model pin.
1356
+ // Fails loud on a missing/unauthed harness binary — populationSize equals
1357
+ // proposers.length, so a dead seat cannot be skipped mid-run.
1358
+ if (config.proposers !== undefined) {
1359
+ const provenance = await captureProposerProvenance(config.proposers)
1360
+ await mkdir(config.outDir, { recursive: true })
1361
+ await writeFile(join(config.outDir, 'proposer-provenance.json'), JSON.stringify(provenance, null, 2))
1362
+ for (const p of provenance.proposers) {
1363
+ log(
1364
+ `proposer ${p.name} (${p.harness}${p.merge ? ', merge seat' : ''}): ` +
1365
+ `model=${p.pinnedModel ?? `cli-default${p.settingsModel ? `:${p.settingsModel}` : ''}`} ` +
1366
+ `version=${p.harnessVersion.split('\n')[0]}`,
1367
+ )
1368
+ }
1369
+ }
1370
+
1371
+ // GEN-4 Pareto parents: materialize the configured prior-run frontier
1372
+ // (commit existence + full diffs) before any authoring.
1373
+ const paretoParents: ParetoParentContext[] =
1374
+ config.paretoParents !== undefined && config.paretoParents.length > 0
1375
+ ? await materializeParetoParents(config.loopsRepo, config.paretoParents)
1376
+ : []
1377
+ if (paretoParents.length > 0) {
1378
+ log(`pareto parents: ${paretoParents.map((p) => `${p.label}@${p.commit.slice(0, 10)}`).join(', ')}`)
1379
+ }
1380
+
1381
+ // The gate's only denominator: a stored prior baseline campaign the LIB
1382
+ // validates (surface hash, seed, reps, split digest, coverage) before
1383
+ // skipping the baseline campaign. A missing artifact = the BOOTSTRAP run —
1384
+ // the baseline is measured (cache-resumable) and the artifact written at
1385
+ // the end of this run.
1386
+ if (typeof config.premeasuredBaselinePath !== 'string' || config.premeasuredBaselinePath.length === 0) {
1387
+ throw new Error('outer-loop: config.premeasuredBaselinePath is required (the bootstrap run writes the artifact there)')
1388
+ }
1389
+ let premeasured: PremeasuredOptimizationBaseline<R4Artifact, Scenario> | undefined
1390
+ if (existsSync(config.premeasuredBaselinePath)) {
1391
+ premeasured = JSON.parse(
1392
+ await readFile(config.premeasuredBaselinePath, 'utf8'),
1393
+ ) as PremeasuredOptimizationBaseline<R4Artifact, Scenario>
1394
+ if (!premeasured || typeof premeasured.surfaceHash !== 'string' || !premeasured.campaign) {
1395
+ throw new Error(`premeasuredBaselinePath: ${config.premeasuredBaselinePath} is not a {surfaceHash, campaign} record`)
1396
+ }
1397
+ log(`premeasured baseline: ${config.premeasuredBaselinePath} (surface ${premeasured.surfaceHash})`)
1398
+ } else {
1399
+ log(
1400
+ `premeasured baseline artifact missing at ${config.premeasuredBaselinePath} — BOOTSTRAP run: ` +
1401
+ 'the baseline campaign will be measured (cache-resumable) and the artifact written there for later runs',
1402
+ )
1403
+ }
1404
+
1405
+ const secrets: SecretsEnv = { secretsDir: config.secretsDir, envFiles: config.envFiles }
1406
+ const excludes = await loadExcludes()
1407
+ const images = await loadInstanceImages(config.instanceImagesPath)
1408
+ const adapter = createSweBenchAdapter()
1409
+ const runId = `r${config.round}-${Date.now().toString(36)}`
1410
+ await mkdir(config.outDir, { recursive: true })
1411
+
1412
+ // GEN-5 public/private split — deterministic (seeded by runId), PERSISTED
1413
+ // per outDir so a resume can never rotate private instances into view.
1414
+ // Scored identically; selection stays combined; proposers + prefilter see
1415
+ // public only.
1416
+ const split: ScoreSplit | null =
1417
+ config.scoreSplit !== undefined
1418
+ ? await loadOrCreateScoreSplit({
1419
+ outDir: config.outDir,
1420
+ runId,
1421
+ instances: config.instances,
1422
+ publicCount: config.scoreSplit.publicCount,
1423
+ })
1424
+ : null
1425
+ const privateIids = new Set(split?.privateInstances ?? [])
1426
+ if (split !== null) {
1427
+ log(
1428
+ `score split (seeded by ${split.seededBy}): public [${split.publicInstances.join(', ')}] + ` +
1429
+ `${split.privateInstances.length} private instance(s) (identities withheld from proposers; ` +
1430
+ `selection uses public+private combined; small-n caveat: 2 private of 6 is a direction check, not certification)`,
1431
+ )
1432
+ }
1433
+
1434
+ // The pre-filter's smoke instance may sit outside the improvement set (e.g.
1435
+ // a designated cheap instance) — it needs the same problem/image/verify
1436
+ // validation and rides the same loaded-task map. Under the gen-5 split the
1437
+ // smoke choice is restricted to PUBLIC instances (the prefilter surfaces
1438
+ // its verdict to the kill log the authors can mine).
1439
+ const smokeIid =
1440
+ config.proposers !== undefined && config.prefilter?.enabled
1441
+ ? resolveSmokeInstance(
1442
+ config.prefilter.smokeInstance,
1443
+ split !== null ? split.publicInstances : config.instances,
1444
+ premeasured ? cellsFromCampaign(premeasured.campaign) : null,
1445
+ )
1446
+ : null
1447
+ const taskIds = [...new Set([...config.instances, ...(smokeIid !== null ? [smokeIid] : [])])]
1448
+ const tasks = await adapter.loadTasks({ ids: taskIds, split: 'test' })
1449
+ const problemById = new Map<string, string>()
1450
+ for (const iid of taskIds) {
1451
+ const task = tasks.find((t) => t.id === iid)
1452
+ if (!task) throw new Error(`outer-loop: ${iid} not found in SWE-bench_Verified`)
1453
+ const problem = String(task.metadata?.problem_statement ?? '')
1454
+ if (!problem) throw new Error(`outer-loop: ${iid} has an empty problem_statement`)
1455
+ if (!images[iid]) throw new Error(`outer-loop: ${iid} has no image mapping`)
1456
+ const verifyScript = join(config.verifyDir, `${iid}.sh`)
1457
+ if (!existsSync(verifyScript)) throw new Error(`outer-loop: missing verify script ${verifyScript}`)
1458
+ problemById.set(iid, problem)
1459
+ }
1460
+ if (smokeIid !== null) log(`prefilter smoke instance: ${smokeIid}`)
1461
+
1462
+ const judge: SerializedJudge = createSerializedJudge(
1463
+ config.judgeTimeoutMs !== undefined ? { timeoutMs: config.judgeTimeoutMs } : {},
1464
+ )
1465
+ await mkdir(config.roundsDir, { recursive: true })
1466
+ const recorder = new RoundRecorder(config.loopsRepo, join(config.outDir, 'candidates'))
1467
+ const analysts: AnalystSpec[] = config.analystModels.map((model, i) => ({ id: `${model}#${i + 1}`, model }))
1468
+
1469
+ // GEN-5 MAP+TOOLBOX briefing: persist the Pareto parent diffs, write the
1470
+ // per-run evidence index (a map — one line per evidence path, private
1471
+ // instances excluded), and resolve the briefing text (the change-space
1472
+ // override at extensions/pi/author-briefing.md wins over the default).
1473
+ let briefingCtx: BriefingContext | undefined
1474
+ if (config.briefing === AUTHOR_BRIEFING_VERSION) {
1475
+ const parentPatches: Array<{ label: string; path: string }> = []
1476
+ if (paretoParents.length > 0) {
1477
+ const parentsDir = join(config.outDir, 'pareto-parents')
1478
+ await mkdir(parentsDir, { recursive: true })
1479
+ for (const parent of paretoParents) {
1480
+ const patchPath = join(parentsDir, `${parent.label}.patch`)
1481
+ await writeFile(patchPath, parent.diff)
1482
+ parentPatches.push({ label: parent.label, path: patchPath })
1483
+ }
1484
+ }
1485
+ const index = await writeEvidenceIndex({
1486
+ outDir: config.outDir,
1487
+ roundsDir: config.roundsDir,
1488
+ seedArtifactRuns: config.seedArtifactRuns,
1489
+ ...(config.priorEvidenceDirs !== undefined ? { priorEvidenceDirs: config.priorEvidenceDirs } : {}),
1490
+ paretoParentPatches: parentPatches,
1491
+ split,
1492
+ })
1493
+ const briefing = await resolveAuthorBriefing(config.loopsRepo, config.loopsBaseRef)
1494
+ briefingCtx = { indexPath: index.path, briefingText: briefing.text, briefingSource: briefing.source }
1495
+ log(
1496
+ `briefing ${AUTHOR_BRIEFING_VERSION}: evidence index → ${index.path} (${index.rows.length} row(s)); ` +
1497
+ `briefing text source: ${briefing.source}`,
1498
+ )
1499
+ }
1500
+
1501
+ // GEN-5 settle-time rollout ledger — tangle.rollout.v1 lines appended live
1502
+ // after each cell judges (label v2); capture failure logs loud but never
1503
+ // kills a cell.
1504
+ const settleCapture: SettleCapture | null =
1505
+ config.rolloutLedger?.enabled === true
1506
+ ? createSettleCapture({
1507
+ ledgerPath: config.rolloutLedger.path ?? join(config.outDir, 'rollout-ledger.jsonl'),
1508
+ runId,
1509
+ instanceCount: config.instances.length,
1510
+ ...(config.rolloutLedger.opencodeDb !== undefined ? { opencodeDb: config.rolloutLedger.opencodeDb } : {}),
1511
+ log,
1512
+ })
1513
+ : null
1514
+ if (settleCapture !== null) log(`rollout-ledger: settle-time capture ON → ${settleCapture.path}`)
1515
+
1516
+ const sweScenarios: Scenario[] = config.instances.map((iid) => ({ id: iid, kind: 'swe-instance' }))
1517
+
1518
+ // Capacity gates on BOTH paths the supervisor arm rides (worker + router).
1519
+ // Shared by every arm dispatch — the improvement cells AND the pre-filter
1520
+ // smoke cell. A cell's WORK clock (config.dispatchTimeoutMs) starts only
1521
+ // after these clear — a capacity hold is never billed to the work budget.
1522
+ const awaitGates = async (gateSignal: AbortSignal | undefined = signal): Promise<void> => {
1523
+ for (const gate of gatesForArmKind('supervisor', secrets, {
1524
+ ...(config.gateWaitCeilingMs !== undefined ? { waitCeilingMs: config.gateWaitCeilingMs } : {}),
1525
+ ...(config.capacityModel !== undefined ? { model: config.capacityModel } : {}),
1526
+ onStatus: log,
1527
+ })) {
1528
+ if (!(await waitForCapacity(gate, gateSignal))) throw new Error(`no capacity on ${gate.name} within ceiling`)
1529
+ }
1530
+ }
1531
+
1532
+ // ── the pre-filter smoke runner: ONE supervisor arm cell + official judge
1533
+ // on the smoke instance, run against the proposer's scratch worktree BEFORE
1534
+ // any full-evaluation spend. A crashed smoke KILLS the candidate (recorded
1535
+ // in the kill reason) rather than the round — the pre-filter is allowed to
1536
+ // be strict; a survivor still faces the full gate. ────────────────────
1537
+ const smokeRunner: SmokeRunner | undefined =
1538
+ smokeIid === null
1539
+ ? undefined
1540
+ : async ({ scratchPath, generation, proposer, costLedger }): Promise<SmokeVerdict> => {
1541
+ const iid = smokeIid
1542
+ const requireResolved = config.prefilter?.requireResolved === true
1543
+ const entry = images[iid]!
1544
+ const armOutDir = join(config.outDir, 'prefilter-smoke', `gen${generation}-${proposer.name}`)
1545
+ const nm = join(scratchPath, 'node_modules')
1546
+ let linked = false
1547
+ const t0 = Date.now()
1548
+ try {
1549
+ if (!existsSync(nm)) {
1550
+ await symlink(join(config.loopsRepo, 'node_modules'), nm, 'dir')
1551
+ linked = true
1552
+ }
1553
+ const work = async (signal: AbortSignal): Promise<{ armRes: SupervisorArmResult; resolved: boolean | null }> => {
1554
+ const spec: SupervisorArmSpec = {
1555
+ kind: 'supervisor',
1556
+ name: config.armName,
1557
+ workerModel: config.arm.workerModel,
1558
+ driverModel: config.arm.driverModel,
1559
+ budget: config.arm.budget,
1560
+ maxSandboxes: config.arm.maxSandboxes,
1561
+ maxUsd: config.arm.maxUsd,
1562
+ maxDepth: config.arm.maxDepth,
1563
+ ...(config.arm.envKnobs ? { envKnobs: config.arm.envKnobs } : {}),
1564
+ loopsRepo: scratchPath,
1565
+ extensionPath: join(scratchPath, 'extensions', 'pi', 'loops.ts'),
1566
+ timeoutMs: config.arm.timeoutMs,
1567
+ }
1568
+ log(`>>> prefilter smoke ${proposer.name} ${iid} gen=${generation}`)
1569
+ const armRes = await runSupervisorArm(spec, {
1570
+ instanceId: iid,
1571
+ image: entry.image,
1572
+ baseCommit: entry.base_commit,
1573
+ problemStatement: problemById.get(iid)!,
1574
+ verifyCmd: `bash ${join(config.verifyDir, `${iid}.sh`)}`,
1575
+ outDir: armOutDir,
1576
+ secrets,
1577
+ excludes,
1578
+ signal,
1579
+ })
1580
+ if (signal.aborted) throw signal.reason
1581
+ const verdict = await judge.judge(
1582
+ iid,
1583
+ armRes.patchPath,
1584
+ `prefilter-g${generation}-${proposer.name}`,
1585
+ signal,
1586
+ )
1587
+ return { armRes, resolved: verdict.resolved }
1588
+ }
1589
+ const runWork = (): Promise<{ armRes: SupervisorArmResult; resolved: boolean | null }> =>
1590
+ runWithPostGateClock({
1591
+ awaitGates,
1592
+ work,
1593
+ timeoutMs: config.dispatchTimeoutMs,
1594
+ label: `prefilter smoke ${proposer.name} ${iid}`,
1595
+ signal,
1596
+ })
1597
+ let outcome: { armRes: SupervisorArmResult; resolved: boolean | null }
1598
+ if (costLedger) {
1599
+ // The smoke's real arm spend reaches the run ledger like any cell.
1600
+ const paid = await costLedger.runPaidCall({
1601
+ channel: 'agent',
1602
+ phase: 'search.prefilter',
1603
+ actor: `prefilter-smoke:${iid}:g${generation}:${proposer.name}`,
1604
+ model: config.arm.workerModel,
1605
+ execute: runWork,
1606
+ receipt: ({ armRes }) => {
1607
+ const spend = armRes.recoveredSpend
1608
+ const usageKnown = (spend?.workerTokIn ?? null) !== null || (spend?.workerTokOut ?? null) !== null
1609
+ return {
1610
+ model: config.arm.workerModel,
1611
+ inputTokens: spend?.workerTokIn ?? 0,
1612
+ outputTokens: spend?.workerTokOut ?? 0,
1613
+ ...(usageKnown ? {} : { usageUnknown: true }),
1614
+ ...(armRes.spentUsd !== null ? { actualCostUsd: armRes.spentUsd } : {}),
1615
+ }
1616
+ },
1617
+ })
1618
+ if (!paid.succeeded) throw paid.error
1619
+ outcome = paid.value
1620
+ } else {
1621
+ outcome = await runWork()
1622
+ }
1623
+ const wallS = Math.round((Date.now() - t0) / 1000)
1624
+ const patchDelivered = outcome.armRes.patch_lines > 0
1625
+ const conclusive = outcome.resolved !== null
1626
+ const pass = requireResolved ? outcome.resolved === true : patchDelivered && conclusive
1627
+ const verdictLine =
1628
+ `smoke ${iid}: resolved=${outcome.resolved} patch_lines=${outcome.armRes.patch_lines} ` +
1629
+ `verify_pass=${outcome.armRes.verify_pass} wall_s=${outcome.armRes.wall_s}`
1630
+ const result: SmokeVerdict = {
1631
+ iid,
1632
+ pass,
1633
+ reason: pass
1634
+ ? verdictLine
1635
+ : `${verdictLine} — below the ${requireResolved ? 'resolved' : 'mechanism (patch + conclusive judge)'} bar`,
1636
+ resolved: outcome.resolved,
1637
+ patchLines: outcome.armRes.patch_lines,
1638
+ wallS,
1639
+ // GEN-6: the GEPA seat's inner-score tiebreak.
1640
+ verifyPass: outcome.armRes.verify_pass,
1641
+ }
1642
+ await mkdir(armOutDir, { recursive: true })
1643
+ await writeFile(join(armOutDir, 'smoke.json'), JSON.stringify(result, null, 2))
1644
+ return result
1645
+ } catch (cause) {
1646
+ if (signal?.aborted) throw signal.reason
1647
+ const wallS = Math.round((Date.now() - t0) / 1000)
1648
+ const result: SmokeVerdict = {
1649
+ iid,
1650
+ pass: false,
1651
+ reason: `smoke errored: ${(cause as Error).message}`,
1652
+ resolved: null,
1653
+ patchLines: 0,
1654
+ wallS,
1655
+ }
1656
+ await mkdir(armOutDir, { recursive: true })
1657
+ await writeFile(join(armOutDir, 'smoke.json'), JSON.stringify(result, null, 2)).catch(() => {})
1658
+ return result
1659
+ } finally {
1660
+ if (linked) await unlink(nm).catch(() => {})
1661
+ }
1662
+ }
1663
+
1664
+ // ── dispatch: one (surface × scenario) cell ──────────────────────────
1665
+ const agent = async (surface: MutableSurface, scenario: Scenario, ctx: DispatchContext): Promise<R4Artifact> => {
1666
+ const cs = asCodeSurface(surface)
1667
+ const rec = await recorder.ensure(cs)
1668
+
1669
+ const iid = scenario.id
1670
+ // FAIL-CLOSED change-space enforcement: an out-of-space candidate must
1671
+ // never reach a model token or a docker container. The thrown cell is the
1672
+ // record (the lib stores it with `error` set — no side bookkeeping).
1673
+ if (rec.violations.length > 0) {
1674
+ throw new Error(`change-space violation (${rec.violations.length} path(s)): ${rec.violations.join(', ')}`)
1675
+ }
1676
+
1677
+ const runCell = async (signal: AbortSignal): Promise<R4Artifact> => {
1678
+ const entry = images[iid]!
1679
+ const evalWt = join(config.outDir, 'eval-wt', `${rec.tag}-${iid}-r${ctx.rep}`)
1680
+ const armOutDir = join(config.outDir, 'arm-runs', rec.tag, `rep-${ctx.rep}`)
1681
+ try {
1682
+ await addEvalWorktree(config.loopsRepo, cs.candidateCommit, evalWt, signal)
1683
+ const spec: SupervisorArmSpec = {
1684
+ kind: 'supervisor',
1685
+ name: config.armName,
1686
+ workerModel: config.arm.workerModel,
1687
+ driverModel: config.arm.driverModel,
1688
+ budget: config.arm.budget,
1689
+ maxSandboxes: config.arm.maxSandboxes,
1690
+ maxUsd: config.arm.maxUsd,
1691
+ maxDepth: config.arm.maxDepth,
1692
+ ...(config.arm.envKnobs ? { envKnobs: config.arm.envKnobs } : {}),
1693
+ loopsRepo: evalWt,
1694
+ extensionPath: join(evalWt, 'extensions', 'pi', 'loops.ts'),
1695
+ timeoutMs: config.arm.timeoutMs,
1696
+ }
1697
+ log(`>>> ${config.armName} ${rec.tag} ${iid} rep=${ctx.rep}`)
1698
+ const armRes: SupervisorArmResult = await runSupervisorArm(spec, {
1699
+ instanceId: iid,
1700
+ image: entry.image,
1701
+ baseCommit: entry.base_commit,
1702
+ problemStatement: problemById.get(iid)!,
1703
+ verifyCmd: `bash ${join(config.verifyDir, `${iid}.sh`)}`,
1704
+ outDir: armOutDir,
1705
+ secrets,
1706
+ excludes,
1707
+ signal,
1708
+ })
1709
+ if (signal.aborted) throw signal.reason
1710
+ const runDir = join(armOutDir, 'runs', iid, config.armName)
1711
+ const { ws: _ws, ...armSummary } = armRes
1712
+ await writeFile(join(runDir, 'result.json'), JSON.stringify(armSummary, null, 1))
1713
+
1714
+ // The official judge is a docker test-suite run — real wall time, zero
1715
+ // LLM spend. Its OWN paid call (channel 'judge', $0 actual) keeps the
1716
+ // run's spend tree attributing judge work per cell without inventing a
1717
+ // token cost; the wall lands on the artifact + judge.json.
1718
+ const judgeT0 = Date.now()
1719
+ const judgePaid = await ctx.cost.runPaidCall({
1720
+ channel: 'judge',
1721
+ actor: `official-judge:${iid}#r${ctx.rep}`,
1722
+ model: OFFICIAL_JUDGE_MODEL,
1723
+ execute: () => {
1724
+ if (signal.aborted) throw signal.reason
1725
+ return judge.judge(iid, armRes.patchPath, `${config.armName}-${rec.tag}`, signal)
1726
+ },
1727
+ receipt: () => ({ model: OFFICIAL_JUDGE_MODEL, inputTokens: 0, outputTokens: 0, actualCostUsd: 0 }),
1728
+ })
1729
+ if (!judgePaid.succeeded) throw judgePaid.error
1730
+ const verdict = judgePaid.value
1731
+ const judgeWallS = Math.round((Date.now() - judgeT0) / 1000)
1732
+ await writeFile(join(runDir, 'judge.json'), JSON.stringify({ ...verdict, wallS: judgeWallS }, null, 1))
1733
+ log(`${config.armName} ${rec.tag} ${iid} judged: resolved=${verdict.resolved} (attempts=${verdict.attempts})`)
1734
+
1735
+ // Deterministic run observability, per cell: steer count, waves, concurrency,
1736
+ // idle, evidence→respawn, cost by role. The headline lands in the run log so
1737
+ // the answers are in the tail without a follow-up command.
1738
+ await writeSupervisorRunReportSafe(runDir, {
1739
+ appendHeadlineTo: join(config.outDir, 'run.log'),
1740
+ patchPath: armRes.patchPath,
1741
+ })
1742
+
1743
+ const spend = armRes.recoveredSpend
1744
+ const recovered =
1745
+ armRes.spentTokens === null && (spend?.workerTokSqlite ?? null) === null
1746
+ ? null
1747
+ : (armRes.spentTokens ?? 0) + (spend?.workerTokSqlite ?? 0)
1748
+ rec.armProvenance = { repo: armRes.provenance.repo, commit: armRes.provenance.commit }
1749
+ await appendFile(
1750
+ join(config.outDir, 'progress.jsonl'),
1751
+ JSON.stringify({
1752
+ at: new Date().toISOString(),
1753
+ runId,
1754
+ candidate: rec.tag,
1755
+ iid,
1756
+ rep: ctx.rep,
1757
+ runDir,
1758
+ resolved: verdict.resolved,
1759
+ verify_pass: armRes.verify_pass,
1760
+ wall_s: armRes.wall_s,
1761
+ spentTokens: armRes.spentTokens,
1762
+ spentUsd: armRes.spentUsd,
1763
+ recoveredTokens: recovered,
1764
+ }) + '\n',
1765
+ )
1766
+ const summaryPath = await ctx.artifacts.writeJson('arm-summary.json', {
1767
+ runDir,
1768
+ patchPath: armRes.patchPath,
1769
+ verdict,
1770
+ })
1771
+
1772
+ // GEN-5 settle-time rollout capture: emit supervisor + worker lines
1773
+ // NOW, while the opencode store still holds the worker transcripts.
1774
+ // Attribution comes from the campaign cell path (never dispatch
1775
+ // order); a capture failure logs loud but never kills the cell.
1776
+ if (settleCapture !== null) {
1777
+ try {
1778
+ const coords = campaignCoordsFromCellPath(summaryPath)
1779
+ if (coords === null) {
1780
+ log(`rollout-ledger: cannot derive campaign coords from ${summaryPath} — cell ${iid} r${ctx.rep} skipped`)
1781
+ } else {
1782
+ const supRunDir = await findSupervisorRunDir(armRes.ws)
1783
+ const deliveredPatch = await readFile(armRes.patchPath, 'utf8').catch(() => '')
1784
+ await settleCapture.captureCell({
1785
+ generation: coords.generation,
1786
+ candidateIndex: coords.candidateIndex,
1787
+ iid,
1788
+ rep: ctx.rep,
1789
+ seed: ctx.seed,
1790
+ splitVisibility: split === null ? null : privateIids.has(iid) ? 'private' : 'public',
1791
+ commit: cs.candidateCommit,
1792
+ resolved: verdict.resolved,
1793
+ judgeVerdict: { ...verdict, wallS: judgeWallS },
1794
+ runDir,
1795
+ patchPath: armRes.patchPath,
1796
+ supRunDir,
1797
+ deliveredPatch,
1798
+ workerModel: config.arm.workerModel,
1799
+ metrics: {
1800
+ resolved: verdict.resolved,
1801
+ verify_pass: armRes.verify_pass,
1802
+ patch_lines: armRes.patch_lines,
1803
+ judge_attempts: verdict.attempts ?? null,
1804
+ judge_wall_s: judgeWallS,
1805
+ spent_tokens: armRes.spentTokens,
1806
+ spent_usd: armRes.spentUsd,
1807
+ recovered_tokens: recovered,
1808
+ sup_status: armRes.sup_status,
1809
+ sup_verdict: armRes.sup_verdict,
1810
+ spawned: armRes.spawned,
1811
+ workers: armRes.workers,
1812
+ settled: armRes.settled,
1813
+ },
1814
+ cost: { usd: armRes.spentUsd, wallS: armRes.wall_s, spentTokens: armRes.spentTokens },
1815
+ })
1816
+ }
1817
+ } catch (cause) {
1818
+ log(`rollout-ledger: settle-time capture FAILED for ${iid} r${ctx.rep}: ${(cause as Error).message}`)
1819
+ }
1820
+ }
1821
+
1822
+ if (verdict.resolved === null) {
1823
+ // Inconclusive judge (double flake / infra) — the cell must FAIL, not
1824
+ // score a fabricated boolean; the candidate becomes coverage-incomplete.
1825
+ throw new Error(`inconclusive judge verdict for ${iid} (${verdict.error ?? 'unknown'})`)
1826
+ }
1827
+ return {
1828
+ kind: 'swe-arm',
1829
+ iid,
1830
+ commit: cs.candidateCommit,
1831
+ resolved: verdict.resolved,
1832
+ verifyPass: armRes.verify_pass,
1833
+ patchLines: armRes.patch_lines,
1834
+ wallS: armRes.wall_s,
1835
+ spentTokens: armRes.spentTokens,
1836
+ spentUsd: armRes.spentUsd,
1837
+ recoveredTokens: recovered,
1838
+ workerTokIn: spend?.workerTokIn ?? null,
1839
+ workerTokOut: spend?.workerTokOut ?? null,
1840
+ judgeAttempts: verdict.attempts ?? null,
1841
+ judgeWallS,
1842
+ runDir,
1843
+ patchPath: armRes.patchPath,
1844
+ }
1845
+ } finally {
1846
+ await removeEvalWorktree(config.loopsRepo, evalWt)
1847
+ }
1848
+ }
1849
+
1850
+ // The arm's real spend reaches the LIB's CostLedger here: one agent-channel
1851
+ // paid call per cell whose receipt carries the recovered worker-session
1852
+ // token split (opencode sqlite join) and the runtime spend-tree dollars
1853
+ // (state.json spentUsd). run-campaign commits it into cell.costUsd /
1854
+ // cell.tokenUsage + durable cost-ledger.jsonl receipts — the stub/$0
1855
+ // rounds this replaces.
1856
+ const paid = await ctx.cost.runPaidCall<R4Artifact>({
1857
+ actor: `${config.armName}:${rec.tag}:${iid}#r${ctx.rep}`,
1858
+ model: config.arm.workerModel,
1859
+ execute: () =>
1860
+ runWithPostGateClock({
1861
+ awaitGates,
1862
+ work: runCell,
1863
+ timeoutMs: config.dispatchTimeoutMs,
1864
+ label: `${config.armName} ${rec.tag} ${iid} r${ctx.rep}`,
1865
+ signal,
1866
+ }),
1867
+ receipt: (artifact) => {
1868
+ if (artifact.kind !== 'swe-arm') throw new Error('swe cell produced a non-arm artifact')
1869
+ const usageKnown = artifact.workerTokIn !== null || artifact.workerTokOut !== null
1870
+ return {
1871
+ model: config.arm.workerModel,
1872
+ inputTokens: artifact.workerTokIn ?? 0,
1873
+ outputTokens: artifact.workerTokOut ?? 0,
1874
+ ...(usageKnown ? {} : { usageUnknown: true }),
1875
+ // The runtime spend-tree usd is the measured bill; without it the
1876
+ // receipt stays honestly unpriced (costUnknown) rather than $0.
1877
+ ...(artifact.spentUsd !== null ? { actualCostUsd: artifact.spentUsd } : {}),
1878
+ }
1879
+ },
1880
+ })
1881
+ if (!paid.succeeded) throw paid.error
1882
+ return paid.value
1883
+ }
1884
+
1885
+ // ── judge config: a deterministic READ of the official verdict the dispatch
1886
+ // already obtained under the serialized-judge lock. ───────────────────
1887
+ const judgeConfig: JudgeConfig<R4Artifact, Scenario> = {
1888
+ name: 'swe-arena-official-judge',
1889
+ dimensions: [{ key: 'resolved', description: 'official SWE-bench judge verdict' }],
1890
+ score: ({ artifact }) => {
1891
+ const v = artifact.resolved ? 1 : 0
1892
+ return {
1893
+ composite: v,
1894
+ dimensions: { resolved: v },
1895
+ notes: `official judge: ${artifact.iid} resolved=${artifact.resolved} (verify_pass=${artifact.verifyPass}, patch_lines=${artifact.patchLines}, wall_s=${artifact.wallS})`,
1896
+ }
1897
+ },
1898
+ }
1899
+
1900
+ // ── diagnosis at the analyzeGeneration seam ──────────────────────────
1901
+ const rawTrace = rawTraceDistiller<Scenario, R4Artifact>({ fallbackFindings: [] })
1902
+ const steeringFinding = makeFinding({
1903
+ analyst_id: 'round4-protocol',
1904
+ severity: 'high',
1905
+ area: 'constraint',
1906
+ confidence: 1,
1907
+ claim:
1908
+ 'Declared change-space: ONLY extensions/pi/** and src/{worker-evidence,best-effort,worker-clone}.ts may change ' +
1909
+ '(plus the .improve/ diagnosis artifact). Judge, verify scripts, task prompts, model ids and budgets are immutable.',
1910
+ recommended_action: 'Keep every edit inside the change-space; out-of-space candidate diffs are rejected before evaluation.',
1911
+ evidence_refs: [],
1912
+ })
1913
+ const analyzeGeneration = async (input: {
1914
+ generation: number
1915
+ runDir: string
1916
+ candidates: Array<{ surfaceHash: string; composite: number; campaign: unknown }>
1917
+ history: unknown[]
1918
+ }): Promise<unknown[]> => {
1919
+ signal?.throwIfAborted()
1920
+ const runs: SupRunArtifacts[] = []
1921
+ if (input.generation === -1) {
1922
+ for (const seed of config.seedArtifactRuns) {
1923
+ if (privateIids.has(seed.iid)) continue // gen-5 split: never surfaced to proposers
1924
+ if (!existsSync(seed.dir)) {
1925
+ // A wiped scratchpad (host reboot) must not feed EMPTY bundles to the
1926
+ // analysts as if they were real artifacts — skip loudly.
1927
+ log(`seed artifact dir missing — skipped from diagnosis: ${seed.dir}`)
1928
+ continue
1929
+ }
1930
+ runs.push({
1931
+ iid: seed.iid,
1932
+ arm: seed.arm,
1933
+ dir: seed.dir,
1934
+ ...(seed.patchPath ? { patchPath: seed.patchPath } : {}),
1935
+ judge: { resolved: seed.resolved, note: 'previous round (seeded)' },
1936
+ })
1937
+ }
1938
+ }
1939
+ // Candidate failure artifacts come from the LIB's campaign cells (the
1940
+ // artifacts name their own runDir/patch) — resume-replayed cells included,
1941
+ // which the old recorder-based lookup silently dropped.
1942
+ const worstFirst = [...input.candidates].sort((a, b) => a.composite - b.composite).slice(0, 4)
1943
+ for (const cand of worstFirst) {
1944
+ const cells = cellsFromCampaign(cand.campaign as CampaignResult<R4Artifact, Scenario>)
1945
+ for (const cell of cells) {
1946
+ const a = cell.artifact
1947
+ if (a === null || a.kind !== 'swe-arm' || !a.runDir) continue
1948
+ if (privateIids.has(a.iid)) continue // gen-5 split: never surfaced to proposers
1949
+ runs.push({
1950
+ iid: a.iid,
1951
+ arm: config.armName,
1952
+ dir: a.runDir,
1953
+ ...(a.patchPath ? { patchPath: a.patchPath } : {}),
1954
+ judge: { resolved: cell.error ? null : a.resolved },
1955
+ })
1956
+ }
1957
+ }
1958
+ let ensembleFindings: unknown[] = []
1959
+ if (runs.length > 0) {
1960
+ try {
1961
+ const scratch = join(config.outDir, 'diagnosis', `gen-${input.generation}`)
1962
+ const ensemble = await runDiagnosisEnsemble({
1963
+ analysts,
1964
+ runs,
1965
+ secrets,
1966
+ scratchDir: scratch,
1967
+ onStatus: log,
1968
+ signal,
1969
+ })
1970
+ signal?.throwIfAborted()
1971
+ await writeFile(
1972
+ join(config.outDir, 'diagnosis', `gen-${input.generation}.json`),
1973
+ JSON.stringify({ reports: ensemble.reports, fused: ensemble.fused }, null, 2),
1974
+ )
1975
+ ensembleFindings = fusedToAnalystFindings(ensemble.fused, {
1976
+ dirs: [...new Set(runs.map((r) => r.dir))],
1977
+ totalAnalysts: analysts.length,
1978
+ })
1979
+ } catch (cause) {
1980
+ if (signal?.aborted) throw signal.reason
1981
+ // A dead router must not kill the round: the raw-trace context below
1982
+ // still grounds the proposer; the failure is logged, never silent.
1983
+ log(`diagnosis ensemble FAILED for gen ${input.generation}: ${(cause as Error).message}`)
1984
+ }
1985
+ }
1986
+ // GEN-5 split: the raw-trace distiller must not hand private-instance
1987
+ // cells' path context to the authors either — censor them out of the
1988
+ // candidates' campaigns before distillation.
1989
+ const censoredInput =
1990
+ split === null
1991
+ ? input
1992
+ : {
1993
+ ...input,
1994
+ candidates: input.candidates.map((cand) => {
1995
+ const campaign = cand.campaign as { cells?: Array<{ scenarioId: string }> } | null
1996
+ if (campaign === null || typeof campaign !== 'object' || !Array.isArray(campaign.cells)) return cand
1997
+ return {
1998
+ ...cand,
1999
+ campaign: { ...campaign, cells: campaign.cells.filter((c) => !privateIids.has(c.scenarioId)) },
2000
+ }
2001
+ }),
2002
+ }
2003
+ signal?.throwIfAborted()
2004
+ const rawFindings = (await rawTrace(censoredInput as Parameters<typeof rawTrace>[0])) as unknown[]
2005
+ signal?.throwIfAborted()
2006
+ return [steeringFinding, ...ensembleFindings, ...rawFindings]
2007
+ }
2008
+
2009
+ // ── protocol_v2: NEVER ships from inside the loop. `budget.holdout:
2010
+ // 'deferred'` makes the LIB dispatch zero holdout cells, force `hold`, omit
2011
+ // `lift`, and record `holdout: 'deferred'` in the provenance record; the
2012
+ // pre-registered holdout run happens later, with operator approval. The
2013
+ // would-be-KEEP operator brief is computed post-run from campaign cells
2014
+ // (see the summary below). ───────────────────────────────────────────
2015
+ const holdoutReps = config.holdoutRepsPerInstance ?? 2
2016
+ const holdoutInstruction =
2017
+ `holdout (${config.holdoutInstances.length} pre-registered instances: ${config.holdoutInstances.join(', ')}) ` +
2018
+ `was NOT run — operator approval required. To certify a would-be KEEP under the ${holdoutReps}-rep ` +
2019
+ 'fail-closed protocol (same-protocol parent comparison): ' +
2020
+ 'tsx src/swe-arena/holdout-certify.mts <config.json> --candidate <winner-loops-commit>'
2021
+ const improveRunDir = join(config.outDir, 'improve-run')
2022
+
2023
+ // ── crash recovery: a killed run leaves its in-flight paid call 'pending'
2024
+ // in the durable cost ledger, and the ledger's fail-closed guard then
2025
+ // refuses ALL new paid work on resume. Under the outDir instance lock
2026
+ // (sole runner), every pending call restored from disk is provably from a
2027
+ // dead process — settle each as a $0 failure receipt (reason
2028
+ // 'process-crash-orphan') so the guard passes without erasing the crash
2029
+ // from the durable record. ───────────────────────────────────────────
2030
+ for (const receipt of reconcileCrashOrphansOnDisk(improveRunDir)) {
2031
+ log(
2032
+ `cost-ledger: reconciled crash-orphaned call '${receipt.callId}' ` +
2033
+ `(${receipt.actor}, ${receipt.phase}) as ${CRASH_ORPHAN_REASON}`,
2034
+ )
2035
+ }
2036
+
2037
+ // ── generator: the gen-3 proposer fan-out (parallel AgentProfile-pinned
2038
+ // authors + pre-filter) when `proposers` is configured; the legacy
2039
+ // single-author generator otherwise. ─────────────────────────────────
2040
+ const fanout =
2041
+ config.proposers !== undefined
2042
+ ? fanOutLoopsGenerator(config, {
2043
+ ...(smokeRunner ? { smokeRunner } : {}),
2044
+ ...(paretoParents.length > 0 ? { parents: paretoParents } : {}),
2045
+ ...(briefingCtx !== undefined ? { briefing: briefingCtx } : {}),
2046
+ // GEN-6: the GEPA seat's inner evaluator rides the SAME public-only
2047
+ // smoke instance; the split guards the never-surfaced invariant at
2048
+ // the bridge boundary too.
2049
+ ...(smokeIid !== null ? { smokeInstanceId: smokeIid } : {}),
2050
+ scoreSplit: split,
2051
+ log,
2052
+ })
2053
+ : null
2054
+ const generator = withParentCancellation(fanout ?? constrainedLoopsGenerator(config), signal)
2055
+
2056
+ // ── the improve() call: the optimizer seat ───────────────────────────
2057
+ // Typed from improve()'s own parameter: the monorepo hoists two
2058
+ // agent-interface majors, so a nominal import can resolve to the wrong one.
2059
+ const profile = { name: 'loops-pi-supervisor' } as Parameters<typeof improve>[0]
2060
+ signal?.throwIfAborted()
2061
+ log(`round ${config.round} runId=${runId}: improve(surface:'code') over ${config.loopsRepo}@${config.loopsBaseRef}`)
2062
+ const result = await improve<Scenario, R4Artifact>(profile, [], {
2063
+ surface: 'code',
2064
+ // analyzeGeneration wins over this flag; the composite above embeds
2065
+ // rawTraceDistiller directly so the raw-trace mechanism stays active.
2066
+ rawTraceContext: true,
2067
+ analyzeGeneration,
2068
+ code: {
2069
+ repoRoot: config.loopsRepo,
2070
+ baseRef: config.loopsBaseRef,
2071
+ worktreeDir: join(config.outDir, 'loops-worktrees'),
2072
+ generator,
2073
+ },
2074
+ scenarios: sweScenarios,
2075
+ judge: judgeConfig,
2076
+ agent,
2077
+ budget: {
2078
+ generations: config.generations,
2079
+ populationSize: config.populationSize,
2080
+ maxConcurrency: 1,
2081
+ reps,
2082
+ maxImprovementShots: config.maxShots,
2083
+ // Deferred with no reserved set: ALL improvement-set scenarios train;
2084
+ // the held-out comparison lives in the separate operator-approved run.
2085
+ holdout: 'deferred',
2086
+ },
2087
+ // TRAINING RECORDER: every scored (artifact, judge score) lands in the
2088
+ // lib's labeled-scenario store as a JSONL corpus under outDir (growth is
2089
+ // outDir-scoped; a handful of cells per round). Records carry the default
2090
+ // 'unverified' trust — corpus-grade, NOT gold-eligible, which is right
2091
+ // until an operator-confirmed holdout verdict upgrades them.
2092
+ labeledStore: new FsLabeledScenarioStore({ root: join(config.outDir, 'labeled-store') }),
2093
+ captureSource: 'eval-run',
2094
+ // Arm/judge/proposer spend reaches the campaign meter through real paid
2095
+ // calls (worker receipt per swe cell, $0 judge receipts, imported
2096
+ // proposer-shot receipts). 'warn' not 'assert': the official judge's $0
2097
+ // receipts are correct-by-design and must not kill the round as "stubs".
2098
+ expectUsage: 'warn',
2099
+ // Widened: covers worst-case capacity-gate holds; the REAL per-cell work
2100
+ // clock (config.dispatchTimeoutMs) starts post-gate inside the dispatch.
2101
+ dispatchTimeoutMs: campaignDispatchCeilingMs(config),
2102
+ runDir: improveRunDir,
2103
+ ...(premeasured ? { premeasuredBaseline: premeasured } : {}),
2104
+ })
2105
+ signal?.throwIfAborted()
2106
+
2107
+ // ── staircase rows + round summary — scored from the LIB's campaign cells
2108
+ // (baselineCampaign + per-generation candidate campaigns), which replay
2109
+ // correctly attributed on resume. The recorder only contributes the
2110
+ // dispatch-time diff/change-space description (recomputed post-run via
2111
+ // ensure() for candidates that were replayed, never dispatched here). ────
2112
+ try {
2113
+ const loop = result.raw.raw
2114
+ const baselineCells = cellsFromCampaign(loop.baselineCampaign)
2115
+ const baselineWallS = sumWallSFromCells(baselineCells)
2116
+ const measuredBaselineCount = resolvedInstanceCount(
2117
+ replicateRunsFromCells(baselineCells),
2118
+ config.instances,
2119
+ reps,
2120
+ )
2121
+ const campaignBySurface = new Map<string, CampaignResult<R4Artifact, Scenario>>()
2122
+ for (const gen of loop.generations) {
2123
+ for (const s of gen.surfaces) campaignBySurface.set(s.surfaceHash, s.campaign)
2124
+ }
2125
+ const resolvedCountOf = (campaign: CampaignResult<R4Artifact, Scenario>): number =>
2126
+ resolvedInstanceCount(replicateRunsFromCells(cellsFromCampaign(campaign)), config.instances, reps)
2127
+
2128
+ // BASELINE-DRIFT: a resumed runDir can still hold baseline cells cached by
2129
+ // an OLDER (pre-artifact) run. When they contradict the lib-validated
2130
+ // premeasured artifact, log loud — the artifact rules, never silently.
2131
+ if (premeasured) {
2132
+ const cachedBaseline = await loadCampaignCells(join(improveRunDir, 'baseline'))
2133
+ if (cachedBaseline.length > 0) {
2134
+ const expected = instanceVerdictsFromCells(baselineCells, config.instances, reps)
2135
+ for (const w of baselineDriftWarnings(
2136
+ expected,
2137
+ replicateRunsFromCells(cachedBaseline),
2138
+ config.instances,
2139
+ reps,
2140
+ )) {
2141
+ log(`BASELINE-DRIFT: ${w}`)
2142
+ }
2143
+ }
2144
+ }
2145
+
2146
+ // Collected per-candidate facts for the gen-5 machinery (activation by
2147
+ // surface hash for the winner brief, lineage nodes, proposer v2 rewards).
2148
+ const activationBySurface = new Map<string, ActivationRecord>()
2149
+ const lineageCandidates: LineageCandidateInput[] = []
2150
+ interface ProposerOutcomeFact {
2151
+ generation: number
2152
+ candidateIndex: number
2153
+ label: string
2154
+ commit: string | null
2155
+ resolvedCount: number
2156
+ diffPath: string | null
2157
+ }
2158
+ const proposerFacts: ProposerOutcomeFact[] = []
2159
+
2160
+ for (let g = 0; g < loop.generations.length; g++) {
2161
+ const gen = loop.generations[g]!
2162
+ const rows: StaircaseRow[] = []
2163
+ for (let candIndex = 0; candIndex < gen.record.candidates.length; candIndex++) {
2164
+ const cand = gen.record.candidates[candIndex]!
2165
+ const surface = gen.surfaces.find((s) => s.surfaceHash === cand.surfaceHash)?.surface
2166
+ const cs = surface && typeof surface === 'object' && surface.kind === 'code' ? surface : null
2167
+ const desc = cs ? await recorder.ensure(cs) : undefined
2168
+ const campaign = campaignBySurface.get(cand.surfaceHash)
2169
+ const cells = campaign ? cellsFromCampaign(campaign) : []
2170
+ const runs = replicateRunsFromCells(cells)
2171
+ const perInstance = perInstanceFromCells(cells)
2172
+ const candResolved = resolvedInstanceCount(runs, config.instances, reps)
2173
+ const wallS = sumWallSFromCells(cells)
2174
+ const coverageComplete =
2175
+ cand.eligibleForPromotion === true && replicateCoverageComplete(runs, config.instances, reps)
2176
+ const costRatio = baselineWallS > 0 ? wallS / baselineWallS : null
2177
+ // Parent's AND-resolved count. A parent hash with no candidate
2178
+ // campaign IS the baseline incumbent — its count comes from the
2179
+ // baseline campaign (the lib-validated premeasured artifact, or the
2180
+ // bootstrap run's measurement; both survive resume, no dispatch-order
2181
+ // guess).
2182
+ const parentCampaign = cand.parentSurfaceHash
2183
+ ? campaignBySurface.get(cand.parentSurfaceHash)
2184
+ : undefined
2185
+ const parentResolvedCount =
2186
+ parentCampaign !== undefined ? resolvedCountOf(parentCampaign) : measuredBaselineCount
2187
+ const violations = desc?.violations ?? []
2188
+
2189
+ // GEN-5 activation gate: run the candidate's own committed predicate
2190
+ // over its own cell run dirs. Fail-closed — a missing/unparseable
2191
+ // predicate (the prefilter should have killed it) quarantines.
2192
+ let activation: ActivationRecord | undefined
2193
+ if (config.activationGate === true && cs !== null) {
2194
+ const committed = await readCommittedPredicate(config.loopsRepo, cs.candidateCommit)
2195
+ if (committed === null || !committed.parsed.ok) {
2196
+ const why =
2197
+ committed === null
2198
+ ? `no ${ACTIVATION_PREDICATE_RELPATH} at ${cs.candidateCommit.slice(0, 10)}`
2199
+ : `unparseable activation predicate: ${committed.parsed.ok ? '' : committed.parsed.error}`
2200
+ activation = {
2201
+ present: false,
2202
+ description: null,
2203
+ fired: false,
2204
+ evidence: [],
2205
+ warnings: [`${why} — fail-closed quarantine`],
2206
+ }
2207
+ } else {
2208
+ const runDirs = [
2209
+ ...new Set(
2210
+ cells
2211
+ .map((c) => (c.artifact !== null && c.artifact.kind === 'swe-arm' ? c.artifact.runDir : null))
2212
+ .filter((d): d is string => typeof d === 'string' && d.length > 0),
2213
+ ),
2214
+ ]
2215
+ const res = await runActivationPredicate(committed.parsed.predicate, runDirs)
2216
+ activation = {
2217
+ present: true,
2218
+ description: committed.parsed.predicate.description,
2219
+ fired: res.fired,
2220
+ evidence: res.evidence,
2221
+ warnings: res.warnings,
2222
+ }
2223
+ }
2224
+ activationBySurface.set(cand.surfaceHash, activation)
2225
+ log(
2226
+ `activation ${cand.label ?? cand.surfaceHash.slice(0, 10)}: present=${activation.present} ` +
2227
+ `fired=${activation.fired}${activation.fired ? ` — ${activation.evidence[0] ?? ''}` : ''}` +
2228
+ `${activation.warnings.length > 0 ? ` (warnings: ${activation.warnings.join('; ')})` : ''}`,
2229
+ )
2230
+ }
2231
+
2232
+ // GEN-5 split sub-scores: both halves logged per candidate; the
2233
+ // selection rule stays combined (candResolved over ALL instances).
2234
+ const verdicts = instanceVerdictsFromCells(cells, config.instances, reps)
2235
+ const splitScores = split !== null ? subScores(verdicts, split) : null
2236
+ if (split !== null && splitScores !== null) {
2237
+ log(
2238
+ `split scores ${cand.label ?? cand.surfaceHash.slice(0, 10)}: ` +
2239
+ `public ${splitScores.publicResolvedCount}/${split.publicInstances.length}, ` +
2240
+ `private ${splitScores.privateResolvedCount}/${split.privateInstances.length} (combined ${candResolved}/${config.instances.length})`,
2241
+ )
2242
+ }
2243
+
2244
+ const verdict = decideVerdict({
2245
+ violations,
2246
+ coverageComplete,
2247
+ resolvedCount: candResolved,
2248
+ parentResolvedCount,
2249
+ costRatio,
2250
+ costGuardRatio: config.costGuardRatio,
2251
+ ...(activation !== undefined ? { activationFired: activation.fired } : {}),
2252
+ })
2253
+ rows.push({
2254
+ schema: STAIRCASE_SCHEMA,
2255
+ round: config.round,
2256
+ generation: g,
2257
+ runId,
2258
+ at: new Date().toISOString(),
2259
+ candidate: cand.surfaceHash,
2260
+ candidateCommit: cs?.candidateCommit ?? null,
2261
+ parent: cand.parentSurfaceHash ?? 'baseline',
2262
+ parentResolvedCount,
2263
+ ...(cand.label ? { label: cand.label } : {}),
2264
+ ...(cand.rationale ? { rationale: cand.rationale } : {}),
2265
+ changedFiles: desc?.changedFiles ?? [],
2266
+ changeSpaceViolations: violations,
2267
+ perInstance,
2268
+ resolvedCount: candResolved,
2269
+ coverageComplete,
2270
+ wallS,
2271
+ baselineWallS,
2272
+ costRatio,
2273
+ costGuardRatio: config.costGuardRatio,
2274
+ internallyPromoted: gen.record.promoted.includes(cand.surfaceHash),
2275
+ verdict,
2276
+ holdout: 'operator-approval-required',
2277
+ armProvenance: desc?.armProvenance ?? null,
2278
+ diffPath: desc?.diffPath ?? null,
2279
+ diffSha256: desc?.diffSha256 ?? null,
2280
+ ...(split !== null && splitScores !== null
2281
+ ? {
2282
+ split: {
2283
+ publicInstances: split.publicInstances,
2284
+ privateInstances: split.privateInstances,
2285
+ ...splitScores,
2286
+ },
2287
+ }
2288
+ : {}),
2289
+ ...(activation !== undefined ? { activation } : {}),
2290
+ })
2291
+
2292
+ const label = cand.label ?? cs?.candidateCommit?.slice(0, 10) ?? cand.surfaceHash.slice(0, 10)
2293
+ lineageCandidates.push({
2294
+ label,
2295
+ commit: cs?.candidateCommit ?? null,
2296
+ resolvedCount: candResolved,
2297
+ verdicts,
2298
+ merge: config.proposers?.find((p) => p.name === cand.label)?.merge === true,
2299
+ verdict,
2300
+ })
2301
+ proposerFacts.push({
2302
+ generation: g,
2303
+ candidateIndex: candIndex,
2304
+ label,
2305
+ commit: cs?.candidateCommit ?? null,
2306
+ resolvedCount: candResolved,
2307
+ diffPath: desc?.diffPath ?? null,
2308
+ })
2309
+ }
2310
+ const genFile = join(config.roundsDir, `gen-${g}.jsonl`)
2311
+ for (const row of rows) await appendFile(genFile, JSON.stringify(row) + '\n')
2312
+ log(`staircase: ${rows.length} row(s) → ${genFile} (${rows.map((r) => r.verdict).join(', ')})`)
2313
+ }
2314
+
2315
+ // Pre-filter kills: candidates the fan-out killed BEFORE evaluation never
2316
+ // became surfaces (zero arm cells), so the loop has no row for them —
2317
+ // each becomes an explicit `rejected-prefilter` staircase dot with its
2318
+ // kill reason and forensics patch.
2319
+ if (fanout) {
2320
+ const kills = fanout.drainPrefilterKills()
2321
+ for (const kill of kills) {
2322
+ const row: StaircaseRow = {
2323
+ schema: STAIRCASE_SCHEMA,
2324
+ round: config.round,
2325
+ generation: kill.generation,
2326
+ runId,
2327
+ at: new Date().toISOString(),
2328
+ candidate: `prefilter-kill:${kill.diffSha256?.slice('sha256:'.length, 'sha256:'.length + 12) ?? kill.proposer}`,
2329
+ candidateCommit: null,
2330
+ parent: premeasured?.surfaceHash ?? 'baseline',
2331
+ parentResolvedCount: measuredBaselineCount,
2332
+ label: kill.proposer,
2333
+ rationale: `prefilter kill at stage '${kill.stage}' (${kill.harness ?? 'engine'})`,
2334
+ changedFiles: [],
2335
+ changeSpaceViolations: kill.stage === 'change-space' ? [kill.reason] : [],
2336
+ perInstance: [],
2337
+ resolvedCount: 0,
2338
+ coverageComplete: false,
2339
+ wallS: kill.smoke?.wallS ?? 0,
2340
+ baselineWallS,
2341
+ costRatio: null,
2342
+ costGuardRatio: config.costGuardRatio,
2343
+ internallyPromoted: false,
2344
+ verdict: 'rejected-prefilter',
2345
+ killReason: `${kill.stage}: ${kill.reason}`,
2346
+ holdout: 'operator-approval-required',
2347
+ armProvenance: null,
2348
+ diffPath: kill.patchPath,
2349
+ diffSha256: kill.diffSha256,
2350
+ }
2351
+ const genFile = join(config.roundsDir, `gen-${kill.generation}.jsonl`)
2352
+ await appendFile(genFile, JSON.stringify(row) + '\n')
2353
+ log(`staircase: prefilter kill dot (${kill.proposer}, ${kill.stage}) → ${genFile}`)
2354
+ }
2355
+ }
2356
+
2357
+ // GEN-5 label-v2 proposer rewards: baseline-relative (candidate − baseline
2358
+ // resolved fraction, improvement positive), one settle-time ledger line
2359
+ // per evaluated candidate now that the round's scores are final.
2360
+ if (settleCapture !== null) {
2361
+ const sanitizeName = (s: string): string => s.replace(/[^a-zA-Z0-9_-]/g, '_')
2362
+ for (const fact of proposerFacts) {
2363
+ try {
2364
+ const flatDir = join(config.outDir, 'proposer-shots')
2365
+ const pattern = new RegExp(`^gen${fact.generation}-cand${fact.candidateIndex}-shot\\d+\\.json$`)
2366
+ const receiptPaths: string[] = []
2367
+ for (const dir of [flatDir, join(flatDir, sanitizeName(fact.label))]) {
2368
+ for (const name of (await readdir(dir).catch(() => [])).sort()) {
2369
+ if (pattern.test(name)) receiptPaths.push(join(dir, name))
2370
+ }
2371
+ }
2372
+ await settleCapture.captureProposer({
2373
+ generation: fact.generation,
2374
+ candidateIndex: fact.candidateIndex,
2375
+ proposer: fact.label,
2376
+ harness: config.proposers?.find((p) => p.name === fact.label)?.harness ?? null,
2377
+ commit: fact.commit,
2378
+ candResolved: fact.resolvedCount,
2379
+ baselineResolved: measuredBaselineCount,
2380
+ shotReceiptPaths: receiptPaths,
2381
+ diffPath: fact.diffPath,
2382
+ })
2383
+ } catch (cause) {
2384
+ log(`rollout-ledger: proposer capture FAILED for ${fact.label}: ${(cause as Error).message}`)
2385
+ }
2386
+ }
2387
+ }
2388
+
2389
+ // GEN-5 lineage DAG: record baseline root + pareto parents + every
2390
+ // evaluated candidate (multi-parent for the merge seat) at the
2391
+ // .evolve-compatible store, and ask the governor for the continuation
2392
+ // decision (recorded below — never acted on inside this run).
2393
+ let lineageResult: Awaited<ReturnType<typeof recordLineageGeneration>> | null = null
2394
+ if (config.lineage === true) {
2395
+ try {
2396
+ const baselineCommit =
2397
+ baselineCells.find((c) => c.artifact !== null && c.artifact.kind === 'swe-arm')?.artifact?.commit ??
2398
+ config.loopsBaseRef
2399
+ lineageResult = await recordLineageGeneration({
2400
+ outDir: config.outDir,
2401
+ runId,
2402
+ instances: config.instances,
2403
+ baseline: {
2404
+ commit: baselineCommit,
2405
+ resolvedCount: measuredBaselineCount,
2406
+ verdicts: instanceVerdictsFromCells(baselineCells, config.instances, reps),
2407
+ },
2408
+ paretoParents: (config.paretoParents ?? []).map((p) => ({
2409
+ label: p.label,
2410
+ commit: p.commit,
2411
+ resolvedInstances: p.resolvedInstances,
2412
+ })),
2413
+ candidates: lineageCandidates,
2414
+ })
2415
+ log(
2416
+ `lineage: ${lineageResult.appended.length} node(s) appended (total ${lineageResult.nodesTotal}) → ` +
2417
+ `${lineageResult.path}; governor decision: ${JSON.stringify(lineageResult.governor)}` +
2418
+ `${lineageResult.skipped.length > 0 ? `; skipped (no commit): ${lineageResult.skipped.join(', ')}` : ''}`,
2419
+ )
2420
+ } catch (cause) {
2421
+ log(`lineage: recording FAILED: ${(cause as Error).message}`)
2422
+ }
2423
+ }
2424
+
2425
+ const winnerSurface = result.raw.winner.surface
2426
+ const winnerCs =
2427
+ typeof winnerSurface === 'object' && winnerSurface !== null && winnerSurface.kind === 'code'
2428
+ ? winnerSurface
2429
+ : null
2430
+ const winnerRec = winnerCs ? await recorder.ensure(winnerCs) : undefined
2431
+ let winnerPatch: string | null = null
2432
+ if (winnerRec?.diffPath) {
2433
+ winnerPatch = join(config.outDir, 'winner.patch')
2434
+ await writeFile(winnerPatch, await readFile(winnerRec.diffPath, 'utf8'))
2435
+ }
2436
+
2437
+ // BOOTSTRAP: persist this run's measured baseline campaign as the
2438
+ // premeasured artifact every later run consumes (and the lib re-validates
2439
+ // by surface hash / seed / reps / split digest). The baseline surface
2440
+ // hash comes from the Pareto frontier's generation −1 entry — the lib's
2441
+ // own record of the baseline measurement.
2442
+ if (!premeasured) {
2443
+ const baselineHash = loop.paretoFrontier.find((p) => p.generation === -1)?.surfaceHash
2444
+ if (baselineHash === undefined) {
2445
+ log('bootstrap: no generation −1 Pareto entry — premeasured baseline artifact NOT written')
2446
+ } else {
2447
+ const artifact: PremeasuredOptimizationBaseline<R4Artifact, Scenario> = {
2448
+ surfaceHash: baselineHash,
2449
+ campaign: loop.baselineCampaign,
2450
+ }
2451
+ await writeFile(config.premeasuredBaselinePath, JSON.stringify(artifact, null, 1))
2452
+ log(`bootstrap: premeasured baseline artifact → ${config.premeasuredBaselinePath} (surface ${baselineHash})`)
2453
+ }
2454
+ }
2455
+
2456
+ // The would-be-KEEP operator brief: winner vs baseline on the improvement
2457
+ // set, from campaign cells. The lib's deferred-holdout gate always holds;
2458
+ // this evidence tells the operator whether the pre-registered holdout run
2459
+ // is worth approving.
2460
+ const winnerHash = winnerCs ? surfaceHash(winnerCs) : null
2461
+ const winnerCampaign = winnerHash !== null ? campaignBySurface.get(winnerHash) : undefined
2462
+ const winnerActivation = winnerHash !== null ? activationBySurface.get(winnerHash) : undefined
2463
+ const improvementSet =
2464
+ winnerCampaign !== undefined && winnerRec !== undefined
2465
+ ? gateEvidenceFromCells({
2466
+ winnerCells: cellsFromCampaign(winnerCampaign),
2467
+ baselineCells,
2468
+ violations: winnerRec.violations,
2469
+ iids: config.instances,
2470
+ reps,
2471
+ costGuardRatio: config.costGuardRatio,
2472
+ ...(winnerActivation !== undefined ? { activationFired: winnerActivation.fired } : {}),
2473
+ })
2474
+ : null
2475
+ const wouldKeep = improvementSet !== null && improvementSet.verdict === 'accepted'
2476
+ if (improvementSet) {
2477
+ log(
2478
+ `improvement set: winner ${improvementSet.candResolved}/${config.instances.length} vs baseline ` +
2479
+ `${improvementSet.baseResolved}/${config.instances.length}; wall ${improvementSet.candWallS}s vs ` +
2480
+ `${improvementSet.baseWallS}s (ratio ${improvementSet.costRatio === null ? 'n/a' : improvementSet.costRatio.toFixed(2)}, ` +
2481
+ `guard ${config.costGuardRatio}); protocol verdict: ${improvementSet.verdict}${wouldKeep ? ' (WOULD-BE KEEP)' : ''}`,
2482
+ )
2483
+ } else {
2484
+ log('improvement set: winner == baseline (no candidate campaign) — nothing to promote')
2485
+ }
2486
+
2487
+ const summary = {
2488
+ schema: 'swe-arena.round-summary.v2',
2489
+ round: config.round,
2490
+ runId,
2491
+ at: new Date().toISOString(),
2492
+ loops: { repo: config.loopsRepo, baseRef: config.loopsBaseRef },
2493
+ // The gate's denominator: the lib-validated premeasured artifact, or
2494
+ // this bootstrap run's freshly measured (and persisted) campaign.
2495
+ baseline: {
2496
+ resolvedCount: measuredBaselineCount,
2497
+ wallS: baselineWallS,
2498
+ perInstance: perInstanceFromCells(baselineCells),
2499
+ premeasured: premeasured !== undefined,
2500
+ artifactPath: config.premeasuredBaselinePath,
2501
+ ...(premeasured ? { surfaceHash: premeasured.surfaceHash } : {}),
2502
+ },
2503
+ winner: winnerCs
2504
+ ? {
2505
+ surfaceHash: winnerHash,
2506
+ commit: winnerCs.candidateCommit,
2507
+ label: result.raw.winner.label ?? null,
2508
+ rationale: result.raw.winner.rationale ?? null,
2509
+ patch: winnerPatch,
2510
+ }
2511
+ : null,
2512
+ // The lib's verdict + reasons: deferred holdout forces `hold` with zero
2513
+ // holdout cells dispatched and no fabricated lift.
2514
+ gateDecision: result.decision,
2515
+ gateReasons: loop.gateResult.reasons,
2516
+ // Improvement-set (search-split) evidence — NOT a held-out measurement.
2517
+ improvementSet: improvementSet === null ? null : { ...improvementSet, wouldKeep },
2518
+ // GEN-5: the public/private split (sub-scores live per candidate in the
2519
+ // staircase rows; selection stays combined; private never surfaced to
2520
+ // proposers — the 2-of-6 private half is a direction check, not a
2521
+ // certification).
2522
+ scoreSplit:
2523
+ split === null
2524
+ ? null
2525
+ : {
2526
+ seededBy: split.seededBy,
2527
+ publicInstances: split.publicInstances,
2528
+ privateInstances: split.privateInstances,
2529
+ },
2530
+ // GEN-5: activation-gate outcomes per candidate surface.
2531
+ activationGate:
2532
+ config.activationGate === true
2533
+ ? {
2534
+ enabled: true,
2535
+ byCandidate: [...activationBySurface.entries()].map(([surface, a]) => ({
2536
+ surface,
2537
+ present: a.present,
2538
+ fired: a.fired,
2539
+ description: a.description,
2540
+ })),
2541
+ }
2542
+ : { enabled: false },
2543
+ // GEN-5: MAP+TOOLBOX briefing provenance.
2544
+ briefing:
2545
+ briefingCtx === undefined
2546
+ ? null
2547
+ : { version: AUTHOR_BRIEFING_VERSION, indexPath: briefingCtx.indexPath, textSource: briefingCtx.briefingSource },
2548
+ // GEN-5: settle-time rollout ledger location (tangle.rollout.v1, label v2).
2549
+ rolloutLedger: settleCapture === null ? null : { path: settleCapture.path, capture: 'settle-time', labels: 'v2' },
2550
+ // GEN-5: lineage DAG + the governor's recorded continuation decision.
2551
+ lineage:
2552
+ lineageResult === null
2553
+ ? null
2554
+ : {
2555
+ path: lineageResult.path,
2556
+ nodesTotal: lineageResult.nodesTotal,
2557
+ appended: lineageResult.appended.length,
2558
+ skipped: lineageResult.skipped,
2559
+ governor: lineageResult.governor,
2560
+ },
2561
+ // Honest run-wide spend from the lib's CostLedger: per-channel rollups
2562
+ // (agent = arm cells, judge = official-judge calls, driver = proposer
2563
+ // shots), token totals, and accounting-completeness flags.
2564
+ cost: {
2565
+ totalCostUsd: result.raw.totalCostUsd,
2566
+ inputTokens: result.raw.cost.inputTokens,
2567
+ outputTokens: result.raw.cost.outputTokens,
2568
+ byChannel: result.raw.cost.byChannel,
2569
+ fullyPriced: result.raw.cost.fullyPriced,
2570
+ usageComplete: result.raw.cost.usageComplete,
2571
+ accountingComplete: result.raw.cost.accountingComplete,
2572
+ incompleteReasons: result.raw.cost.incompleteReasons,
2573
+ receipts: result.raw.receipts.length,
2574
+ },
2575
+ holdout: {
2576
+ instances: config.holdoutInstances,
2577
+ mode: 'deferred',
2578
+ status: 'operator-approval-required',
2579
+ // The certification protocol the operator run must use — 2-rep
2580
+ // fail-closed with a same-protocol parent (gen-2 postmortem).
2581
+ protocol: {
2582
+ repsPerInstance: holdoutReps,
2583
+ resolvedRule: 'all-reps',
2584
+ parentBaseline: config.holdoutBaseline ?? 'measure',
2585
+ },
2586
+ instruction: holdoutInstruction,
2587
+ },
2588
+ }
2589
+ const summaryPath = join(config.roundsDir, `round${config.round}-summary-${runId}.json`)
2590
+ await writeFile(summaryPath, JSON.stringify(summary, null, 2))
2591
+ log(`round summary → ${summaryPath}`)
2592
+
2593
+ // Round rollup at gate time: every cell's orchestration/economics in one table,
2594
+ // written next to the round summary and echoed into the run log.
2595
+ await reportSupervisorRound(join(config.outDir, 'arm-runs'), {
2596
+ appendHeadlineTo: join(config.outDir, 'run.log'),
2597
+ reportDir: config.roundsDir,
2598
+ title: `Round ${config.round} rollup — ${runId}`,
2599
+ echo: true,
2600
+ }).catch((err: unknown) => {
2601
+ log(`round rollup failed: ${err instanceof Error ? err.message : String(err)}`)
2602
+ })
2603
+
2604
+ log(`gate: ${result.decision} — ${loop.gateResult.reasons[0] ?? ''}`)
2605
+ } finally {
2606
+ await result.dispose()
2607
+ }
2608
+ signal?.throwIfAborted()
2609
+ }
2610
+
2611
+ // ---------------------------------------------------------------------------
2612
+ // Calibration smoke — the ensemble over the REAL round-2 django SUP2 run
2613
+ // (known truth: the worker authored a LOCAL idna helper inside the mail module
2614
+ // while the gold fix adds punycode() in django/utils/encoding.py — a fix
2615
+ // PLACEMENT failure). Cheap (a few k tokens/analyst); grades whether each
2616
+ // blind analyst independently surfaces placement.
2617
+ // ---------------------------------------------------------------------------
2618
+
2619
+ export interface SmokeArgs {
2620
+ supRunDir?: string
2621
+ patchPath?: string
2622
+ analysts?: number
2623
+ model?: string
2624
+ /** 'router' (default) or 'zai' — the z.ai coding endpoint is the proven
2625
+ * fallback when router.tangle.tools 524-storms (a measured infra class). */
2626
+ endpoint?: 'router' | 'zai'
2627
+ /** Transport retries per analyst. Default 4 in the smoke (storms pass). */
2628
+ retries?: number
2629
+ secrets?: SecretsEnv
2630
+ scratchDir?: string
2631
+ }
2632
+
2633
+ export async function calibrationSmoke(args: SmokeArgs = {}): Promise<{
2634
+ perAnalyst: Array<{ analystId: string; ok: boolean; surfacesPlacement: boolean; findings: number; error?: string }>
2635
+ fusedTop: string[]
2636
+ }> {
2637
+ const supRunDir = args.supRunDir ?? join(DEFAULT_HH_SCRATCHPAD, 'runs', 'django__django-11532', 'SUP2')
2638
+ const patchPath = args.patchPath ?? join(DEFAULT_HH_SCRATCHPAD, 'patches', 'django__django-11532.sup2.patch')
2639
+ const secrets: SecretsEnv = args.secrets ?? {
2640
+ secretsDir: '/home/drew/company/devops/secrets',
2641
+ envFiles: ['agent-state.env', 'tangle-router.env'],
2642
+ }
2643
+ const scratchDir = args.scratchDir ?? join(supRunDir, '..', '..', '..', 'r4', 'calibration-smoke')
2644
+ const analysts: AnalystSpec[] = defaultAnalysts(args.analysts ?? 3, args.model ?? 'glm-5.2').map((s) =>
2645
+ args.endpoint === 'zai' ? { ...s, url: ZAI_CODING_ENDPOINT, apiKeyEnv: 'ZAI_API_KEY' } : s,
2646
+ )
2647
+ const runs: SupRunArtifacts[] = [
2648
+ {
2649
+ iid: 'django__django-11532',
2650
+ arm: 'SUP2',
2651
+ dir: supRunDir,
2652
+ ...(existsSync(patchPath) ? { patchPath } : {}),
2653
+ judge: { resolved: false, note: 'round-2 official judge: unresolved while the self-verify passed' },
2654
+ },
2655
+ ]
2656
+ const ensemble = await runDiagnosisEnsemble({
2657
+ analysts,
2658
+ runs,
2659
+ secrets,
2660
+ scratchDir,
2661
+ retriesPerAnalyst: args.retries ?? 4,
2662
+ retryDelayMs: 15_000,
2663
+ onStatus: log,
2664
+ })
2665
+ const placement = surfacesPlacementRegex()
2666
+ const perAnalyst = ensemble.reports.map((r) => ({
2667
+ analystId: r.analystId,
2668
+ ok: r.ok,
2669
+ surfacesPlacement: r.findings.some((f) =>
2670
+ placement.test(`${f.failure_class} ${f.evidence_quote} ${f.proposed_direction}`),
2671
+ ),
2672
+ findings: r.findings.length,
2673
+ ...(r.error ? { error: r.error } : {}),
2674
+ }))
2675
+ console.log('\n=== CALIBRATION SMOKE (django__django-11532 SUP2, truth = fix placement) ===')
2676
+ console.log(`bundle: ${ensemble.bundleChars} chars; analysts: ${analysts.map((a) => a.model).join(', ')}`)
2677
+ for (const r of ensemble.reports) {
2678
+ const grade = perAnalyst.find((p) => p.analystId === r.analystId)!
2679
+ console.log(`\n--- ${r.analystId} ok=${r.ok} placement-surfaced=${grade.surfacesPlacement}${r.error ? ` error=${r.error}` : ''}` +
2680
+ (r.tokens ? ` tokens(in=${r.tokens.input},out=${r.tokens.output})` : ''))
2681
+ for (const f of r.findings) {
2682
+ console.log(` [${f.confidence.toFixed(2)}] ${f.failure_class} → ${f.proposed_direction.slice(0, 160)}`)
2683
+ if (f.evidence_quote) console.log(` evidence: ${f.evidence_quote.slice(0, 160)}`)
2684
+ }
2685
+ }
2686
+ console.log('\n--- fused (agreement-ranked) ---')
2687
+ for (const f of ensemble.fused) {
2688
+ console.log(
2689
+ ` agreement=${f.agreement}${f.competingHypothesis ? ' [competing hypothesis]' : ''} conf=${f.meanConfidence.toFixed(2)} — ${f.failure_class}`,
2690
+ )
2691
+ }
2692
+ const outPath = join(scratchDir, 'calibration-smoke.json')
2693
+ await mkdir(scratchDir, { recursive: true })
2694
+ await writeFile(outPath, JSON.stringify({ perAnalyst, ensemble }, null, 2))
2695
+ console.log(`\nfull output → ${outPath}`)
2696
+ return { perAnalyst, fusedTop: ensemble.fused.map((f) => f.failure_class) }
2697
+ }
2698
+
2699
+ // ---------------------------------------------------------------------------
2700
+ // CLI.
2701
+ // ---------------------------------------------------------------------------
2702
+
2703
+ const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
2704
+
2705
+ if (isMain) {
2706
+ const argv = process.argv.slice(2)
2707
+ // Config execution owns long-lived workers and therefore needs cooperative
2708
+ // cleanup. Utility modes keep the terminal's default signal behavior because
2709
+ // their analyst APIs do not yet accept AbortSignal; installing a handler there
2710
+ // would swallow Ctrl-C while the model call continued.
2711
+ const interrupt = argv[0] && !argv[0].startsWith('--')
2712
+ ? installProcessSignalAbort('outer-loop')
2713
+ : undefined
2714
+ try {
2715
+ const flag = (name: string): string | undefined => {
2716
+ const i = argv.indexOf(name)
2717
+ return i !== -1 ? argv[i + 1] : undefined
2718
+ }
2719
+ if (argv[0] === '--write-config') {
2720
+ const path = argv[1]
2721
+ if (!path || path.startsWith('--')) {
2722
+ console.error('usage: outer-loop.mts --write-config <path> [--out-name <dirname>] [--gen3|--gen4|--gen5]')
2723
+ process.exit(2)
2724
+ }
2725
+ const outDirName = flag('--out-name')
2726
+ const gen3 = argv.includes('--gen3')
2727
+ const gen4 = argv.includes('--gen4')
2728
+ const gen5 = argv.includes('--gen5')
2729
+ let config: OuterLoopConfig
2730
+ let flavor: string
2731
+ if (gen4 || gen5) {
2732
+ // The codex seat rides only when the CLI is actually present — a config
2733
+ // naming a missing harness would fail the whole launch at t=0.
2734
+ const codexProbe = await run('codex', ['--version'])
2735
+ const includeCodex = codexProbe.code === 0
2736
+ if (!includeCodex) {
2737
+ console.log(
2738
+ `codex CLI unavailable (rc=${codexProbe.code}) — ${gen5 ? 'gen-5' : 'gen-4'} config written WITHOUT the codex-author seat`,
2739
+ )
2740
+ }
2741
+ const make = gen5 ? defaultGen5Config : defaultGen4Config
2742
+ config = make(undefined, { ...(outDirName ? { outDirName } : {}), includeCodex })
2743
+ flavor = gen5 ? 'gen-5' : 'gen-4'
2744
+ } else {
2745
+ const make = gen3 ? defaultGen3Config : defaultRound4Config
2746
+ config = make(undefined, outDirName ? { outDirName } : {})
2747
+ flavor = gen3 ? 'gen-3' : 'round-4'
2748
+ }
2749
+ await writeFile(path, JSON.stringify(config, null, 2) + '\n')
2750
+ console.log(`default ${flavor} config → ${path}`)
2751
+ } else if (argv[0] === '--calibration-smoke') {
2752
+ const dir = argv[1] && !argv[1].startsWith('--') ? argv[1] : undefined
2753
+ const n = flag('--analysts')
2754
+ const model = flag('--model')
2755
+ const endpoint = flag('--endpoint')
2756
+ const retries = flag('--retries')
2757
+ if (endpoint !== undefined && endpoint !== 'router' && endpoint !== 'zai') {
2758
+ console.error(`--endpoint must be 'router' or 'zai', got ${JSON.stringify(endpoint)}`)
2759
+ process.exit(2)
2760
+ }
2761
+ await calibrationSmoke({
2762
+ ...(dir ? { supRunDir: dir } : {}),
2763
+ ...(n ? { analysts: Number(n) } : {}),
2764
+ ...(model ? { model } : {}),
2765
+ ...(endpoint ? { endpoint } : {}),
2766
+ ...(retries ? { retries: Number(retries) } : {}),
2767
+ })
2768
+ } else if (argv[0] && !argv[0].startsWith('--')) {
2769
+ const config = JSON.parse(await readFile(argv[0], 'utf8')) as OuterLoopConfig
2770
+ // Launch guards BEFORE any spend: keys present (dotenvx forgotten = hours
2771
+ // of confusing downstream failures) and exactly one loop per outDir.
2772
+ assertLaunchEnv()
2773
+ interrupt!.signal.throwIfAborted()
2774
+ const lock = await acquireInstanceLock(config.outDir)
2775
+ try {
2776
+ await runRound(config, interrupt!.signal)
2777
+ } finally {
2778
+ await lock.release()
2779
+ }
2780
+ } else {
2781
+ console.error(
2782
+ 'usage: tsx src/swe-arena/outer-loop.mts <config.json> # SPENDS: arms + judges + proposer\n' +
2783
+ ' tsx src/swe-arena/outer-loop.mts --write-config <path> [--out-name <dirname>] [--gen3|--gen4|--gen5]\n' +
2784
+ ' tsx src/swe-arena/outer-loop.mts --calibration-smoke [supRunDir] [--analysts N] [--model M] [--endpoint router|zai] [--retries N]',
2785
+ )
2786
+ process.exit(2)
2787
+ }
2788
+ } catch (cause) {
2789
+ if (!interrupt?.signal.aborted) throw cause
2790
+ const detail = cause instanceof Error ? cause.message : String(cause)
2791
+ console.error(`outer-loop stopped after cleanup: ${detail}`)
2792
+ } finally {
2793
+ interrupt?.dispose()
2794
+ }
2795
+ }