@tangle-network/agent-bench 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/HARNESS.md +43 -0
- package/dist/adapters.js +24 -24
- package/dist/benchmarks/_harness.d.ts +1 -1
- package/dist/benchmarks/_harness.js +1 -1
- package/dist/benchmarks/aec-bench.js +2 -2
- package/dist/benchmarks/agentbench.js +2 -2
- package/dist/benchmarks/appworld.js +2 -2
- package/dist/benchmarks/bfcl.js +2 -2
- package/dist/benchmarks/commit0.js +2 -2
- package/dist/benchmarks/crag.js +2 -2
- package/dist/benchmarks/dabstep.js +2 -2
- package/dist/benchmarks/enterpriseops-gym.js +2 -2
- package/dist/benchmarks/finresearchbench.js +2 -2
- package/dist/benchmarks/humaneval.d.ts +10 -1
- package/dist/benchmarks/humaneval.js +5 -3
- package/dist/benchmarks/nomiracl.js +2 -2
- package/dist/benchmarks/open-rag-bench.js +2 -2
- package/dist/benchmarks/programbench.js +2 -2
- package/dist/benchmarks/ragbench.js +2 -2
- package/dist/benchmarks/swe-bench.js +2 -2
- package/dist/benchmarks/t2-ragbench.js +2 -2
- package/dist/benchmarks/tau-bench-shared.js +2 -2
- package/dist/benchmarks/tau2-bench.js +3 -3
- package/dist/benchmarks/tau3-banking.js +3 -3
- package/dist/benchmarks/terminal-bench.js +2 -2
- package/dist/benchmarks/toollm.js +2 -2
- package/dist/benchmarks/webarena-verified.js +2 -2
- package/dist/{chunk-CKUVRZ2T.js → chunk-3U5TXJZS.js} +2 -2
- package/dist/{chunk-PPYSEKFM.js → chunk-5H5XV76F.js} +73 -15
- package/dist/chunk-5H5XV76F.js.map +1 -0
- package/dist/{chunk-YCGY7UIZ.js → chunk-7GRVHU22.js} +2 -2
- package/dist/{chunk-Z7ML6L77.js → chunk-HWST3SED.js} +2 -2
- package/dist/{chunk-SYDW647C.js → chunk-IA2FBTWC.js} +2 -2
- package/dist/{chunk-R67DFVLO.js → chunk-IFVINJ4B.js} +2 -2
- package/dist/{chunk-R36V2VP7.js → chunk-IZ5M6OAC.js} +2 -2
- package/dist/{chunk-ODT47UAY.js → chunk-K3BQGZCT.js} +2 -2
- package/dist/{chunk-IFAV6KEM.js → chunk-KP5KD6EN.js} +2 -2
- package/dist/{chunk-ZEWMTR5M.js → chunk-MQMRLGOG.js} +2 -2
- package/dist/{chunk-TSWPNOYM.js → chunk-NQG5XDSB.js} +2 -2
- package/dist/{chunk-7WSD27QQ.js → chunk-PB64GYIG.js} +2 -2
- package/dist/{chunk-HBSWHQNJ.js → chunk-RCYQEFNX.js} +3 -3
- package/dist/{chunk-J3KDJNX2.js → chunk-RH5F53JT.js} +2 -2
- package/dist/{chunk-UAIOHCUK.js → chunk-SFLA7OH3.js} +3 -3
- package/dist/{chunk-Y6O2OCUO.js → chunk-SHM6MRRF.js} +2 -2
- package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js} +2 -2
- package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js} +2 -2
- package/dist/{chunk-5SBJCB6W.js → chunk-V7AEBY6U.js} +22 -22
- package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js} +18 -2
- package/dist/chunk-WSKWVEQB.js.map +1 -0
- package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js} +2 -2
- package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js} +2 -2
- package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js} +2 -2
- package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js} +2 -2
- package/dist/index.js +24 -24
- package/package.json +6 -5
- package/scripts/run-package-tests.mjs +30 -8
- package/scripts/verify-packed-consumer.mjs +1 -1
- package/scripts/verify-pier-agent.mts +1 -0
- package/src/benchmarks/_harness.ts +20 -2
- package/src/benchmarks/humaneval.test.mts +122 -0
- package/src/benchmarks/humaneval.ts +100 -27
- package/src/david-attribution.mts +78 -0
- package/src/david-goliath.mts +149 -0
- package/src/hev-improve.mts +25 -6
- package/src/humaneval-object-ablation.mts +201 -0
- package/src/live-improve-campaign-mbpp.mts +641 -0
- package/src/live-improve-campaign.mts +500 -0
- package/src/mbpp-structural.mts +12 -7
- package/src/quant-arena/README.md +144 -0
- package/src/quant-arena/backtest.test.mts +135 -0
- package/src/quant-arena/backtest.ts +218 -0
- package/src/quant-arena/data.test.mts +44 -0
- package/src/quant-arena/data.ts +141 -0
- package/src/quant-arena/driver.test.mts +253 -0
- package/src/quant-arena/driver.ts +219 -0
- package/src/quant-arena/fixtures/data/PROVENANCE.md +26 -0
- package/src/quant-arena/fixtures/data/holdout/IDX.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S01.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S02.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S03.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S04.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S05.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S06.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S07.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S08.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S09.csv +523 -0
- package/src/quant-arena/fixtures/data/holdout/S10.csv +523 -0
- package/src/quant-arena/fixtures/data/insample/IDX.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S01.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S02.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S03.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S04.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S05.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S06.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S07.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S08.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S09.csv +2087 -0
- package/src/quant-arena/fixtures/data/insample/S10.csv +2087 -0
- package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +16 -0
- package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +5 -0
- package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +171 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +119 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +119 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +105 -0
- package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +102 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +4 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +2 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +84 -0
- package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +117 -0
- package/src/quant-arena/holdout-certify.mts +206 -0
- package/src/quant-arena/holdout-certify.test.mts +82 -0
- package/src/quant-arena/leak-audit.test.mts +79 -0
- package/src/quant-arena/leak-audit.ts +95 -0
- package/src/quant-arena/make-fixtures.mts +161 -0
- package/src/quant-arena/multiplicity.test.mts +68 -0
- package/src/quant-arena/multiplicity.ts +87 -0
- package/src/quant-arena/nautilus-certify.ts +31 -0
- package/src/quant-arena/oms.ts +90 -0
- package/src/quant-arena/profiles/quant-researcher.profile.json +7 -0
- package/src/quant-arena/python/pyproject.toml +8 -0
- package/src/quant-arena/python/uv.lock +1297 -0
- package/src/quant-arena/python/vbt-worker.py +192 -0
- package/src/quant-arena/quant-loop.mts +813 -0
- package/src/quant-arena/quant-loop.test.mts +75 -0
- package/src/quant-arena/strategies/buy-hold-index/strategy.ts +11 -0
- package/src/quant-arena/strategies/equal-weight/strategy.ts +20 -0
- package/src/quant-arena/strategies/sma-crossover/strategy.ts +42 -0
- package/src/quant-arena/types.ts +133 -0
- package/src/quant-arena/vbt-client.ts +321 -0
- package/src/quant-arena/vbt-parity.test.mts +183 -0
- package/src/quant-arena/windows.test.mts +45 -0
- package/src/quant-arena/windows.ts +54 -0
- package/src/rollout-ledger/backfill-swe-arena.mts +606 -0
- package/src/rollout-ledger/backfill-swe-arena.test.mts +338 -0
- package/src/rollout-ledger/settle-capture.mts +442 -0
- package/src/rollout-ledger/settle-capture.test.mts +270 -0
- package/src/stream-observe.py +45 -0
- package/src/stream-observe.tpl.html +247 -0
- package/src/supervisor-arena.mts +816 -0
- package/src/swe-arena/activation.mts +228 -0
- package/src/swe-arena/activation.test.mts +303 -0
- package/src/swe-arena/analyze.ts +211 -0
- package/src/swe-arena/arms.ts +804 -0
- package/src/swe-arena/bootstrap-meta.mts +188 -0
- package/src/swe-arena/bootstrap-meta.test.mts +51 -0
- package/src/swe-arena/briefing.mts +217 -0
- package/src/swe-arena/briefing.test.mts +178 -0
- package/src/swe-arena/calibrate.ts +217 -0
- package/src/swe-arena/capabilities.mts +76 -0
- package/src/swe-arena/capabilities.test.mts +57 -0
- package/src/swe-arena/capacity.ts +194 -0
- package/src/swe-arena/cell-evidence.mts +437 -0
- package/src/swe-arena/cell-evidence.test.mts +248 -0
- package/src/swe-arena/diagnosis-ensemble.test.mts +210 -0
- package/src/swe-arena/diagnosis-ensemble.ts +520 -0
- package/src/swe-arena/execution.test.mts +1170 -0
- package/src/swe-arena/factory-command-container.ts +284 -0
- package/src/swe-arena/factory-judge-child.mts +228 -0
- package/src/swe-arena/factory.test.mts +643 -0
- package/src/swe-arena/fixtures/analyze.py +80 -0
- package/src/swe-arena/fixtures/excludes.txt +8 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +51 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +29 -0
- package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +64 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +48 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +29 -0
- package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +48 -0
- package/src/swe-arena/fixtures/factory/loops-28/calibration.md +47 -0
- package/src/swe-arena/fixtures/factory/loops-28/manifest.json +30 -0
- package/src/swe-arena/fixtures/factory/loops-28/spec.md +50 -0
- package/src/swe-arena/fixtures/gen1-salvage/README.md +45 -0
- package/src/swe-arena/fixtures/gen1-salvage/cand0-e6d7361.diff +116 -0
- package/src/swe-arena/fixtures/gen1-salvage/cand1-76a8590.diff +293 -0
- package/src/swe-arena/fixtures/holdout-preregister.log +12 -0
- package/src/swe-arena/fixtures/holdout.json +44 -0
- package/src/swe-arena/fixtures/instances.json +146 -0
- package/src/swe-arena/fixtures/ledger.jsonl +12 -0
- package/src/swe-arena/fixtures/patches/pallets__flask-5014.solo.patch +36 -0
- package/src/swe-arena/fixtures/patches/pydata__xarray-4687.sup.patch +33 -0
- package/src/swe-arena/fixtures/rejudge.jsonl +15 -0
- package/src/swe-arena/fixtures/rematch.jsonl +3 -0
- package/src/swe-arena/fixtures/rematch2.jsonl +3 -0
- package/src/swe-arena/fixtures/rematch3.jsonl +3 -0
- package/src/swe-arena/fixtures/run-report/README.md +43 -0
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +173 -0
- package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +100 -0
- package/src/swe-arena/fixtures/run-report/gen3-rollup.json +551 -0
- package/src/swe-arena/fixtures/run-report/gen3-rollup.md +64 -0
- package/src/swe-arena/fixtures/sup-journal-true.json +19 -0
- package/src/swe-arena/fixtures/verify/astropy__astropy-13033.sh +48 -0
- package/src/swe-arena/fixtures/verify/django__django-11532.sh +50 -0
- package/src/swe-arena/fixtures/verify/matplotlib__matplotlib-20826.sh +76 -0
- package/src/swe-arena/fixtures/verify/pydata__xarray-4687.sh +44 -0
- package/src/swe-arena/fixtures/verify/pytest-dev__pytest-6197.sh +32 -0
- package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh +51 -0
- package/src/swe-arena/fixtures/worker-tokens.json +42 -0
- package/src/swe-arena/fixtures.ts +237 -0
- package/src/swe-arena/gepa-seat.mts +583 -0
- package/src/swe-arena/gepa-seat.test.mts +635 -0
- package/src/swe-arena/holdout-certify.mts +408 -0
- package/src/swe-arena/holdout-certify.test.mts +160 -0
- package/src/swe-arena/judge-child.mts +37 -0
- package/src/swe-arena/ledger-orphans.mts +77 -0
- package/src/swe-arena/ledger-orphans.test.mts +147 -0
- package/src/swe-arena/lineage-record.mts +164 -0
- package/src/swe-arena/lineage-record.test.mts +115 -0
- package/src/swe-arena/manifest.mts +293 -0
- package/src/swe-arena/manifest.test.mts +169 -0
- package/src/swe-arena/materialize.ts +142 -0
- package/src/swe-arena/outer-loop.mts +2795 -0
- package/src/swe-arena/outer-loop.test.mts +696 -0
- package/src/swe-arena/parity.test.mts +87 -0
- package/src/swe-arena/premeasured-from-cells.mts +281 -0
- package/src/swe-arena/premeasured-from-cells.test.mts +180 -0
- package/src/swe-arena/proc.test.mts +174 -0
- package/src/swe-arena/proc.ts +260 -0
- package/src/swe-arena/profiles/default-author.profile.json +4 -0
- package/src/swe-arena/proposer-fanout.mts +770 -0
- package/src/swe-arena/proposer-fanout.test.mts +619 -0
- package/src/swe-arena/proposer-provenance.mts +177 -0
- package/src/swe-arena/proposer-provenance.test.mts +106 -0
- package/src/swe-arena/reconcile.ts +0 -0
- package/src/swe-arena/replay.mts +183 -0
- package/src/swe-arena/replay.test.mts +300 -0
- package/src/swe-arena/run-experiment.mts +727 -0
- package/src/swe-arena/run-report.mts +75 -0
- package/src/swe-arena/run-supervisor.mjs +297 -0
- package/src/swe-arena/run-supervisor.test.mts +500 -0
- package/src/swe-arena/score-split.mts +140 -0
- package/src/swe-arena/score-split.test.mts +123 -0
- package/src/swe-arena/serialized-judge.ts +414 -0
- package/src/swe-arena/types.ts +218 -0
- package/src/swe-code-improve.mts +328 -0
- package/src/swe-emit-patch.mts +104 -0
- package/src/swe-improve.mts +232 -0
- package/src/swe-jail.ts +2 -2
- package/src/swe-local-proof.mts +169 -0
- package/src/swe-repro-calibrate.mts +446 -0
- package/src/swe-stream.mts +1497 -0
- package/src/swe-structural.mts +245 -837
- package/dist/chunk-LRRD7NAG.js.map +0 -1
- package/dist/chunk-PPYSEKFM.js.map +0 -1
- /package/dist/{chunk-CKUVRZ2T.js.map → chunk-3U5TXJZS.js.map} +0 -0
- /package/dist/{chunk-YCGY7UIZ.js.map → chunk-7GRVHU22.js.map} +0 -0
- /package/dist/{chunk-Z7ML6L77.js.map → chunk-HWST3SED.js.map} +0 -0
- /package/dist/{chunk-SYDW647C.js.map → chunk-IA2FBTWC.js.map} +0 -0
- /package/dist/{chunk-R67DFVLO.js.map → chunk-IFVINJ4B.js.map} +0 -0
- /package/dist/{chunk-R36V2VP7.js.map → chunk-IZ5M6OAC.js.map} +0 -0
- /package/dist/{chunk-ODT47UAY.js.map → chunk-K3BQGZCT.js.map} +0 -0
- /package/dist/{chunk-IFAV6KEM.js.map → chunk-KP5KD6EN.js.map} +0 -0
- /package/dist/{chunk-ZEWMTR5M.js.map → chunk-MQMRLGOG.js.map} +0 -0
- /package/dist/{chunk-TSWPNOYM.js.map → chunk-NQG5XDSB.js.map} +0 -0
- /package/dist/{chunk-7WSD27QQ.js.map → chunk-PB64GYIG.js.map} +0 -0
- /package/dist/{chunk-HBSWHQNJ.js.map → chunk-RCYQEFNX.js.map} +0 -0
- /package/dist/{chunk-J3KDJNX2.js.map → chunk-RH5F53JT.js.map} +0 -0
- /package/dist/{chunk-UAIOHCUK.js.map → chunk-SFLA7OH3.js.map} +0 -0
- /package/dist/{chunk-Y6O2OCUO.js.map → chunk-SHM6MRRF.js.map} +0 -0
- /package/dist/{chunk-KDIKRJGB.js.map → chunk-SHYIRB7I.js.map} +0 -0
- /package/dist/{chunk-HHXFIHXC.js.map → chunk-SVR2LKYI.js.map} +0 -0
- /package/dist/{chunk-5SBJCB6W.js.map → chunk-V7AEBY6U.js.map} +0 -0
- /package/dist/{chunk-2PVVP7GN.js.map → chunk-XKEFIFIC.js.map} +0 -0
- /package/dist/{chunk-JRWWGMK7.js.map → chunk-XYA4XSNU.js.map} +0 -0
- /package/dist/{chunk-X5YKXC6V.js.map → chunk-YSMEKBTD.js.map} +0 -0
- /package/dist/{chunk-2XU6OGEN.js.map → chunk-Z4TZ76N7.js.map} +0 -0
|
@@ -0,0 +1,770 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gen-3 proposer fan-out — N configured proposers author candidates
|
|
3
|
+
* CONCURRENTLY, each as an `AgentProfile`-pinned harness invocation in its own
|
|
4
|
+
* scratch loops worktree, followed by a CHEAP PRE-FILTER so only survivors
|
|
5
|
+
* reach the expensive full evaluation (reps × instances arm cells).
|
|
6
|
+
*
|
|
7
|
+
* Shape (gen3_design, supervisor-lab/.evolve/state.json):
|
|
8
|
+
* stage 1 — parallel authoring. The improvement driver calls
|
|
9
|
+
* `generate()` once per candidate index, serially. The FIRST call of a
|
|
10
|
+
* generation triggers the fan-out barrier: every proposer authors at once
|
|
11
|
+
* (Promise.all) in scratch worktrees created from the incumbent commit;
|
|
12
|
+
* later calls consume the finished outcomes and apply each proposer's
|
|
13
|
+
* patch onto the driver-owned candidate worktree. Authoring parallelism is
|
|
14
|
+
* free; eval bandwidth stays the binding resource.
|
|
15
|
+
* stage 2 — pre-filter per candidate BEFORE arm spend: change-space check +
|
|
16
|
+
* `tsc` run inside the authoring loop's verifier (existing), plus ONE
|
|
17
|
+
* smoke cell (cheapest instance, 1 rep) through the injected
|
|
18
|
+
* `SmokeRunner`. Killed candidates never become surfaces (the driver
|
|
19
|
+
* discards their worktree — zero arm cells); each kill is recorded and
|
|
20
|
+
* drained into a staircase dot with verdict `rejected-prefilter`.
|
|
21
|
+
* stage 3 — survivors proceed through the unchanged pinned-baseline
|
|
22
|
+
* reps-fail-closed gate.
|
|
23
|
+
*
|
|
24
|
+
* Proposer identity: each proposer is an `AgentProfile` invocation
|
|
25
|
+
* (`agenticGenerator`'s `profile` option → `harnessInvocation`, which threads
|
|
26
|
+
* `prompt.systemPrompt`/`instructions` into the composed prompt and
|
|
27
|
+
* `model.default` into the harness's `-m` flag — verified supported for the
|
|
28
|
+
* claude CLI; a bare profile is byte-identical to the legacy prompt-only
|
|
29
|
+
* invocation). Profile `resources` are only materialized on the reproducible
|
|
30
|
+
* Codex path, so a non-codex proposer with resources fails loud here instead
|
|
31
|
+
* of silently dropping them.
|
|
32
|
+
*
|
|
33
|
+
* NOTE on the import cycle with outer-loop.mts: outer-loop imports this
|
|
34
|
+
* module's generator factory; this module imports outer-loop's change-space +
|
|
35
|
+
* verifier primitives. Every cross-reference is inside a function body (never
|
|
36
|
+
* at module-evaluation time), which ESM resolves correctly.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { createHash } from 'node:crypto'
|
|
40
|
+
import { readFileSync } from 'node:fs'
|
|
41
|
+
import { mkdir, readFile as readFilePromise, rm, writeFile } from 'node:fs/promises'
|
|
42
|
+
import { join } from 'node:path'
|
|
43
|
+
import { fileURLToPath } from 'node:url'
|
|
44
|
+
import {
|
|
45
|
+
agenticGenerator,
|
|
46
|
+
type CandidateGenerator,
|
|
47
|
+
type AgenticGeneratorShotExecution,
|
|
48
|
+
type AgenticGeneratorShotReceipt,
|
|
49
|
+
} from '@tangle-network/agent-runtime'
|
|
50
|
+
import { runLocalHarness } from '@tangle-network/agent-runtime/mcp'
|
|
51
|
+
import type { AgentProfile } from '@tangle-network/agent-interface'
|
|
52
|
+
import type { AnalystFinding, CostLedgerHandle } from '@tangle-network/agent-eval'
|
|
53
|
+
import {
|
|
54
|
+
changeSpaceViolations,
|
|
55
|
+
loopsCandidateVerifier,
|
|
56
|
+
proposerShotEnv,
|
|
57
|
+
purgeIgnoredArtifacts,
|
|
58
|
+
round4BuildPrompt,
|
|
59
|
+
type OuterLoopConfig,
|
|
60
|
+
} from './outer-loop.mts'
|
|
61
|
+
import {
|
|
62
|
+
ACTIVATION_PREDICATE_RELPATH,
|
|
63
|
+
activationPredicateInstruction,
|
|
64
|
+
parseActivationPredicate,
|
|
65
|
+
} from './activation.mts'
|
|
66
|
+
import { briefingPromptSection, type BriefingContext } from './briefing.mts'
|
|
67
|
+
import { gepaSeatAuthor, isGepaSeat, validateGepaSeat, type GepaMethodFactory } from './gepa-seat.mts'
|
|
68
|
+
import type { ScoreSplit } from './score-split.mts'
|
|
69
|
+
import { run, runOk } from './proc.ts'
|
|
70
|
+
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
// Config types.
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
export interface ProposerSpec {
|
|
76
|
+
/** Unique short name — becomes the candidate's staircase `label`. */
|
|
77
|
+
name: string
|
|
78
|
+
/** Path to an `AgentProfile` JSON. Absolute, or relative to this module's
|
|
79
|
+
* `profiles/` directory. Omitted = bare profile (legacy invocation). */
|
|
80
|
+
profile?: string
|
|
81
|
+
/** Required for harness-authored seats. ABSENT on a GEN-6 engine seat
|
|
82
|
+
* (`engine` set) — enforced both ways at generator construction. */
|
|
83
|
+
harness?: 'claude' | 'codex' | 'opencode'
|
|
84
|
+
/** GEN-4 pinned model id, threaded to the harness CLI as `-m <model>` via
|
|
85
|
+
* the author profile's `model.default` (harnessInvocation maps it for all
|
|
86
|
+
* three harnesses). Unset = the CLI's own resolved model (its login/settings
|
|
87
|
+
* default) — recorded per run by the proposer-provenance capture, so the
|
|
88
|
+
* seat identity is pinned in provenance even when the flag is absent. */
|
|
89
|
+
model?: string
|
|
90
|
+
/** GEN-4 merge seat: this proposer's task is to MERGE the configured
|
|
91
|
+
* Pareto parents' diffs into one coherent surface (see
|
|
92
|
+
* `mergeAuthorPrompt`). Requires ≥2 materialized parents — enforced at
|
|
93
|
+
* generator construction. */
|
|
94
|
+
merge?: boolean
|
|
95
|
+
/** Free-text authoring lens appended to the task prompt (e.g. a mechanics
|
|
96
|
+
* or reviewer stance). Never overrides the protocol change-space text. */
|
|
97
|
+
lens?: string
|
|
98
|
+
/** Which diagnosis findings this proposer sees. Protocol/steering and
|
|
99
|
+
* raw-trace-context findings always pass through. Default 'all'. */
|
|
100
|
+
diagnosisSlice?: 'all' | 'mechanics' | 'prompts'
|
|
101
|
+
/** GEN-6 GEPA seat: this seat is an ENGINE invocation (agent-eval's
|
|
102
|
+
* external-GEPA adapter), not a harness CLI. `gepa` = one bounded engine
|
|
103
|
+
* run; `omni` = GEPA's published best-of-then-continue shape. Validation +
|
|
104
|
+
* authoring live in gepa-seat.mts. */
|
|
105
|
+
engine?: 'gepa' | 'omni'
|
|
106
|
+
/** GEN-6 GEPA seat: the ONE repo-relative change-space file GEPA optimizes
|
|
107
|
+
* as a string; the rest of the loops repo stays at the incumbent commit. */
|
|
108
|
+
surface?: string
|
|
109
|
+
/** GEN-6 GEPA seat: total inner-evaluation budget (each inner call is one
|
|
110
|
+
* real smoke arm cell). Default 10. */
|
|
111
|
+
maxMetricCalls?: number
|
|
112
|
+
/** GEN-6 GEPA seat: requested GEPA proposer spend cap in USD (the adapter
|
|
113
|
+
* reports GEPA's own model/CLI spend without an agent-eval receipt). */
|
|
114
|
+
maxProposerCostUsd?: number
|
|
115
|
+
/** GEN-6 GEPA seat: python executable for the bridge. Default 'python3'. */
|
|
116
|
+
python?: string
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface PrefilterConfig {
|
|
120
|
+
enabled: boolean
|
|
121
|
+
/** Instance for the single smoke cell: an explicit iid (e.g.
|
|
122
|
+
* 'pallets__flask-5014' once its verify fixture is committed) or
|
|
123
|
+
* 'cheapest-of-set' — the improvement-set instance with the smallest
|
|
124
|
+
* measured baseline wall time. */
|
|
125
|
+
smokeInstance: string
|
|
126
|
+
/** When true the smoke cell must RESOLVE to survive (right for a designated
|
|
127
|
+
* easy instance). Default false: the mechanism bar — the arm must complete,
|
|
128
|
+
* deliver a non-empty patch, and draw a conclusive judge verdict — so a
|
|
129
|
+
* hard cheapest-of-set instance cannot kill every candidate. */
|
|
130
|
+
requireResolved?: boolean
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export interface SmokeVerdict {
|
|
134
|
+
iid: string
|
|
135
|
+
pass: boolean
|
|
136
|
+
reason: string
|
|
137
|
+
resolved: boolean | null
|
|
138
|
+
patchLines: number
|
|
139
|
+
wallS: number
|
|
140
|
+
/** Committed verify fixture passed (GEN-6: the GEPA seat's inner-score
|
|
141
|
+
* tiebreak). Absent on errored smokes and pre-gen-6 records. */
|
|
142
|
+
verifyPass?: boolean
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type SmokeRunner = (args: {
|
|
146
|
+
scratchPath: string
|
|
147
|
+
generation: number
|
|
148
|
+
proposer: ProposerSpec
|
|
149
|
+
costLedger?: CostLedgerHandle
|
|
150
|
+
}) => Promise<SmokeVerdict>
|
|
151
|
+
|
|
152
|
+
export interface PrefilterKill {
|
|
153
|
+
generation: number
|
|
154
|
+
candidateIndex: number
|
|
155
|
+
proposer: string
|
|
156
|
+
harness: ProposerSpec['harness']
|
|
157
|
+
stage: 'change-space' | 'activation-predicate' | 'smoke'
|
|
158
|
+
reason: string
|
|
159
|
+
diffSha256: string | null
|
|
160
|
+
/** Persisted kill-forensics patch under `<outDir>/proposer-patches/`. */
|
|
161
|
+
patchPath: string | null
|
|
162
|
+
smoke?: SmokeVerdict
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
// Author profiles.
|
|
167
|
+
// ---------------------------------------------------------------------------
|
|
168
|
+
|
|
169
|
+
export const PROFILES_DIR = fileURLToPath(new URL('./profiles', import.meta.url))
|
|
170
|
+
|
|
171
|
+
/** Load + validate a proposer's `AgentProfile`. Fail-closed on resources for
|
|
172
|
+
* non-codex harnesses: `agenticGenerator` only materializes profile resource
|
|
173
|
+
* files on the reproducible Codex path, so accepting them here would drop
|
|
174
|
+
* them silently. */
|
|
175
|
+
export function loadAuthorProfile(spec: ProposerSpec): AgentProfile | undefined {
|
|
176
|
+
if (!spec.profile) return undefined
|
|
177
|
+
const path = spec.profile.startsWith('/') ? spec.profile : join(PROFILES_DIR, spec.profile)
|
|
178
|
+
const profile = JSON.parse(readFileSync(path, 'utf8')) as AgentProfile
|
|
179
|
+
if (typeof profile.name !== 'string' || profile.name.length === 0) {
|
|
180
|
+
throw new Error(`proposer ${spec.name}: profile ${path} has no name`)
|
|
181
|
+
}
|
|
182
|
+
if (profile.resources && spec.harness !== 'codex') {
|
|
183
|
+
throw new Error(
|
|
184
|
+
`proposer ${spec.name}: profile ${path} declares resources, which only materialize on the ` +
|
|
185
|
+
`reproducible codex path — the ${spec.harness} harness would silently drop them`,
|
|
186
|
+
)
|
|
187
|
+
}
|
|
188
|
+
return profile
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** The gen-2 author, codified: one bare-profile claude proposer. */
|
|
192
|
+
export function defaultProposers(): ProposerSpec[] {
|
|
193
|
+
return [{ name: 'default-author', profile: 'default-author.profile.json', harness: 'claude' }]
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** The profile the author shot actually runs: the loaded profile (if any) with
|
|
197
|
+
* the spec's PINNED MODEL merged into `model.default` — harnessInvocation
|
|
198
|
+
* turns that into the CLI's `-m <model>` flag (claude/codex/opencode all map
|
|
199
|
+
* it). Without a pinned model this is byte-identical to `loadAuthorProfile`,
|
|
200
|
+
* so gen-3 seats keep their exact invocation. A pinned model with no profile
|
|
201
|
+
* path synthesizes a minimal named profile carrying only the pin. */
|
|
202
|
+
export function resolveAuthorProfile(spec: ProposerSpec): AgentProfile | undefined {
|
|
203
|
+
const profile = loadAuthorProfile(spec)
|
|
204
|
+
if (!spec.model) return profile
|
|
205
|
+
const base: AgentProfile = profile ?? { name: `${spec.name}-pinned` }
|
|
206
|
+
return { ...base, model: { ...base.model, default: spec.model } }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// GEN-4 Pareto parents — cross-run seeding at the buildPrompt seam.
|
|
211
|
+
//
|
|
212
|
+
// The LIB's own `ctx.paretoParents` (runOptimization → SurfaceProposer) only
|
|
213
|
+
// accumulates surfaces scored WITHIN one run: the frontier starts from this
|
|
214
|
+
// run's baseline and generations, and a prior campaign's surfaces cannot be
|
|
215
|
+
// injected without replaying that campaign's runDir + cost ledger (the cached
|
|
216
|
+
// cells refuse to load without their ledger receipts). Gen-4 runs in a fresh
|
|
217
|
+
// outDir, so the gen-3 winners are seeded HERE — their diffs + per-instance
|
|
218
|
+
// results enter every author's task prompt, and the merge seat gets both
|
|
219
|
+
// diffs as its explicit merge input. This is our seam, not the lib path;
|
|
220
|
+
// noted in the gen-4 config docs.
|
|
221
|
+
// ---------------------------------------------------------------------------
|
|
222
|
+
|
|
223
|
+
/** One prior-run frontier member, as configured (commits live in loopsRepo). */
|
|
224
|
+
export interface ParetoParentSeed {
|
|
225
|
+
/** Loops commit of the parent candidate (must exist in `loopsRepo`). */
|
|
226
|
+
commit: string
|
|
227
|
+
/** Staircase label, e.g. 'default-author'. */
|
|
228
|
+
label: string
|
|
229
|
+
/** Instances the parent resolved under the fail-closed all-reps rule. */
|
|
230
|
+
resolvedInstances: string[]
|
|
231
|
+
/** Free-text evidence note (discordant replicates, mechanism summary). */
|
|
232
|
+
note?: string
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** A seed materialized against the loops repo: the parent's full diff. */
|
|
236
|
+
export interface ParetoParentContext extends ParetoParentSeed {
|
|
237
|
+
diff: string
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export const PARENT_DIFF_MAX_CHARS = 60_000
|
|
241
|
+
|
|
242
|
+
/** Materialize parent seeds: verify each commit exists in `loopsRepo` and
|
|
243
|
+
* capture its full diff (`git show`). Fails loud on a missing commit — a
|
|
244
|
+
* silently absent parent would turn the merge seat into a no-op. */
|
|
245
|
+
export async function materializeParetoParents(
|
|
246
|
+
loopsRepo: string,
|
|
247
|
+
seeds: ParetoParentSeed[],
|
|
248
|
+
maxDiffChars = PARENT_DIFF_MAX_CHARS,
|
|
249
|
+
): Promise<ParetoParentContext[]> {
|
|
250
|
+
const parents: ParetoParentContext[] = []
|
|
251
|
+
for (const seed of seeds) {
|
|
252
|
+
const exists = await run('git', ['-C', loopsRepo, 'cat-file', '-e', `${seed.commit}^{commit}`])
|
|
253
|
+
if (exists.code !== 0) {
|
|
254
|
+
throw new Error(`pareto parent ${seed.label}: commit ${seed.commit} not found in ${loopsRepo}`)
|
|
255
|
+
}
|
|
256
|
+
const show = await runOk('git', ['-C', loopsRepo, 'show', '--no-color', seed.commit])
|
|
257
|
+
const diff =
|
|
258
|
+
show.stdout.length > maxDiffChars
|
|
259
|
+
? `${show.stdout.slice(0, maxDiffChars)}\n[... diff truncated at ${maxDiffChars} chars ...]`
|
|
260
|
+
: show.stdout
|
|
261
|
+
parents.push({ ...seed, diff })
|
|
262
|
+
}
|
|
263
|
+
return parents
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function parentEvidenceLine(parent: ParetoParentContext): string {
|
|
267
|
+
const resolved = parent.resolvedInstances.length > 0 ? parent.resolvedInstances.join(', ') : 'none'
|
|
268
|
+
return `${parent.label} (${parent.commit.slice(0, 10)}) — resolved: ${resolved}${parent.note ? `; ${parent.note}` : ''}`
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** The parents section appended to every NON-merge author's prompt: measured
|
|
272
|
+
* evidence of what worked, never an instruction to copy. */
|
|
273
|
+
export function parentsPromptSection(parents: ParetoParentContext[]): string {
|
|
274
|
+
const lines: string[] = [
|
|
275
|
+
'PARETO PARENTS — the prior generation’s frontier candidates, measured on the SAME 6-instance set.',
|
|
276
|
+
'Each beat the baseline on different instances; treat their diffs as measured evidence of what works.',
|
|
277
|
+
'You may build on either (or both), but your candidate is measured on the full set — do not blindly copy.',
|
|
278
|
+
]
|
|
279
|
+
for (const parent of parents) {
|
|
280
|
+
lines.push('', `--- parent: ${parentEvidenceLine(parent)}`, parent.diff.trimEnd())
|
|
281
|
+
}
|
|
282
|
+
return lines.join('\n')
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** The merge seat's task prompt: the change-space contract + findings context
|
|
286
|
+
* stay (round4BuildPrompt), and the TASK is replaced with an explicit
|
|
287
|
+
* coherent-union merge of the parents' diffs. */
|
|
288
|
+
export function mergeAuthorPrompt(
|
|
289
|
+
args: { report: unknown; findings: Array<Record<string, unknown>> },
|
|
290
|
+
spec: ProposerSpec,
|
|
291
|
+
parents: ParetoParentContext[],
|
|
292
|
+
): string {
|
|
293
|
+
if (parents.length < 2) {
|
|
294
|
+
throw new Error(`merge proposer ${spec.name}: needs >=2 materialized pareto parents, got ${parents.length}`)
|
|
295
|
+
}
|
|
296
|
+
const lines: string[] = [
|
|
297
|
+
round4BuildPrompt(args),
|
|
298
|
+
'',
|
|
299
|
+
`YOUR TASK (${spec.name} — MERGE SEAT):`,
|
|
300
|
+
'The prior generation produced the frontier candidates below. Each beat the baseline on DIFFERENT',
|
|
301
|
+
'instances, so their lessons are complementary. Produce ONE coherent surface that is the UNION of the',
|
|
302
|
+
'parent diffs:',
|
|
303
|
+
...parents.map((p) => ` - ${parentEvidenceLine(p)}`),
|
|
304
|
+
'',
|
|
305
|
+
'Merge rules:',
|
|
306
|
+
'- Apply BOTH parents’ behaviors. Where the diffs touch the same file/section (e.g. the supervisor',
|
|
307
|
+
' reviewer bullet or the worker self-test rules), resolve the conflict by KEEPING BOTH BEHAVIORS —',
|
|
308
|
+
' write one merged passage that carries every constraint from each side, never by dropping one side.',
|
|
309
|
+
'- Keep each parent’s mechanical changes intact (code paths, exported helpers, threading) alongside the',
|
|
310
|
+
' other parent’s prompt/goal-authoring changes.',
|
|
311
|
+
'- Do NOT invent new mechanisms beyond the union; the smallest coherent merge wins.',
|
|
312
|
+
'- The merged surface must typecheck and stay inside the declared change-space above.',
|
|
313
|
+
'',
|
|
314
|
+
'Parent diffs (full):',
|
|
315
|
+
]
|
|
316
|
+
for (const parent of parents) {
|
|
317
|
+
lines.push('', `=== parent ${parent.label} (${parent.commit.slice(0, 10)}) ===`, parent.diff.trimEnd())
|
|
318
|
+
}
|
|
319
|
+
return lines.join('\n')
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// ---------------------------------------------------------------------------
|
|
323
|
+
// Diagnosis slicing + prompt lens.
|
|
324
|
+
// ---------------------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
const PROMPTY = /prompt|instruction|wording|message text|phrasing/i
|
|
327
|
+
|
|
328
|
+
/** Findings that must reach EVERY proposer regardless of slice: the protocol
|
|
329
|
+
* steering finding and the raw-trace-context findings (they are contract +
|
|
330
|
+
* grounding, not diagnosis). */
|
|
331
|
+
function isSteeringOrRawTrace(f: Record<string, unknown>): boolean {
|
|
332
|
+
return (
|
|
333
|
+
f.analyst_id === 'round4-protocol' ||
|
|
334
|
+
f.analyst_id === 'raw-trace-distiller' ||
|
|
335
|
+
f.area === 'raw-trace-context' ||
|
|
336
|
+
f.area === 'constraint'
|
|
337
|
+
)
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** Slice the diagnosis findings for one proposer. Pure. */
|
|
341
|
+
export function sliceFindings(findings: AnalystFinding[], slice: ProposerSpec['diagnosisSlice']): AnalystFinding[] {
|
|
342
|
+
if (slice === undefined || slice === 'all') return findings
|
|
343
|
+
return findings.filter((finding) => {
|
|
344
|
+
const f = finding as unknown as Record<string, unknown>
|
|
345
|
+
if (isSteeringOrRawTrace(f)) return true
|
|
346
|
+
const text = `${String(f.area ?? '')} ${String(f.subject ?? '')} ${String(f.claim ?? '')} ${String(f.recommended_action ?? '')}`
|
|
347
|
+
return slice === 'prompts' ? PROMPTY.test(text) : !PROMPTY.test(text)
|
|
348
|
+
})
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** GEN-5 prompt extensions shared by every seat (merge included): the
|
|
352
|
+
* MAP+TOOLBOX+PERMISSION briefing (briefing.mts) and the activation-predicate
|
|
353
|
+
* deliverable contract (activation.mts). */
|
|
354
|
+
export interface Gen5PromptExtras {
|
|
355
|
+
briefing?: BriefingContext
|
|
356
|
+
activationGate?: boolean
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function appendGen5Sections(prompt: string, extras: Gen5PromptExtras): string {
|
|
360
|
+
let out = prompt
|
|
361
|
+
if (extras.briefing) out = `${out}\n\n${briefingPromptSection(extras.briefing)}`
|
|
362
|
+
if (extras.activationGate) out = `${out}\n\n${activationPredicateInstruction()}`
|
|
363
|
+
return out
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** The proposer's task prompt: the shared round prompt plus this proposer's
|
|
367
|
+
* authoring lens (appended so the protocol change-space text stays intact),
|
|
368
|
+
* plus the gen-4 Pareto-parents section when parents are seeded, plus the
|
|
369
|
+
* gen-5 briefing/activation sections when configured. A merge-seat spec gets
|
|
370
|
+
* the dedicated merge prompt instead (gen-5 sections still apply). */
|
|
371
|
+
export function proposerBuildPrompt(
|
|
372
|
+
args: { report: unknown; findings: Array<Record<string, unknown>> },
|
|
373
|
+
spec: ProposerSpec,
|
|
374
|
+
parents: ParetoParentContext[] = [],
|
|
375
|
+
extras: Gen5PromptExtras = {},
|
|
376
|
+
): string {
|
|
377
|
+
if (spec.merge) return appendGen5Sections(mergeAuthorPrompt(args, spec, parents), extras)
|
|
378
|
+
let prompt = round4BuildPrompt(args)
|
|
379
|
+
if (spec.lens) prompt = `${prompt}\n\nYOUR AUTHORING LENS (${spec.name}):\n${spec.lens}`
|
|
380
|
+
if (parents.length > 0) prompt = `${prompt}\n\n${parentsPromptSection(parents)}`
|
|
381
|
+
return appendGen5Sections(prompt, extras)
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// ---------------------------------------------------------------------------
|
|
385
|
+
// Shared shot persistence + spend settlement (used by the fan-out authors AND
|
|
386
|
+
// the legacy single-proposer generator in outer-loop.mts — one implementation).
|
|
387
|
+
// ---------------------------------------------------------------------------
|
|
388
|
+
|
|
389
|
+
export function proposerShotHooks(opts: {
|
|
390
|
+
shotDir: string
|
|
391
|
+
harness: ProposerSpec['harness']
|
|
392
|
+
ledger: () => CostLedgerHandle | undefined
|
|
393
|
+
phase: () => string | undefined
|
|
394
|
+
}): (receipt: AgenticGeneratorShotReceipt, execution: AgenticGeneratorShotExecution | null) => Promise<void> {
|
|
395
|
+
return async (receipt, execution) => {
|
|
396
|
+
const tail = (s: string | undefined): string | null =>
|
|
397
|
+
s === undefined ? null : s.length > 20_000 ? s.slice(-20_000) : s
|
|
398
|
+
await mkdir(opts.shotDir, { recursive: true })
|
|
399
|
+
const name = `gen${receipt.generation ?? 'x'}-cand${receipt.candidateIndex ?? 'x'}-shot${receipt.shot}.json`
|
|
400
|
+
await writeFile(
|
|
401
|
+
join(opts.shotDir, name),
|
|
402
|
+
JSON.stringify({ receipt, stdoutTail: tail(execution?.stdout), stderrTail: tail(execution?.stderr) }, null, 2),
|
|
403
|
+
)
|
|
404
|
+
// Proposer-shot spend → the lib's run ledger. The generator only settles
|
|
405
|
+
// its own receipts on the codexReproducible path (costCallId non-null);
|
|
406
|
+
// the claude/opencode author path otherwise leaves every shot as $0 in
|
|
407
|
+
// the run's spend summary. Import the shot receipt's measured usage.
|
|
408
|
+
const ledger = opts.ledger()
|
|
409
|
+
if (ledger && receipt.costCallId === null && (receipt.usage || receipt.costUsdKnown)) {
|
|
410
|
+
const usage = receipt.usage
|
|
411
|
+
const paid = await ledger.runPaidCall({
|
|
412
|
+
channel: 'driver',
|
|
413
|
+
phase: opts.phase() ?? 'search.proposal',
|
|
414
|
+
actor: `proposer-shot:${opts.harness}`,
|
|
415
|
+
model: receipt.model ?? `${opts.harness}-cli`,
|
|
416
|
+
tags: {
|
|
417
|
+
generation: String(receipt.generation ?? -1),
|
|
418
|
+
candidateIndex: String(receipt.candidateIndex ?? -1),
|
|
419
|
+
shot: String(receipt.shot),
|
|
420
|
+
},
|
|
421
|
+
execute: async () => receipt,
|
|
422
|
+
receipt: () => ({
|
|
423
|
+
model: receipt.model ?? `${opts.harness}-cli`,
|
|
424
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
425
|
+
outputTokens: usage ? usage.outputTokens + usage.reasoningOutputTokens : 0,
|
|
426
|
+
...(usage ? { cachedTokens: usage.cachedInputTokens } : { usageUnknown: true }),
|
|
427
|
+
...(receipt.costUsdKnown && receipt.costUsd !== null ? { actualCostUsd: receipt.costUsd } : {}),
|
|
428
|
+
}),
|
|
429
|
+
})
|
|
430
|
+
if (!paid.succeeded) throw paid.error
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// ---------------------------------------------------------------------------
|
|
436
|
+
// Scratch authoring worktrees (no node_modules symlink — the verifier and the
|
|
437
|
+
// smoke runner link one in temporarily when they need it).
|
|
438
|
+
// ---------------------------------------------------------------------------
|
|
439
|
+
|
|
440
|
+
async function addAuthoringWorktree(loopsRepo: string, commit: string, dest: string): Promise<void> {
|
|
441
|
+
await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest])
|
|
442
|
+
await rm(dest, { recursive: true, force: true })
|
|
443
|
+
await run('git', ['-C', loopsRepo, 'worktree', 'prune'])
|
|
444
|
+
await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', dest, commit])
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
async function removeAuthoringWorktree(loopsRepo: string, dest: string): Promise<void> {
|
|
448
|
+
const res = await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest])
|
|
449
|
+
if (res.code !== 0) {
|
|
450
|
+
await rm(dest, { recursive: true, force: true })
|
|
451
|
+
await run('git', ['-C', loopsRepo, 'worktree', 'prune'])
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const sanitize = (s: string): string => s.replace(/[^a-zA-Z0-9_-]/g, '_')
|
|
456
|
+
|
|
457
|
+
// ---------------------------------------------------------------------------
|
|
458
|
+
// The fan-out generator.
|
|
459
|
+
// ---------------------------------------------------------------------------
|
|
460
|
+
|
|
461
|
+
/** One proposer's authoring + pre-filter outcome for a generation. */
|
|
462
|
+
interface AuthorOutcome {
|
|
463
|
+
proposer: ProposerSpec
|
|
464
|
+
applied: boolean
|
|
465
|
+
summary: string
|
|
466
|
+
/** Full staged diff (tracked edits + untracked deliverables) vs the
|
|
467
|
+
* incumbent commit. Null when not applied or killed. */
|
|
468
|
+
patch: string | null
|
|
469
|
+
kill: PrefilterKill | null
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Authoring seam — test-injectable. The default runs the proposer's
|
|
473
|
+
* `agenticGenerator` (profile-pinned harness invocation) in the scratch
|
|
474
|
+
* worktree. */
|
|
475
|
+
export type AuthorFn = (
|
|
476
|
+
proposer: ProposerSpec,
|
|
477
|
+
args: Parameters<CandidateGenerator['generate']>[0],
|
|
478
|
+
) => Promise<{ applied: boolean; summary: string }>
|
|
479
|
+
|
|
480
|
+
export interface FanOutDeps {
|
|
481
|
+
author?: AuthorFn
|
|
482
|
+
smokeRunner?: SmokeRunner
|
|
483
|
+
/** GEN-4: materialized Pareto parents seeded into every author's prompt (and
|
|
484
|
+
* the merge seat's explicit merge input). Empty/omitted = gen-3 behavior. */
|
|
485
|
+
parents?: ParetoParentContext[]
|
|
486
|
+
/** GEN-5: MAP+TOOLBOX briefing context threaded into every author prompt. */
|
|
487
|
+
briefing?: BriefingContext
|
|
488
|
+
/** GEN-6: the resolved PUBLIC smoke instance — the GEPA seat's inner
|
|
489
|
+
* evaluator target. Required (with `smokeRunner`) when a gepa seat is
|
|
490
|
+
* configured and no custom `author` is injected. */
|
|
491
|
+
smokeInstanceId?: string
|
|
492
|
+
/** GEN-6: the gen-5 score split; private instance ids never reach the GEPA
|
|
493
|
+
* bridge (gepa-seat.mts asserts, fail-closed). Null/omitted = no split. */
|
|
494
|
+
scoreSplit?: Pick<ScoreSplit, 'privateInstances'> | null
|
|
495
|
+
/** GEN-6 test seam: the adapter factory (default: checked dynamic import of
|
|
496
|
+
* agent-eval's gepaOptimizationMethod, loud when the install predates it). */
|
|
497
|
+
gepaMethodFactory?: GepaMethodFactory
|
|
498
|
+
log?: (msg: string) => void
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
export interface FanOutGenerator extends CandidateGenerator {
|
|
502
|
+
/** Prefilter kills recorded so far; draining clears the buffer. The caller
|
|
503
|
+
* turns each into a `rejected-prefilter` staircase dot. */
|
|
504
|
+
drainPrefilterKills(): PrefilterKill[]
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn {
|
|
508
|
+
// One agenticGenerator per proposer, created lazily and cached: each carries
|
|
509
|
+
// its own profile, lens prompt, and shot-receipt home.
|
|
510
|
+
const inners = new Map<string, CandidateGenerator>()
|
|
511
|
+
const ledgers = new Map<string, { ledger?: CostLedgerHandle; phase?: string }>()
|
|
512
|
+
// GEN-6: the GEPA seat authors through the agent-eval adapter, not a
|
|
513
|
+
// harness CLI. Its inner evaluator is the SAME injected smoke runner the
|
|
514
|
+
// pre-filter uses (presence enforced at generator construction).
|
|
515
|
+
let gepaAuthor: AuthorFn | undefined
|
|
516
|
+
return (proposer, args) => {
|
|
517
|
+
if (isGepaSeat(proposer)) {
|
|
518
|
+
gepaAuthor ??= gepaSeatAuthor(config, {
|
|
519
|
+
smokeRunner: deps.smokeRunner!,
|
|
520
|
+
smokeInstanceId: deps.smokeInstanceId!,
|
|
521
|
+
scoreSplit: deps.scoreSplit ?? null,
|
|
522
|
+
...(deps.gepaMethodFactory ? { methodFactory: deps.gepaMethodFactory } : {}),
|
|
523
|
+
...(deps.log ? { log: deps.log } : {}),
|
|
524
|
+
})
|
|
525
|
+
return gepaAuthor(proposer, args)
|
|
526
|
+
}
|
|
527
|
+
const harness = proposer.harness
|
|
528
|
+
if (harness === undefined) {
|
|
529
|
+
throw new Error(`proposer ${proposer.name}: harness is required for a non-engine seat`)
|
|
530
|
+
}
|
|
531
|
+
let inner = inners.get(proposer.name)
|
|
532
|
+
const slot = ledgers.get(proposer.name) ?? {}
|
|
533
|
+
ledgers.set(proposer.name, slot)
|
|
534
|
+
if (!inner) {
|
|
535
|
+
// GEN-4: the resolved profile carries the spec's pinned model
|
|
536
|
+
// (`model.default` → the harness CLI's `-m` flag).
|
|
537
|
+
const profile = resolveAuthorProfile(proposer)
|
|
538
|
+
inner = agenticGenerator({
|
|
539
|
+
harness,
|
|
540
|
+
...(profile ? { profile } : {}),
|
|
541
|
+
timeoutMs: config.proposerTimeoutMs,
|
|
542
|
+
buildPrompt: (a) =>
|
|
543
|
+
proposerBuildPrompt(
|
|
544
|
+
a as unknown as { report: unknown; findings: Array<Record<string, unknown>> },
|
|
545
|
+
proposer,
|
|
546
|
+
deps.parents ?? [],
|
|
547
|
+
{
|
|
548
|
+
...(deps.briefing ? { briefing: deps.briefing } : {}),
|
|
549
|
+
...(config.activationGate === true ? { activationGate: true } : {}),
|
|
550
|
+
},
|
|
551
|
+
),
|
|
552
|
+
verify: loopsCandidateVerifier(config.loopsRepo),
|
|
553
|
+
runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(harness) }),
|
|
554
|
+
onShotCompleted: proposerShotHooks({
|
|
555
|
+
shotDir: join(config.outDir, 'proposer-shots', sanitize(proposer.name)),
|
|
556
|
+
harness,
|
|
557
|
+
ledger: () => slot.ledger,
|
|
558
|
+
phase: () => slot.phase,
|
|
559
|
+
}),
|
|
560
|
+
})
|
|
561
|
+
inners.set(proposer.name, inner)
|
|
562
|
+
}
|
|
563
|
+
slot.ledger = args.costLedger
|
|
564
|
+
slot.phase = args.costPhase
|
|
565
|
+
return inner.generate(args)
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/**
|
|
570
|
+
* Build the fan-out `CandidateGenerator` for `config.proposers`. The driver
|
|
571
|
+
* still owns every candidate worktree; this generator authors in its OWN
|
|
572
|
+
* scratch worktrees (in parallel, one per proposer) and applies each
|
|
573
|
+
* surviving proposer's patch onto the driver worktree for that candidate
|
|
574
|
+
* index. `populationSize` must equal `proposers.length` — enforced by the
|
|
575
|
+
* caller (runRound) and re-checked per call here.
|
|
576
|
+
*/
|
|
577
|
+
export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps = {}): FanOutGenerator {
|
|
578
|
+
const proposers = config.proposers ?? []
|
|
579
|
+
if (proposers.length === 0) throw new Error('fanOutLoopsGenerator: config.proposers is empty')
|
|
580
|
+
const names = new Set(proposers.map((p) => p.name))
|
|
581
|
+
if (names.size !== proposers.length) throw new Error('fanOutLoopsGenerator: duplicate proposer names')
|
|
582
|
+
const mergeSeats = proposers.filter((p) => p.merge === true)
|
|
583
|
+
if (mergeSeats.length > 0 && (deps.parents ?? []).length < 2) {
|
|
584
|
+
throw new Error(
|
|
585
|
+
`fanOutLoopsGenerator: merge proposer(s) ${mergeSeats.map((p) => p.name).join(', ')} configured but ` +
|
|
586
|
+
`only ${(deps.parents ?? []).length} pareto parent(s) materialized — a merge seat needs >=2`,
|
|
587
|
+
)
|
|
588
|
+
}
|
|
589
|
+
// GEN-6: engine seats are validated fail-closed at construction, and the
|
|
590
|
+
// default author path requires the pre-filter smoke runner — it IS the GEPA
|
|
591
|
+
// seat's inner evaluator (spec: score = smoke resolve + verify-pass
|
|
592
|
+
// tiebreak). A custom injected `author` owns its own evaluator.
|
|
593
|
+
const gepaSeats = proposers.filter(isGepaSeat)
|
|
594
|
+
for (const seat of gepaSeats) validateGepaSeat(seat)
|
|
595
|
+
for (const seat of proposers) {
|
|
596
|
+
if (!isGepaSeat(seat) && seat.harness === undefined) {
|
|
597
|
+
throw new Error(`fanOutLoopsGenerator: proposer ${seat.name} has neither a harness nor an engine`)
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
if (gepaSeats.length > 0 && deps.author === undefined && (deps.smokeRunner === undefined || deps.smokeInstanceId === undefined)) {
|
|
601
|
+
throw new Error(
|
|
602
|
+
`fanOutLoopsGenerator: gepa seat(s) ${gepaSeats.map((p) => p.name).join(', ')} need the pre-filter smoke ` +
|
|
603
|
+
'cell as their inner evaluator — enable config.prefilter and provide smokeRunner + smokeInstanceId',
|
|
604
|
+
)
|
|
605
|
+
}
|
|
606
|
+
const author = deps.author ?? defaultAuthor(config, deps)
|
|
607
|
+
const log = deps.log ?? (() => {})
|
|
608
|
+
const kills: PrefilterKill[] = []
|
|
609
|
+
const byGeneration = new Map<number, Promise<AuthorOutcome[]>>()
|
|
610
|
+
|
|
611
|
+
const runFanOut = async (
|
|
612
|
+
generation: number,
|
|
613
|
+
args: Parameters<CandidateGenerator['generate']>[0],
|
|
614
|
+
): Promise<AuthorOutcome[]> => {
|
|
615
|
+
const head = await runOk('git', ['-C', args.worktreePath, 'rev-parse', 'HEAD'])
|
|
616
|
+
const baseCommit = head.stdout.trim()
|
|
617
|
+
const patchesDir = join(config.outDir, 'proposer-patches')
|
|
618
|
+
await mkdir(patchesDir, { recursive: true })
|
|
619
|
+
log(`fan-out gen ${generation}: ${proposers.length} proposer(s) authoring in parallel from ${baseCommit.slice(0, 10)}`)
|
|
620
|
+
return Promise.all(
|
|
621
|
+
proposers.map(async (proposer, index): Promise<AuthorOutcome> => {
|
|
622
|
+
const scratch = join(config.outDir, 'proposer-wt', `gen${generation}-cand${index}-${sanitize(proposer.name)}`)
|
|
623
|
+
await addAuthoringWorktree(config.loopsRepo, baseCommit, scratch)
|
|
624
|
+
try {
|
|
625
|
+
const authored = await author(proposer, {
|
|
626
|
+
...args,
|
|
627
|
+
worktreePath: scratch,
|
|
628
|
+
findings: sliceFindings(args.findings, proposer.diagnosisSlice),
|
|
629
|
+
candidateIndex: index,
|
|
630
|
+
})
|
|
631
|
+
if (!authored.applied) {
|
|
632
|
+
log(`fan-out gen ${generation} ${proposer.name}: no verified candidate (applied=false)`)
|
|
633
|
+
return { proposer, applied: false, summary: authored.summary, patch: null, kill: null }
|
|
634
|
+
}
|
|
635
|
+
// Anti-tamper purge BEFORE the diff: ignored artifacts (dep installs)
|
|
636
|
+
// must never reach the candidate worktree or the change-space check.
|
|
637
|
+
await purgeIgnoredArtifacts(scratch)
|
|
638
|
+
await runOk('git', ['-C', scratch, 'add', '-A'])
|
|
639
|
+
const diff = (await runOk('git', ['-C', scratch, 'diff', '--cached'])).stdout
|
|
640
|
+
if (diff.trim().length === 0) {
|
|
641
|
+
return { proposer, applied: false, summary: authored.summary, patch: null, kill: null }
|
|
642
|
+
}
|
|
643
|
+
const namesOut = await runOk('git', ['-C', scratch, 'diff', '--cached', '--name-only'])
|
|
644
|
+
const changed = namesOut.stdout.split('\n').map((s) => s.trim()).filter(Boolean)
|
|
645
|
+
const diffSha256 = `sha256:${createHash('sha256').update(diff).digest('hex')}`
|
|
646
|
+
const patchPath = join(patchesDir, `gen${generation}-cand${index}-${sanitize(proposer.name)}.patch`)
|
|
647
|
+
await writeFile(patchPath, diff)
|
|
648
|
+
|
|
649
|
+
// Pre-filter stage A — change-space (defense-in-depth: the authoring
|
|
650
|
+
// verifier already enforced it shot-by-shot; the final diff rules).
|
|
651
|
+
const violations = changeSpaceViolations(changed)
|
|
652
|
+
if (violations.length > 0) {
|
|
653
|
+
const kill: PrefilterKill = {
|
|
654
|
+
generation,
|
|
655
|
+
candidateIndex: index,
|
|
656
|
+
proposer: proposer.name,
|
|
657
|
+
harness: proposer.harness,
|
|
658
|
+
stage: 'change-space',
|
|
659
|
+
reason: `out-of-space paths: ${violations.join(', ')}`,
|
|
660
|
+
diffSha256,
|
|
661
|
+
patchPath,
|
|
662
|
+
}
|
|
663
|
+
kills.push(kill)
|
|
664
|
+
log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
|
|
665
|
+
return { proposer, applied: false, summary: authored.summary, patch: null, kill }
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// Pre-filter stage A.5 — GEN-5 activation gate: a candidate without
|
|
669
|
+
// a parseable machine-checkable predicate is killed before any
|
|
670
|
+
// evaluation spend (the post-eval quarantine needs it to exist).
|
|
671
|
+
if (config.activationGate === true) {
|
|
672
|
+
const raw = await readFilePromise(join(scratch, ACTIVATION_PREDICATE_RELPATH), 'utf8').catch(() => null)
|
|
673
|
+
const parsed = raw === null ? null : parseActivationPredicate(raw)
|
|
674
|
+
if (parsed === null || !parsed.ok) {
|
|
675
|
+
const kill: PrefilterKill = {
|
|
676
|
+
generation,
|
|
677
|
+
candidateIndex: index,
|
|
678
|
+
proposer: proposer.name,
|
|
679
|
+
harness: proposer.harness,
|
|
680
|
+
stage: 'activation-predicate',
|
|
681
|
+
reason:
|
|
682
|
+
parsed === null
|
|
683
|
+
? `missing ${ACTIVATION_PREDICATE_RELPATH} (required activation predicate)`
|
|
684
|
+
: `invalid ${ACTIVATION_PREDICATE_RELPATH}: ${parsed.error}`,
|
|
685
|
+
diffSha256,
|
|
686
|
+
patchPath,
|
|
687
|
+
}
|
|
688
|
+
kills.push(kill)
|
|
689
|
+
log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
|
|
690
|
+
return { proposer, applied: false, summary: authored.summary, patch: null, kill }
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// Pre-filter stage B — one smoke cell before any full-arm spend.
|
|
695
|
+
if (config.prefilter?.enabled && deps.smokeRunner) {
|
|
696
|
+
const smoke = await deps.smokeRunner({
|
|
697
|
+
scratchPath: scratch,
|
|
698
|
+
generation,
|
|
699
|
+
proposer,
|
|
700
|
+
...(args.costLedger ? { costLedger: args.costLedger } : {}),
|
|
701
|
+
})
|
|
702
|
+
if (!smoke.pass) {
|
|
703
|
+
const kill: PrefilterKill = {
|
|
704
|
+
generation,
|
|
705
|
+
candidateIndex: index,
|
|
706
|
+
proposer: proposer.name,
|
|
707
|
+
harness: proposer.harness,
|
|
708
|
+
stage: 'smoke',
|
|
709
|
+
reason: smoke.reason,
|
|
710
|
+
diffSha256,
|
|
711
|
+
patchPath,
|
|
712
|
+
smoke,
|
|
713
|
+
}
|
|
714
|
+
kills.push(kill)
|
|
715
|
+
log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
|
|
716
|
+
return { proposer, applied: false, summary: authored.summary, patch: null, kill }
|
|
717
|
+
}
|
|
718
|
+
log(`prefilter pass gen ${generation} ${proposer.name}: ${smoke.reason}`)
|
|
719
|
+
}
|
|
720
|
+
return { proposer, applied: true, summary: authored.summary, patch: diff, kill: null }
|
|
721
|
+
} finally {
|
|
722
|
+
await removeAuthoringWorktree(config.loopsRepo, scratch)
|
|
723
|
+
}
|
|
724
|
+
}),
|
|
725
|
+
)
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
return {
|
|
729
|
+
kind: `gen3-fanout:${proposers.map((p) => `${p.name}@${p.engine ? `engine:${p.engine}` : p.harness}${p.model ? `:${p.model}` : ''}${p.merge ? ':merge' : ''}`).join('+')}`,
|
|
730
|
+
proposesWithoutFindings: true,
|
|
731
|
+
drainPrefilterKills() {
|
|
732
|
+
return kills.splice(0, kills.length)
|
|
733
|
+
},
|
|
734
|
+
async generate(args) {
|
|
735
|
+
const generation = args.generation ?? 0
|
|
736
|
+
const index = args.candidateIndex ?? 0
|
|
737
|
+
if (index >= proposers.length) {
|
|
738
|
+
throw new Error(
|
|
739
|
+
`fan-out: candidate index ${index} has no proposer (proposers=${proposers.length}) — ` +
|
|
740
|
+
'config.populationSize must equal config.proposers.length',
|
|
741
|
+
)
|
|
742
|
+
}
|
|
743
|
+
let pending = byGeneration.get(generation)
|
|
744
|
+
if (!pending) {
|
|
745
|
+
pending = runFanOut(generation, args)
|
|
746
|
+
byGeneration.set(generation, pending)
|
|
747
|
+
}
|
|
748
|
+
const outcomes = await pending
|
|
749
|
+
const outcome = outcomes[index]!
|
|
750
|
+
if (!outcome.applied || outcome.patch === null) return { applied: false, summary: outcome.summary }
|
|
751
|
+
|
|
752
|
+
// Apply the survivor's patch onto the driver-owned candidate worktree.
|
|
753
|
+
const patchFile = join(config.outDir, 'proposer-patches', `.apply-gen${generation}-cand${index}.patch`)
|
|
754
|
+
await writeFile(patchFile, outcome.patch)
|
|
755
|
+
await runOk('git', ['-C', args.worktreePath, 'apply', '--whitespace=nowarn', patchFile])
|
|
756
|
+
await rm(patchFile, { force: true })
|
|
757
|
+
await purgeIgnoredArtifacts(args.worktreePath)
|
|
758
|
+
return {
|
|
759
|
+
applied: true,
|
|
760
|
+
summary: outcome.summary,
|
|
761
|
+
label: outcome.proposer.name,
|
|
762
|
+
rationale:
|
|
763
|
+
`proposer ${outcome.proposer.name} (${outcome.proposer.engine ? `engine ${outcome.proposer.engine}` : outcome.proposer.harness}` +
|
|
764
|
+
`${outcome.proposer.profile ? `, profile ${outcome.proposer.profile}` : ''}` +
|
|
765
|
+
`${outcome.proposer.diagnosisSlice && outcome.proposer.diagnosisSlice !== 'all' ? `, slice ${outcome.proposer.diagnosisSlice}` : ''}` +
|
|
766
|
+
`): ${outcome.summary || 'authored change'}`,
|
|
767
|
+
}
|
|
768
|
+
},
|
|
769
|
+
}
|
|
770
|
+
}
|