@tangle-network/agent-bench 0.3.7 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/HARNESS.md +43 -0
  3. package/README.md +7 -0
  4. package/dist/adapters.js +23 -23
  5. package/dist/benchmarks/_harness.d.ts +1 -1
  6. package/dist/benchmarks/_harness.js +1 -1
  7. package/dist/benchmarks/aec-bench.js +2 -2
  8. package/dist/benchmarks/agentbench.js +2 -2
  9. package/dist/benchmarks/appworld.js +2 -2
  10. package/dist/benchmarks/bfcl.js +2 -2
  11. package/dist/benchmarks/commit0.js +2 -2
  12. package/dist/benchmarks/crag.js +2 -2
  13. package/dist/benchmarks/dabstep.js +2 -2
  14. package/dist/benchmarks/enterpriseops-gym.js +2 -2
  15. package/dist/benchmarks/finresearchbench.js +2 -2
  16. package/dist/benchmarks/nomiracl.js +2 -2
  17. package/dist/benchmarks/open-rag-bench.js +2 -2
  18. package/dist/benchmarks/programbench.js +2 -2
  19. package/dist/benchmarks/ragbench.js +2 -2
  20. package/dist/benchmarks/swe-bench.js +2 -2
  21. package/dist/benchmarks/t2-ragbench.js +2 -2
  22. package/dist/benchmarks/tau-bench-shared.js +2 -2
  23. package/dist/benchmarks/tau2-bench.js +3 -3
  24. package/dist/benchmarks/tau3-banking.js +3 -3
  25. package/dist/benchmarks/terminal-bench.js +2 -2
  26. package/dist/benchmarks/toollm.js +2 -2
  27. package/dist/benchmarks/webarena-verified.js +2 -2
  28. package/dist/{chunk-CKUVRZ2T.js → chunk-3U5TXJZS.js} +2 -2
  29. package/dist/{chunk-YCGY7UIZ.js → chunk-7GRVHU22.js} +2 -2
  30. package/dist/{chunk-Z7ML6L77.js → chunk-HWST3SED.js} +2 -2
  31. package/dist/{chunk-SYDW647C.js → chunk-IA2FBTWC.js} +2 -2
  32. package/dist/{chunk-R67DFVLO.js → chunk-IFVINJ4B.js} +2 -2
  33. package/dist/{chunk-R36V2VP7.js → chunk-IZ5M6OAC.js} +2 -2
  34. package/dist/{chunk-ODT47UAY.js → chunk-K3BQGZCT.js} +2 -2
  35. package/dist/{chunk-IFAV6KEM.js → chunk-KP5KD6EN.js} +2 -2
  36. package/dist/{chunk-ZEWMTR5M.js → chunk-MQMRLGOG.js} +2 -2
  37. package/dist/{chunk-TSWPNOYM.js → chunk-NQG5XDSB.js} +2 -2
  38. package/dist/{chunk-7WSD27QQ.js → chunk-PB64GYIG.js} +2 -2
  39. package/dist/{chunk-HBSWHQNJ.js → chunk-RCYQEFNX.js} +3 -3
  40. package/dist/{chunk-J3KDJNX2.js → chunk-RH5F53JT.js} +2 -2
  41. package/dist/{chunk-UAIOHCUK.js → chunk-SFLA7OH3.js} +3 -3
  42. package/dist/{chunk-Y6O2OCUO.js → chunk-SHM6MRRF.js} +2 -2
  43. package/dist/{chunk-KDIKRJGB.js → chunk-SHYIRB7I.js} +2 -2
  44. package/dist/{chunk-HHXFIHXC.js → chunk-SVR2LKYI.js} +2 -2
  45. package/dist/{chunk-PWQVGAJB.js → chunk-V7AEBY6U.js} +21 -21
  46. package/dist/{chunk-LRRD7NAG.js → chunk-WSKWVEQB.js} +18 -2
  47. package/dist/chunk-WSKWVEQB.js.map +1 -0
  48. package/dist/{chunk-2PVVP7GN.js → chunk-XKEFIFIC.js} +2 -2
  49. package/dist/{chunk-JRWWGMK7.js → chunk-XYA4XSNU.js} +2 -2
  50. package/dist/{chunk-X5YKXC6V.js → chunk-YSMEKBTD.js} +2 -2
  51. package/dist/{chunk-2XU6OGEN.js → chunk-Z4TZ76N7.js} +2 -2
  52. package/dist/index.js +23 -23
  53. package/package.json +5 -5
  54. package/scripts/trata-hedge/README.md +6 -5
  55. package/scripts/verify-packed-consumer.mjs +1 -1
  56. package/src/benchmarks/_harness.ts +20 -2
  57. package/src/benchmarks/humaneval.test.mts +2 -2
  58. package/src/gate.ts +1 -1
  59. package/src/hev-eval.mts +5 -2
  60. package/src/hev-improve.mts +118 -73
  61. package/src/official-optimizer-config.mts +89 -0
  62. package/src/official-optimizer-config.test.mts +88 -0
  63. package/src/profiles.ts +2 -2
  64. package/src/quant-arena/README.md +144 -0
  65. package/src/quant-arena/backtest.test.mts +135 -0
  66. package/src/quant-arena/backtest.ts +218 -0
  67. package/src/quant-arena/data.test.mts +44 -0
  68. package/src/quant-arena/data.ts +141 -0
  69. package/src/quant-arena/driver.test.mts +253 -0
  70. package/src/quant-arena/driver.ts +219 -0
  71. package/src/quant-arena/fixtures/data/PROVENANCE.md +26 -0
  72. package/src/quant-arena/fixtures/data/holdout/IDX.csv +523 -0
  73. package/src/quant-arena/fixtures/data/holdout/S01.csv +523 -0
  74. package/src/quant-arena/fixtures/data/holdout/S02.csv +523 -0
  75. package/src/quant-arena/fixtures/data/holdout/S03.csv +523 -0
  76. package/src/quant-arena/fixtures/data/holdout/S04.csv +523 -0
  77. package/src/quant-arena/fixtures/data/holdout/S05.csv +523 -0
  78. package/src/quant-arena/fixtures/data/holdout/S06.csv +523 -0
  79. package/src/quant-arena/fixtures/data/holdout/S07.csv +523 -0
  80. package/src/quant-arena/fixtures/data/holdout/S08.csv +523 -0
  81. package/src/quant-arena/fixtures/data/holdout/S09.csv +523 -0
  82. package/src/quant-arena/fixtures/data/holdout/S10.csv +523 -0
  83. package/src/quant-arena/fixtures/data/insample/IDX.csv +2087 -0
  84. package/src/quant-arena/fixtures/data/insample/S01.csv +2087 -0
  85. package/src/quant-arena/fixtures/data/insample/S02.csv +2087 -0
  86. package/src/quant-arena/fixtures/data/insample/S03.csv +2087 -0
  87. package/src/quant-arena/fixtures/data/insample/S04.csv +2087 -0
  88. package/src/quant-arena/fixtures/data/insample/S05.csv +2087 -0
  89. package/src/quant-arena/fixtures/data/insample/S06.csv +2087 -0
  90. package/src/quant-arena/fixtures/data/insample/S07.csv +2087 -0
  91. package/src/quant-arena/fixtures/data/insample/S08.csv +2087 -0
  92. package/src/quant-arena/fixtures/data/insample/S09.csv +2087 -0
  93. package/src/quant-arena/fixtures/data/insample/S10.csv +2087 -0
  94. package/src/quant-arena/fixtures/demo-campaign/cost-ledger.jsonl +16 -0
  95. package/src/quant-arena/fixtures/demo-campaign/notebook.jsonl +5 -0
  96. package/src/quant-arena/fixtures/demo-campaign/rollout-manifest.json +171 -0
  97. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-001-default-author/strategy.ts +119 -0
  98. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-002-default-author/strategy.ts +119 -0
  99. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-003-quant-researcher/strategy.ts +105 -0
  100. package/src/quant-arena/fixtures/demo-campaign/strategies/cand-004-quant-researcher/strategy.ts +102 -0
  101. package/src/quant-arena/fixtures/demo-campaign-v2/cost-ledger.jsonl +4 -0
  102. package/src/quant-arena/fixtures/demo-campaign-v2/notebook.jsonl +2 -0
  103. package/src/quant-arena/fixtures/demo-campaign-v2/rollout-manifest.json +84 -0
  104. package/src/quant-arena/fixtures/demo-campaign-v2/strategies/cand-001-quant-researcher/strategy.ts +117 -0
  105. package/src/quant-arena/holdout-certify.mts +206 -0
  106. package/src/quant-arena/holdout-certify.test.mts +82 -0
  107. package/src/quant-arena/leak-audit.test.mts +79 -0
  108. package/src/quant-arena/leak-audit.ts +95 -0
  109. package/src/quant-arena/make-fixtures.mts +161 -0
  110. package/src/quant-arena/multiplicity.test.mts +68 -0
  111. package/src/quant-arena/multiplicity.ts +87 -0
  112. package/src/quant-arena/nautilus-certify.ts +31 -0
  113. package/src/quant-arena/oms.ts +90 -0
  114. package/src/quant-arena/profiles/quant-researcher.profile.json +7 -0
  115. package/src/quant-arena/python/pyproject.toml +8 -0
  116. package/src/quant-arena/python/uv.lock +1297 -0
  117. package/src/quant-arena/python/vbt-worker.py +192 -0
  118. package/src/quant-arena/quant-loop.mts +813 -0
  119. package/src/quant-arena/quant-loop.test.mts +75 -0
  120. package/src/quant-arena/strategies/buy-hold-index/strategy.ts +11 -0
  121. package/src/quant-arena/strategies/equal-weight/strategy.ts +20 -0
  122. package/src/quant-arena/strategies/sma-crossover/strategy.ts +42 -0
  123. package/src/quant-arena/types.ts +133 -0
  124. package/src/quant-arena/vbt-client.ts +321 -0
  125. package/src/quant-arena/vbt-parity.test.mts +183 -0
  126. package/src/quant-arena/windows.test.mts +45 -0
  127. package/src/quant-arena/windows.ts +54 -0
  128. package/src/rollout-ledger/backfill-swe-arena.mts +606 -0
  129. package/src/rollout-ledger/backfill-swe-arena.test.mts +342 -0
  130. package/src/rollout-ledger/settle-capture.mts +442 -0
  131. package/src/rollout-ledger/settle-capture.test.mts +270 -0
  132. package/src/smoke-structural-rollout.mts +15 -9
  133. package/src/swe-arena/activation.mts +225 -0
  134. package/src/swe-arena/activation.test.mts +300 -0
  135. package/src/swe-arena/arms.ts +30 -14
  136. package/src/swe-arena/briefing.mts +217 -0
  137. package/src/swe-arena/briefing.test.mts +178 -0
  138. package/src/swe-arena/calibrate.ts +102 -1
  139. package/src/swe-arena/cell-evidence.mts +41 -9
  140. package/src/swe-arena/factory-command-container.ts +284 -0
  141. package/src/swe-arena/factory-judge-child.mts +228 -0
  142. package/src/swe-arena/factory.test.mts +643 -0
  143. package/src/swe-arena/fixtures/factory/agent-eval-309/calibration.md +51 -0
  144. package/src/swe-arena/fixtures/factory/agent-eval-309/manifest.json +29 -0
  145. package/src/swe-arena/fixtures/factory/agent-eval-309/spec.md +64 -0
  146. package/src/swe-arena/fixtures/factory/agent-runtime-232/calibration.md +48 -0
  147. package/src/swe-arena/fixtures/factory/agent-runtime-232/manifest.json +29 -0
  148. package/src/swe-arena/fixtures/factory/agent-runtime-232/spec.md +48 -0
  149. package/src/swe-arena/fixtures/factory/loops-28/calibration.md +47 -0
  150. package/src/swe-arena/fixtures/factory/loops-28/manifest.json +30 -0
  151. package/src/swe-arena/fixtures/factory/loops-28/spec.md +50 -0
  152. package/src/swe-arena/fixtures/run-report/README.md +43 -0
  153. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json +173 -0
  154. package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md +100 -0
  155. package/src/swe-arena/fixtures/run-report/gen3-rollup.json +551 -0
  156. package/src/swe-arena/fixtures/run-report/gen3-rollup.md +64 -0
  157. package/src/swe-arena/fixtures.ts +135 -2
  158. package/src/swe-arena/gepa-seat.mts +877 -0
  159. package/src/swe-arena/gepa-seat.test.mts +1059 -0
  160. package/src/swe-arena/implementation-ref.test.mts +64 -0
  161. package/src/swe-arena/implementation-ref.ts +62 -0
  162. package/src/swe-arena/ledger-orphans.mts +77 -0
  163. package/src/swe-arena/ledger-orphans.test.mts +147 -0
  164. package/src/swe-arena/outer-loop.mts +710 -33
  165. package/src/swe-arena/premeasured-from-cells.mts +281 -0
  166. package/src/swe-arena/premeasured-from-cells.test.mts +180 -0
  167. package/src/swe-arena/proposer-fanout.mts +327 -31
  168. package/src/swe-arena/proposer-fanout.test.mts +246 -0
  169. package/src/swe-arena/proposer-provenance.mts +172 -0
  170. package/src/swe-arena/proposer-provenance.test.mts +106 -0
  171. package/src/swe-arena/run-experiment.mts +367 -1
  172. package/src/swe-arena/run-report.mts +75 -0
  173. package/src/swe-arena/run-supervisor.test.mts +5 -3
  174. package/src/swe-arena/score-split.mts +140 -0
  175. package/src/swe-arena/score-split.test.mts +123 -0
  176. package/src/swe-arena/scratch-worktree.test.mts +55 -0
  177. package/src/swe-arena/scratch-worktree.ts +34 -0
  178. package/src/swe-arena/types.ts +52 -0
  179. package/src/swe-code-improve.mts +24 -25
  180. package/src/swe-improve.mts +129 -96
  181. package/src/swe-local-proof.mts +6 -1
  182. package/src/swe-stream.mts +4 -2
  183. package/src/swe-structural.mts +245 -837
  184. package/src/tb-container-executor.test.mts +30 -6
  185. package/src/tb-supervisor-sidecar.mts +2 -1
  186. package/src/trata-gepa.mts +182 -245
  187. package/dist/chunk-LRRD7NAG.js.map +0 -1
  188. package/src/live-improve-campaign-mbpp.mts +0 -641
  189. package/src/live-improve-campaign.mts +0 -500
  190. /package/dist/{chunk-CKUVRZ2T.js.map → chunk-3U5TXJZS.js.map} +0 -0
  191. /package/dist/{chunk-YCGY7UIZ.js.map → chunk-7GRVHU22.js.map} +0 -0
  192. /package/dist/{chunk-Z7ML6L77.js.map → chunk-HWST3SED.js.map} +0 -0
  193. /package/dist/{chunk-SYDW647C.js.map → chunk-IA2FBTWC.js.map} +0 -0
  194. /package/dist/{chunk-R67DFVLO.js.map → chunk-IFVINJ4B.js.map} +0 -0
  195. /package/dist/{chunk-R36V2VP7.js.map → chunk-IZ5M6OAC.js.map} +0 -0
  196. /package/dist/{chunk-ODT47UAY.js.map → chunk-K3BQGZCT.js.map} +0 -0
  197. /package/dist/{chunk-IFAV6KEM.js.map → chunk-KP5KD6EN.js.map} +0 -0
  198. /package/dist/{chunk-ZEWMTR5M.js.map → chunk-MQMRLGOG.js.map} +0 -0
  199. /package/dist/{chunk-TSWPNOYM.js.map → chunk-NQG5XDSB.js.map} +0 -0
  200. /package/dist/{chunk-7WSD27QQ.js.map → chunk-PB64GYIG.js.map} +0 -0
  201. /package/dist/{chunk-HBSWHQNJ.js.map → chunk-RCYQEFNX.js.map} +0 -0
  202. /package/dist/{chunk-J3KDJNX2.js.map → chunk-RH5F53JT.js.map} +0 -0
  203. /package/dist/{chunk-UAIOHCUK.js.map → chunk-SFLA7OH3.js.map} +0 -0
  204. /package/dist/{chunk-Y6O2OCUO.js.map → chunk-SHM6MRRF.js.map} +0 -0
  205. /package/dist/{chunk-KDIKRJGB.js.map → chunk-SHYIRB7I.js.map} +0 -0
  206. /package/dist/{chunk-HHXFIHXC.js.map → chunk-SVR2LKYI.js.map} +0 -0
  207. /package/dist/{chunk-PWQVGAJB.js.map → chunk-V7AEBY6U.js.map} +0 -0
  208. /package/dist/{chunk-2PVVP7GN.js.map → chunk-XKEFIFIC.js.map} +0 -0
  209. /package/dist/{chunk-JRWWGMK7.js.map → chunk-XYA4XSNU.js.map} +0 -0
  210. /package/dist/{chunk-X5YKXC6V.js.map → chunk-YSMEKBTD.js.map} +0 -0
  211. /package/dist/{chunk-2XU6OGEN.js.map → chunk-Z4TZ76N7.js.map} +0 -0
@@ -38,7 +38,7 @@
38
38
 
39
39
  import { createHash } from 'node:crypto'
40
40
  import { readFileSync } from 'node:fs'
41
- import { mkdir, rm, writeFile } from 'node:fs/promises'
41
+ import { mkdir, readFile as readFilePromise, rm, writeFile } from 'node:fs/promises'
42
42
  import { join } from 'node:path'
43
43
  import { fileURLToPath } from 'node:url'
44
44
  import {
@@ -58,7 +58,26 @@ import {
58
58
  round4BuildPrompt,
59
59
  type OuterLoopConfig,
60
60
  } from './outer-loop.mts'
61
+ import {
62
+ ACTIVATION_PREDICATE_RELPATH,
63
+ activationPredicateInstruction,
64
+ parseActivationPredicate,
65
+ } from './activation.mts'
66
+ import { briefingPromptSection, type BriefingContext } from './briefing.mts'
67
+ import {
68
+ gepaSeatAuthor,
69
+ isGepaSeat,
70
+ validateGepaSeat,
71
+ type GepaMethodFactory,
72
+ type GepaSeatDeps,
73
+ } from './gepa-seat.mts'
74
+ import type { ScoreSplit } from './score-split.mts'
61
75
  import { run, runOk } from './proc.ts'
76
+ import {
77
+ createDetachedWorktree,
78
+ pruneDetachedWorktrees,
79
+ removeDetachedWorktree,
80
+ } from './scratch-worktree.ts'
62
81
 
63
82
  // ---------------------------------------------------------------------------
64
83
  // Config types.
@@ -70,13 +89,41 @@ export interface ProposerSpec {
70
89
  /** Path to an `AgentProfile` JSON. Absolute, or relative to this module's
71
90
  * `profiles/` directory. Omitted = bare profile (legacy invocation). */
72
91
  profile?: string
73
- harness: 'claude' | 'codex' | 'opencode'
92
+ /** Required for harness-authored seats. Absent on an engine seat
93
+ * (`engine` set) — enforced both ways at generator construction. */
94
+ harness?: 'claude' | 'codex' | 'opencode'
95
+ /** GEN-4 pinned model id, threaded to the harness CLI as `-m <model>` via
96
+ * the author profile's `model.default` (harnessInvocation maps it for all
97
+ * three harnesses). Unset = the CLI's own resolved model (its login/settings
98
+ * default) — recorded per run by the proposer-provenance capture, so the
99
+ * seat identity is pinned in provenance even when the flag is absent. */
100
+ model?: string
101
+ /** GEN-4 merge seat: this proposer's task is to MERGE the configured
102
+ * Pareto parents' diffs into one coherent surface (see
103
+ * `mergeAuthorPrompt`). Requires ≥2 materialized parents — enforced at
104
+ * generator construction. */
105
+ merge?: boolean
74
106
  /** Free-text authoring lens appended to the task prompt (e.g. a mechanics
75
107
  * or reviewer stance). Never overrides the protocol change-space text. */
76
108
  lens?: string
77
109
  /** Which diagnosis findings this proposer sees. Protocol/steering and
78
110
  * raw-trace-context findings always pass through. Default 'all'. */
79
111
  diagnosisSlice?: 'all' | 'mechanics' | 'prompts'
112
+ /** GEPA seat: this seat is an engine invocation (agent-eval's
113
+ * external-GEPA adapter), not a harness CLI. `gepa` = one bounded engine
114
+ * run; `omni` = GEPA's official Omni recipe. Validation +
115
+ * authoring live in gepa-seat.mts. */
116
+ engine?: 'gepa' | 'omni'
117
+ /** GEPA seat: the one repo-relative change-space file GEPA optimizes
118
+ * as a string; the rest of the loops repo stays at the incumbent commit. */
119
+ surface?: string
120
+ /** GEPA seat: total inner-evaluation budget (each inner call is one
121
+ * real smoke arm cell). Default 10. */
122
+ maxMetricCalls?: number
123
+ /** GEPA seat: hard cap for metered optimizer-model spend. */
124
+ maxProposerCostUsd?: number
125
+ /** GEPA seat: Python executable for the bridge. Default 'python3'. */
126
+ python?: string
80
127
  }
81
128
 
82
129
  export interface PrefilterConfig {
@@ -100,12 +147,16 @@ export interface SmokeVerdict {
100
147
  resolved: boolean | null
101
148
  patchLines: number
102
149
  wallS: number
150
+ /** Committed verify fixture passed, used as the GEPA seat's inner-score
151
+ * tiebreak. Absent on errored smokes and older records. */
152
+ verifyPass?: boolean
103
153
  }
104
154
 
105
155
  export type SmokeRunner = (args: {
106
156
  scratchPath: string
107
157
  generation: number
108
158
  proposer: ProposerSpec
159
+ evaluationKey: string
109
160
  costLedger?: CostLedgerHandle
110
161
  }) => Promise<SmokeVerdict>
111
162
 
@@ -114,7 +165,7 @@ export interface PrefilterKill {
114
165
  candidateIndex: number
115
166
  proposer: string
116
167
  harness: ProposerSpec['harness']
117
- stage: 'change-space' | 'smoke'
168
+ stage: 'change-space' | 'activation-predicate' | 'smoke'
118
169
  reason: string
119
170
  diffSha256: string | null
120
171
  /** Persisted kill-forensics patch under `<outDir>/proposer-patches/`. */
@@ -153,6 +204,132 @@ export function defaultProposers(): ProposerSpec[] {
153
204
  return [{ name: 'default-author', profile: 'default-author.profile.json', harness: 'claude' }]
154
205
  }
155
206
 
207
+ /** The profile the author shot actually runs: the loaded profile (if any) with
208
+ * the spec's PINNED MODEL merged into `model.default` — harnessInvocation
209
+ * turns that into the CLI's `-m <model>` flag (claude/codex/opencode all map
210
+ * it). Without a pinned model this is byte-identical to `loadAuthorProfile`,
211
+ * so gen-3 seats keep their exact invocation. A pinned model with no profile
212
+ * path synthesizes a minimal named profile carrying only the pin. */
213
+ export function resolveAuthorProfile(spec: ProposerSpec): AgentProfile | undefined {
214
+ const profile = loadAuthorProfile(spec)
215
+ if (!spec.model) return profile
216
+ const base: AgentProfile = profile ?? { name: `${spec.name}-pinned` }
217
+ return { ...base, model: { ...base.model, default: spec.model } }
218
+ }
219
+
220
+ // ---------------------------------------------------------------------------
221
+ // GEN-4 Pareto parents — cross-run seeding at the buildPrompt seam.
222
+ //
223
+ // The LIB's own `ctx.paretoParents` (runOptimization → SurfaceProposer) only
224
+ // accumulates surfaces scored WITHIN one run: the frontier starts from this
225
+ // run's baseline and generations, and a prior campaign's surfaces cannot be
226
+ // injected without replaying that campaign's runDir + cost ledger (the cached
227
+ // cells refuse to load without their ledger receipts). Gen-4 runs in a fresh
228
+ // outDir, so the gen-3 winners are seeded HERE — their diffs + per-instance
229
+ // results enter every author's task prompt, and the merge seat gets both
230
+ // diffs as its explicit merge input. This is our seam, not the lib path;
231
+ // noted in the gen-4 config docs.
232
+ // ---------------------------------------------------------------------------
233
+
234
+ /** One prior-run frontier member, as configured (commits live in loopsRepo). */
235
+ export interface ParetoParentSeed {
236
+ /** Loops commit of the parent candidate (must exist in `loopsRepo`). */
237
+ commit: string
238
+ /** Staircase label, e.g. 'default-author'. */
239
+ label: string
240
+ /** Instances the parent resolved under the fail-closed all-reps rule. */
241
+ resolvedInstances: string[]
242
+ /** Free-text evidence note (discordant replicates, mechanism summary). */
243
+ note?: string
244
+ }
245
+
246
+ /** A seed materialized against the loops repo: the parent's full diff. */
247
+ export interface ParetoParentContext extends ParetoParentSeed {
248
+ diff: string
249
+ }
250
+
251
+ export const PARENT_DIFF_MAX_CHARS = 60_000
252
+
253
+ /** Materialize parent seeds: verify each commit exists in `loopsRepo` and
254
+ * capture its full diff (`git show`). Fails loud on a missing commit — a
255
+ * silently absent parent would turn the merge seat into a no-op. */
256
+ export async function materializeParetoParents(
257
+ loopsRepo: string,
258
+ seeds: ParetoParentSeed[],
259
+ maxDiffChars = PARENT_DIFF_MAX_CHARS,
260
+ ): Promise<ParetoParentContext[]> {
261
+ const parents: ParetoParentContext[] = []
262
+ for (const seed of seeds) {
263
+ const exists = await run('git', ['-C', loopsRepo, 'cat-file', '-e', `${seed.commit}^{commit}`])
264
+ if (exists.code !== 0) {
265
+ throw new Error(`pareto parent ${seed.label}: commit ${seed.commit} not found in ${loopsRepo}`)
266
+ }
267
+ const show = await runOk('git', ['-C', loopsRepo, 'show', '--no-color', seed.commit])
268
+ const diff =
269
+ show.stdout.length > maxDiffChars
270
+ ? `${show.stdout.slice(0, maxDiffChars)}\n[... diff truncated at ${maxDiffChars} chars ...]`
271
+ : show.stdout
272
+ parents.push({ ...seed, diff })
273
+ }
274
+ return parents
275
+ }
276
+
277
+ function parentEvidenceLine(parent: ParetoParentContext): string {
278
+ const resolved = parent.resolvedInstances.length > 0 ? parent.resolvedInstances.join(', ') : 'none'
279
+ return `${parent.label} (${parent.commit.slice(0, 10)}) — resolved: ${resolved}${parent.note ? `; ${parent.note}` : ''}`
280
+ }
281
+
282
+ /** The parents section appended to every NON-merge author's prompt: measured
283
+ * evidence of what worked, never an instruction to copy. */
284
+ export function parentsPromptSection(parents: ParetoParentContext[]): string {
285
+ const lines: string[] = [
286
+ 'PARETO PARENTS — the prior generation’s frontier candidates, measured on the SAME 6-instance set.',
287
+ 'Each beat the baseline on different instances; treat their diffs as measured evidence of what works.',
288
+ 'You may build on either (or both), but your candidate is measured on the full set — do not blindly copy.',
289
+ ]
290
+ for (const parent of parents) {
291
+ lines.push('', `--- parent: ${parentEvidenceLine(parent)}`, parent.diff.trimEnd())
292
+ }
293
+ return lines.join('\n')
294
+ }
295
+
296
+ /** The merge seat's task prompt: the change-space contract + findings context
297
+ * stay (round4BuildPrompt), and the TASK is replaced with an explicit
298
+ * coherent-union merge of the parents' diffs. */
299
+ export function mergeAuthorPrompt(
300
+ args: { report: unknown; findings: Array<Record<string, unknown>> },
301
+ spec: ProposerSpec,
302
+ parents: ParetoParentContext[],
303
+ ): string {
304
+ if (parents.length < 2) {
305
+ throw new Error(`merge proposer ${spec.name}: needs >=2 materialized pareto parents, got ${parents.length}`)
306
+ }
307
+ const lines: string[] = [
308
+ round4BuildPrompt(args),
309
+ '',
310
+ `YOUR TASK (${spec.name} — MERGE SEAT):`,
311
+ 'The prior generation produced the frontier candidates below. Each beat the baseline on DIFFERENT',
312
+ 'instances, so their lessons are complementary. Produce ONE coherent surface that is the UNION of the',
313
+ 'parent diffs:',
314
+ ...parents.map((p) => ` - ${parentEvidenceLine(p)}`),
315
+ '',
316
+ 'Merge rules:',
317
+ '- Apply BOTH parents’ behaviors. Where the diffs touch the same file/section (e.g. the supervisor',
318
+ ' reviewer bullet or the worker self-test rules), resolve the conflict by KEEPING BOTH BEHAVIORS —',
319
+ ' write one merged passage that carries every constraint from each side, never by dropping one side.',
320
+ '- Keep each parent’s mechanical changes intact (code paths, exported helpers, threading) alongside the',
321
+ ' other parent’s prompt/goal-authoring changes.',
322
+ '- Do NOT invent new mechanisms beyond the union; the smallest coherent merge wins.',
323
+ '- The merged surface must typecheck and stay inside the declared change-space above.',
324
+ '',
325
+ 'Parent diffs (full):',
326
+ ]
327
+ for (const parent of parents) {
328
+ lines.push('', `=== parent ${parent.label} (${parent.commit.slice(0, 10)}) ===`, parent.diff.trimEnd())
329
+ }
330
+ return lines.join('\n')
331
+ }
332
+
156
333
  // ---------------------------------------------------------------------------
157
334
  // Diagnosis slicing + prompt lens.
158
335
  // ---------------------------------------------------------------------------
@@ -182,15 +359,37 @@ export function sliceFindings(findings: AnalystFinding[], slice: ProposerSpec['d
182
359
  })
183
360
  }
184
361
 
362
+ /** GEN-5 prompt extensions shared by every seat (merge included): the
363
+ * MAP+TOOLBOX+PERMISSION briefing (briefing.mts) and the activation-predicate
364
+ * deliverable contract (activation.mts). */
365
+ export interface Gen5PromptExtras {
366
+ briefing?: BriefingContext
367
+ activationGate?: boolean
368
+ }
369
+
370
+ function appendGen5Sections(prompt: string, extras: Gen5PromptExtras): string {
371
+ let out = prompt
372
+ if (extras.briefing) out = `${out}\n\n${briefingPromptSection(extras.briefing)}`
373
+ if (extras.activationGate) out = `${out}\n\n${activationPredicateInstruction()}`
374
+ return out
375
+ }
376
+
185
377
  /** The proposer's task prompt: the shared round prompt plus this proposer's
186
- * authoring lens (appended so the protocol change-space text stays intact). */
378
+ * authoring lens (appended so the protocol change-space text stays intact),
379
+ * plus the gen-4 Pareto-parents section when parents are seeded, plus the
380
+ * gen-5 briefing/activation sections when configured. A merge-seat spec gets
381
+ * the dedicated merge prompt instead (gen-5 sections still apply). */
187
382
  export function proposerBuildPrompt(
188
383
  args: { report: unknown; findings: Array<Record<string, unknown>> },
189
384
  spec: ProposerSpec,
385
+ parents: ParetoParentContext[] = [],
386
+ extras: Gen5PromptExtras = {},
190
387
  ): string {
191
- const base = round4BuildPrompt(args)
192
- if (!spec.lens) return base
193
- return `${base}\n\nYOUR AUTHORING LENS (${spec.name}):\n${spec.lens}`
388
+ if (spec.merge) return appendGen5Sections(mergeAuthorPrompt(args, spec, parents), extras)
389
+ let prompt = round4BuildPrompt(args)
390
+ if (spec.lens) prompt = `${prompt}\n\nYOUR AUTHORING LENS (${spec.name}):\n${spec.lens}`
391
+ if (parents.length > 0) prompt = `${prompt}\n\n${parentsPromptSection(parents)}`
392
+ return appendGen5Sections(prompt, extras)
194
393
  }
195
394
 
196
395
  // ---------------------------------------------------------------------------
@@ -249,21 +448,6 @@ export function proposerShotHooks(opts: {
249
448
  // smoke runner link one in temporarily when they need it).
250
449
  // ---------------------------------------------------------------------------
251
450
 
252
- async function addAuthoringWorktree(loopsRepo: string, commit: string, dest: string): Promise<void> {
253
- await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest])
254
- await rm(dest, { recursive: true, force: true })
255
- await run('git', ['-C', loopsRepo, 'worktree', 'prune'])
256
- await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', dest, commit])
257
- }
258
-
259
- async function removeAuthoringWorktree(loopsRepo: string, dest: string): Promise<void> {
260
- const res = await run('git', ['-C', loopsRepo, 'worktree', 'remove', '--force', '--', dest])
261
- if (res.code !== 0) {
262
- await rm(dest, { recursive: true, force: true })
263
- await run('git', ['-C', loopsRepo, 'worktree', 'prune'])
264
- }
265
- }
266
-
267
451
  const sanitize = (s: string): string => s.replace(/[^a-zA-Z0-9_-]/g, '_')
268
452
 
269
453
  // ---------------------------------------------------------------------------
@@ -292,6 +476,26 @@ export type AuthorFn = (
292
476
  export interface FanOutDeps {
293
477
  author?: AuthorFn
294
478
  smokeRunner?: SmokeRunner
479
+ /** Immutable digest of the smoke runner plus every captured execution input. */
480
+ runnerImplementationRef?: string
481
+ /** Immutable digest of the official judge implementation and configuration. */
482
+ judgeImplementationRef?: string
483
+ /** GEN-4: materialized Pareto parents seeded into every author's prompt (and
484
+ * the merge seat's explicit merge input). Empty/omitted = gen-3 behavior. */
485
+ parents?: ParetoParentContext[]
486
+ /** GEN-5: MAP+TOOLBOX briefing context threaded into every author prompt. */
487
+ briefing?: BriefingContext
488
+ /** The resolved public smoke instance used by the GEPA seat's inner
489
+ * evaluator target. Required (with `smokeRunner`) when a gepa seat is
490
+ * configured and no custom `author` is injected. */
491
+ smokeInstanceId?: string
492
+ /** The score split; private instance ids never reach the GEPA
493
+ * bridge (gepa-seat.mts asserts, fail-closed). Null/omitted = no split. */
494
+ scoreSplit?: Pick<ScoreSplit, 'privateInstances'> | null
495
+ /** GEPA method override for tests. Default: agent-eval's official GEPA method. */
496
+ gepaMethodFactory?: GepaMethodFactory
497
+ /** Explicit GEPA optimizer model override, primarily for isolated tests. */
498
+ gepaOptimizer?: GepaSeatDeps['optimizer']
295
499
  log?: (msg: string) => void
296
500
  }
297
501
 
@@ -306,23 +510,54 @@ function defaultAuthor(config: OuterLoopConfig, deps: FanOutDeps): AuthorFn {
306
510
  // its own profile, lens prompt, and shot-receipt home.
307
511
  const inners = new Map<string, CandidateGenerator>()
308
512
  const ledgers = new Map<string, { ledger?: CostLedgerHandle; phase?: string }>()
513
+ // The GEPA seat authors through the agent-eval adapter, not a
514
+ // harness CLI. Its inner evaluator is the SAME injected smoke runner the
515
+ // pre-filter uses (presence enforced at generator construction).
516
+ let gepaAuthor: AuthorFn | undefined
309
517
  return (proposer, args) => {
518
+ if (isGepaSeat(proposer)) {
519
+ gepaAuthor ??= gepaSeatAuthor(config, {
520
+ smokeRunner: deps.smokeRunner!,
521
+ runnerImplementationRef: deps.runnerImplementationRef!,
522
+ judgeImplementationRef: deps.judgeImplementationRef!,
523
+ smokeInstanceId: deps.smokeInstanceId!,
524
+ scoreSplit: deps.scoreSplit ?? null,
525
+ ...(deps.gepaMethodFactory ? { methodFactory: deps.gepaMethodFactory } : {}),
526
+ ...(deps.gepaOptimizer ? { optimizer: deps.gepaOptimizer } : {}),
527
+ ...(deps.log ? { log: deps.log } : {}),
528
+ })
529
+ return gepaAuthor(proposer, args)
530
+ }
531
+ const harness = proposer.harness
532
+ if (harness === undefined) {
533
+ throw new Error(`proposer ${proposer.name}: harness is required for a non-engine seat`)
534
+ }
310
535
  let inner = inners.get(proposer.name)
311
536
  const slot = ledgers.get(proposer.name) ?? {}
312
537
  ledgers.set(proposer.name, slot)
313
538
  if (!inner) {
314
- const profile = loadAuthorProfile(proposer)
539
+ // GEN-4: the resolved profile carries the spec's pinned model
540
+ // (`model.default` → the harness CLI's `-m` flag).
541
+ const profile = resolveAuthorProfile(proposer)
315
542
  inner = agenticGenerator({
316
- harness: proposer.harness,
543
+ harness,
317
544
  ...(profile ? { profile } : {}),
318
545
  timeoutMs: config.proposerTimeoutMs,
319
546
  buildPrompt: (a) =>
320
- proposerBuildPrompt(a as unknown as { report: unknown; findings: Array<Record<string, unknown>> }, proposer),
547
+ proposerBuildPrompt(
548
+ a as unknown as { report: unknown; findings: Array<Record<string, unknown>> },
549
+ proposer,
550
+ deps.parents ?? [],
551
+ {
552
+ ...(deps.briefing ? { briefing: deps.briefing } : {}),
553
+ ...(config.activationGate === true ? { activationGate: true } : {}),
554
+ },
555
+ ),
321
556
  verify: loopsCandidateVerifier(config.loopsRepo),
322
- runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(proposer.harness) }),
557
+ runHarness: (options) => runLocalHarness({ ...options, env: proposerShotEnv(harness) }),
323
558
  onShotCompleted: proposerShotHooks({
324
559
  shotDir: join(config.outDir, 'proposer-shots', sanitize(proposer.name)),
325
- harness: proposer.harness,
560
+ harness,
326
561
  ledger: () => slot.ledger,
327
562
  phase: () => slot.phase,
328
563
  }),
@@ -348,6 +583,39 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps =
348
583
  if (proposers.length === 0) throw new Error('fanOutLoopsGenerator: config.proposers is empty')
349
584
  const names = new Set(proposers.map((p) => p.name))
350
585
  if (names.size !== proposers.length) throw new Error('fanOutLoopsGenerator: duplicate proposer names')
586
+ const mergeSeats = proposers.filter((p) => p.merge === true)
587
+ if (mergeSeats.length > 0 && (deps.parents ?? []).length < 2) {
588
+ throw new Error(
589
+ `fanOutLoopsGenerator: merge proposer(s) ${mergeSeats.map((p) => p.name).join(', ')} configured but ` +
590
+ `only ${(deps.parents ?? []).length} pareto parent(s) materialized — a merge seat needs >=2`,
591
+ )
592
+ }
593
+ // Engine seats are validated at construction, and the
594
+ // default author path requires the pre-filter smoke runner — it IS the GEPA
595
+ // seat's inner evaluator (spec: score = smoke resolve + verify-pass
596
+ // tiebreak). A custom injected `author` owns its own evaluator.
597
+ const gepaSeats = proposers.filter(isGepaSeat)
598
+ for (const seat of gepaSeats) validateGepaSeat(seat)
599
+ for (const seat of proposers) {
600
+ if (!isGepaSeat(seat) && seat.harness === undefined) {
601
+ throw new Error(`fanOutLoopsGenerator: proposer ${seat.name} has neither a harness nor an engine`)
602
+ }
603
+ }
604
+ if (
605
+ gepaSeats.length > 0 &&
606
+ deps.author === undefined &&
607
+ (
608
+ deps.smokeRunner === undefined ||
609
+ deps.smokeInstanceId === undefined ||
610
+ deps.runnerImplementationRef === undefined ||
611
+ deps.judgeImplementationRef === undefined
612
+ )
613
+ ) {
614
+ throw new Error(
615
+ `fanOutLoopsGenerator: gepa seat(s) ${gepaSeats.map((p) => p.name).join(', ')} need the pre-filter smoke ` +
616
+ 'cell plus immutable runner and judge references as their inner evaluator',
617
+ )
618
+ }
351
619
  const author = deps.author ?? defaultAuthor(config, deps)
352
620
  const log = deps.log ?? (() => {})
353
621
  const kills: PrefilterKill[] = []
@@ -361,11 +629,12 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps =
361
629
  const baseCommit = head.stdout.trim()
362
630
  const patchesDir = join(config.outDir, 'proposer-patches')
363
631
  await mkdir(patchesDir, { recursive: true })
632
+ await pruneDetachedWorktrees(config.loopsRepo)
364
633
  log(`fan-out gen ${generation}: ${proposers.length} proposer(s) authoring in parallel from ${baseCommit.slice(0, 10)}`)
365
634
  return Promise.all(
366
635
  proposers.map(async (proposer, index): Promise<AuthorOutcome> => {
367
636
  const scratch = join(config.outDir, 'proposer-wt', `gen${generation}-cand${index}-${sanitize(proposer.name)}`)
368
- await addAuthoringWorktree(config.loopsRepo, baseCommit, scratch)
637
+ await createDetachedWorktree(config.loopsRepo, baseCommit, scratch)
369
638
  try {
370
639
  const authored = await author(proposer, {
371
640
  ...args,
@@ -410,12 +679,39 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps =
410
679
  return { proposer, applied: false, summary: authored.summary, patch: null, kill }
411
680
  }
412
681
 
682
+ // Pre-filter stage A.5 — GEN-5 activation gate: a candidate without
683
+ // a parseable machine-checkable predicate is killed before any
684
+ // evaluation spend (the post-eval quarantine needs it to exist).
685
+ if (config.activationGate === true) {
686
+ const raw = await readFilePromise(join(scratch, ACTIVATION_PREDICATE_RELPATH), 'utf8').catch(() => null)
687
+ const parsed = raw === null ? null : parseActivationPredicate(raw)
688
+ if (parsed === null || !parsed.ok) {
689
+ const kill: PrefilterKill = {
690
+ generation,
691
+ candidateIndex: index,
692
+ proposer: proposer.name,
693
+ harness: proposer.harness,
694
+ stage: 'activation-predicate',
695
+ reason:
696
+ parsed === null
697
+ ? `missing ${ACTIVATION_PREDICATE_RELPATH} (required activation predicate)`
698
+ : `invalid ${ACTIVATION_PREDICATE_RELPATH}: ${parsed.error}`,
699
+ diffSha256,
700
+ patchPath,
701
+ }
702
+ kills.push(kill)
703
+ log(`prefilter KILL gen ${generation} ${proposer.name}: ${kill.reason}`)
704
+ return { proposer, applied: false, summary: authored.summary, patch: null, kill }
705
+ }
706
+ }
707
+
413
708
  // Pre-filter stage B — one smoke cell before any full-arm spend.
414
709
  if (config.prefilter?.enabled && deps.smokeRunner) {
415
710
  const smoke = await deps.smokeRunner({
416
711
  scratchPath: scratch,
417
712
  generation,
418
713
  proposer,
714
+ evaluationKey: `candidate-${index}`,
419
715
  ...(args.costLedger ? { costLedger: args.costLedger } : {}),
420
716
  })
421
717
  if (!smoke.pass) {
@@ -438,14 +734,14 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps =
438
734
  }
439
735
  return { proposer, applied: true, summary: authored.summary, patch: diff, kill: null }
440
736
  } finally {
441
- await removeAuthoringWorktree(config.loopsRepo, scratch)
737
+ await removeDetachedWorktree(config.loopsRepo, scratch)
442
738
  }
443
739
  }),
444
740
  )
445
741
  }
446
742
 
447
743
  return {
448
- kind: `gen3-fanout:${proposers.map((p) => `${p.name}@${p.harness}`).join('+')}`,
744
+ kind: `gen3-fanout:${proposers.map((p) => `${p.name}@${p.engine ? `engine:${p.engine}` : p.harness}${p.model ? `:${p.model}` : ''}${p.merge ? ':merge' : ''}`).join('+')}`,
449
745
  proposesWithoutFindings: true,
450
746
  drainPrefilterKills() {
451
747
  return kills.splice(0, kills.length)
@@ -479,7 +775,7 @@ export function fanOutLoopsGenerator(config: OuterLoopConfig, deps: FanOutDeps =
479
775
  summary: outcome.summary,
480
776
  label: outcome.proposer.name,
481
777
  rationale:
482
- `proposer ${outcome.proposer.name} (${outcome.proposer.harness}` +
778
+ `proposer ${outcome.proposer.name} (${outcome.proposer.engine ? `engine ${outcome.proposer.engine}` : outcome.proposer.harness}` +
483
779
  `${outcome.proposer.profile ? `, profile ${outcome.proposer.profile}` : ''}` +
484
780
  `${outcome.proposer.diagnosisSlice && outcome.proposer.diagnosisSlice !== 'all' ? `, slice ${outcome.proposer.diagnosisSlice}` : ''}` +
485
781
  `): ${outcome.summary || 'authored change'}`,