@tangle-network/agent-eval 0.94.0 → 0.95.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.md +44 -30
  3. package/dist/adapters/http.d.ts +8 -7
  4. package/dist/adapters/http.js.map +1 -1
  5. package/dist/adapters/langchain.d.ts +3 -2
  6. package/dist/adapters/otel.d.ts +5 -4
  7. package/dist/analyst/index.d.ts +11 -31
  8. package/dist/analyst/index.js +5 -65
  9. package/dist/analyst/index.js.map +1 -1
  10. package/dist/{analyze-runs-B6Ljo_dI.d.ts → analyze-runs-DtT6F_6T.d.ts} +3 -3
  11. package/dist/belief-state/index.d.ts +4 -3
  12. package/dist/benchmarks/index.d.ts +3 -2
  13. package/dist/campaign/index.d.ts +727 -616
  14. package/dist/campaign/index.js +1863 -1316
  15. package/dist/campaign/index.js.map +1 -1
  16. package/dist/{chunk-2K6UUZ7P.js → chunk-2T4EZACH.js} +1 -1
  17. package/dist/chunk-2T4EZACH.js.map +1 -0
  18. package/dist/{chunk-CTBHKLEU.js → chunk-77T4STFI.js} +59 -86
  19. package/dist/chunk-77T4STFI.js.map +1 -0
  20. package/dist/{chunk-EGPMSBEZ.js → chunk-7QTQKIDD.js} +178 -177
  21. package/dist/chunk-7QTQKIDD.js.map +1 -0
  22. package/dist/{chunk-MIFZUPEK.js → chunk-AQ5WQAIV.js} +21 -6
  23. package/dist/chunk-AQ5WQAIV.js.map +1 -0
  24. package/dist/chunk-DJWX3GVS.js +81 -0
  25. package/dist/chunk-DJWX3GVS.js.map +1 -0
  26. package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
  27. package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
  28. package/dist/{chunk-TBDR6PAI.js → chunk-IZCEK2HR.js} +2 -2
  29. package/dist/{chunk-SD2YFWQQ.js → chunk-KKWJD5E6.js} +20 -20
  30. package/dist/chunk-KKWJD5E6.js.map +1 -0
  31. package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
  32. package/dist/chunk-LO6IOIJ2.js.map +1 -0
  33. package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
  34. package/dist/chunk-NZEQVRH5.js.map +1 -0
  35. package/dist/{chunk-MPQWFX6Y.js → chunk-PSWWQXHF.js} +13 -88
  36. package/dist/chunk-PSWWQXHF.js.map +1 -0
  37. package/dist/{chunk-Q5LIB7BC.js → chunk-S4SYLDFX.js} +2 -2
  38. package/dist/chunk-S4SYLDFX.js.map +1 -0
  39. package/dist/{chunk-KW53MSA5.js → chunk-X74V6ESX.js} +2 -2
  40. package/dist/{chunk-QMUEXQJS.js → chunk-YBIGNSCZ.js} +81 -4
  41. package/dist/chunk-YBIGNSCZ.js.map +1 -0
  42. package/dist/{chunk-2KNZHH3P.js → chunk-Z6L6YSU6.js} +2 -2
  43. package/dist/{code-agent-session-BO8nCnv3.d.ts → code-agent-session-CPHRCb4-.d.ts} +1 -1
  44. package/dist/contract/index.d.ts +91 -43
  45. package/dist/contract/index.js +127 -17
  46. package/dist/contract/index.js.map +1 -1
  47. package/dist/{control-D6qwHXIR.d.ts → control-Doncu-B_.d.ts} +2 -2
  48. package/dist/control.d.ts +3 -2
  49. package/dist/control.js +2 -2
  50. package/dist/{corpus-B8A4BDR3.d.ts → corpus-D4YW9UoJ.d.ts} +1 -1
  51. package/dist/{default-registry-6dhErQbs.d.ts → default-registry-GyE8X5SP.d.ts} +3 -3
  52. package/dist/diagnose.d.ts +4 -3
  53. package/dist/diagnose.js +1 -1
  54. package/dist/{run-improvement-loop-DBahB8Ax.d.ts → gepa-C1NCIZ9o.d.ts} +117 -130
  55. package/dist/hosted/index.d.ts +5 -4
  56. package/dist/{index-Bx3gZ8xl.d.ts → index-_Y4oNOOb.d.ts} +1 -1
  57. package/dist/index.d.ts +76 -81
  58. package/dist/index.js +66 -31
  59. package/dist/index.js.map +1 -1
  60. package/dist/{insight-report-DWl3z9tl.d.ts → insight-report-BnRjTibG.d.ts} +1 -1
  61. package/dist/{kind-factory-0BhLSI27.d.ts → kind-factory-X3eDYbKn.d.ts} +2 -3
  62. package/dist/matrix/index.d.ts +1 -1
  63. package/dist/meta-eval/index.d.ts +3 -2
  64. package/dist/multishot/index.d.ts +4 -4
  65. package/dist/multishot/index.js.map +1 -1
  66. package/dist/openapi.json +1 -1
  67. package/dist/{pre-registration-mAnCugl9.d.ts → pre-registration-nfUdc9EQ.d.ts} +2 -42
  68. package/dist/{provenance-P-bCL2Fo.d.ts → provenance-CncDq9qE.d.ts} +26 -41
  69. package/dist/{release-report-BEbWmVYj.d.ts → release-report-pidWUMZ2.d.ts} +2 -2
  70. package/dist/reporting.d.ts +5 -4
  71. package/dist/{researcher-B0C2_fVO.d.ts → researcher-Jr8ME1dZ.d.ts} +2 -2
  72. package/dist/rl.d.ts +516 -515
  73. package/dist/rl.js +612 -612
  74. package/dist/rl.js.map +1 -1
  75. package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +1 -1
  76. package/dist/{run-campaign-7WNXMDSN.js → run-campaign-WXY7KI67.js} +2 -2
  77. package/dist/{run-record-e7vj1uZQ.d.ts → run-record-CP2ObebC.d.ts} +14 -18
  78. package/dist/{runtime-trajectory-BDgfGZSr.d.ts → runtime-trajectory-BOUUjI0y.d.ts} +1 -1
  79. package/dist/{semantic-concept-judge-B9MgmBnM.d.ts → semantic-concept-judge-DSBB2Cfp.d.ts} +2 -2
  80. package/dist/{summary-report-BDOFevaT.d.ts → summary-report-CInXwsza.d.ts} +1 -1
  81. package/dist/testing-C21CHsq2.d.ts +20 -0
  82. package/dist/testing.d.ts +1 -0
  83. package/dist/testing.js +8 -0
  84. package/dist/testing.js.map +1 -0
  85. package/dist/traces.d.ts +26 -10
  86. package/dist/traces.js +41 -11
  87. package/dist/{types-Ce17tDlG.d.ts → types-B5x54y6n.d.ts} +1 -1
  88. package/dist/{types-mn5Aqk7x.d.ts → types-BUxNaJ8c.d.ts} +2 -4
  89. package/dist/{types-BU-7W85F.d.ts → types-DQRY8ZT-.d.ts} +60 -58
  90. package/dist/workflow/index.d.ts +5 -4
  91. package/dist/workflow/index.js +1 -1
  92. package/docs/campaign-proposers.md +170 -0
  93. package/docs/concepts.md +8 -4
  94. package/docs/customer-journeys.md +15 -13
  95. package/docs/design/loop-taxonomy.md +34 -66
  96. package/docs/distributed-driver.md +14 -14
  97. package/docs/feature-guide.md +1 -1
  98. package/docs/hosted-ingest-spec.md +2 -3
  99. package/docs/multi-shot-optimization.md +8 -8
  100. package/docs/product-eval-adoption.md +1 -1
  101. package/docs/self-improvement-map.md +33 -29
  102. package/package.json +8 -14
  103. package/dist/chunk-2K6UUZ7P.js.map +0 -1
  104. package/dist/chunk-CTBHKLEU.js.map +0 -1
  105. package/dist/chunk-E4GH6USR.js.map +0 -1
  106. package/dist/chunk-EGPMSBEZ.js.map +0 -1
  107. package/dist/chunk-KWRRMR3J.js.map +0 -1
  108. package/dist/chunk-MIFZUPEK.js.map +0 -1
  109. package/dist/chunk-MPQWFX6Y.js.map +0 -1
  110. package/dist/chunk-Q5LIB7BC.js.map +0 -1
  111. package/dist/chunk-QMUEXQJS.js.map +0 -1
  112. package/dist/chunk-SD2YFWQQ.js.map +0 -1
  113. package/docs/design/external-agent-wedge.md +0 -89
  114. package/docs/design/phase-d-rfc.md +0 -125
  115. package/docs/design/phase4-consumer-migration.md +0 -70
  116. package/docs/design/primitives-integration-spec.md +0 -393
  117. package/docs/design/product-self-improvement-loop.md +0 -146
  118. package/docs/design/self-improvement-engine.md +0 -140
  119. package/docs/design/self-improvement-protocol.md +0 -223
  120. package/docs/design/self-improvement-roadmap.md +0 -106
  121. package/docs/design/substrate-gaps.md +0 -118
  122. package/docs/phase-b-pairing-kit.md +0 -188
  123. package/docs/phase-b-runbook.md +0 -176
  124. package/docs/pilot/README.md +0 -62
  125. package/docs/pilot/customer-checklist.md +0 -90
  126. package/docs/pilot/integration-foreign-stack.md +0 -296
  127. package/docs/pilot/integration-tangle-stack.md +0 -248
  128. package/docs/pilot/one-pager.md +0 -161
  129. package/docs/pilot/sample-insight-report.json +0 -172
  130. package/docs/quickstart-external.md +0 -229
  131. package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
  132. package/docs/research/research-roadmap.md +0 -205
  133. package/docs/specs/driver-honest-spec.md +0 -251
  134. package/docs/specs/hermes-self-improvement-audit.md +0 -93
  135. package/docs/specs/profile-versioning.md +0 -291
  136. package/docs/three-package-architecture.md +0 -168
  137. /package/dist/{chunk-TBDR6PAI.js.map → chunk-IZCEK2HR.js.map} +0 -0
  138. /package/dist/{chunk-KW53MSA5.js.map → chunk-X74V6ESX.js.map} +0 -0
  139. /package/dist/{chunk-2KNZHH3P.js.map → chunk-Z6L6YSU6.js.map} +0 -0
  140. /package/dist/{run-campaign-7WNXMDSN.js.map → run-campaign-WXY7KI67.js.map} +0 -0
@@ -4,18 +4,20 @@ import {
4
4
  defaultProductionGate,
5
5
  detectScale,
6
6
  dimensionRegressions,
7
- evolutionaryDriver,
7
+ evolutionaryProposer,
8
8
  heldoutSignificance,
9
9
  pairHoldout,
10
10
  paretoPolicy,
11
11
  paretoSignificanceGate,
12
12
  runEval
13
- } from "../chunk-SD2YFWQQ.js";
13
+ } from "../chunk-KKWJD5E6.js";
14
14
  import {
15
15
  agentProfileHash,
16
+ agentProfileId,
17
+ agentProfileModelId,
16
18
  extractProducedState,
17
19
  verifyCompletion
18
- } from "../chunk-S6OZEZQK.js";
20
+ } from "../chunk-HMA63UEO.js";
19
21
  import {
20
22
  buildLoopProvenanceRecord,
21
23
  campaignBreakdown,
@@ -24,7 +26,7 @@ import {
24
26
  defaultRenderDiff,
25
27
  emitLoopProvenance,
26
28
  extractH2Sections,
27
- gepaDriver,
29
+ gepaProposer,
28
30
  heldOutGate,
29
31
  isProposedCandidate,
30
32
  labelTrustRank,
@@ -37,14 +39,14 @@ import {
37
39
  runOptimization,
38
40
  surfaceContentHash,
39
41
  surfaceHash
40
- } from "../chunk-EGPMSBEZ.js";
42
+ } from "../chunk-7QTQKIDD.js";
41
43
  import {
42
44
  assertRealBackend,
43
45
  fsCampaignStorage,
44
46
  inMemoryCampaignStorage,
45
47
  runCampaign,
46
48
  summarizeBackendIntegrity
47
- } from "../chunk-2K6UUZ7P.js";
49
+ } from "../chunk-2T4EZACH.js";
48
50
  import {
49
51
  estimateCost,
50
52
  isModelPriced
@@ -53,7 +55,7 @@ import {
53
55
  AnalystRegistry,
54
56
  DEFAULT_TRACE_ANALYST_KINDS,
55
57
  createTraceAnalystKind
56
- } from "../chunk-Q5LIB7BC.js";
58
+ } from "../chunk-S4SYLDFX.js";
57
59
  import "../chunk-5HRORJQY.js";
58
60
  import {
59
61
  callLlm
@@ -64,17 +66,17 @@ import {
64
66
  mulberry32,
65
67
  pairedBootstrap
66
68
  } from "../chunk-56GC6VYO.js";
67
- import "../chunk-GGE4NNQT.js";
68
69
  import {
69
70
  analyzeTraces
70
- } from "../chunk-TBDR6PAI.js";
71
+ } from "../chunk-IZCEK2HR.js";
72
+ import "../chunk-GGE4NNQT.js";
71
73
  import {
72
74
  OtlpFileTraceStore
73
- } from "../chunk-QMUEXQJS.js";
75
+ } from "../chunk-YBIGNSCZ.js";
74
76
  import "../chunk-PC4UYEBM.js";
75
77
  import {
76
78
  validateRunRecord
77
- } from "../chunk-KWRRMR3J.js";
79
+ } from "../chunk-LO6IOIJ2.js";
78
80
  import {
79
81
  canonicalize
80
82
  } from "../chunk-VSMTAMNK.js";
@@ -145,383 +147,1068 @@ function failureModeRecallJudge(opts = {}) {
145
147
  };
146
148
  }
147
149
 
148
- // src/campaign/drivers/_findings-text.ts
149
- function findingToLesson(f) {
150
- if (typeof f === "string") return f.trim() || null;
151
- if (f && typeof f === "object") {
152
- const o = f;
153
- const cand = o.recommended_action ?? o.claim ?? o.lesson ?? o.text ?? o.message;
154
- if (typeof cand === "string" && cand.trim()) return cand.trim();
150
+ // src/campaign/gates/sequential.ts
151
+ import { createHash } from "crypto";
152
+ function verifyManifestSync(m) {
153
+ if (m.algo !== void 0 && m.algo !== "sha256-content") {
154
+ throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
155
155
  }
156
- return null;
157
- }
158
- function normKey(s) {
159
- return s.toLowerCase().replace(/\s+/g, " ").replace(/[.;:!?\s]+$/, "").trim();
160
- }
161
- function surfaceToText2(surface) {
162
- if (typeof surface === "string") return surface;
163
- throw new Error(
164
- `curator driver: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
165
- );
166
- }
167
- function extractBlockBody(text, startMarker, endMarker) {
168
- const start = text.indexOf(startMarker);
169
- const end = text.indexOf(endMarker);
170
- if (start === -1 || end === -1 || end < start) return "";
171
- return text.slice(start + startMarker.length, end);
172
- }
173
- function stripBlock(text, startMarker, endMarker) {
174
- const start = text.indexOf(startMarker);
175
- const end = text.indexOf(endMarker);
176
- if (start === -1 || end === -1 || end < start) return text.trimEnd();
177
- return (text.slice(0, start) + text.slice(end + endMarker.length)).trimEnd();
156
+ const { contentHash, algo: _algo, ...rest } = m;
157
+ void _algo;
158
+ const bytes = JSON.stringify(canonicalize(rest));
159
+ const hash = createHash("sha256").update(bytes, "utf8").digest("hex");
160
+ return hash === contentHash;
178
161
  }
179
-
180
- // src/campaign/drivers/ace.ts
181
- var BLOCK_START = "<!-- BEGIN ace-playbook (auto-managed by aceDriver) -->";
182
- var BLOCK_END = "<!-- END ace-playbook -->";
183
- var DEFAULT_HEADING = "## Playbook (accumulated lessons \u2014 append-only)";
184
- function parsePlaybook(surface) {
185
- const body = extractBlockBody(surface, BLOCK_START, BLOCK_END);
186
- const out = [];
187
- for (const raw of body.split("\n")) {
188
- const line = raw.trim();
189
- if (!line.startsWith("- ")) continue;
190
- const item = line.slice(2).trim();
191
- const tag = /^\[g(-?\d+)\]\s*(.*)$/.exec(item);
192
- if (tag) out.push({ gen: Number(tag[1]), text: tag[2].trim() });
193
- else out.push({ gen: -1, text: item });
162
+ function resolveConfig(opts) {
163
+ const m = opts.preRegistration;
164
+ let alpha;
165
+ let maxN;
166
+ let direction;
167
+ let minEffect;
168
+ if (m) {
169
+ if (!verifyManifestSync(m)) {
170
+ throw new Error(
171
+ `sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
172
+ );
173
+ }
174
+ if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
175
+ throw new Error(
176
+ `sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
177
+ );
178
+ }
179
+ if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
180
+ throw new Error(
181
+ `sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
182
+ );
183
+ }
184
+ alpha = m.alpha;
185
+ maxN = m.preRegisteredN;
186
+ direction = m.direction;
187
+ minEffect = m.minEffect;
188
+ } else {
189
+ alpha = opts.alpha ?? 0.05;
190
+ if (opts.maxN === void 0) {
191
+ throw new Error(
192
+ "sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
193
+ );
194
+ }
195
+ maxN = opts.maxN;
196
+ direction = "increase";
197
+ minEffect = 0;
194
198
  }
195
- return out;
199
+ const scale = opts.scale ?? 1;
200
+ if (!Number.isFinite(scale) || scale <= 0) {
201
+ throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
202
+ }
203
+ if (!Number.isInteger(maxN) || maxN < 1) {
204
+ throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
205
+ }
206
+ const minN = opts.minN ?? 5;
207
+ if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
208
+ throw new Error(
209
+ `sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
210
+ );
211
+ }
212
+ if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
213
+ throw new Error(
214
+ `sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
215
+ );
216
+ }
217
+ const nullMean = 0.5 + minEffect / (2 * scale);
218
+ return {
219
+ alpha,
220
+ minN,
221
+ maxN,
222
+ maxBet: opts.maxBet ?? 0.5,
223
+ scale,
224
+ shuffleSeed: opts.shuffleSeed ?? 1337,
225
+ direction,
226
+ nullMean,
227
+ minEffect
228
+ };
196
229
  }
197
- function aceDriver(opts = {}) {
198
- const maxEntries = opts.maxEntries ?? 50;
199
- if (maxEntries < 1) throw new Error("aceDriver: maxEntries must be >= 1");
200
- const heading = opts.sectionHeading ?? DEFAULT_HEADING;
230
+ function makeStream(cfg) {
231
+ const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
232
+ const threshold = 1 / cfg.alpha;
233
+ let terminal;
201
234
  return {
202
- kind: "ace",
203
- async propose(ctx) {
204
- const parent = surfaceToText2(ctx.currentSurface);
205
- const existing = parsePlaybook(parent);
206
- const seen = new Set(existing.map((b) => normKey(b.text)));
207
- const fresh = [];
208
- for (const f of ctx.findings ?? []) {
209
- const lesson = findingToLesson(f);
210
- if (!lesson) continue;
211
- const k = normKey(lesson);
212
- if (!k || seen.has(k)) continue;
213
- seen.add(k);
214
- fresh.push({ gen: ctx.generation, text: lesson });
235
+ observe(delta) {
236
+ if (terminal === "undecided-at-maxN") {
237
+ throw new Error(
238
+ `sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
239
+ );
215
240
  }
216
- if (fresh.length === 0) return [];
217
- const all = [...existing, ...fresh].slice(-maxEntries);
218
- const block = [
219
- BLOCK_START,
220
- heading,
221
- ...all.map((b) => `- [g${b.gen}] ${b.text}`),
222
- BLOCK_END
223
- ].join("\n");
224
- const base = stripBlock(parent, BLOCK_START, BLOCK_END);
225
- const surface = base ? `${base}
226
-
227
- ${block}` : block;
228
- return [
229
- {
230
- surface,
231
- label: `ace-playbook +${fresh.length}`,
232
- rationale: `appended ${fresh.length} new lesson(s) from gen ${ctx.generation} findings (playbook now ${all.length} bullet(s), append-only)`
233
- }
234
- ];
241
+ if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
242
+ throw new Error(
243
+ `sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
244
+ );
245
+ }
246
+ const d = cfg.direction === "decrease" ? -delta : delta;
247
+ const step = proc.update((d / cfg.scale + 1) / 2);
248
+ if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
249
+ terminal = "promote";
250
+ }
251
+ if (terminal === "promote") {
252
+ return {
253
+ decision: "promote",
254
+ eValue: step.wealth,
255
+ n: step.n,
256
+ reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
257
+ };
258
+ }
259
+ if (step.n >= cfg.maxN) {
260
+ terminal = "undecided-at-maxN";
261
+ return {
262
+ decision: "undecided-at-maxN",
263
+ eValue: step.wealth,
264
+ n: step.n,
265
+ reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
266
+ };
267
+ }
268
+ return {
269
+ decision: "continue",
270
+ eValue: step.wealth,
271
+ n: step.n,
272
+ reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
273
+ };
274
+ },
275
+ state() {
276
+ return { ...proc.state(), decision: terminal ?? "continue" };
235
277
  }
236
278
  };
237
279
  }
238
-
239
- // src/campaign/drivers/halo.ts
240
- import { execFile } from "child_process";
241
- import { promisify } from "util";
242
-
243
- // src/campaign/drivers/analysis-edit.ts
244
- import { mkdtempSync, writeFileSync } from "fs";
245
- import { tmpdir } from "os";
246
- import { join } from "path";
247
- var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
248
- function surfaceToPromptText(surface) {
249
- return typeof surface === "string" ? surface : JSON.stringify(surface);
280
+ function seededShuffle(items, seed) {
281
+ const rng = mulberry32(seed);
282
+ for (let i = items.length - 1; i > 0; i--) {
283
+ const j = Math.floor(rng() * (i + 1));
284
+ const tmp = items[i];
285
+ items[i] = items[j];
286
+ items[j] = tmp;
287
+ }
288
+ return items;
250
289
  }
251
- function analysisEditDriver(opts) {
290
+ function sequentialPairedGate(options) {
291
+ const cfg = resolveConfig(options);
292
+ const name = options.name ?? "sequentialPairedGate";
293
+ const manifest = options.preRegistration;
294
+ const observeStream = makeStream(cfg);
252
295
  return {
253
- kind: opts.kind,
254
- async propose(ctx) {
255
- const parent = surfaceToPromptText(ctx.currentSurface);
256
- const traces = await opts.resolveTraces(ctx) ?? "";
257
- if (!traces.trim()) throw new Error(opts.noTracesError);
258
- const dir = mkdtempSync(join(tmpdir(), `${opts.kind}-driver-`));
259
- const tracePath = join(dir, "traces.jsonl");
260
- writeFileSync(tracePath, traces.endsWith("\n") ? traces : `${traces}
261
- `);
262
- const report = await opts.analyze(tracePath, ctx);
263
- const applied = await callLlm(
264
- {
265
- model: opts.applyModel,
266
- messages: [
267
- { role: "system", content: APPLY_SYSTEM },
268
- {
269
- role: "user",
270
- content: `CURRENT PROMPT:
271
- ${parent}
272
-
273
- TRACE-ANALYSIS REPORT:
274
- ${report}
275
-
276
- Return the full revised prompt.`
277
- }
278
- ]
279
- },
280
- { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
281
- );
282
- const text = applied.content.trim();
283
- if (!text || text === parent) return [];
284
- return [{ surface: text, label: opts.label, rationale: opts.rationale(report) }];
285
- }
286
- };
287
- }
288
-
289
- // src/campaign/drivers/halo.ts
290
- var execFileAsync = promisify(execFile);
291
- var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
292
- function haloDriver(opts) {
293
- const haloBin = opts.haloBin ?? "halo";
294
- const model = opts.model ?? "gpt-5.4-mini";
295
- return analysisEditDriver({
296
- kind: "halo",
297
- label: "halo",
298
- baseUrl: opts.baseUrl,
299
- apiKey: opts.apiKey,
300
- applyModel: opts.applyModel ?? model,
301
- fetchImpl: opts.fetchImpl,
302
- resolveTraces: opts.resolveTraces,
303
- noTracesError: "haloDriver: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze",
304
- // HALO's real findings are preserved verbatim in the rationale (attribution).
305
- rationale: (findings) => `halo-engine findings:
306
- ${findings.slice(0, 800)}`,
307
- analyze: async (tracePath, ctx) => {
308
- const args = [
309
- tracePath,
310
- "-p",
311
- opts.analysisPrompt ?? DEFAULT_ANALYSIS_PROMPT,
312
- "-m",
313
- model,
314
- ...opts.maxDepth !== void 0 ? ["--max-depth", String(opts.maxDepth)] : [],
315
- ...opts.maxTurns !== void 0 ? ["--max-turns", String(opts.maxTurns)] : []
316
- ];
317
- let findings;
318
- try {
319
- const { stdout } = await execFileAsync(haloBin, args, {
320
- maxBuffer: 64 * 1024 * 1024,
321
- signal: ctx.signal,
322
- env: {
323
- ...process.env,
324
- ...opts.apiKey ? { OPENAI_API_KEY: opts.apiKey } : {},
325
- OPENAI_BASE_URL: opts.baseUrl
326
- }
327
- });
328
- findings = stdout.trim();
329
- } catch (e) {
296
+ name,
297
+ observe(delta) {
298
+ return observeStream.observe(delta);
299
+ },
300
+ state() {
301
+ return observeStream.state();
302
+ },
303
+ async decide(ctx) {
304
+ if (!ctx.baselineJudgeScores) {
330
305
  throw new Error(
331
- `haloDriver: halo-engine ('${haloBin}') failed \u2014 ${e instanceof Error ? e.message : String(e)}`
306
+ `${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
332
307
  );
333
308
  }
334
- if (!findings) throw new Error("haloDriver: halo-engine produced no findings");
335
- return findings;
309
+ const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
310
+ const paired = pairHoldout(
311
+ ctx.judgeScores,
312
+ ctx.baselineJudgeScores,
313
+ scenarioIds,
314
+ (s) => s.composite
315
+ );
316
+ const deltas = paired.after.map((a, i) => a - paired.before[i]);
317
+ seededShuffle(deltas, cfg.shuffleSeed);
318
+ const stream = makeStream(cfg);
319
+ let last;
320
+ for (const d of deltas) {
321
+ last = stream.observe(d);
322
+ if (last.decision !== "continue") break;
323
+ }
324
+ const detail = {
325
+ ...stream.state(),
326
+ minN: cfg.minN,
327
+ maxN: cfg.maxN,
328
+ scale: cfg.scale,
329
+ shuffleSeed: cfg.shuffleSeed,
330
+ direction: cfg.direction,
331
+ minEffect: cfg.minEffect,
332
+ pairedN: deltas.length,
333
+ preRegisteredId: manifest?.id,
334
+ metric: manifest?.metric
335
+ };
336
+ const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
337
+ if (last === void 0) {
338
+ return {
339
+ decision: "need_more_work",
340
+ reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
341
+ contributingGates: [{ name, passed: false, detail }]
342
+ };
343
+ }
344
+ const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
345
+ return {
346
+ decision,
347
+ reasons: [`${name}: ${last.reason}`],
348
+ contributingGates: [{ name, passed: decision === "ship", detail }],
349
+ delta: meanDelta
350
+ };
336
351
  }
337
- });
338
- }
339
-
340
- // src/campaign/drivers/memory.ts
341
- var BLOCK_START2 = "<!-- BEGIN curated-memory (auto-managed by memoryCurationDriver) -->";
342
- var BLOCK_END2 = "<!-- END curated-memory -->";
343
- var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
344
- var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
345
- function extractExistingLessons(text) {
346
- return extractBlockBody(text, BLOCK_START2, BLOCK_END2).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
352
+ };
347
353
  }
348
- async function distillLessons(raw, distill) {
349
- const res = await callLlm(
350
- {
351
- model: distill.model,
352
- messages: [
353
- { role: "system", content: DISTILL_SYSTEM },
354
- { role: "user", content: `Findings:
355
- ${raw.map((r) => `- ${r}`).join("\n")}` }
356
- ]
357
- },
358
- { baseUrl: distill.baseUrl, apiKey: distill.apiKey, fetch: distill.fetchImpl }
359
- );
360
- try {
361
- const parsed = JSON.parse(res.content.trim());
362
- if (Array.isArray(parsed)) {
363
- const lessons = parsed.filter(
364
- (x) => typeof x === "string" && x.trim().length > 0
354
+ function sequentialDecide(options = {}) {
355
+ const alpha = options.alpha ?? 0.05;
356
+ const minN = options.minN ?? 5;
357
+ const scale = options.scale ?? 1;
358
+ if (!Number.isFinite(scale) || scale <= 0) {
359
+ throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
360
+ }
361
+ const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
362
+ const threshold = 1 / alpha;
363
+ let processedGenerations = 0;
364
+ let reference;
365
+ let stopped;
366
+ const topCandidate = (record) => {
367
+ if (record.candidates.length === 0) {
368
+ throw new Error(
369
+ `sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
365
370
  );
366
- if (lessons.length > 0) return lessons;
367
371
  }
368
- } catch {
369
- }
370
- return raw;
371
- }
372
- function memoryCurationDriver(opts = {}) {
373
- const maxEntries = opts.maxEntries ?? 12;
374
- const heading = opts.sectionHeading ?? DEFAULT_HEADING2;
375
- return {
376
- kind: "memory-curation",
377
- async propose(ctx) {
378
- const parent = surfaceToText2(ctx.currentSurface);
379
- const fresh = [];
380
- for (const f of ctx.findings ?? []) {
381
- const l = findingToLesson(f);
382
- if (l) fresh.push(l);
383
- }
384
- const carried = extractExistingLessons(parent);
385
- if (fresh.length === 0 && carried.length === 0) return [];
386
- const distilled = opts.distill && fresh.length > 0 ? await distillLessons(fresh, opts.distill) : fresh;
387
- const byKey = /* @__PURE__ */ new Map();
388
- for (const l of carried) {
389
- const k = normKey(l);
390
- if (k) byKey.set(k, { text: l, count: 1 });
391
- }
392
- for (const l of distilled) {
393
- const k = normKey(l);
394
- if (!k) continue;
395
- const e = byKey.get(k);
396
- if (e) e.count += 1;
397
- else byKey.set(k, { text: l, count: 1 });
398
- }
399
- const ranked = [...byKey.values()].sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)).slice(0, maxEntries);
400
- if (ranked.length === 0) return [];
401
- const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
402
- "\n"
372
+ return record.candidates.reduce((best, c) => c.composite > best.composite ? c : best);
373
+ };
374
+ const decide = ({ history }) => {
375
+ if (stopped) return stopped;
376
+ if (history.length === 0) return { stop: false };
377
+ if (reference === void 0) {
378
+ reference = new Map(
379
+ topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
403
380
  );
404
- const next = `${stripBlock(parent, BLOCK_START2, BLOCK_END2)}
405
-
406
- ${block}
407
- `;
408
- if (next === parent) return [];
409
- return [
410
- {
411
- surface: next,
412
- label: "memory-curation",
413
- rationale: `curated ${ranked.length} lessons (from ${fresh.length} new finding(s) + ${carried.length} carried)`
381
+ }
382
+ for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
383
+ const top = topCandidate(history[g]);
384
+ const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
385
+ for (const [scenarioId, refComposite] of reference) {
386
+ const candComposite = byScenario.get(scenarioId);
387
+ if (candComposite === void 0) {
388
+ throw new Error(
389
+ `sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
390
+ );
414
391
  }
415
- ];
392
+ const delta = candComposite - refComposite;
393
+ if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
394
+ throw new Error(
395
+ `sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
396
+ );
397
+ }
398
+ const step = proc.update((delta / scale + 1) / 2);
399
+ if (step.n >= minN && step.wealth >= threshold) {
400
+ stopped = {
401
+ stop: true,
402
+ reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
403
+ };
404
+ processedGenerations = history.length;
405
+ return stopped;
406
+ }
407
+ }
416
408
  }
409
+ processedGenerations = history.length;
410
+ return { stop: false };
417
411
  };
412
+ decide.state = () => proc.state();
413
+ return decide;
418
414
  }
419
415
 
420
- // src/campaign/skill-patch.ts
421
- function applySkillPatch(surface, patch) {
422
- let lines = surface.split("\n");
423
- let applied = 0;
424
- const rejected = [];
425
- const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
426
- for (const op of patch.ops) {
427
- if (op.op === "add") {
428
- if (typeof op.text !== "string" || op.text.trim() === "") {
429
- rejected.push({ op, reason: "empty add text" });
430
- continue;
431
- }
432
- const insert = op.text.split("\n");
433
- if (op.after === void 0 || op.after === "") {
434
- lines = [...lines, ...insert];
435
- applied++;
436
- continue;
437
- }
438
- const idx = findLine(op.after);
439
- if (idx === -1) {
440
- rejected.push({ op, reason: `add anchor not found: ${truncate(op.after)}` });
441
- continue;
442
- }
443
- lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
444
- applied++;
445
- } else if (op.op === "delete") {
446
- const idx = findLine(op.anchor);
447
- if (idx === -1) {
448
- rejected.push({ op, reason: `delete anchor not found: ${truncate(op.anchor)}` });
449
- continue;
450
- }
451
- lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
452
- applied++;
453
- } else {
454
- const idx = findLine(op.anchor);
455
- if (idx === -1) {
456
- rejected.push({ op, reason: `replace anchor not found: ${truncate(op.anchor)}` });
457
- continue;
458
- }
459
- if (typeof op.text !== "string") {
460
- rejected.push({ op, reason: "replace text missing" });
461
- continue;
416
+ // src/campaign/labeled-store/fs-adapter.ts
417
+ import { createHash as createHash2 } from "crypto";
418
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
419
+ import { join } from "path";
420
+ var LabeledScenarioStoreError = class extends Error {
421
+ constructor(code, message) {
422
+ super(message);
423
+ this.code = code;
424
+ this.name = "LabeledScenarioStoreError";
425
+ }
426
+ code;
427
+ };
428
+ var FsLabeledScenarioStore = class {
429
+ constructor(options) {
430
+ this.options = options;
431
+ if (!existsSync(options.root)) mkdirSync(options.root, { recursive: true });
432
+ this.now = options.now ?? Date.now;
433
+ }
434
+ options;
435
+ now;
436
+ rateLimits = /* @__PURE__ */ new Map();
437
+ async observe(write) {
438
+ this.assertProvenance(write);
439
+ this.assertRateLimit(write);
440
+ const record = this.toRecord(write);
441
+ const path = this.pathForSource(write.source);
442
+ const line = `${JSON.stringify(record)}
443
+ `;
444
+ appendLine(path, line);
445
+ }
446
+ async sample(args) {
447
+ if (!args.split) {
448
+ throw new LabeledScenarioStoreError(
449
+ "split_required",
450
+ "sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
451
+ );
452
+ }
453
+ if (!args.capturedBefore) {
454
+ throw new LabeledScenarioStoreError(
455
+ "capturedBefore_required",
456
+ "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
457
+ );
458
+ }
459
+ const all = [];
460
+ for (const source of ALL_SOURCES) {
461
+ if (args.split === "train" && source === "production-trace") {
462
+ const explicit = sourceFilterContains(args.filter?.source, "production-trace");
463
+ if (!explicit) continue;
464
+ }
465
+ const path = this.pathForSource(source);
466
+ if (!existsSync(path)) continue;
467
+ const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
468
+ for (const line of lines) {
469
+ let record;
470
+ try {
471
+ record = JSON.parse(line);
472
+ } catch {
473
+ continue;
474
+ }
475
+ if (!matchesFilter(record, args, source)) continue;
476
+ all.push(record);
477
+ }
478
+ }
479
+ all.sort((a, b) => {
480
+ if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
481
+ return a.recordHash.localeCompare(b.recordHash);
482
+ });
483
+ return all.slice(0, args.count);
484
+ }
485
+ async size() {
486
+ const bySource = {};
487
+ const byTrust = {
488
+ unverified: 0,
489
+ "verified-signal": 0,
490
+ "human-rated": 0
491
+ };
492
+ let total = 0;
493
+ for (const source of ALL_SOURCES) {
494
+ const path = this.pathForSource(source);
495
+ if (!existsSync(path)) {
496
+ bySource[source] = 0;
497
+ continue;
498
+ }
499
+ const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
500
+ bySource[source] = lines.length;
501
+ total += lines.length;
502
+ for (const line of lines) {
503
+ let trust = "unverified";
504
+ try {
505
+ trust = JSON.parse(line).labelTrust ?? "unverified";
506
+ } catch {
507
+ }
508
+ byTrust[trust] += 1;
462
509
  }
463
- lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
464
- applied++;
465
510
  }
511
+ return { train: total, test: total, bySource, byTrust };
466
512
  }
467
- return { surface: lines.join("\n"), applied, rejected };
513
+ assertProvenance(write) {
514
+ if (!write.source) {
515
+ throw new LabeledScenarioStoreError(
516
+ "missing_source",
517
+ "LabeledScenarioWrite requires `source`"
518
+ );
519
+ }
520
+ if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
521
+ throw new LabeledScenarioStoreError(
522
+ "missing_source_version",
523
+ "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
524
+ );
525
+ }
526
+ if (!write.capturedAt) {
527
+ throw new LabeledScenarioStoreError(
528
+ "missing_captured_at",
529
+ "LabeledScenarioWrite requires `capturedAt` ISO timestamp"
530
+ );
531
+ }
532
+ if (!write.redactionStatus) {
533
+ throw new LabeledScenarioStoreError(
534
+ "missing_redaction_status",
535
+ "LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
536
+ );
537
+ }
538
+ if (!ALL_SOURCES.includes(write.source)) {
539
+ throw new LabeledScenarioStoreError(
540
+ "unknown_source",
541
+ `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
542
+ );
543
+ }
544
+ }
545
+ assertRateLimit(write) {
546
+ const cap = this.options.maxWritesPerMinutePerBucket;
547
+ if (!cap || !write.rateLimitBucket) return;
548
+ const now = this.now();
549
+ const windowMs = 6e4;
550
+ let state = this.rateLimits.get(write.rateLimitBucket);
551
+ if (!state || now - state.windowStartMs >= windowMs) {
552
+ state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
553
+ this.rateLimits.set(write.rateLimitBucket, state);
554
+ }
555
+ if (state.count >= cap) {
556
+ throw new LabeledScenarioStoreError(
557
+ "rate_limit_exceeded",
558
+ `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
559
+ );
560
+ }
561
+ state.count += 1;
562
+ }
563
+ toRecord(write) {
564
+ const recordHash = sha256(
565
+ JSON.stringify({
566
+ id: write.scenario.id,
567
+ src: write.source,
568
+ at: write.capturedAt,
569
+ ver: write.sourceVersionHash
570
+ })
571
+ );
572
+ return {
573
+ ...write,
574
+ recordHash,
575
+ split: "train"
576
+ };
577
+ }
578
+ pathForSource(source) {
579
+ return join(this.options.root, `${source}.jsonl`);
580
+ }
581
+ };
582
+ var ALL_SOURCES = [
583
+ "production-trace",
584
+ "eval-run",
585
+ "manual",
586
+ "red-team",
587
+ "synthetic"
588
+ ];
589
+ function sourceFilterContains(filter, needle) {
590
+ if (!filter) return false;
591
+ if (Array.isArray(filter)) return filter.includes(needle);
592
+ return filter === needle;
468
593
  }
469
- function patchEditCount(patch) {
470
- return patch.ops.length;
594
+ function matchesFilter(record, args, source) {
595
+ if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
596
+ if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
597
+ const f = args.filter;
598
+ if (!f) return true;
599
+ if (f.kind && record.scenario.kind !== f.kind) return false;
600
+ if (f.source) {
601
+ const sources = Array.isArray(f.source) ? f.source : [f.source];
602
+ if (!sources.includes(source)) return false;
603
+ }
604
+ if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
605
+ const composites = Object.values(record.judgeScores).map((s) => s.composite);
606
+ const max = composites.length === 0 ? 0 : Math.max(...composites);
607
+ if (f.minComposite !== void 0 && max < f.minComposite) return false;
608
+ if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
609
+ }
610
+ if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
611
+ return false;
612
+ }
613
+ return true;
471
614
  }
472
- function truncate(s, max = 48) {
473
- return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
615
+ function sha256(input) {
616
+ return createHash2("sha256").update(input).digest("hex").slice(0, 16);
617
+ }
618
+ function appendLine(path, line) {
619
+ if (existsSync(path)) {
620
+ const existing = readFileSync(path, "utf8");
621
+ writeFileSync(path, existing + line);
622
+ } else {
623
+ writeFileSync(path, line);
624
+ }
474
625
  }
475
626
 
476
- // src/campaign/drivers/skill-opt.ts
477
- var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
478
- function skillOptDriver(opts) {
479
- const evidenceK = opts.evidenceK ?? 3;
480
- const defaultBudget = opts.editBudget ?? 3;
481
- async function proposePatches(args) {
482
- const userPrompt = buildPatchPrompt({
483
- target: opts.target,
484
- surface: args.surface,
485
- evidence: args.evidence,
486
- editBudget: args.editBudget,
487
- rejectedBuffer: args.rejectedBuffer,
488
- metaNote: args.metaNote,
489
- findingsNote: args.findingsNote,
490
- count: args.count
491
- });
492
- const result = await callLlm(
493
- {
494
- model: opts.model,
495
- messages: [
496
- { role: "system", content: SKILLOPT_SYSTEM },
497
- { role: "user", content: userPrompt }
498
- ],
499
- jsonMode: true,
500
- temperature: opts.temperature ?? 0.6,
501
- maxTokens: opts.maxTokens ?? 4e3
502
- },
503
- opts.llm
504
- );
505
- return parseSkillPatchResponse(result.content, args.count, args.editBudget);
627
+ // src/campaign/proposers/fapo.ts
628
+ var FAPO_LEVELS = ["prompt", "parameter", "structural"];
629
+ var MAX_FINDING_DEPTH = 16;
630
+ var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
631
+ function fapoProposer(opts) {
632
+ const proposers = levelProposers(opts);
633
+ const allowed = allowedLevels(opts, proposers);
634
+ const plateauWindow = opts.plateauWindow ?? 3;
635
+ const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
636
+ const minImprovement = opts.minImprovement ?? 0;
637
+ const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
638
+ if (allowed.length === 0) {
639
+ throw new Error("fapoProposer: at least one allowed level must have a proposer");
506
640
  }
641
+ if (plateauWindow < 1) throw new Error("fapoProposer: plateauWindow must be >= 1");
642
+ if (minDistinctStrategies < 1) {
643
+ throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
644
+ }
645
+ if (proposalsPerCycle < 1) throw new Error("fapoProposer: proposalsPerCycle must be >= 1");
507
646
  return {
508
- kind: "skill-opt",
509
- proposePatches,
647
+ kind: "fapo",
510
648
  async propose(ctx) {
511
- if (typeof ctx.currentSurface !== "string") {
649
+ const state = policyState(ctx.history, ctx.findings, {
650
+ plateauWindow,
651
+ minDistinctStrategies,
652
+ minImprovement
653
+ });
654
+ const decision = chooseLevel({
655
+ allowed,
656
+ available: allowed.filter((level) => !state.exhausted[level]),
657
+ proposers,
658
+ state,
659
+ promptFirst: opts.promptFirst ?? true,
660
+ parameterBeforeStructural: opts.parameterBeforeStructural ?? true
661
+ });
662
+ if (!decision) return [];
663
+ const proposer = proposers[decision.level];
664
+ if (!proposer) {
665
+ throw new Error(`fapoProposer: selected ${decision.level} but no proposer is configured`);
666
+ }
667
+ const requested = Math.min(ctx.populationSize, proposalsPerCycle);
668
+ const raw = await proposer.propose({ ...ctx, populationSize: requested });
669
+ const wrapped = raw.map((candidate) => normalizeProposal(candidate)).map((candidate) => wrapProposal(candidate, decision.level, decision.reason));
670
+ const reviewed = [];
671
+ for (const candidate of wrapped) {
672
+ const review = opts.reviewCandidate ? await opts.reviewCandidate({
673
+ level: decision.level,
674
+ candidate,
675
+ context: ctx,
676
+ reason: decision.reason
677
+ }) : { verdict: "pass" };
678
+ if (review.verdict === "fail") continue;
679
+ const warn = review.verdict === "warn" ? renderReviewWarnings(review) : "";
680
+ reviewed.push(
681
+ warn ? { ...candidate, rationale: `${candidate.rationale}
682
+ ${warn}` } : candidate
683
+ );
684
+ }
685
+ if (wrapped.length > 0 && reviewed.length === 0) {
512
686
  throw new Error(
513
- "skillOptDriver: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
687
+ `fapoProposer: reviewer blocked every ${decision.level} candidate; refusing to evaluate an unreviewed variant`
514
688
  );
515
689
  }
516
- const surface = ctx.currentSurface;
517
- const patches = await proposePatches({
518
- surface,
519
- evidence: evidenceFromHistory(ctx, evidenceK),
520
- editBudget: defaultBudget,
521
- rejectedBuffer: [],
522
- findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
523
- count: ctx.populationSize,
524
- signal: ctx.signal
690
+ return reviewed;
691
+ },
692
+ decide({ history }) {
693
+ const last = history[history.length - 1];
694
+ if (last && last.candidates.length === 0) {
695
+ return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
696
+ }
697
+ const state = policyState(history, [], {
698
+ plateauWindow,
699
+ minDistinctStrategies,
700
+ minImprovement
701
+ });
702
+ const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
703
+ return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
704
+ }
705
+ };
706
+ }
707
+ function levelProposers(opts) {
708
+ return {
709
+ ...opts.proposers ?? {},
710
+ ...opts.promptProposer ? { prompt: opts.promptProposer } : {},
711
+ ...opts.parameterProposer ? { parameter: opts.parameterProposer } : {},
712
+ ...opts.structuralProposer ? { structural: opts.structuralProposer } : {}
713
+ };
714
+ }
715
+ function allowedLevels(opts, proposers) {
716
+ const explicit = opts.scope?.allowedLevels;
717
+ const forbidden = new Set(opts.scope?.forbiddenLevels ?? []);
718
+ return FAPO_LEVELS.filter((level) => {
719
+ if (!proposers[level]) return false;
720
+ if (forbidden.has(level)) return false;
721
+ return explicit ? explicit.includes(level) : true;
722
+ });
723
+ }
724
+ function policyState(history, findings, opts) {
725
+ const attempts = extractAttempts(history);
726
+ return {
727
+ attempts,
728
+ exhausted: {
729
+ prompt: isLevelExhausted("prompt", attempts, opts),
730
+ parameter: isLevelExhausted("parameter", attempts, opts),
731
+ structural: isLevelExhausted("structural", attempts, opts)
732
+ },
733
+ signals: extractFapoAttributionSignals(findings)
734
+ };
735
+ }
736
+ function extractAttempts(history) {
737
+ const attempts = [];
738
+ for (const generation of history) {
739
+ for (const candidate of generation.candidates) {
740
+ const level = candidateLevel(candidate);
741
+ if (!level) continue;
742
+ attempts.push({
743
+ level,
744
+ composite: candidate.composite,
745
+ strategy: candidateStrategy(candidate)
746
+ });
747
+ }
748
+ }
749
+ return attempts;
750
+ }
751
+ function candidateLevel(candidate) {
752
+ const label = candidate.label ?? "";
753
+ const rationale = candidate.rationale ?? "";
754
+ return parseLevel(label) ?? parseLevel(rationale);
755
+ }
756
+ function parseLevel(text) {
757
+ const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
758
+ return match ? match[1] : null;
759
+ }
760
+ function candidateStrategy(candidate) {
761
+ const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
762
+ return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
763
+ }
764
+ function isLevelExhausted(level, attempts, opts) {
765
+ const own = attempts.filter((attempt) => attempt.level === level);
766
+ if (own.length < opts.plateauWindow) return false;
767
+ const distinctStrategies = new Set(own.map((attempt) => attempt.strategy)).size;
768
+ if (distinctStrategies < opts.minDistinctStrategies) return false;
769
+ let best = Number.NEGATIVE_INFINITY;
770
+ let nonImproving = 0;
771
+ for (const attempt of own) {
772
+ if (attempt.composite > best + opts.minImprovement) {
773
+ best = attempt.composite;
774
+ nonImproving = 0;
775
+ } else {
776
+ nonImproving += 1;
777
+ }
778
+ }
779
+ return nonImproving >= opts.plateauWindow;
780
+ }
781
+ function chooseLevel(args) {
782
+ const { available, state } = args;
783
+ if (available.length === 0) return null;
784
+ const triedPrompt = state.attempts.some((attempt) => attempt.level === "prompt");
785
+ if (args.promptFirst && available.includes("prompt") && !triedPrompt) {
786
+ return {
787
+ level: "prompt",
788
+ reason: "prompt-first policy: no prompt-level variant has been tried yet"
789
+ };
790
+ }
791
+ const supported = available.filter((level) => state.signals.counts[level] > 0);
792
+ if (supported.includes("prompt")) {
793
+ return {
794
+ level: "prompt",
795
+ reason: `attribution has ${state.signals.counts.prompt} prompt-addressable failure(s)`
796
+ };
797
+ }
798
+ const strongest = supported.sort(
799
+ (a, b) => state.signals.counts[b] - state.signals.counts[a] || levelRank(a) - levelRank(b)
800
+ )[0];
801
+ if (strongest) {
802
+ if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
803
+ return {
804
+ level: "parameter",
805
+ reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
806
+ };
807
+ }
808
+ return {
809
+ level: strongest,
810
+ reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
811
+ };
812
+ }
813
+ if (state.attempts.length === 0) {
814
+ const first = available.slice().sort((a, b) => levelRank(a) - levelRank(b))[0];
815
+ return { level: first, reason: "no attribution yet; starting at the cheapest allowed level" };
816
+ }
817
+ return null;
818
+ }
819
+ function levelRank(level) {
820
+ return level === "prompt" ? 0 : level === "parameter" ? 1 : 2;
821
+ }
822
+ function normalizeProposal(value) {
823
+ return isProposedCandidate(value) ? value : { surface: value, label: "candidate", rationale: "bare candidate from level proposer" };
824
+ }
825
+ function wrapProposal(candidate, level, reason) {
826
+ const label = candidate.label ? `fapo:${level}:${candidate.label}` : `fapo:${level}`;
827
+ return {
828
+ ...candidate,
829
+ label,
830
+ rationale: [
831
+ `fapo:${level} selected by reviewed escalation policy`,
832
+ `Reason: ${reason}`,
833
+ candidate.rationale ? `Level proposer rationale: ${candidate.rationale}` : ""
834
+ ].filter(Boolean).join("\n")
835
+ };
836
+ }
837
+ function renderReviewWarnings(review) {
838
+ const issues = review.issues?.filter((issue) => issue.severity === "warn") ?? [];
839
+ if (issues.length === 0) return "";
840
+ return `Reviewer warnings:
841
+ ${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
842
+ }
843
+ function extractFapoAttributionSignals(findings) {
844
+ const signals = {
845
+ counts: { prompt: 0, parameter: 0, structural: 0 },
846
+ clusters: []
847
+ };
848
+ const seen = /* @__PURE__ */ new WeakSet();
849
+ for (const finding of findings) {
850
+ collectFinding(signals, finding, seen, 0);
851
+ }
852
+ return signals;
853
+ }
854
+ function collectFinding(signals, finding, seen, depth) {
855
+ if (depth > MAX_FINDING_DEPTH) return;
856
+ if (!finding) return;
857
+ if (typeof finding === "string") {
858
+ const level = inferLevelFromText(finding);
859
+ if (level) addCluster(signals, { label: finding, level, count: 1 });
860
+ return;
861
+ }
862
+ if (Array.isArray(finding)) {
863
+ if (seen.has(finding)) return;
864
+ seen.add(finding);
865
+ for (const item of finding) collectFinding(signals, item, seen, depth + 1);
866
+ return;
867
+ }
868
+ if (typeof finding !== "object") return;
869
+ if (seen.has(finding)) return;
870
+ seen.add(finding);
871
+ const obj = finding;
872
+ collectLevelPartition(signals, obj.level_partition ?? obj.levelPartition, seen, depth + 1);
873
+ collectCounts(signals, obj);
874
+ collectClusters(signals, obj.clusters, true, seen, depth + 1);
875
+ const explicit = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel);
876
+ const text = textFields(obj);
877
+ const inferred = explicit ?? inferLevelFromText(text);
878
+ if (inferred) {
879
+ addCluster(signals, {
880
+ label: text || inferred,
881
+ level: inferred,
882
+ count: numberField(obj.count) ?? 1,
883
+ confidence: confidenceField(obj.confidence),
884
+ suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
885
+ caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
886
+ });
887
+ }
888
+ }
889
+ function collectLevelPartition(signals, raw, seen, depth) {
890
+ if (!raw || typeof raw !== "object") return;
891
+ if (seen.has(raw)) return;
892
+ seen.add(raw);
893
+ const partition = raw;
894
+ for (const level of FAPO_LEVELS) {
895
+ const bucket = partition[level];
896
+ if (!bucket || typeof bucket !== "object") continue;
897
+ const obj = bucket;
898
+ const count = numberField(obj.count) ?? 0;
899
+ if (count > 0) signals.counts[level] += count;
900
+ collectClusters(signals, obj.clusters, false, seen, depth + 1);
901
+ }
902
+ }
903
+ function collectCounts(signals, obj) {
904
+ const prompt = numberField(obj.prompt_addressable ?? obj.promptAddressable);
905
+ const structural = numberField(obj.structural_addressable ?? obj.structuralAddressable);
906
+ const tool = numberField(obj.tool_addressable ?? obj.toolAddressable);
907
+ if (prompt) signals.counts.prompt += prompt;
908
+ if (structural) signals.counts.structural += structural;
909
+ if (tool) signals.counts.structural += tool;
910
+ }
911
+ function collectClusters(signals, raw, countClusters, seen, depth) {
912
+ if (!Array.isArray(raw)) return;
913
+ if (seen.has(raw)) return;
914
+ seen.add(raw);
915
+ for (const item of raw) {
916
+ if (countClusters) {
917
+ collectFinding(signals, item, seen, depth + 1);
918
+ continue;
919
+ }
920
+ const cluster = parseCluster(item);
921
+ if (cluster) signals.clusters.push(cluster);
922
+ }
923
+ }
924
+ function addCluster(signals, cluster) {
925
+ signals.counts[cluster.level] += Math.max(1, cluster.count);
926
+ signals.clusters.push(cluster);
927
+ }
928
+ function parseCluster(raw) {
929
+ if (!raw || typeof raw !== "object") return null;
930
+ const obj = raw;
931
+ const text = textFields(obj);
932
+ const level = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel) ?? inferLevelFromText(text);
933
+ if (!level) return null;
934
+ return {
935
+ label: text || level,
936
+ level,
937
+ count: numberField(obj.count) ?? 1,
938
+ confidence: confidenceField(obj.confidence),
939
+ suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
940
+ caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
941
+ };
942
+ }
943
+ function parseFindingLevel(raw) {
944
+ if (typeof raw !== "string") return null;
945
+ const lower = raw.toLowerCase();
946
+ if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
947
+ if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
948
+ if (lower === "config" || lower === "params") return "parameter";
949
+ return null;
950
+ }
951
+ function inferLevelFromText(text) {
952
+ const lower = text.toLowerCase();
953
+ if (/\b(retrieval_k|temperature|top_p|max_tokens|max_completion_tokens|reasoning_effort|config|parameter)\b/.test(
954
+ lower
955
+ )) {
956
+ return "parameter";
957
+ }
958
+ if (/\b(retriev\w*|search\w*|bm25|hop|evidence|cascade|tool|node|chain|state|structur\w*|topology|import)\b/.test(
959
+ lower
960
+ )) {
961
+ return "structural";
962
+ }
963
+ if (/\b(format\w*|verbose|brevity|abstain\w*|reasoning|instruction|prompt|output|answer)\b/.test(
964
+ lower
965
+ )) {
966
+ return "prompt";
967
+ }
968
+ return null;
969
+ }
970
+ function textFields(obj) {
971
+ return [
972
+ obj.label,
973
+ obj.claim,
974
+ obj.recommended_action,
975
+ obj.suggested_fix,
976
+ obj.suggestedFix,
977
+ obj.message,
978
+ obj.text,
979
+ obj.heuristic,
980
+ obj.area
981
+ ].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
982
+ }
983
+ function numberField(value) {
984
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
985
+ }
986
+ function stringField(value) {
987
+ return typeof value === "string" && value.trim() ? value.trim() : void 0;
988
+ }
989
+ function confidenceField(value) {
990
+ return value === "high" || value === "medium" || value === "low" ? value : void 0;
991
+ }
992
+ function stringArrayField(value) {
993
+ return Array.isArray(value) ? value.filter((item) => typeof item === "string") : void 0;
994
+ }
995
+ function parameterSweepProposer(opts) {
996
+ if (opts.candidates.length === 0) {
997
+ throw new Error("parameterSweepProposer: candidates must not be empty");
998
+ }
999
+ return {
1000
+ kind: "parameter-sweep",
1001
+ async propose(ctx) {
1002
+ if (typeof ctx.currentSurface !== "string") {
1003
+ throw new Error(
1004
+ "parameterSweepProposer: currentSurface must be a JSON string config surface"
1005
+ );
1006
+ }
1007
+ const parse = opts.parse ?? parseJsonObject;
1008
+ const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
1009
+ const current = parse(ctx.currentSurface);
1010
+ const currentCanonical = canonicalJson(current);
1011
+ const tried = triedLabels(ctx.history);
1012
+ const out = [];
1013
+ for (const candidate of opts.candidates) {
1014
+ if (tried.has(candidate.label)) continue;
1015
+ const next = applyParameterCandidate(current, candidate);
1016
+ const surface = stringify(next);
1017
+ if (surface === ctx.currentSurface || canonicalJson(next) === currentCanonical) continue;
1018
+ out.push({ surface, label: candidate.label, rationale: candidate.rationale });
1019
+ if (out.length >= ctx.populationSize) break;
1020
+ }
1021
+ return out;
1022
+ }
1023
+ };
1024
+ }
1025
+ function parseJsonObject(surface) {
1026
+ let parsed;
1027
+ try {
1028
+ parsed = JSON.parse(surface);
1029
+ } catch (error) {
1030
+ throw new Error(
1031
+ `parameterSweepProposer: currentSurface must be valid JSON object config (${error instanceof Error ? error.message : String(error)})`
1032
+ );
1033
+ }
1034
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
1035
+ throw new Error("parameterSweepProposer: JSON surface must parse to an object");
1036
+ }
1037
+ return parsed;
1038
+ }
1039
+ function triedLabels(history) {
1040
+ const tried = /* @__PURE__ */ new Set();
1041
+ for (const generation of history) {
1042
+ for (const candidate of generation.candidates) {
1043
+ if (!candidate.label) continue;
1044
+ tried.add(candidate.label);
1045
+ tried.add(candidateStrategy(candidate));
1046
+ }
1047
+ }
1048
+ return tried;
1049
+ }
1050
+ function applyParameterCandidate(current, candidate) {
1051
+ const next = cloneJsonObject(current);
1052
+ if (candidate.patch) deepMerge(next, candidate.patch);
1053
+ for (const change of candidate.changes ?? []) {
1054
+ setPath(
1055
+ next,
1056
+ typeof change.path === "string" ? change.path.split(".") : [...change.path],
1057
+ change.value
1058
+ );
1059
+ }
1060
+ return next;
1061
+ }
1062
+ function cloneJsonObject(value) {
1063
+ return JSON.parse(JSON.stringify(value));
1064
+ }
1065
+ function deepMerge(target, patch) {
1066
+ for (const [key, value] of Object.entries(patch)) {
1067
+ assertSafeJsonKey(key);
1068
+ if (isPlainObject(value) && isPlainObject(target[key])) {
1069
+ deepMerge(target[key], value);
1070
+ } else {
1071
+ target[key] = value;
1072
+ }
1073
+ }
1074
+ }
1075
+ function setPath(target, path, value) {
1076
+ if (path.length === 0) throw new Error("parameterSweepProposer: change path must not be empty");
1077
+ for (const part of path) assertSafeJsonKey(part);
1078
+ let cursor = target;
1079
+ for (const part of path.slice(0, -1)) {
1080
+ const existing = cursor[part];
1081
+ if (!isPlainObject(existing)) {
1082
+ cursor[part] = {};
1083
+ }
1084
+ cursor = cursor[part];
1085
+ }
1086
+ cursor[path[path.length - 1]] = value;
1087
+ }
1088
+ function assertSafeJsonKey(key) {
1089
+ if (!key.trim()) throw new Error("parameterSweepProposer: change path contains an empty key");
1090
+ if (UNSAFE_JSON_KEYS.has(key)) {
1091
+ throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
1092
+ }
1093
+ }
1094
+ function canonicalJson(value) {
1095
+ if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
1096
+ if (isPlainObject(value)) {
1097
+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
1098
+ }
1099
+ return JSON.stringify(value);
1100
+ }
1101
+ function isPlainObject(value) {
1102
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
1103
+ const proto = Object.getPrototypeOf(value);
1104
+ return proto === Object.prototype || proto === null;
1105
+ }
1106
+
1107
+ // src/campaign/skill-patch.ts
1108
+ function applySkillPatch(surface, patch) {
1109
+ let lines = surface.split("\n");
1110
+ let applied = 0;
1111
+ const rejected = [];
1112
+ const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
1113
+ for (const op of patch.ops) {
1114
+ if (op.op === "add") {
1115
+ if (typeof op.text !== "string" || op.text.trim() === "") {
1116
+ rejected.push({ op, reason: "empty add text" });
1117
+ continue;
1118
+ }
1119
+ const insert = op.text.split("\n");
1120
+ if (op.after === void 0 || op.after === "") {
1121
+ lines = [...lines, ...insert];
1122
+ applied++;
1123
+ continue;
1124
+ }
1125
+ const idx = findLine(op.after);
1126
+ if (idx === -1) {
1127
+ rejected.push({ op, reason: `add anchor not found: ${truncate(op.after)}` });
1128
+ continue;
1129
+ }
1130
+ lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
1131
+ applied++;
1132
+ } else if (op.op === "delete") {
1133
+ const idx = findLine(op.anchor);
1134
+ if (idx === -1) {
1135
+ rejected.push({ op, reason: `delete anchor not found: ${truncate(op.anchor)}` });
1136
+ continue;
1137
+ }
1138
+ lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
1139
+ applied++;
1140
+ } else {
1141
+ const idx = findLine(op.anchor);
1142
+ if (idx === -1) {
1143
+ rejected.push({ op, reason: `replace anchor not found: ${truncate(op.anchor)}` });
1144
+ continue;
1145
+ }
1146
+ if (typeof op.text !== "string") {
1147
+ rejected.push({ op, reason: "replace text missing" });
1148
+ continue;
1149
+ }
1150
+ lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
1151
+ applied++;
1152
+ }
1153
+ }
1154
+ return { surface: lines.join("\n"), applied, rejected };
1155
+ }
1156
+ function patchEditCount(patch) {
1157
+ return patch.ops.length;
1158
+ }
1159
+ function truncate(s, max = 48) {
1160
+ return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
1161
+ }
1162
+
1163
+ // src/campaign/proposers/skill-opt.ts
1164
+ var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
1165
+ function skillOptProposer(opts) {
1166
+ const evidenceK = opts.evidenceK ?? 3;
1167
+ const defaultBudget = opts.editBudget ?? 3;
1168
+ async function proposePatches(args) {
1169
+ const userPrompt = buildPatchPrompt({
1170
+ target: opts.target,
1171
+ surface: args.surface,
1172
+ evidence: args.evidence,
1173
+ editBudget: args.editBudget,
1174
+ rejectedBuffer: args.rejectedBuffer,
1175
+ metaNote: args.metaNote,
1176
+ findingsNote: args.findingsNote,
1177
+ count: args.count
1178
+ });
1179
+ const result = await callLlm(
1180
+ {
1181
+ model: opts.model,
1182
+ messages: [
1183
+ { role: "system", content: SKILLOPT_SYSTEM },
1184
+ { role: "user", content: userPrompt }
1185
+ ],
1186
+ jsonMode: true,
1187
+ temperature: opts.temperature ?? 0.6,
1188
+ maxTokens: opts.maxTokens ?? 4e3
1189
+ },
1190
+ opts.llm
1191
+ );
1192
+ return parseSkillPatchResponse(result.content, args.count, args.editBudget);
1193
+ }
1194
+ return {
1195
+ kind: "skill-opt",
1196
+ proposePatches,
1197
+ async propose(ctx) {
1198
+ if (typeof ctx.currentSurface !== "string") {
1199
+ throw new Error(
1200
+ "skillOptProposer: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
1201
+ );
1202
+ }
1203
+ const surface = ctx.currentSurface;
1204
+ const patches = await proposePatches({
1205
+ surface,
1206
+ evidence: evidenceFromHistory(ctx, evidenceK),
1207
+ editBudget: defaultBudget,
1208
+ rejectedBuffer: [],
1209
+ findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
1210
+ count: ctx.populationSize,
1211
+ signal: ctx.signal
525
1212
  });
526
1213
  const out = [];
527
1214
  const seen = /* @__PURE__ */ new Set();
@@ -655,770 +1342,236 @@ function snippet(s, max = 120) {
655
1342
  return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
656
1343
  }
657
1344
 
658
- // src/campaign/drivers/trace-analyst.ts
659
- import { ai } from "@ax-llm/ax";
660
- function renderFindings(findings) {
661
- return findings.map((f, i) => {
662
- const action = f.recommended_action ? `
663
- FIX: ${f.recommended_action}` : "";
664
- const subject = f.subject ? ` (${f.subject})` : "";
665
- return `${i + 1}. [${f.severity}/${f.area}]${subject} ${f.claim}${action}`;
666
- }).join("\n");
667
- }
668
- function traceAnalystDriver(opts) {
669
- if (!opts.apiKey) throw new Error("traceAnalystDriver: apiKey is required");
670
- if (!opts.model) throw new Error("traceAnalystDriver: model is required");
671
- const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
672
- const produceFindings = opts.analyze ?? (async (path, c) => {
673
- const aiService = ai({
674
- name: opts.provider ?? "openai",
675
- apiKey: opts.apiKey,
676
- apiURL: opts.baseUrl,
677
- config: { model: opts.model }
678
- });
679
- const registry = new AnalystRegistry();
680
- for (const spec of kinds) {
681
- registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
682
- }
683
- const result = await registry.run(
684
- `trace-analyst-gen-${c.generation}`,
685
- { traceStore: new OtlpFileTraceStore({ path }) },
686
- { signal: c.signal }
1345
+ // src/campaign/presets/run-skill-opt.ts
1346
+ async function runSkillOpt(opts) {
1347
+ if (opts.trainScenarios.length === 0) throw new Error("runSkillOpt: trainScenarios is empty");
1348
+ if (opts.holdoutScenarios.length === 0) throw new Error("runSkillOpt: holdoutScenarios is empty");
1349
+ if (!opts.judges || opts.judges.length === 0) {
1350
+ throw new Error(
1351
+ "runSkillOpt: at least one judge is required \u2014 scoring (and therefore acceptance) is meaningless without one, and would report a silent zero lift."
687
1352
  );
688
- return result.findings;
689
- });
690
- return analysisEditDriver({
691
- kind: "trace-analyst",
692
- label: "trace-analyst",
693
- baseUrl: opts.baseUrl,
694
- apiKey: opts.apiKey,
695
- applyModel: opts.applyModel ?? opts.model,
696
- fetchImpl: opts.fetchImpl,
697
- resolveTraces: opts.resolveTraces,
698
- noTracesError: "traceAnalystDriver: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read",
699
- rationale: (report) => `trace-analyst findings:
700
- ${report.slice(0, 800)}`,
701
- analyze: async (tracePath, ctx) => {
702
- let findings;
703
- try {
704
- findings = await produceFindings(tracePath, ctx);
705
- } catch (e) {
706
- throw new Error(
707
- `traceAnalystDriver: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
708
- );
709
- }
710
- if (findings.length === 0) {
711
- throw new Error("traceAnalystDriver: analyst engine produced no findings");
712
- }
713
- return renderFindings(findings);
714
- }
715
- });
716
- }
717
-
718
- // src/campaign/gates/sequential.ts
719
- import { createHash } from "crypto";
720
- function verifyManifestSync(m) {
721
- if (m.algo !== void 0 && m.algo !== "sha256-content") {
722
- throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
723
- }
724
- const { contentHash, algo: _algo, ...rest } = m;
725
- void _algo;
726
- const bytes = JSON.stringify(canonicalize(rest));
727
- const hash = createHash("sha256").update(bytes, "utf8").digest("hex");
728
- return hash === contentHash;
729
- }
730
- function resolveConfig(opts) {
731
- const m = opts.preRegistration;
732
- let alpha;
733
- let maxN;
734
- let direction;
735
- let minEffect;
736
- if (m) {
737
- if (!verifyManifestSync(m)) {
738
- throw new Error(
739
- `sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
740
- );
741
- }
742
- if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
743
- throw new Error(
744
- `sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
745
- );
746
- }
747
- if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
748
- throw new Error(
749
- `sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
750
- );
751
- }
752
- alpha = m.alpha;
753
- maxN = m.preRegisteredN;
754
- direction = m.direction;
755
- minEffect = m.minEffect;
756
- } else {
757
- alpha = opts.alpha ?? 0.05;
758
- if (opts.maxN === void 0) {
759
- throw new Error(
760
- "sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
761
- );
762
- }
763
- maxN = opts.maxN;
764
- direction = "increase";
765
- minEffect = 0;
766
- }
767
- const scale = opts.scale ?? 1;
768
- if (!Number.isFinite(scale) || scale <= 0) {
769
- throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
770
- }
771
- if (!Number.isInteger(maxN) || maxN < 1) {
772
- throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
773
1353
  }
774
- const minN = opts.minN ?? 5;
775
- if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
1354
+ const holdoutIds = new Set(opts.holdoutScenarios.map((s) => s.id));
1355
+ const overlap = opts.trainScenarios.filter((s) => holdoutIds.has(s.id)).map((s) => s.id);
1356
+ if (overlap.length > 0) {
776
1357
  throw new Error(
777
- `sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
1358
+ `runSkillOpt: trainScenarios and holdoutScenarios must be disjoint (overlap: [${overlap.join(
1359
+ ", "
1360
+ )}]) \u2014 a shared scenario leaks the held-out acceptance axis into the proposal evidence.`
778
1361
  );
779
1362
  }
780
- if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
1363
+ const patchesPerEpoch = opts.patchesPerEpoch ?? 2;
1364
+ const initialBudget = opts.editBudget ?? 3;
1365
+ const minImprovement = opts.minImprovement ?? 0;
1366
+ if (minImprovement < 0) {
781
1367
  throw new Error(
782
- `sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
1368
+ "runSkillOpt: minImprovement must be >= 0 \u2014 a negative threshold would accept held-out regressions, breaking the monotonic-lift contract."
783
1369
  );
784
1370
  }
785
- const nullMean = 0.5 + minEffect / (2 * scale);
786
- return {
787
- alpha,
788
- minN,
789
- maxN,
790
- maxBet: opts.maxBet ?? 0.5,
791
- scale,
792
- shuffleSeed: opts.shuffleSeed ?? 1337,
793
- direction,
794
- nullMean,
795
- minEffect
1371
+ const patience = opts.patience ?? opts.maxEpochs;
1372
+ const budgetAnneal = opts.budgetAnneal ?? true;
1373
+ const rejectedBufferSize = opts.rejectedBufferSize ?? 12;
1374
+ const slowMetaEvery = opts.slowMetaEvery ?? 2;
1375
+ let totalCostUsd = 0;
1376
+ const scoreHoldout = async (surface, tag) => {
1377
+ const campaign = await runScoringCampaign(opts, opts.holdoutScenarios, surface, tag);
1378
+ totalCostUsd += campaign.aggregates.totalCostUsd;
1379
+ return campaignMeanComposite(campaign);
796
1380
  };
797
- }
798
- function makeStream(cfg) {
799
- const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
800
- const threshold = 1 / cfg.alpha;
801
- let terminal;
802
- return {
803
- observe(delta) {
804
- if (terminal === "undecided-at-maxN") {
805
- throw new Error(
806
- `sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
807
- );
808
- }
809
- if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
810
- throw new Error(
811
- `sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
812
- );
813
- }
814
- const d = cfg.direction === "decrease" ? -delta : delta;
815
- const step = proc.update((d / cfg.scale + 1) / 2);
816
- if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
817
- terminal = "promote";
818
- }
819
- if (terminal === "promote") {
820
- return {
821
- decision: "promote",
822
- eValue: step.wealth,
823
- n: step.n,
824
- reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
825
- };
1381
+ const evidenceK = opts.evidenceK ?? 3;
1382
+ const trainEvidence = async (surface, tag) => {
1383
+ const campaign = await runScoringCampaign(opts, opts.trainScenarios, surface, tag);
1384
+ totalCostUsd += campaign.aggregates.totalCostUsd;
1385
+ return toEvidence(campaign, evidenceK);
1386
+ };
1387
+ let current = opts.baselineSurface;
1388
+ let currentEvidence = await trainEvidence(current, "baseline-train");
1389
+ const baselineHoldout = await scoreHoldout(current, "baseline-holdout");
1390
+ let currentHoldout = baselineHoldout;
1391
+ const buffer = [];
1392
+ const acceptedEdits = [];
1393
+ const rejectedAll = [];
1394
+ const history = [];
1395
+ let budget = initialBudget;
1396
+ let sinceAccept = 0;
1397
+ let metaNote;
1398
+ let epochsRun = 0;
1399
+ for (let epoch = 0; epoch < opts.maxEpochs; epoch++) {
1400
+ epochsRun++;
1401
+ const patches = await opts.proposer.proposePatches({
1402
+ surface: current,
1403
+ evidence: currentEvidence,
1404
+ editBudget: budget,
1405
+ rejectedBuffer: buffer,
1406
+ metaNote,
1407
+ count: patchesPerEpoch,
1408
+ signal: opts.signal ?? new AbortController().signal
1409
+ });
1410
+ let accepted = null;
1411
+ const rejectedThisEpoch = [];
1412
+ for (let i = 0; i < patches.length; i++) {
1413
+ const patch = patches[i];
1414
+ const { surface: candidate, applied } = applySkillPatch(current, patch);
1415
+ if (applied === 0 || candidate === current) {
1416
+ rejectedThisEpoch.push({
1417
+ label: patch.label,
1418
+ rationale: patch.rationale,
1419
+ reason: "no-op (unanchored or zero-change)"
1420
+ });
1421
+ continue;
826
1422
  }
827
- if (step.n >= cfg.maxN) {
828
- terminal = "undecided-at-maxN";
829
- return {
830
- decision: "undecided-at-maxN",
831
- eValue: step.wealth,
832
- n: step.n,
833
- reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
1423
+ const candidateHoldout = await scoreHoldout(candidate, `epoch-${epoch}-cand-${i}-holdout`);
1424
+ if (candidateHoldout > currentHoldout + minImprovement) {
1425
+ accepted = {
1426
+ epoch,
1427
+ label: patch.label,
1428
+ rationale: patch.rationale,
1429
+ holdoutDelta: candidateHoldout - currentHoldout
834
1430
  };
1431
+ current = candidate;
1432
+ currentHoldout = candidateHoldout;
1433
+ currentEvidence = await trainEvidence(current, `epoch-${epoch}-train`);
1434
+ break;
835
1435
  }
836
- return {
837
- decision: "continue",
838
- eValue: step.wealth,
839
- n: step.n,
840
- reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
841
- };
842
- },
843
- state() {
844
- return { ...proc.state(), decision: terminal ?? "continue" };
1436
+ rejectedThisEpoch.push({
1437
+ label: patch.label,
1438
+ rationale: patch.rationale,
1439
+ reason: `held-out ${candidateHoldout.toFixed(3)} \u2264 current ${currentHoldout.toFixed(3)}`
1440
+ });
1441
+ }
1442
+ if (accepted) {
1443
+ acceptedEdits.push(accepted);
1444
+ sinceAccept = 0;
1445
+ } else {
1446
+ sinceAccept++;
1447
+ if (budgetAnneal && sinceAccept >= 2 && budget > 1) budget--;
1448
+ }
1449
+ for (const r of rejectedThisEpoch) {
1450
+ buffer.push(r);
1451
+ rejectedAll.push(r);
1452
+ }
1453
+ while (buffer.length > rejectedBufferSize) buffer.shift();
1454
+ if (slowMetaEvery > 0 && (epoch + 1) % slowMetaEvery === 0) {
1455
+ metaNote = buildMetaNote(acceptedEdits, buffer);
845
1456
  }
1457
+ history.push({
1458
+ epoch,
1459
+ editBudget: budget,
1460
+ proposed: patches.length,
1461
+ accepted,
1462
+ rejected: rejectedThisEpoch,
1463
+ holdoutComposite: currentHoldout
1464
+ });
1465
+ if (sinceAccept >= patience) break;
1466
+ }
1467
+ return {
1468
+ winnerSurface: current,
1469
+ baselineHoldoutComposite: baselineHoldout,
1470
+ winnerHoldoutComposite: currentHoldout,
1471
+ lift: currentHoldout - baselineHoldout,
1472
+ acceptedEdits,
1473
+ rejectedEdits: rejectedAll,
1474
+ epochsRun,
1475
+ history,
1476
+ totalCostUsd
846
1477
  };
847
1478
  }
848
- function seededShuffle(items, seed) {
849
- const rng = mulberry32(seed);
850
- for (let i = items.length - 1; i > 0; i--) {
851
- const j = Math.floor(rng() * (i + 1));
852
- const tmp = items[i];
853
- items[i] = items[j];
854
- items[j] = tmp;
855
- }
856
- return items;
1479
+ function runScoringCampaign(opts, scenarios, surface, tag) {
1480
+ return runCampaign({
1481
+ ...opts,
1482
+ scenarios,
1483
+ dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
1484
+ runDir: `${opts.runDir}/${tag}`
1485
+ });
857
1486
  }
858
- function sequentialPairedGate(options) {
859
- const cfg = resolveConfig(options);
860
- const name = options.name ?? "sequentialPairedGate";
861
- const manifest = options.preRegistration;
862
- const observeStream = makeStream(cfg);
863
- return {
864
- name,
865
- observe(delta) {
866
- return observeStream.observe(delta);
867
- },
868
- state() {
869
- return observeStream.state();
870
- },
871
- async decide(ctx) {
872
- if (!ctx.baselineJudgeScores) {
873
- throw new Error(
874
- `${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
875
- );
876
- }
877
- const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
878
- const paired = pairHoldout(
879
- ctx.judgeScores,
880
- ctx.baselineJudgeScores,
881
- scenarioIds,
882
- (s) => s.composite
883
- );
884
- const deltas = paired.after.map((a, i) => a - paired.before[i]);
885
- seededShuffle(deltas, cfg.shuffleSeed);
886
- const stream = makeStream(cfg);
887
- let last;
888
- for (const d of deltas) {
889
- last = stream.observe(d);
890
- if (last.decision !== "continue") break;
891
- }
892
- const detail = {
893
- ...stream.state(),
894
- minN: cfg.minN,
895
- maxN: cfg.maxN,
896
- scale: cfg.scale,
897
- shuffleSeed: cfg.shuffleSeed,
898
- direction: cfg.direction,
899
- minEffect: cfg.minEffect,
900
- pairedN: deltas.length,
901
- preRegisteredId: manifest?.id,
902
- metric: manifest?.metric
903
- };
904
- const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
905
- if (last === void 0) {
906
- return {
907
- decision: "need_more_work",
908
- reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
909
- contributingGates: [{ name, passed: false, detail }]
910
- };
911
- }
912
- const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
913
- return {
914
- decision,
915
- reasons: [`${name}: ${last.reason}`],
916
- contributingGates: [{ name, passed: decision === "ship", detail }],
917
- delta: meanDelta
918
- };
919
- }
920
- };
1487
+ function toEvidence(campaign, k) {
1488
+ const { dimensions, scenarios } = campaignBreakdown(campaign);
1489
+ const weakScenarios = [...scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
1490
+ const weakDimensions = Object.entries(dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
1491
+ return { weakScenarios, weakDimensions };
921
1492
  }
922
- function sequentialDecide(options = {}) {
923
- const alpha = options.alpha ?? 0.05;
924
- const minN = options.minN ?? 5;
925
- const scale = options.scale ?? 1;
926
- if (!Number.isFinite(scale) || scale <= 0) {
927
- throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
1493
+ function buildMetaNote(accepted, rejected) {
1494
+ const parts = [];
1495
+ if (accepted.length > 0) {
1496
+ parts.push(
1497
+ `Edits that improved held-out so far: ${accepted.map((a) => `"${a.label}" (+${a.holdoutDelta.toFixed(3)})`).join("; ")}. Build on these.`
1498
+ );
928
1499
  }
929
- const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
930
- const threshold = 1 / alpha;
931
- let processedGenerations = 0;
932
- let reference;
933
- let stopped;
934
- const topCandidate = (record) => {
935
- if (record.candidates.length === 0) {
936
- throw new Error(
937
- `sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
938
- );
939
- }
940
- return record.candidates.reduce((best, c) => c.composite > best.composite ? c : best);
941
- };
942
- const decide = ({ history }) => {
943
- if (stopped) return stopped;
944
- if (history.length === 0) return { stop: false };
945
- if (reference === void 0) {
946
- reference = new Map(
947
- topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
948
- );
949
- }
950
- for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
951
- const top = topCandidate(history[g]);
952
- const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
953
- for (const [scenarioId, refComposite] of reference) {
954
- const candComposite = byScenario.get(scenarioId);
955
- if (candComposite === void 0) {
956
- throw new Error(
957
- `sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
958
- );
959
- }
960
- const delta = candComposite - refComposite;
961
- if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
962
- throw new Error(
963
- `sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
964
- );
965
- }
966
- const step = proc.update((delta / scale + 1) / 2);
967
- if (step.n >= minN && step.wealth >= threshold) {
968
- stopped = {
969
- stop: true,
970
- reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
971
- };
972
- processedGenerations = history.length;
973
- return stopped;
974
- }
975
- }
976
- }
977
- processedGenerations = history.length;
978
- return { stop: false };
979
- };
980
- decide.state = () => proc.state();
981
- return decide;
1500
+ if (rejected.length > 0) {
1501
+ const labels = [...new Set(rejected.map((r) => r.label))].slice(0, 5);
1502
+ parts.push(`Dead ends to avoid: ${labels.join(", ")}. Try a different anchor or rule.`);
1503
+ }
1504
+ parts.push("Keep edits small and anchored to existing lines.");
1505
+ return parts.join(" ");
982
1506
  }
983
1507
 
984
- // src/campaign/labeled-store/fs-adapter.ts
985
- import { createHash as createHash2 } from "crypto";
986
- import { existsSync, mkdirSync, readFileSync, writeFileSync as writeFileSync2 } from "fs";
987
- import { join as join2 } from "path";
988
- var LabeledScenarioStoreError = class extends Error {
989
- constructor(code, message) {
990
- super(message);
991
- this.code = code;
992
- this.name = "LabeledScenarioStoreError";
993
- }
994
- code;
995
- };
996
- var FsLabeledScenarioStore = class {
997
- constructor(options) {
998
- this.options = options;
999
- if (!existsSync(options.root)) mkdirSync(options.root, { recursive: true });
1000
- this.now = options.now ?? Date.now;
1001
- }
1002
- options;
1003
- now;
1004
- rateLimits = /* @__PURE__ */ new Map();
1005
- async observe(write) {
1006
- this.assertProvenance(write);
1007
- this.assertRateLimit(write);
1008
- const record = this.toRecord(write);
1009
- const path = this.pathForSource(write.source);
1010
- const line = `${JSON.stringify(record)}
1011
- `;
1012
- appendLine(path, line);
1013
- }
1014
- async sample(args) {
1015
- if (!args.split) {
1016
- throw new LabeledScenarioStoreError(
1017
- "split_required",
1018
- "sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
1019
- );
1508
+ // src/campaign/presets/compare-proposers.ts
1509
+ async function compareProposers(opts) {
1510
+ return compareProposerEntries(opts);
1511
+ }
1512
+ async function compareProposerEntries(opts) {
1513
+ if (opts.proposers.length === 0) throw new Error("compareProposers: no proposers to compare");
1514
+ const seed = opts.seed ?? 42;
1515
+ const resamples = opts.resamples ?? 2e3;
1516
+ const confidence = opts.confidence ?? 0.95;
1517
+ const scoreOnHoldout = async (surface, tag) => {
1518
+ const campaign = await runCampaign({
1519
+ ...opts,
1520
+ scenarios: opts.holdoutScenarios,
1521
+ dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
1522
+ runDir: `${opts.runDir}/${tag}`
1523
+ });
1524
+ const byScenario = {};
1525
+ for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
1526
+ byScenario[scenarioId] = composite;
1020
1527
  }
1021
- if (!args.capturedBefore) {
1022
- throw new LabeledScenarioStoreError(
1023
- "capturedBefore_required",
1024
- "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
1528
+ return byScenario;
1529
+ };
1530
+ const scenarioIds = [...new Set(opts.holdoutScenarios.map((s) => s.id))].sort();
1531
+ if (scenarioIds.length === 0) throw new Error("compareProposers: holdoutScenarios is empty");
1532
+ const align = (byScenario, label) => {
1533
+ const missing = scenarioIds.filter((id) => !(id in byScenario));
1534
+ if (missing.length > 0) {
1535
+ throw new Error(
1536
+ `compareProposers: ${label} produced no held-out score for scenario(s) [${missing.join(
1537
+ ", "
1538
+ )}] \u2014 a cell errored or its judges returned nothing. Refusing to fabricate a 0 (it would corrupt the lift comparison). Fix the dispatch/judge or drop the scenario.`
1025
1539
  );
1026
1540
  }
1027
- const all = [];
1028
- for (const source of ALL_SOURCES) {
1029
- if (args.split === "train" && source === "production-trace") {
1030
- const explicit = sourceFilterContains(args.filter?.source, "production-trace");
1031
- if (!explicit) continue;
1032
- }
1033
- const path = this.pathForSource(source);
1034
- if (!existsSync(path)) continue;
1035
- const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
1036
- for (const line of lines) {
1037
- let record;
1038
- try {
1039
- record = JSON.parse(line);
1040
- } catch {
1041
- continue;
1042
- }
1043
- if (!matchesFilter(record, args, source)) continue;
1044
- all.push(record);
1045
- }
1046
- }
1047
- all.sort((a, b) => {
1048
- if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
1049
- return a.recordHash.localeCompare(b.recordHash);
1541
+ return scenarioIds.map((id) => byScenario[id]);
1542
+ };
1543
+ const baselineArr = align(
1544
+ await scoreOnHoldout(opts.baselineSurface, "compare-baseline"),
1545
+ "baseline"
1546
+ );
1547
+ const winners = [];
1548
+ for (const proposer of opts.proposers) {
1549
+ const out = await proposer.optimize();
1550
+ const byScenario = await scoreOnHoldout(out.winnerSurface, `compare-${slug(proposer.name)}`);
1551
+ winners.push({
1552
+ name: proposer.name,
1553
+ winnerSurface: out.winnerSurface,
1554
+ costUsd: out.costUsd,
1555
+ durationMs: out.durationMs,
1556
+ arr: align(byScenario, `proposer "${proposer.name}"`)
1050
1557
  });
1051
- return all.slice(0, args.count);
1052
1558
  }
1053
- async size() {
1054
- const bySource = {};
1055
- const byTrust = {
1056
- unverified: 0,
1057
- "verified-signal": 0,
1058
- "human-rated": 0
1059
- };
1060
- let total = 0;
1061
- for (const source of ALL_SOURCES) {
1062
- const path = this.pathForSource(source);
1063
- if (!existsSync(path)) {
1064
- bySource[source] = 0;
1065
- continue;
1066
- }
1067
- const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
1068
- bySource[source] = lines.length;
1069
- total += lines.length;
1070
- for (const line of lines) {
1071
- let trust = "unverified";
1072
- try {
1073
- trust = JSON.parse(line).labelTrust ?? "unverified";
1074
- } catch {
1075
- }
1076
- byTrust[trust] += 1;
1077
- }
1078
- }
1079
- return { train: total, test: total, bySource, byTrust };
1080
- }
1081
- assertProvenance(write) {
1082
- if (!write.source) {
1083
- throw new LabeledScenarioStoreError(
1084
- "missing_source",
1085
- "LabeledScenarioWrite requires `source`"
1086
- );
1087
- }
1088
- if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
1089
- throw new LabeledScenarioStoreError(
1090
- "missing_source_version",
1091
- "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
1092
- );
1093
- }
1094
- if (!write.capturedAt) {
1095
- throw new LabeledScenarioStoreError(
1096
- "missing_captured_at",
1097
- "LabeledScenarioWrite requires `capturedAt` ISO timestamp"
1098
- );
1099
- }
1100
- if (!write.redactionStatus) {
1101
- throw new LabeledScenarioStoreError(
1102
- "missing_redaction_status",
1103
- "LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
1104
- );
1105
- }
1106
- if (!ALL_SOURCES.includes(write.source)) {
1107
- throw new LabeledScenarioStoreError(
1108
- "unknown_source",
1109
- `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
1110
- );
1111
- }
1112
- }
1113
- assertRateLimit(write) {
1114
- const cap = this.options.maxWritesPerMinutePerBucket;
1115
- if (!cap || !write.rateLimitBucket) return;
1116
- const now = this.now();
1117
- const windowMs = 6e4;
1118
- let state = this.rateLimits.get(write.rateLimitBucket);
1119
- if (!state || now - state.windowStartMs >= windowMs) {
1120
- state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
1121
- this.rateLimits.set(write.rateLimitBucket, state);
1122
- }
1123
- if (state.count >= cap) {
1124
- throw new LabeledScenarioStoreError(
1125
- "rate_limit_exceeded",
1126
- `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
1127
- );
1128
- }
1129
- state.count += 1;
1130
- }
1131
- toRecord(write) {
1132
- const recordHash = sha256(
1133
- JSON.stringify({
1134
- id: write.scenario.id,
1135
- src: write.source,
1136
- at: write.capturedAt,
1137
- ver: write.sourceVersionHash
1138
- })
1139
- );
1140
- return {
1141
- ...write,
1142
- recordHash,
1143
- split: "train"
1144
- };
1145
- }
1146
- pathForSource(source) {
1147
- return join2(this.options.root, `${source}.jsonl`);
1148
- }
1149
- };
1150
- var ALL_SOURCES = [
1151
- "production-trace",
1152
- "eval-run",
1153
- "manual",
1154
- "red-team",
1155
- "synthetic"
1156
- ];
1157
- function sourceFilterContains(filter, needle) {
1158
- if (!filter) return false;
1159
- if (Array.isArray(filter)) return filter.includes(needle);
1160
- return filter === needle;
1161
- }
1162
- function matchesFilter(record, args, source) {
1163
- if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
1164
- if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
1165
- const f = args.filter;
1166
- if (!f) return true;
1167
- if (f.kind && record.scenario.kind !== f.kind) return false;
1168
- if (f.source) {
1169
- const sources = Array.isArray(f.source) ? f.source : [f.source];
1170
- if (!sources.includes(source)) return false;
1171
- }
1172
- if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
1173
- const composites = Object.values(record.judgeScores).map((s) => s.composite);
1174
- const max = composites.length === 0 ? 0 : Math.max(...composites);
1175
- if (f.minComposite !== void 0 && max < f.minComposite) return false;
1176
- if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
1177
- }
1178
- if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
1179
- return false;
1180
- }
1181
- return true;
1182
- }
1183
- function sha256(input) {
1184
- return createHash2("sha256").update(input).digest("hex").slice(0, 16);
1185
- }
1186
- function appendLine(path, line) {
1187
- if (existsSync(path)) {
1188
- const existing = readFileSync(path, "utf8");
1189
- writeFileSync2(path, existing + line);
1190
- } else {
1191
- writeFileSync2(path, line);
1192
- }
1193
- }
1194
-
1195
- // src/campaign/presets/run-skill-opt.ts
1196
- async function runSkillOpt(opts) {
1197
- if (opts.trainScenarios.length === 0) throw new Error("runSkillOpt: trainScenarios is empty");
1198
- if (opts.holdoutScenarios.length === 0) throw new Error("runSkillOpt: holdoutScenarios is empty");
1199
- if (!opts.judges || opts.judges.length === 0) {
1200
- throw new Error(
1201
- "runSkillOpt: at least one judge is required \u2014 scoring (and therefore acceptance) is meaningless without one, and would report a silent zero lift."
1202
- );
1203
- }
1204
- const holdoutIds = new Set(opts.holdoutScenarios.map((s) => s.id));
1205
- const overlap = opts.trainScenarios.filter((s) => holdoutIds.has(s.id)).map((s) => s.id);
1206
- if (overlap.length > 0) {
1207
- throw new Error(
1208
- `runSkillOpt: trainScenarios and holdoutScenarios must be disjoint (overlap: [${overlap.join(
1209
- ", "
1210
- )}]) \u2014 a shared scenario leaks the held-out acceptance axis into the proposal evidence.`
1211
- );
1212
- }
1213
- const patchesPerEpoch = opts.patchesPerEpoch ?? 2;
1214
- const initialBudget = opts.editBudget ?? 3;
1215
- const minImprovement = opts.minImprovement ?? 0;
1216
- if (minImprovement < 0) {
1217
- throw new Error(
1218
- "runSkillOpt: minImprovement must be >= 0 \u2014 a negative threshold would accept held-out regressions, breaking the monotonic-lift contract."
1219
- );
1220
- }
1221
- const patience = opts.patience ?? opts.maxEpochs;
1222
- const budgetAnneal = opts.budgetAnneal ?? true;
1223
- const rejectedBufferSize = opts.rejectedBufferSize ?? 12;
1224
- const slowMetaEvery = opts.slowMetaEvery ?? 2;
1225
- let totalCostUsd = 0;
1226
- const scoreHoldout = async (surface, tag) => {
1227
- const campaign = await runScoringCampaign(opts, opts.holdoutScenarios, surface, tag);
1228
- totalCostUsd += campaign.aggregates.totalCostUsd;
1229
- return campaignMeanComposite(campaign);
1230
- };
1231
- const evidenceK = opts.evidenceK ?? 3;
1232
- const trainEvidence = async (surface, tag) => {
1233
- const campaign = await runScoringCampaign(opts, opts.trainScenarios, surface, tag);
1234
- totalCostUsd += campaign.aggregates.totalCostUsd;
1235
- return toEvidence(campaign, evidenceK);
1236
- };
1237
- let current = opts.baselineSurface;
1238
- let currentEvidence = await trainEvidence(current, "baseline-train");
1239
- const baselineHoldout = await scoreHoldout(current, "baseline-holdout");
1240
- let currentHoldout = baselineHoldout;
1241
- const buffer = [];
1242
- const acceptedEdits = [];
1243
- const rejectedAll = [];
1244
- const history = [];
1245
- let budget = initialBudget;
1246
- let sinceAccept = 0;
1247
- let metaNote;
1248
- let epochsRun = 0;
1249
- for (let epoch = 0; epoch < opts.maxEpochs; epoch++) {
1250
- epochsRun++;
1251
- const patches = await opts.driver.proposePatches({
1252
- surface: current,
1253
- evidence: currentEvidence,
1254
- editBudget: budget,
1255
- rejectedBuffer: buffer,
1256
- metaNote,
1257
- count: patchesPerEpoch,
1258
- signal: opts.signal ?? new AbortController().signal
1259
- });
1260
- let accepted = null;
1261
- const rejectedThisEpoch = [];
1262
- for (let i = 0; i < patches.length; i++) {
1263
- const patch = patches[i];
1264
- const { surface: candidate, applied } = applySkillPatch(current, patch);
1265
- if (applied === 0 || candidate === current) {
1266
- rejectedThisEpoch.push({
1267
- label: patch.label,
1268
- rationale: patch.rationale,
1269
- reason: "no-op (unanchored or zero-change)"
1270
- });
1271
- continue;
1272
- }
1273
- const candidateHoldout = await scoreHoldout(candidate, `epoch-${epoch}-cand-${i}-holdout`);
1274
- if (candidateHoldout > currentHoldout + minImprovement) {
1275
- accepted = {
1276
- epoch,
1277
- label: patch.label,
1278
- rationale: patch.rationale,
1279
- holdoutDelta: candidateHoldout - currentHoldout
1280
- };
1281
- current = candidate;
1282
- currentHoldout = candidateHoldout;
1283
- currentEvidence = await trainEvidence(current, `epoch-${epoch}-train`);
1284
- break;
1285
- }
1286
- rejectedThisEpoch.push({
1287
- label: patch.label,
1288
- rationale: patch.rationale,
1289
- reason: `held-out ${candidateHoldout.toFixed(3)} \u2264 current ${currentHoldout.toFixed(3)}`
1290
- });
1291
- }
1292
- if (accepted) {
1293
- acceptedEdits.push(accepted);
1294
- sinceAccept = 0;
1295
- } else {
1296
- sinceAccept++;
1297
- if (budgetAnneal && sinceAccept >= 2 && budget > 1) budget--;
1298
- }
1299
- for (const r of rejectedThisEpoch) {
1300
- buffer.push(r);
1301
- rejectedAll.push(r);
1302
- }
1303
- while (buffer.length > rejectedBufferSize) buffer.shift();
1304
- if (slowMetaEvery > 0 && (epoch + 1) % slowMetaEvery === 0) {
1305
- metaNote = buildMetaNote(acceptedEdits, buffer);
1306
- }
1307
- history.push({
1308
- epoch,
1309
- editBudget: budget,
1310
- proposed: patches.length,
1311
- accepted,
1312
- rejected: rejectedThisEpoch,
1313
- holdoutComposite: currentHoldout
1314
- });
1315
- if (sinceAccept >= patience) break;
1316
- }
1317
- return {
1318
- winnerSurface: current,
1319
- baselineHoldoutComposite: baselineHoldout,
1320
- winnerHoldoutComposite: currentHoldout,
1321
- lift: currentHoldout - baselineHoldout,
1322
- acceptedEdits,
1323
- rejectedEdits: rejectedAll,
1324
- epochsRun,
1325
- history,
1326
- totalCostUsd
1327
- };
1328
- }
1329
- function runScoringCampaign(opts, scenarios, surface, tag) {
1330
- return runCampaign({
1331
- ...opts,
1332
- scenarios,
1333
- dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
1334
- runDir: `${opts.runDir}/${tag}`
1335
- });
1336
- }
1337
- function toEvidence(campaign, k) {
1338
- const { dimensions, scenarios } = campaignBreakdown(campaign);
1339
- const weakScenarios = [...scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
1340
- const weakDimensions = Object.entries(dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
1341
- return { weakScenarios, weakDimensions };
1342
- }
1343
- function buildMetaNote(accepted, rejected) {
1344
- const parts = [];
1345
- if (accepted.length > 0) {
1346
- parts.push(
1347
- `Edits that improved held-out so far: ${accepted.map((a) => `"${a.label}" (+${a.holdoutDelta.toFixed(3)})`).join("; ")}. Build on these.`
1348
- );
1349
- }
1350
- if (rejected.length > 0) {
1351
- const labels = [...new Set(rejected.map((r) => r.label))].slice(0, 5);
1352
- parts.push(`Dead ends to avoid: ${labels.join(", ")}. Try a different anchor or rule.`);
1353
- }
1354
- parts.push("Keep edits small and anchored to existing lines.");
1355
- return parts.join(" ");
1356
- }
1357
-
1358
- // src/campaign/presets/compare-drivers.ts
1359
- async function compareDrivers(opts) {
1360
- if (opts.drivers.length === 0) throw new Error("compareDrivers: no drivers to compare");
1361
- const seed = opts.seed ?? 42;
1362
- const resamples = opts.resamples ?? 2e3;
1363
- const confidence = opts.confidence ?? 0.95;
1364
- const scoreOnHoldout = async (surface, tag) => {
1365
- const campaign = await runCampaign({
1366
- ...opts,
1367
- scenarios: opts.holdoutScenarios,
1368
- dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
1369
- runDir: `${opts.runDir}/${tag}`
1370
- });
1371
- const byScenario = {};
1372
- for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
1373
- byScenario[scenarioId] = composite;
1374
- }
1375
- return byScenario;
1376
- };
1377
- const scenarioIds = [...new Set(opts.holdoutScenarios.map((s) => s.id))].sort();
1378
- if (scenarioIds.length === 0) throw new Error("compareDrivers: holdoutScenarios is empty");
1379
- const align = (byScenario, label) => {
1380
- const missing = scenarioIds.filter((id) => !(id in byScenario));
1381
- if (missing.length > 0) {
1382
- throw new Error(
1383
- `compareDrivers: ${label} produced no held-out score for scenario(s) [${missing.join(
1384
- ", "
1385
- )}] \u2014 a cell errored or its judges returned nothing. Refusing to fabricate a 0 (it would corrupt the lift comparison). Fix the dispatch/judge or drop the scenario.`
1386
- );
1387
- }
1388
- return scenarioIds.map((id) => byScenario[id]);
1389
- };
1390
- const baselineArr = align(
1391
- await scoreOnHoldout(opts.baselineSurface, "compare-baseline"),
1392
- "baseline"
1393
- );
1394
- const winners = [];
1395
- for (const d of opts.drivers) {
1396
- const out = await d.optimize();
1397
- const byScenario = await scoreOnHoldout(out.winnerSurface, `compare-${slug(d.name)}`);
1398
- winners.push({
1399
- name: d.name,
1400
- winnerSurface: out.winnerSurface,
1401
- costUsd: out.costUsd,
1402
- durationMs: out.durationMs,
1403
- arr: align(byScenario, `driver "${d.name}"`)
1404
- });
1405
- }
1406
- const scores = winners.map((w) => {
1407
- const boot = pairedBootstrap(baselineArr, w.arr, {
1408
- seed,
1409
- resamples,
1410
- confidence,
1411
- statistic: "mean"
1412
- });
1413
- const score = {
1414
- name: w.name,
1415
- baselineComposite: mean(baselineArr),
1416
- winnerComposite: mean(w.arr),
1417
- lift: boot.mean,
1418
- liftCi: { low: boot.low, high: boot.high },
1419
- costUsd: w.costUsd,
1420
- winnerSurface: w.winnerSurface,
1421
- rank: 0
1559
+ const scores = winners.map((w) => {
1560
+ const boot = pairedBootstrap(baselineArr, w.arr, {
1561
+ seed,
1562
+ resamples,
1563
+ confidence,
1564
+ statistic: "mean"
1565
+ });
1566
+ const score = {
1567
+ name: w.name,
1568
+ baselineComposite: mean(baselineArr),
1569
+ winnerComposite: mean(w.arr),
1570
+ lift: boot.mean,
1571
+ liftCi: { low: boot.low, high: boot.high },
1572
+ costUsd: w.costUsd,
1573
+ winnerSurface: w.winnerSurface,
1574
+ rank: 0
1422
1575
  };
1423
1576
  if (w.durationMs !== void 0) score.durationMs = w.durationMs;
1424
1577
  return score;
@@ -1467,7 +1620,7 @@ function gepaEntry(config, combineParents, name) {
1467
1620
  name,
1468
1621
  async optimize() {
1469
1622
  const started = Date.now();
1470
- const driver = gepaDriver({
1623
+ const proposer = gepaProposer({
1471
1624
  llm: config.llm,
1472
1625
  model: config.model,
1473
1626
  target: config.target,
@@ -1480,7 +1633,7 @@ function gepaEntry(config, combineParents, name) {
1480
1633
  baselineSurface: config.baselineSurface,
1481
1634
  dispatchWithSurface: config.dispatchWithSurface,
1482
1635
  judges: config.judges,
1483
- driver,
1636
+ proposer,
1484
1637
  populationSize: config.populationSize ?? 2,
1485
1638
  maxGenerations: config.maxGenerations ?? 3,
1486
1639
  gate: defaultProductionGate({
@@ -1490,9 +1643,9 @@ function gepaEntry(config, combineParents, name) {
1490
1643
  autoOnPromote: "none",
1491
1644
  runDir: `${config.runDir}/${slug(name)}-loop`,
1492
1645
  ...config.seed !== void 0 ? { seed: config.seed } : {},
1493
- // EYES→HANDS: flow findings to the driver's propose(). These reach
1494
- // runOptimization unchanged (runImprovementLoop extends RunOptimizationOptions
1495
- // and forwards {...opts}); ctx.findings/report/analyzeGeneration are consumed there.
1646
+ // Flow findings to the proposer's propose(). These reach runOptimization
1647
+ // unchanged (runImprovementLoop extends RunOptimizationOptions and
1648
+ // forwards {...opts}); ctx.findings/report/analyzeGeneration are consumed there.
1496
1649
  ...config.findings !== void 0 ? { findings: config.findings } : {},
1497
1650
  ...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
1498
1651
  ...config.report !== void 0 ? { report: config.report } : {}
@@ -1510,12 +1663,16 @@ function skillOptEntry(config, name = "skill-opt") {
1510
1663
  name,
1511
1664
  async optimize() {
1512
1665
  const started = Date.now();
1513
- const driver = skillOptDriver({ llm: config.llm, model: config.model, target: config.target });
1666
+ const proposer = skillOptProposer({
1667
+ llm: config.llm,
1668
+ model: config.model,
1669
+ target: config.target
1670
+ });
1514
1671
  const result = await runSkillOpt({
1515
1672
  baselineSurface: config.baselineSurface,
1516
1673
  dispatchWithSurface: config.dispatchWithSurface,
1517
1674
  judges: config.judges,
1518
- driver,
1675
+ proposer,
1519
1676
  trainScenarios: config.trainScenarios,
1520
1677
  holdoutScenarios: config.holdoutScenarios,
1521
1678
  maxEpochs: config.maxEpochs ?? 6,
@@ -1530,6 +1687,54 @@ function skillOptEntry(config, name = "skill-opt") {
1530
1687
  }
1531
1688
  };
1532
1689
  }
1690
+ function fapoEscalationEntry(config, name = "fapo-escalation") {
1691
+ return {
1692
+ name,
1693
+ async optimize() {
1694
+ const started = Date.now();
1695
+ const promptProposer = config.promptProposer ?? gepaProposer({
1696
+ llm: config.llm,
1697
+ model: config.model,
1698
+ target: config.target,
1699
+ combineParents: true,
1700
+ ...config.mutationPrimitives ? { mutationPrimitives: config.mutationPrimitives } : {}
1701
+ });
1702
+ const parameterProposer = config.parameterProposer ?? (config.parameterCandidates ? parameterSweepProposer({ candidates: config.parameterCandidates }) : void 0);
1703
+ const structuralProposer = config.structuralProposer;
1704
+ const proposer = fapoProposer({
1705
+ ...config.fapo ?? {},
1706
+ promptProposer,
1707
+ ...parameterProposer ? { parameterProposer } : {},
1708
+ ...structuralProposer ? { structuralProposer } : {}
1709
+ });
1710
+ const result = await runImprovementLoop({
1711
+ scenarios: config.trainScenarios,
1712
+ holdoutScenarios: config.holdoutScenarios,
1713
+ baselineSurface: config.baselineSurface,
1714
+ dispatchWithSurface: config.dispatchWithSurface,
1715
+ judges: config.judges,
1716
+ proposer,
1717
+ populationSize: config.populationSize ?? 2,
1718
+ maxGenerations: config.maxGenerations ?? 3,
1719
+ gate: defaultProductionGate({
1720
+ holdoutScenarios: config.holdoutScenarios,
1721
+ deltaThreshold: 0
1722
+ }),
1723
+ autoOnPromote: "none",
1724
+ runDir: `${config.runDir}/${slug(name)}-loop`,
1725
+ ...config.seed !== void 0 ? { seed: config.seed } : {},
1726
+ ...config.findings !== void 0 ? { findings: config.findings } : {},
1727
+ ...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
1728
+ ...config.report !== void 0 ? { report: config.report } : {}
1729
+ });
1730
+ const costUsd = result.baselineCampaign.aggregates.totalCostUsd + result.generations.reduce(
1731
+ (sum, g) => sum + g.surfaces.reduce((s, sf) => s + sf.campaign.aggregates.totalCostUsd, 0),
1732
+ 0
1733
+ );
1734
+ return { winnerSurface: result.winnerSurface, costUsd, durationMs: Date.now() - started };
1735
+ }
1736
+ };
1737
+ }
1533
1738
 
1534
1739
  // src/campaign/presets/playback.ts
1535
1740
  function makePlaybackDispatch(driver) {
@@ -1626,236 +1831,574 @@ function renderScoreboardMarkdown(rows, opts = {}) {
1626
1831
  "| Requirement | Status | Evidence |",
1627
1832
  "| --- | --- | --- |"
1628
1833
  );
1629
- for (const r of storyRows) {
1630
- out.push(
1631
- `| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
1632
- );
1633
- }
1634
- out.push("");
1834
+ for (const r of storyRows) {
1835
+ out.push(
1836
+ `| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
1837
+ );
1838
+ }
1839
+ out.push("");
1840
+ }
1841
+ return out.join("\n");
1842
+ }
1843
+
1844
+ // src/campaign/presets/run-profile-matrix.ts
1845
+ import { createHash as createHash3 } from "crypto";
1846
+ import { join as join2 } from "path";
1847
+ var ProfileMatrixError = class extends AgentEvalError {
1848
+ constructor(message) {
1849
+ super("profile_matrix", message);
1850
+ }
1851
+ };
1852
+ function sanitize(id) {
1853
+ return id.replace(/[^a-zA-Z0-9_-]/g, "_");
1854
+ }
1855
+ function sha(input) {
1856
+ return createHash3("sha256").update(JSON.stringify(input)).digest("hex");
1857
+ }
1858
+ function mean2(xs) {
1859
+ return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
1860
+ }
1861
+ function cellComposite(cell) {
1862
+ const composites = Object.values(cell.judgeScores).map((s) => s.composite);
1863
+ return composites.length === 0 ? 0 : mean2(composites);
1864
+ }
1865
+ function buildRunRecord(args) {
1866
+ const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
1867
+ const profileId = agentProfileId(profile);
1868
+ const model = agentProfileModelId(profile);
1869
+ const composite = cellComposite(cell);
1870
+ const raw = { composite };
1871
+ const perJudge = {};
1872
+ const dimAccum = {};
1873
+ const notes = [];
1874
+ for (const [judgeName, js] of Object.entries(cell.judgeScores)) {
1875
+ perJudge[judgeName] = { ...js.dimensions };
1876
+ for (const [dim, value] of Object.entries(js.dimensions)) {
1877
+ raw[`${judgeName}.${dim}`] = value;
1878
+ dimAccum[dim] ??= [];
1879
+ dimAccum[dim].push(value);
1880
+ }
1881
+ if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
1882
+ }
1883
+ const perDimMean = {};
1884
+ for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean2(values);
1885
+ let costUsd = cell.costUsd;
1886
+ let costEstimated = false;
1887
+ if (costUsd === 0 && cell.tokenUsage.output > 0 && isModelPriced(model)) {
1888
+ costUsd = estimateCost(cell.tokenUsage.input, cell.tokenUsage.output, model);
1889
+ costEstimated = costUsd > 0;
1890
+ }
1891
+ raw.cost_usd = costUsd;
1892
+ raw.cost_estimated = costEstimated ? 1 : 0;
1893
+ raw.tokens_input = cell.tokenUsage.input;
1894
+ raw.tokens_output = cell.tokenUsage.output;
1895
+ if (typeof cell.tokenUsage.cached === "number") raw.tokens_cached = cell.tokenUsage.cached;
1896
+ raw.latency_ms = cell.durationMs;
1897
+ if (costUsd > 0) {
1898
+ raw.tokens_per_dollar = (cell.tokenUsage.input + cell.tokenUsage.output) / costUsd;
1899
+ }
1900
+ if (composite > 0.01) raw.cost_per_quality = costUsd / composite;
1901
+ const outcome = splitTag === "holdout" ? { holdoutScore: composite, raw } : { searchScore: composite, raw };
1902
+ if (Object.keys(perJudge).length > 0) {
1903
+ outcome.judgeScores = {
1904
+ perJudge,
1905
+ perDimMean,
1906
+ composite,
1907
+ ...notes.length > 0 ? { notes: notes.join(" | ") } : {}
1908
+ };
1909
+ }
1910
+ const record = {
1911
+ runId: `${matrixId}:${profileId}:${cell.cellId}`,
1912
+ experimentId,
1913
+ candidateId: profileId,
1914
+ seed: cell.seed,
1915
+ model,
1916
+ promptHash: profileHash,
1917
+ configHash,
1918
+ commitSha,
1919
+ wallMs: cell.durationMs,
1920
+ costUsd,
1921
+ tokenUsage: cell.tokenUsage,
1922
+ outcome,
1923
+ splitTag,
1924
+ scenarioId: cell.scenarioId,
1925
+ ...cell.error ? { failureMode: cell.error } : {}
1926
+ };
1927
+ if (args.corpusText && args.scenario) {
1928
+ try {
1929
+ const text = args.corpusText(cell.artifact, args.scenario);
1930
+ if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
1931
+ record.prompt = text.prompt;
1932
+ record.completion = text.completion;
1933
+ }
1934
+ } catch {
1935
+ }
1936
+ }
1937
+ return record;
1938
+ }
1939
+ async function runProfileMatrix(opts) {
1940
+ if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
1941
+ if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
1942
+ const splitTag = opts.splitTag ?? "search";
1943
+ const seed = opts.seed ?? 42;
1944
+ const validate = opts.validate ?? true;
1945
+ const integrityMode = opts.integrity ?? "assert";
1946
+ const profileIds = opts.profiles.map(agentProfileId);
1947
+ const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
1948
+ const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
1949
+ const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
1950
+ for (const profile of opts.profiles) {
1951
+ const profileHash = agentProfileHash(profile);
1952
+ const profileId = agentProfileId(profile);
1953
+ const model = agentProfileModelId(profile);
1954
+ try {
1955
+ validateRunRecord({
1956
+ runId: `${matrixId}:${profileId}:probe`,
1957
+ experimentId,
1958
+ candidateId: profileId,
1959
+ seed,
1960
+ model,
1961
+ promptHash: profileHash,
1962
+ configHash: profileHash,
1963
+ commitSha: opts.commitSha,
1964
+ wallMs: 0,
1965
+ costUsd: 0,
1966
+ tokenUsage: { input: 0, output: 0 },
1967
+ outcome: splitTag === "holdout" ? { holdoutScore: 0, raw: {} } : { searchScore: 0, raw: {} },
1968
+ splitTag
1969
+ });
1970
+ } catch (err) {
1971
+ throw new ProfileMatrixError(
1972
+ `profile '${profileId}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
1973
+ );
1974
+ }
1975
+ }
1976
+ const records = [];
1977
+ const campaigns = {};
1978
+ const byProfile = {};
1979
+ for (const profile of opts.profiles) {
1980
+ const profileHash = agentProfileHash(profile);
1981
+ const profileId = agentProfileId(profile);
1982
+ const model = agentProfileModelId(profile);
1983
+ const configHash = sha({
1984
+ profile: profileHash,
1985
+ judges: (opts.judges ?? []).map((j) => j.name),
1986
+ seed,
1987
+ splitTag
1988
+ });
1989
+ const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
1990
+ Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
1991
+ const campaign = await runCampaign({
1992
+ scenarios: opts.scenarios,
1993
+ dispatch,
1994
+ judges: opts.judges,
1995
+ seed,
1996
+ reps: opts.reps,
1997
+ maxConcurrency: opts.maxConcurrency,
1998
+ costCeiling: opts.costCeiling,
1999
+ labeledStore: opts.labeledStore,
2000
+ captureSource: opts.captureSource,
2001
+ storage: opts.storage,
2002
+ now: opts.now,
2003
+ runDir: join2(opts.runDir, sanitize(profileId))
2004
+ });
2005
+ const profileRecords = [];
2006
+ for (const cell of campaign.cells) {
2007
+ const record = buildRunRecord({
2008
+ cell,
2009
+ profile,
2010
+ profileHash,
2011
+ configHash,
2012
+ experimentId,
2013
+ splitTag,
2014
+ commitSha: opts.commitSha,
2015
+ matrixId,
2016
+ scenario: scenarioById.get(cell.scenarioId),
2017
+ corpusText: opts.corpusText
2018
+ });
2019
+ if (validate) validateRunRecord(record);
2020
+ profileRecords.push(record);
2021
+ records.push(record);
2022
+ }
2023
+ const pricedTotalCostUsd = profileRecords.reduce((a, r) => a + r.costUsd, 0);
2024
+ campaigns[profileId] = {
2025
+ ...campaign,
2026
+ aggregates: { ...campaign.aggregates, totalCostUsd: pricedTotalCostUsd }
2027
+ };
2028
+ byProfile[profileId] = {
2029
+ profileId,
2030
+ profileHash,
2031
+ model,
2032
+ records: profileRecords.length,
2033
+ meanComposite: mean2(profileRecords.map(compositeOf)),
2034
+ totalCostUsd: pricedTotalCostUsd,
2035
+ integrity: summarizeBackendIntegrity(profileRecords)
2036
+ };
2037
+ }
2038
+ const integrity = summarizeBackendIntegrity(records);
2039
+ if (integrityMode === "assert") {
2040
+ assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
2041
+ } else if (integrityMode === "warn" && integrity.verdict !== "real") {
2042
+ console.warn(
2043
+ `[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
2044
+ );
1635
2045
  }
1636
- return out.join("\n");
2046
+ const byScenario = rollup(records, (r) => r.scenarioId);
2047
+ const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
2048
+ return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
2049
+ }
2050
+ function compositeOf(r) {
2051
+ return r.outcome.holdoutScore ?? r.outcome.searchScore ?? 0;
2052
+ }
2053
+ function rollup(records, keyOf) {
2054
+ const groups = /* @__PURE__ */ new Map();
2055
+ for (const r of records) {
2056
+ const key = keyOf(r);
2057
+ if (key === void 0) continue;
2058
+ const arr = groups.get(key) ?? [];
2059
+ arr.push(compositeOf(r));
2060
+ groups.set(key, arr);
2061
+ }
2062
+ const out = {};
2063
+ for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
2064
+ return out;
2065
+ }
2066
+ function rollupByPersona(records, scenarios, personaOf) {
2067
+ const personaByScenarioId = /* @__PURE__ */ new Map();
2068
+ for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
2069
+ return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
1637
2070
  }
1638
2071
 
1639
- // src/campaign/presets/run-profile-matrix.ts
1640
- import { createHash as createHash3 } from "crypto";
1641
- import { join as join3 } from "path";
1642
- var ProfileMatrixError = class extends AgentEvalError {
1643
- constructor(message) {
1644
- super("profile_matrix", message);
2072
+ // src/campaign/proposers/_findings-text.ts
2073
+ function findingToLesson(f) {
2074
+ if (typeof f === "string") return f.trim() || null;
2075
+ if (f && typeof f === "object") {
2076
+ const o = f;
2077
+ const cand = o.recommended_action ?? o.claim ?? o.lesson ?? o.text ?? o.message;
2078
+ if (typeof cand === "string" && cand.trim()) return cand.trim();
1645
2079
  }
1646
- };
1647
- function sanitize(id) {
1648
- return id.replace(/[^a-zA-Z0-9_-]/g, "_");
2080
+ return null;
1649
2081
  }
1650
- function sha(input) {
1651
- return createHash3("sha256").update(JSON.stringify(input)).digest("hex");
2082
+ function normKey(s) {
2083
+ return s.toLowerCase().replace(/\s+/g, " ").replace(/[.;:!?\s]+$/, "").trim();
1652
2084
  }
1653
- function mean2(xs) {
1654
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
2085
+ function surfaceToText2(surface) {
2086
+ if (typeof surface === "string") return surface;
2087
+ throw new Error(
2088
+ `curator proposer: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
2089
+ );
1655
2090
  }
1656
- function cellComposite(cell) {
1657
- const composites = Object.values(cell.judgeScores).map((s) => s.composite);
1658
- return composites.length === 0 ? 0 : mean2(composites);
2091
+ function extractBlockBody(text, startMarker, endMarker) {
2092
+ const start = text.indexOf(startMarker);
2093
+ const end = text.indexOf(endMarker);
2094
+ if (start === -1 || end === -1 || end < start) return "";
2095
+ return text.slice(start + startMarker.length, end);
1659
2096
  }
1660
- function buildRunRecord(args) {
1661
- const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
1662
- const composite = cellComposite(cell);
1663
- const raw = { composite };
1664
- const perJudge = {};
1665
- const dimAccum = {};
1666
- const notes = [];
1667
- for (const [judgeName, js] of Object.entries(cell.judgeScores)) {
1668
- perJudge[judgeName] = { ...js.dimensions };
1669
- for (const [dim, value] of Object.entries(js.dimensions)) {
1670
- raw[`${judgeName}.${dim}`] = value;
1671
- dimAccum[dim] ??= [];
1672
- dimAccum[dim].push(value);
1673
- }
1674
- if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
1675
- }
1676
- const perDimMean = {};
1677
- for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean2(values);
1678
- let costUsd = cell.costUsd;
1679
- let costEstimated = false;
1680
- if (costUsd === 0 && cell.tokenUsage.output > 0 && isModelPriced(profile.model)) {
1681
- costUsd = estimateCost(cell.tokenUsage.input, cell.tokenUsage.output, profile.model);
1682
- costEstimated = costUsd > 0;
1683
- }
1684
- raw.cost_usd = costUsd;
1685
- raw.cost_estimated = costEstimated ? 1 : 0;
1686
- raw.tokens_input = cell.tokenUsage.input;
1687
- raw.tokens_output = cell.tokenUsage.output;
1688
- if (typeof cell.tokenUsage.cached === "number") raw.tokens_cached = cell.tokenUsage.cached;
1689
- raw.latency_ms = cell.durationMs;
1690
- if (costUsd > 0) {
1691
- raw.tokens_per_dollar = (cell.tokenUsage.input + cell.tokenUsage.output) / costUsd;
1692
- }
1693
- if (composite > 0.01) raw.cost_per_quality = costUsd / composite;
1694
- const outcome = splitTag === "holdout" ? { holdoutScore: composite, raw } : { searchScore: composite, raw };
1695
- if (Object.keys(perJudge).length > 0) {
1696
- outcome.judgeScores = {
1697
- perJudge,
1698
- perDimMean,
1699
- composite,
1700
- ...notes.length > 0 ? { notes: notes.join(" | ") } : {}
1701
- };
2097
+ function stripBlock(text, startMarker, endMarker) {
2098
+ const start = text.indexOf(startMarker);
2099
+ const end = text.indexOf(endMarker);
2100
+ if (start === -1 || end === -1 || end < start) return text.trimEnd();
2101
+ return (text.slice(0, start) + text.slice(end + endMarker.length)).trimEnd();
2102
+ }
2103
+
2104
+ // src/campaign/proposers/ace.ts
2105
+ var BLOCK_START = "<!-- BEGIN ace-playbook (auto-managed by aceProposer) -->";
2106
+ var BLOCK_END = "<!-- END ace-playbook -->";
2107
+ var DEFAULT_HEADING = "## Playbook (accumulated lessons \u2014 append-only)";
2108
+ function parsePlaybook(surface) {
2109
+ const body = extractBlockBody(surface, BLOCK_START, BLOCK_END);
2110
+ const out = [];
2111
+ for (const raw of body.split("\n")) {
2112
+ const line = raw.trim();
2113
+ if (!line.startsWith("- ")) continue;
2114
+ const item = line.slice(2).trim();
2115
+ const tag = /^\[g(-?\d+)\]\s*(.*)$/.exec(item);
2116
+ if (tag) out.push({ gen: Number(tag[1]), text: tag[2].trim() });
2117
+ else out.push({ gen: -1, text: item });
1702
2118
  }
1703
- const record = {
1704
- runId: `${matrixId}:${profile.id}:${cell.cellId}`,
1705
- experimentId,
1706
- candidateId: profile.id,
1707
- seed: cell.seed,
1708
- model: profile.model,
1709
- promptHash: profileHash,
1710
- configHash,
1711
- commitSha,
1712
- wallMs: cell.durationMs,
1713
- costUsd,
1714
- tokenUsage: cell.tokenUsage,
1715
- outcome,
1716
- splitTag,
1717
- scenarioId: cell.scenarioId,
1718
- ...cell.error ? { failureMode: cell.error } : {}
2119
+ return out;
2120
+ }
2121
+ function aceProposer(opts = {}) {
2122
+ const maxEntries = opts.maxEntries ?? 50;
2123
+ if (maxEntries < 1) throw new Error("aceProposer: maxEntries must be >= 1");
2124
+ const heading = opts.sectionHeading ?? DEFAULT_HEADING;
2125
+ return {
2126
+ kind: "ace",
2127
+ async propose(ctx) {
2128
+ const parent = surfaceToText2(ctx.currentSurface);
2129
+ const existing = parsePlaybook(parent);
2130
+ const seen = new Set(existing.map((b) => normKey(b.text)));
2131
+ const fresh = [];
2132
+ for (const f of ctx.findings ?? []) {
2133
+ const lesson = findingToLesson(f);
2134
+ if (!lesson) continue;
2135
+ const k = normKey(lesson);
2136
+ if (!k || seen.has(k)) continue;
2137
+ seen.add(k);
2138
+ fresh.push({ gen: ctx.generation, text: lesson });
2139
+ }
2140
+ if (fresh.length === 0) return [];
2141
+ const all = [...existing, ...fresh].slice(-maxEntries);
2142
+ const block = [
2143
+ BLOCK_START,
2144
+ heading,
2145
+ ...all.map((b) => `- [g${b.gen}] ${b.text}`),
2146
+ BLOCK_END
2147
+ ].join("\n");
2148
+ const base = stripBlock(parent, BLOCK_START, BLOCK_END);
2149
+ const surface = base ? `${base}
2150
+
2151
+ ${block}` : block;
2152
+ return [
2153
+ {
2154
+ surface,
2155
+ label: `ace-playbook +${fresh.length}`,
2156
+ rationale: `appended ${fresh.length} new lesson(s) from gen ${ctx.generation} findings (playbook now ${all.length} bullet(s), append-only)`
2157
+ }
2158
+ ];
2159
+ }
1719
2160
  };
1720
- if (args.corpusText && args.scenario) {
1721
- try {
1722
- const text = args.corpusText(cell.artifact, args.scenario);
1723
- if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
1724
- record.prompt = text.prompt;
1725
- record.completion = text.completion;
2161
+ }
2162
+
2163
+ // src/campaign/proposers/halo.ts
2164
+ import { execFile } from "child_process";
2165
+ import { promisify } from "util";
2166
+
2167
+ // src/campaign/proposers/analysis-edit.ts
2168
+ import { mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
2169
+ import { tmpdir } from "os";
2170
+ import { join as join3 } from "path";
2171
+ var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
2172
+ function surfaceToPromptText(surface) {
2173
+ return typeof surface === "string" ? surface : JSON.stringify(surface);
2174
+ }
2175
+ function analysisEditProposer(opts) {
2176
+ return {
2177
+ kind: opts.kind,
2178
+ async propose(ctx) {
2179
+ const parent = surfaceToPromptText(ctx.currentSurface);
2180
+ const traces = await opts.resolveTraces(ctx) ?? "";
2181
+ if (!traces.trim()) throw new Error(opts.noTracesError);
2182
+ const dir = mkdtempSync(join3(tmpdir(), `${opts.kind}-proposer-`));
2183
+ const tracePath = join3(dir, "traces.jsonl");
2184
+ writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
2185
+ `);
2186
+ const report = await opts.analyze(tracePath, ctx);
2187
+ const applied = await callLlm(
2188
+ {
2189
+ model: opts.applyModel,
2190
+ messages: [
2191
+ { role: "system", content: APPLY_SYSTEM },
2192
+ {
2193
+ role: "user",
2194
+ content: `CURRENT PROMPT:
2195
+ ${parent}
2196
+
2197
+ TRACE-ANALYSIS REPORT:
2198
+ ${report}
2199
+
2200
+ Return the full revised prompt.`
2201
+ }
2202
+ ]
2203
+ },
2204
+ { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
2205
+ );
2206
+ const text = applied.content.trim();
2207
+ if (!text || text === parent) return [];
2208
+ return [{ surface: text, label: opts.label, rationale: opts.rationale(report) }];
2209
+ }
2210
+ };
2211
+ }
2212
+
2213
+ // src/campaign/proposers/halo.ts
2214
+ var execFileAsync = promisify(execFile);
2215
+ var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
2216
+ function haloProposer(opts) {
2217
+ const haloBin = opts.haloBin ?? "halo";
2218
+ const model = opts.model ?? "gpt-5.4-mini";
2219
+ return analysisEditProposer({
2220
+ kind: "halo",
2221
+ label: "halo",
2222
+ baseUrl: opts.baseUrl,
2223
+ apiKey: opts.apiKey,
2224
+ applyModel: opts.applyModel ?? model,
2225
+ fetchImpl: opts.fetchImpl,
2226
+ resolveTraces: opts.resolveTraces,
2227
+ noTracesError: "haloProposer: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze",
2228
+ // HALO's real findings are preserved verbatim in the rationale (attribution).
2229
+ rationale: (findings) => `halo-engine findings:
2230
+ ${findings.slice(0, 800)}`,
2231
+ analyze: async (tracePath, ctx) => {
2232
+ const args = [
2233
+ tracePath,
2234
+ "-p",
2235
+ opts.analysisPrompt ?? DEFAULT_ANALYSIS_PROMPT,
2236
+ "-m",
2237
+ model,
2238
+ ...opts.maxDepth !== void 0 ? ["--max-depth", String(opts.maxDepth)] : [],
2239
+ ...opts.maxTurns !== void 0 ? ["--max-turns", String(opts.maxTurns)] : []
2240
+ ];
2241
+ let findings;
2242
+ try {
2243
+ const { stdout } = await execFileAsync(haloBin, args, {
2244
+ maxBuffer: 64 * 1024 * 1024,
2245
+ signal: ctx.signal,
2246
+ env: {
2247
+ ...process.env,
2248
+ ...opts.apiKey ? { OPENAI_API_KEY: opts.apiKey } : {},
2249
+ OPENAI_BASE_URL: opts.baseUrl
2250
+ }
2251
+ });
2252
+ findings = stdout.trim();
2253
+ } catch (e) {
2254
+ throw new Error(
2255
+ `haloProposer: halo-engine ('${haloBin}') failed \u2014 ${e instanceof Error ? e.message : String(e)}`
2256
+ );
1726
2257
  }
1727
- } catch {
2258
+ if (!findings) throw new Error("haloProposer: halo-engine produced no findings");
2259
+ return findings;
1728
2260
  }
1729
- }
1730
- return record;
2261
+ });
1731
2262
  }
1732
- async function runProfileMatrix(opts) {
1733
- if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
1734
- if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
1735
- const splitTag = opts.splitTag ?? "search";
1736
- const seed = opts.seed ?? 42;
1737
- const validate = opts.validate ?? true;
1738
- const integrityMode = opts.integrity ?? "assert";
1739
- const profileIds = opts.profiles.map((p) => p.id);
1740
- const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
1741
- const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
1742
- const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
1743
- for (const profile of opts.profiles) {
1744
- const profileHash = agentProfileHash(profile);
1745
- try {
1746
- validateRunRecord({
1747
- runId: `${matrixId}:${profile.id}:probe`,
1748
- experimentId,
1749
- candidateId: profile.id,
1750
- seed,
1751
- model: profile.model,
1752
- promptHash: profileHash,
1753
- configHash: profileHash,
1754
- commitSha: opts.commitSha,
1755
- wallMs: 0,
1756
- costUsd: 0,
1757
- tokenUsage: { input: 0, output: 0 },
1758
- outcome: splitTag === "holdout" ? { holdoutScore: 0, raw: {} } : { searchScore: 0, raw: {} },
1759
- splitTag
1760
- });
1761
- } catch (err) {
1762
- throw new ProfileMatrixError(
1763
- `profile '${profile.id}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
2263
+
2264
+ // src/campaign/proposers/memory.ts
2265
+ var BLOCK_START2 = "<!-- BEGIN curated-memory (auto-managed by memoryCurationProposer) -->";
2266
+ var BLOCK_END2 = "<!-- END curated-memory -->";
2267
+ var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
2268
+ var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
2269
+ function extractExistingLessons(text) {
2270
+ return extractBlockBody(text, BLOCK_START2, BLOCK_END2).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
2271
+ }
2272
+ async function distillLessons(raw, distill) {
2273
+ const res = await callLlm(
2274
+ {
2275
+ model: distill.model,
2276
+ messages: [
2277
+ { role: "system", content: DISTILL_SYSTEM },
2278
+ { role: "user", content: `Findings:
2279
+ ${raw.map((r) => `- ${r}`).join("\n")}` }
2280
+ ]
2281
+ },
2282
+ { baseUrl: distill.baseUrl, apiKey: distill.apiKey, fetch: distill.fetchImpl }
2283
+ );
2284
+ try {
2285
+ const parsed = JSON.parse(res.content.trim());
2286
+ if (Array.isArray(parsed)) {
2287
+ const lessons = parsed.filter(
2288
+ (x) => typeof x === "string" && x.trim().length > 0
1764
2289
  );
2290
+ if (lessons.length > 0) return lessons;
1765
2291
  }
2292
+ } catch {
1766
2293
  }
1767
- const records = [];
1768
- const campaigns = {};
1769
- const byProfile = {};
1770
- for (const profile of opts.profiles) {
1771
- const profileHash = agentProfileHash(profile);
1772
- const configHash = sha({
1773
- profile: profileHash,
1774
- judges: (opts.judges ?? []).map((j) => j.name),
1775
- seed,
1776
- splitTag
1777
- });
1778
- const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
1779
- Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profile.id)}` });
1780
- const campaign = await runCampaign({
1781
- scenarios: opts.scenarios,
1782
- dispatch,
1783
- judges: opts.judges,
1784
- seed,
1785
- reps: opts.reps,
1786
- maxConcurrency: opts.maxConcurrency,
1787
- costCeiling: opts.costCeiling,
1788
- labeledStore: opts.labeledStore,
1789
- captureSource: opts.captureSource,
1790
- storage: opts.storage,
1791
- now: opts.now,
1792
- runDir: join3(opts.runDir, sanitize(profile.id))
1793
- });
1794
- const profileRecords = [];
1795
- for (const cell of campaign.cells) {
1796
- const record = buildRunRecord({
1797
- cell,
1798
- profile,
1799
- profileHash,
1800
- configHash,
1801
- experimentId,
1802
- splitTag,
1803
- commitSha: opts.commitSha,
1804
- matrixId,
1805
- scenario: scenarioById.get(cell.scenarioId),
1806
- corpusText: opts.corpusText
1807
- });
1808
- if (validate) validateRunRecord(record);
1809
- profileRecords.push(record);
1810
- records.push(record);
1811
- }
1812
- const pricedTotalCostUsd = profileRecords.reduce((a, r) => a + r.costUsd, 0);
1813
- campaigns[profile.id] = {
1814
- ...campaign,
1815
- aggregates: { ...campaign.aggregates, totalCostUsd: pricedTotalCostUsd }
1816
- };
1817
- byProfile[profile.id] = {
1818
- profileId: profile.id,
1819
- profileHash,
1820
- model: profile.model,
1821
- records: profileRecords.length,
1822
- meanComposite: mean2(profileRecords.map(compositeOf)),
1823
- totalCostUsd: pricedTotalCostUsd,
1824
- integrity: summarizeBackendIntegrity(profileRecords)
1825
- };
1826
- }
1827
- const integrity = summarizeBackendIntegrity(records);
1828
- if (integrityMode === "assert") {
1829
- assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
1830
- } else if (integrityMode === "warn" && integrity.verdict !== "real") {
1831
- console.warn(
1832
- `[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
1833
- );
1834
- }
1835
- const byScenario = rollup(records, (r) => r.scenarioId);
1836
- const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
1837
- return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
2294
+ return raw;
1838
2295
  }
1839
- function compositeOf(r) {
1840
- return r.outcome.holdoutScore ?? r.outcome.searchScore ?? 0;
2296
+ function memoryCurationProposer(opts = {}) {
2297
+ const maxEntries = opts.maxEntries ?? 12;
2298
+ const heading = opts.sectionHeading ?? DEFAULT_HEADING2;
2299
+ return {
2300
+ kind: "memory-curation",
2301
+ async propose(ctx) {
2302
+ const parent = surfaceToText2(ctx.currentSurface);
2303
+ const fresh = [];
2304
+ for (const f of ctx.findings ?? []) {
2305
+ const l = findingToLesson(f);
2306
+ if (l) fresh.push(l);
2307
+ }
2308
+ const carried = extractExistingLessons(parent);
2309
+ if (fresh.length === 0 && carried.length === 0) return [];
2310
+ const distilled = opts.distill && fresh.length > 0 ? await distillLessons(fresh, opts.distill) : fresh;
2311
+ const byKey = /* @__PURE__ */ new Map();
2312
+ for (const l of carried) {
2313
+ const k = normKey(l);
2314
+ if (k) byKey.set(k, { text: l, count: 1 });
2315
+ }
2316
+ for (const l of distilled) {
2317
+ const k = normKey(l);
2318
+ if (!k) continue;
2319
+ const e = byKey.get(k);
2320
+ if (e) e.count += 1;
2321
+ else byKey.set(k, { text: l, count: 1 });
2322
+ }
2323
+ const ranked = [...byKey.values()].sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)).slice(0, maxEntries);
2324
+ if (ranked.length === 0) return [];
2325
+ const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
2326
+ "\n"
2327
+ );
2328
+ const next = `${stripBlock(parent, BLOCK_START2, BLOCK_END2)}
2329
+
2330
+ ${block}
2331
+ `;
2332
+ if (next === parent) return [];
2333
+ return [
2334
+ {
2335
+ surface: next,
2336
+ label: "memory-curation",
2337
+ rationale: `curated ${ranked.length} lessons (from ${fresh.length} new finding(s) + ${carried.length} carried)`
2338
+ }
2339
+ ];
2340
+ }
2341
+ };
1841
2342
  }
1842
- function rollup(records, keyOf) {
1843
- const groups = /* @__PURE__ */ new Map();
1844
- for (const r of records) {
1845
- const key = keyOf(r);
1846
- if (key === void 0) continue;
1847
- const arr = groups.get(key) ?? [];
1848
- arr.push(compositeOf(r));
1849
- groups.set(key, arr);
1850
- }
1851
- const out = {};
1852
- for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
1853
- return out;
2343
+
2344
+ // src/campaign/proposers/trace-analyst.ts
2345
+ import { ai } from "@ax-llm/ax";
2346
+ function renderFindings(findings) {
2347
+ return findings.map((f, i) => {
2348
+ const action = f.recommended_action ? `
2349
+ FIX: ${f.recommended_action}` : "";
2350
+ const subject = f.subject ? ` (${f.subject})` : "";
2351
+ return `${i + 1}. [${f.severity}/${f.area}]${subject} ${f.claim}${action}`;
2352
+ }).join("\n");
1854
2353
  }
1855
- function rollupByPersona(records, scenarios, personaOf) {
1856
- const personaByScenarioId = /* @__PURE__ */ new Map();
1857
- for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
1858
- return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
2354
+ function traceAnalystProposer(opts) {
2355
+ if (!opts.apiKey) throw new Error("traceAnalystProposer: apiKey is required");
2356
+ if (!opts.model) throw new Error("traceAnalystProposer: model is required");
2357
+ const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
2358
+ const produceFindings = opts.analyze ?? (async (path, c) => {
2359
+ const aiService = ai({
2360
+ name: opts.provider ?? "openai",
2361
+ apiKey: opts.apiKey,
2362
+ apiURL: opts.baseUrl,
2363
+ config: { model: opts.model }
2364
+ });
2365
+ const registry = new AnalystRegistry();
2366
+ for (const spec of kinds) {
2367
+ registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
2368
+ }
2369
+ const result = await registry.run(
2370
+ `trace-analyst-gen-${c.generation}`,
2371
+ { traceStore: new OtlpFileTraceStore({ path }) },
2372
+ { signal: c.signal }
2373
+ );
2374
+ return result.findings;
2375
+ });
2376
+ return analysisEditProposer({
2377
+ kind: "trace-analyst",
2378
+ label: "trace-analyst",
2379
+ baseUrl: opts.baseUrl,
2380
+ apiKey: opts.apiKey,
2381
+ applyModel: opts.applyModel ?? opts.model,
2382
+ fetchImpl: opts.fetchImpl,
2383
+ resolveTraces: opts.resolveTraces,
2384
+ noTracesError: "traceAnalystProposer: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read",
2385
+ rationale: (report) => `trace-analyst findings:
2386
+ ${report.slice(0, 800)}`,
2387
+ analyze: async (tracePath, ctx) => {
2388
+ let findings;
2389
+ try {
2390
+ findings = await produceFindings(tracePath, ctx);
2391
+ } catch (e) {
2392
+ throw new Error(
2393
+ `traceAnalystProposer: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
2394
+ );
2395
+ }
2396
+ if (findings.length === 0) {
2397
+ throw new Error("traceAnalystProposer: analyst engine produced no findings");
2398
+ }
2399
+ return renderFindings(findings);
2400
+ }
2401
+ });
1859
2402
  }
1860
2403
 
1861
2404
  // src/campaign/worktree/index.ts
@@ -1923,14 +2466,14 @@ export {
1923
2466
  ProfileMatrixError,
1924
2467
  SkillPatchParseError,
1925
2468
  WorktreeAdapterError,
1926
- aceDriver,
2469
+ aceProposer,
1927
2470
  applySkillPatch,
1928
2471
  buildAnalystSurfaceDispatch,
1929
2472
  buildEvidenceVector,
1930
2473
  buildLoopProvenanceRecord,
1931
2474
  campaignBreakdown,
1932
2475
  campaignMeanComposite,
1933
- compareDrivers,
2476
+ compareProposers,
1934
2477
  composeGate,
1935
2478
  countSentenceEdits,
1936
2479
  defaultProductionGate,
@@ -1938,15 +2481,18 @@ export {
1938
2481
  detectScale,
1939
2482
  dimensionRegressions,
1940
2483
  emitLoopProvenance,
1941
- evolutionaryDriver,
2484
+ evolutionaryProposer,
2485
+ extractFapoAttributionSignals,
1942
2486
  extractH2Sections,
1943
2487
  failureModeRecallJudge,
2488
+ fapoEscalationEntry,
2489
+ fapoProposer,
1944
2490
  fsCampaignStorage,
1945
- gepaDriver,
1946
2491
  gepaParetoEntry,
2492
+ gepaProposer,
1947
2493
  gepaReflectionEntry,
1948
2494
  gitWorktreeAdapter,
1949
- haloDriver,
2495
+ haloProposer,
1950
2496
  heldOutGate,
1951
2497
  heldoutSignificance,
1952
2498
  inMemoryCampaignStorage,
@@ -1954,9 +2500,10 @@ export {
1954
2500
  labelTrustRank,
1955
2501
  loopProvenanceSpans,
1956
2502
  makePlaybackDispatch,
1957
- memoryCurationDriver,
2503
+ memoryCurationProposer,
1958
2504
  openAutoPr,
1959
2505
  pairHoldout,
2506
+ parameterSweepProposer,
1960
2507
  paretoPolicy,
1961
2508
  paretoSignificanceGate,
1962
2509
  parseSkillPatchResponse,
@@ -1975,11 +2522,11 @@ export {
1975
2522
  scoreboardSummary,
1976
2523
  sequentialDecide,
1977
2524
  sequentialPairedGate,
1978
- skillOptDriver,
1979
2525
  skillOptEntry,
2526
+ skillOptProposer,
1980
2527
  surfaceContentHash,
1981
2528
  surfaceHash,
1982
- traceAnalystDriver,
2529
+ traceAnalystProposer,
1983
2530
  userStoryScoreboard
1984
2531
  };
1985
2532
  //# sourceMappingURL=index.js.map