@tangle-network/agent-eval 0.123.5 → 0.123.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ import {
15
15
  runImprovementLoop,
16
16
  surfaceContentHash,
17
17
  surfaceHash
18
- } from "./chunk-SUN7QLPB.js";
18
+ } from "./chunk-KKPPFIDS.js";
19
19
  import {
20
20
  SearchLedgerConflictError,
21
21
  SearchLedgerError,
@@ -2405,1427 +2405,1875 @@ function sequentialDecide(options = {}) {
2405
2405
  return decide;
2406
2406
  }
2407
2407
 
2408
- // src/campaign/grounded-reflection.ts
2409
- function rolloutArgumentDiff(rollouts, opts = {}) {
2410
- const passThreshold = opts.passThreshold ?? 1;
2411
- const maxValues = opts.maxValuesPerField ?? 4;
2412
- const collect = (pass) => {
2413
- const byField = /* @__PURE__ */ new Map();
2414
- for (const r of rollouts) {
2415
- if (pass !== r.score >= passThreshold) continue;
2416
- for (const c of r.calls) {
2417
- for (const [k, v] of Object.entries(c.args)) {
2418
- const set = byField.get(k) ?? /* @__PURE__ */ new Set();
2419
- set.add(String(v));
2420
- byField.set(k, set);
2421
- }
2422
- }
2408
+ // src/campaign/gepa-optimization-method.ts
2409
+ import { spawn } from "child_process";
2410
+ import { randomBytes } from "crypto";
2411
+ import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises";
2412
+ import { createServer } from "http";
2413
+ import { tmpdir } from "os";
2414
+ import { join as join2 } from "path";
2415
+
2416
+ // src/campaign/presets/compare-optimization-methods.ts
2417
+ import { randomUUID } from "crypto";
2418
+ async function compareOptimizationMethods(opts) {
2419
+ assertOptimizationMethods(opts.methods);
2420
+ assertComparisonPartitions(opts);
2421
+ const seed = opts.seed ?? 42;
2422
+ const confidence = opts.confidence ?? 0.95;
2423
+ assertConfidence(confidence);
2424
+ const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
2425
+ const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
2426
+ const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
2427
+ const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
2428
+ const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
2429
+ assertComparisonControls(opts, seed, resamples, confidence);
2430
+ const storage = opts.storage ?? fsCampaignStorage();
2431
+ const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
2432
+ const testCostPhase = `compareOptimizationMethods:test:${randomUUID()}`;
2433
+ const testCostLedger = opts.costLedger ?? createRunCostLedger({
2434
+ storage,
2435
+ runDir: `${resolvedRunDir}/test/cost`,
2436
+ costCeilingUsd: opts.costCeiling
2437
+ });
2438
+ const scoreOnTest = async (surface, tag) => {
2439
+ const campaign = await runCampaign({
2440
+ ...opts,
2441
+ storage,
2442
+ costLedger: testCostLedger,
2443
+ costPhase: testCostPhase,
2444
+ scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
2445
+ dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
2446
+ runDir: `${resolvedRunDir}/${tag}`
2447
+ });
2448
+ const byScenario = {};
2449
+ for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
2450
+ byScenario[scenarioId] = composite;
2423
2451
  }
2424
- return byField;
2425
- };
2426
- const passing = collect(true);
2427
- const failing = collect(false);
2428
- const lines = [];
2429
- for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
2430
- const pv = [...passing.get(field) ?? []].slice(0, maxValues);
2431
- const fv = [...failing.get(field) ?? []].slice(0, maxValues);
2432
- const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
2433
- lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
2434
- }
2435
- const lower = (m) => {
2436
- const out = /* @__PURE__ */ new Set();
2437
- for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
2438
- return out;
2439
- };
2440
- return {
2441
- text: lines.join("\n") || " (no calls observed)",
2442
- passingValues: lower(passing),
2443
- failingValues: lower(failing)
2444
- };
2445
- }
2446
- function classifyUngroundedLiterals(text, diff) {
2447
- const ungrounded = /* @__PURE__ */ new Set();
2448
- for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
2449
- const w = m[1].toLowerCase();
2450
- if (!diff.passingValues.has(w)) ungrounded.add(w);
2451
- }
2452
- const all = [...ungrounded];
2453
- return {
2454
- ungrounded: all,
2455
- harmful: all.filter((w) => diff.failingValues.has(w))
2452
+ return byScenario;
2456
2453
  };
2457
- }
2458
-
2459
- // src/campaign/labeled-store/fs-adapter.ts
2460
- import { createHash as createHash4 } from "crypto";
2461
- import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
2462
- import { join as join2 } from "path";
2463
- var LabeledScenarioStoreError = class extends Error {
2464
- constructor(code, message) {
2465
- super(message);
2466
- this.code = code;
2467
- this.name = "LabeledScenarioStoreError";
2468
- }
2469
- code;
2470
- };
2471
- var FsLabeledScenarioStore = class {
2472
- constructor(options) {
2473
- this.options = options;
2474
- if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
2475
- this.now = options.now ?? Date.now;
2476
- }
2477
- options;
2478
- now;
2479
- rateLimits = /* @__PURE__ */ new Map();
2480
- async observe(write) {
2481
- this.assertProvenance(write);
2482
- this.assertRateLimit(write);
2483
- const record = this.toRecord(write);
2484
- const path = this.pathForSource(write.source);
2485
- const line = `${JSON.stringify(record)}
2486
- `;
2487
- appendLine(path, line);
2488
- }
2489
- async sample(args) {
2490
- if (!args.split) {
2491
- throw new LabeledScenarioStoreError(
2492
- "split_required",
2493
- "sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
2454
+ const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
2455
+ const align = (byScenario, label) => {
2456
+ const missing = scenarioIds.filter((id) => !(id in byScenario));
2457
+ if (missing.length > 0) {
2458
+ throw new Error(
2459
+ `compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(
2460
+ ", "
2461
+ )}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`
2494
2462
  );
2495
2463
  }
2496
- if (!args.capturedBefore) {
2497
- throw new LabeledScenarioStoreError(
2498
- "capturedBefore_required",
2499
- "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
2464
+ return scenarioIds.map((id) => byScenario[id]);
2465
+ };
2466
+ const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
2467
+ const out = await method.optimize(
2468
+ createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed)
2469
+ );
2470
+ assertOptimizationResult(method.name, out);
2471
+ const winnerSurface = structuredClone(out.winnerSurface);
2472
+ return {
2473
+ name: method.name,
2474
+ winnerSurface,
2475
+ cost: out.cost,
2476
+ durationMs: out.durationMs
2477
+ };
2478
+ });
2479
+ const baselineArr = align(await scoreOnTest(opts.baselineSurface, "test/baseline"), "baseline");
2480
+ const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(opts.baselineSurface), baselineArr]]);
2481
+ const winners = [];
2482
+ for (const winner of optimized) {
2483
+ const surfaceKey = surfaceContentHash(winner.winnerSurface);
2484
+ let arr = testScoresBySurface.get(surfaceKey);
2485
+ if (!arr) {
2486
+ const byScenario = await scoreOnTest(
2487
+ winner.winnerSurface,
2488
+ `test/methods/${slug(winner.name)}`
2500
2489
  );
2490
+ arr = align(byScenario, `method "${winner.name}"`);
2491
+ testScoresBySurface.set(surfaceKey, arr);
2501
2492
  }
2502
- const all = [];
2503
- for (const source of ALL_SOURCES) {
2504
- if (args.split === "train" && source === "production-trace") {
2505
- const explicit = sourceFilterContains(args.filter?.source, "production-trace");
2506
- if (!explicit) continue;
2507
- }
2508
- const path = this.pathForSource(source);
2509
- if (!existsSync2(path)) continue;
2510
- const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
2511
- for (const line of lines) {
2512
- let record;
2513
- try {
2514
- record = JSON.parse(line);
2515
- } catch {
2516
- continue;
2517
- }
2518
- if (!matchesFilter(record, args, source)) continue;
2519
- all.push(record);
2520
- }
2521
- }
2522
- all.sort((a, b) => {
2523
- if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
2524
- return a.recordHash.localeCompare(b.recordHash);
2493
+ winners.push({
2494
+ ...winner,
2495
+ arr
2525
2496
  });
2526
- return all.slice(0, args.count);
2527
2497
  }
2528
- async size() {
2529
- const bySource = {};
2530
- const byTrust = {
2531
- unverified: 0,
2532
- "verified-signal": 0,
2533
- "human-rated": 0
2498
+ const scores = winners.map((w) => {
2499
+ const boot = pairedBootstrap(baselineArr, w.arr, {
2500
+ seed,
2501
+ resamples,
2502
+ confidence: intervalConfidence,
2503
+ statistic: "mean"
2504
+ });
2505
+ const score = {
2506
+ name: w.name,
2507
+ baselineComposite: mean2(baselineArr),
2508
+ winnerComposite: mean2(w.arr),
2509
+ lift: boot.mean,
2510
+ liftCi: { low: boot.low, high: boot.high },
2511
+ optimizationCost: w.cost,
2512
+ scenarioScores: scenarioIds.map((scenarioId, index) => ({
2513
+ scenarioId,
2514
+ baselineComposite: baselineArr[index],
2515
+ winnerComposite: w.arr[index],
2516
+ lift: w.arr[index] - baselineArr[index]
2517
+ })),
2518
+ winnerSurface: w.winnerSurface,
2519
+ rank: 0
2534
2520
  };
2535
- let total = 0;
2536
- for (const source of ALL_SOURCES) {
2537
- const path = this.pathForSource(source);
2538
- if (!existsSync2(path)) {
2539
- bySource[source] = 0;
2540
- continue;
2541
- }
2542
- const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
2543
- bySource[source] = lines.length;
2544
- total += lines.length;
2545
- for (const line of lines) {
2546
- let trust = "unverified";
2547
- try {
2548
- trust = JSON.parse(line).labelTrust ?? "unverified";
2549
- } catch {
2550
- }
2551
- byTrust[trust] += 1;
2552
- }
2521
+ if (w.durationMs !== void 0) score.durationMs = w.durationMs;
2522
+ return score;
2523
+ });
2524
+ scores.sort((a, b) => b.lift - a.lift);
2525
+ for (let start = 0; start < scores.length; ) {
2526
+ let end = start + 1;
2527
+ while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
2528
+ const tied = scores.slice(start, end);
2529
+ if (tied.every((score) => score.optimizationCost.accountingComplete)) {
2530
+ tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
2531
+ scores.splice(start, tied.length, ...tied);
2553
2532
  }
2554
- return { train: total, test: total, bySource, byTrust };
2533
+ start = end;
2555
2534
  }
2556
- assertProvenance(write) {
2557
- if (!write.source) {
2558
- throw new LabeledScenarioStoreError(
2559
- "missing_source",
2560
- "LabeledScenarioWrite requires `source`"
2561
- );
2562
- }
2563
- if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
2564
- throw new LabeledScenarioStoreError(
2565
- "missing_source_version",
2566
- "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
2567
- );
2535
+ scores.forEach((s, i) => {
2536
+ s.rank = i + 1;
2537
+ });
2538
+ const best = scores[0];
2539
+ const byName = new Map(winners.map((w) => [w.name, w]));
2540
+ const bestArr = byName.get(best.name).arr;
2541
+ const pairwise = scores.slice(1).map((other) => {
2542
+ const otherArr = byName.get(other.name).arr;
2543
+ const boot = pairedBootstrap(otherArr, bestArr, {
2544
+ seed,
2545
+ resamples,
2546
+ confidence: intervalConfidence,
2547
+ statistic: "mean"
2548
+ });
2549
+ const favored = boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
2550
+ return {
2551
+ a: best.name,
2552
+ b: other.name,
2553
+ deltaMean: boot.mean,
2554
+ low: boot.low,
2555
+ high: boot.high,
2556
+ favored
2557
+ };
2558
+ });
2559
+ const optimizationCost = combineCosts(
2560
+ scores.map((score) => ({ label: `method '${score.name}'`, cost: score.optimizationCost }))
2561
+ );
2562
+ const testCost = costFromLedgerSummary(testCostLedger.summary({ phase: testCostPhase }));
2563
+ const totalCost = combineCosts([
2564
+ { label: "optimization", cost: optimizationCost },
2565
+ { label: "final test", cost: testCost }
2566
+ ]);
2567
+ return {
2568
+ scores,
2569
+ best,
2570
+ pairwise,
2571
+ testScenarioIds: scenarioIds,
2572
+ optimizationCost,
2573
+ testCost,
2574
+ totalCost,
2575
+ confidence,
2576
+ intervalConfidence,
2577
+ comparisonCount,
2578
+ seed,
2579
+ resamples,
2580
+ reps: opts.reps ?? 1
2581
+ };
2582
+ }
2583
+ function assertOptimizationMethods(methods) {
2584
+ if (!Array.isArray(methods) || methods.length === 0) {
2585
+ throw new Error("compareOptimizationMethods: no methods to compare");
2586
+ }
2587
+ const names = /* @__PURE__ */ new Set();
2588
+ const pathOwners = /* @__PURE__ */ new Map();
2589
+ for (const method of methods) {
2590
+ if (!method || typeof method !== "object" || typeof method.optimize !== "function") {
2591
+ throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
2568
2592
  }
2569
- if (!write.capturedAt) {
2570
- throw new LabeledScenarioStoreError(
2571
- "missing_captured_at",
2572
- "LabeledScenarioWrite requires `capturedAt` ISO timestamp"
2573
- );
2593
+ if (!method.name || method.name.trim() !== method.name) {
2594
+ throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
2574
2595
  }
2575
- if (!write.redactionStatus) {
2576
- throw new LabeledScenarioStoreError(
2577
- "missing_redaction_status",
2578
- "LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
2579
- );
2596
+ if (names.has(method.name)) {
2597
+ throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
2580
2598
  }
2581
- if (!ALL_SOURCES.includes(write.source)) {
2582
- throw new LabeledScenarioStoreError(
2583
- "unknown_source",
2584
- `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
2599
+ names.add(method.name);
2600
+ const pathKey = slug(method.name);
2601
+ const prior = pathOwners.get(pathKey);
2602
+ if (prior) {
2603
+ throw new Error(
2604
+ `compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`
2585
2605
  );
2586
2606
  }
2607
+ pathOwners.set(pathKey, method.name);
2587
2608
  }
2588
- assertRateLimit(write) {
2589
- const cap = this.options.maxWritesPerMinutePerBucket;
2590
- if (!cap || !write.rateLimitBucket) return;
2591
- const now = this.now();
2592
- const windowMs = 6e4;
2593
- let state = this.rateLimits.get(write.rateLimitBucket);
2594
- if (!state || now - state.windowStartMs >= windowMs) {
2595
- state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
2596
- this.rateLimits.set(write.rateLimitBucket, state);
2597
- }
2598
- if (state.count >= cap) {
2599
- throw new LabeledScenarioStoreError(
2600
- "rate_limit_exceeded",
2601
- `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
2602
- );
2603
- }
2604
- state.count += 1;
2609
+ }
2610
+ function assertOptimizationResult(name, result) {
2611
+ if (!result || typeof result !== "object") {
2612
+ throw new Error(`compareOptimizationMethods: method '${name}' returned no result`);
2605
2613
  }
2606
- toRecord(write) {
2607
- const recordHash = sha256(
2608
- JSON.stringify({
2609
- id: write.scenario.id,
2610
- src: write.source,
2611
- at: write.capturedAt,
2612
- ver: write.sourceVersionHash
2613
- })
2614
+ try {
2615
+ surfaceContentHash(result.winnerSurface);
2616
+ } catch (cause) {
2617
+ throw new Error(
2618
+ `compareOptimizationMethods: method '${name}' returned an invalid winnerSurface`,
2619
+ { cause }
2614
2620
  );
2615
- return {
2616
- ...write,
2617
- recordHash,
2618
- split: "train"
2619
- };
2620
2621
  }
2621
- pathForSource(source) {
2622
- return join2(this.options.root, `${source}.jsonl`);
2622
+ assertComparisonCost(result.cost, `method '${name}'`);
2623
+ if (result.durationMs !== void 0 && (!Number.isFinite(result.durationMs) || result.durationMs < 0)) {
2624
+ throw new Error(`compareOptimizationMethods: method '${name}' returned an invalid durationMs`);
2623
2625
  }
2624
- };
2625
- var ALL_SOURCES = [
2626
- "production-trace",
2627
- "eval-run",
2628
- "manual",
2629
- "red-team",
2630
- "synthetic"
2631
- ];
2632
- function sourceFilterContains(filter, needle) {
2633
- if (!filter) return false;
2634
- if (Array.isArray(filter)) return filter.includes(needle);
2635
- return filter === needle;
2636
2626
  }
2637
- function matchesFilter(record, args, source) {
2638
- if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
2639
- if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
2640
- const f = args.filter;
2641
- if (!f) return true;
2642
- if (f.kind && record.scenario.kind !== f.kind) return false;
2643
- if (f.source) {
2644
- const sources = Array.isArray(f.source) ? f.source : [f.source];
2645
- if (!sources.includes(source)) return false;
2627
+ function assertComparisonControls(opts, seed, resamples, confidence) {
2628
+ if (!opts.judges || opts.judges.length === 0) {
2629
+ throw new Error("compareOptimizationMethods: at least one judge is required");
2646
2630
  }
2647
- if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
2648
- const composites = Object.values(record.judgeScores).map((s) => s.composite);
2649
- const max = composites.length === 0 ? 0 : Math.max(...composites);
2650
- if (f.minComposite !== void 0 && max < f.minComposite) return false;
2651
- if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
2631
+ if (typeof opts.dispatchWithSurface !== "function") {
2632
+ throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
2652
2633
  }
2653
- if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
2654
- return false;
2634
+ try {
2635
+ surfaceContentHash(opts.baselineSurface);
2636
+ } catch (cause) {
2637
+ throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
2655
2638
  }
2656
- return true;
2657
- }
2658
- function sha256(input) {
2659
- return createHash4("sha256").update(input).digest("hex").slice(0, 16);
2660
- }
2661
- function appendLine(path, line) {
2662
- if (existsSync2(path)) {
2663
- const existing = readFileSync2(path, "utf8");
2664
- writeFileSync(path, existing + line);
2665
- } else {
2666
- writeFileSync(path, line);
2639
+ const judgeNames = /* @__PURE__ */ new Set();
2640
+ for (const judge of opts.judges) {
2641
+ if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) {
2642
+ throw new Error(
2643
+ "compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)"
2644
+ );
2645
+ }
2646
+ if (judgeNames.has(judge.name)) {
2647
+ throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
2648
+ }
2649
+ judgeNames.add(judge.name);
2650
+ const dimensionKeys = /* @__PURE__ */ new Set();
2651
+ for (const dimension of judge.dimensions) {
2652
+ if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) {
2653
+ throw new Error(
2654
+ `compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`
2655
+ );
2656
+ }
2657
+ if (dimensionKeys.has(dimension.key)) {
2658
+ throw new Error(
2659
+ `compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`
2660
+ );
2661
+ }
2662
+ dimensionKeys.add(dimension.key);
2663
+ }
2667
2664
  }
2668
- }
2669
-
2670
- // src/campaign/neutralize.ts
2671
- var FILLER = "#";
2672
- function neutralizeText(content) {
2673
- return content.replace(/\S/g, FILLER);
2674
- }
2675
-
2676
- // src/campaign/proposers/fapo.ts
2677
- var FAPO_LEVELS = ["prompt", "parameter", "structural"];
2678
- var MAX_FINDING_DEPTH = 16;
2679
- var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
2680
- function fapoProposer(opts) {
2681
- const proposers = levelProposers(opts);
2682
- const allowed = allowedLevels(opts, proposers);
2683
- const plateauWindow = opts.plateauWindow ?? 3;
2684
- const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
2685
- const minImprovement = opts.minImprovement ?? 0;
2686
- const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
2687
- if (allowed.length === 0) {
2688
- throw new Error("fapoProposer: at least one allowed level must have a proposer");
2665
+ if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
2666
+ throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
2689
2667
  }
2690
- if (plateauWindow < 1) throw new Error("fapoProposer: plateauWindow must be >= 1");
2691
- if (minDistinctStrategies < 1) {
2692
- throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
2668
+ if (!Number.isSafeInteger(seed)) {
2669
+ throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
2693
2670
  }
2694
- if (proposalsPerCycle < 1) throw new Error("fapoProposer: proposalsPerCycle must be >= 1");
2695
- return {
2696
- kind: "fapo",
2697
- async propose(ctx) {
2698
- const state = policyState(ctx.history, ctx.findings, {
2699
- plateauWindow,
2700
- minDistinctStrategies,
2701
- minImprovement
2702
- });
2703
- const decision = chooseLevel({
2704
- allowed,
2705
- available: allowed.filter((level) => !state.exhausted[level]),
2706
- proposers,
2707
- state,
2708
- promptFirst: opts.promptFirst ?? true,
2709
- parameterBeforeStructural: opts.parameterBeforeStructural ?? true
2710
- });
2711
- if (!decision) return [];
2712
- const proposer = proposers[decision.level];
2713
- if (!proposer) {
2714
- throw new Error(`fapoProposer: selected ${decision.level} but no proposer is configured`);
2715
- }
2716
- const requested = Math.min(ctx.populationSize, proposalsPerCycle);
2717
- const raw = await proposer.propose({ ...ctx, populationSize: requested });
2718
- const wrapped = raw.map((candidate) => normalizeProposal(candidate)).map((candidate) => wrapProposal(candidate, decision.level, decision.reason));
2719
- const reviewed = [];
2720
- for (const candidate of wrapped) {
2721
- const review = opts.reviewCandidate ? await opts.reviewCandidate({
2722
- level: decision.level,
2723
- candidate,
2724
- context: ctx,
2725
- reason: decision.reason
2726
- }) : { verdict: "pass" };
2727
- if (review.verdict === "fail") continue;
2728
- const warn = review.verdict === "warn" ? renderReviewWarnings(review) : "";
2729
- reviewed.push(
2730
- warn ? { ...candidate, rationale: `${candidate.rationale}
2731
- ${warn}` } : candidate
2732
- );
2671
+ if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) {
2672
+ throw new Error(
2673
+ `compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`
2674
+ );
2675
+ }
2676
+ if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
2677
+ throw new Error(
2678
+ `compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
2679
+ );
2680
+ }
2681
+ const minimumResamples = minimumBootstrapResamples(
2682
+ confidence,
2683
+ opts.methods.length * (opts.methods.length + 1) / 2
2684
+ );
2685
+ if (resamples < minimumResamples) {
2686
+ throw new Error(
2687
+ `compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`
2688
+ );
2689
+ }
2690
+ if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) {
2691
+ throw new Error(
2692
+ "compareOptimizationMethods: optimizationConcurrency must be a positive safe integer"
2693
+ );
2694
+ }
2695
+ if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) {
2696
+ throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
2697
+ }
2698
+ if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) {
2699
+ throw new Error(
2700
+ "compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647"
2701
+ );
2702
+ }
2703
+ if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) {
2704
+ throw new Error(
2705
+ "compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0"
2706
+ );
2707
+ }
2708
+ if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) {
2709
+ throw new Error(
2710
+ "compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling"
2711
+ );
2712
+ }
2713
+ }
2714
+ function assertComparisonPartitions(opts) {
2715
+ const legacy = opts;
2716
+ if (legacy.holdoutScenarios !== void 0) {
2717
+ throw new Error(
2718
+ "compareOptimizationMethods: holdoutScenarios is ambiguous and no longer accepted. Provide disjoint trainScenarios, selectionScenarios, and testScenarios; selection may be reused adaptively, test must remain untouched."
2719
+ );
2720
+ }
2721
+ const partitions = [
2722
+ { name: "trainScenarios", scenarios: opts.trainScenarios },
2723
+ { name: "selectionScenarios", scenarios: opts.selectionScenarios },
2724
+ { name: "testScenarios", scenarios: opts.testScenarios }
2725
+ ];
2726
+ const owner = /* @__PURE__ */ new Map();
2727
+ for (const partition of partitions) {
2728
+ if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) {
2729
+ throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
2730
+ }
2731
+ if (partition.name === "testScenarios" && partition.scenarios.length < 2) {
2732
+ throw new Error(
2733
+ "compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty"
2734
+ );
2735
+ }
2736
+ const local = /* @__PURE__ */ new Set();
2737
+ const duplicates = /* @__PURE__ */ new Set();
2738
+ const overlaps = /* @__PURE__ */ new Map();
2739
+ for (const scenario of partition.scenarios) {
2740
+ if (local.has(scenario.id)) duplicates.add(scenario.id);
2741
+ local.add(scenario.id);
2742
+ const prior = owner.get(scenario.id);
2743
+ if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
2744
+ }
2745
+ if (duplicates.size > 0) {
2746
+ throw new Error(
2747
+ `compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[
2748
+ ...duplicates
2749
+ ].join(", ")}]`
2750
+ );
2751
+ }
2752
+ if (overlaps.size > 0) {
2753
+ const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} \u2229 ${partition.name})`).join(", ");
2754
+ throw new Error(
2755
+ `compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`
2756
+ );
2757
+ }
2758
+ assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
2759
+ for (const id of local) owner.set(id, partition.name);
2760
+ }
2761
+ }
2762
+ function mean2(xs) {
2763
+ return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
2764
+ }
2765
+ function slug(name) {
2766
+ return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
2767
+ }
2768
+ function assertConfidence(confidence) {
2769
+ if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
2770
+ throw new Error(
2771
+ `compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
2772
+ );
2773
+ }
2774
+ }
2775
+ function minimumBootstrapResamples(confidence, comparisonCount) {
2776
+ const exact = 2 * comparisonCount / (1 - confidence);
2777
+ return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
2778
+ }
2779
+ function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed) {
2780
+ const cloneScenarios = (scenarios) => Object.freeze(scenarios.map((scenario) => structuredClone(scenario)));
2781
+ const judges = opts.judges.map(
2782
+ (judge) => Object.freeze({
2783
+ ...judge,
2784
+ dimensions: Object.freeze(
2785
+ judge.dimensions.map((dimension) => Object.freeze({ ...dimension }))
2786
+ )
2787
+ })
2788
+ );
2789
+ return Object.freeze({
2790
+ baselineSurface: structuredClone(opts.baselineSurface),
2791
+ trainScenarios: cloneScenarios(opts.trainScenarios),
2792
+ selectionScenarios: cloneScenarios(opts.selectionScenarios),
2793
+ dispatchWithSurface: opts.dispatchWithSurface,
2794
+ judges: Object.freeze(judges),
2795
+ runDir: `${resolvedRunDir}/optimization/${slug(methodName)}`,
2796
+ seed,
2797
+ runOptions: Object.freeze({ ...opts.optimizationRunOptions ?? {} })
2798
+ });
2799
+ }
2800
+ function costFromLedgerSummary(summary) {
2801
+ const cost = {
2802
+ totalCostUsd: summary.totalCostUsd,
2803
+ accountingComplete: summary.accountingComplete,
2804
+ incompleteReasons: [...summary.incompleteReasons]
2805
+ };
2806
+ assertComparisonCost(cost, "cost ledger");
2807
+ return cost;
2808
+ }
2809
+ function combineCosts(entries) {
2810
+ return {
2811
+ totalCostUsd: entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0),
2812
+ accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
2813
+ incompleteReasons: entries.flatMap(
2814
+ (entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`)
2815
+ )
2816
+ };
2817
+ }
2818
+ function assertComparisonCost(cost, label) {
2819
+ if (!cost || typeof cost !== "object") {
2820
+ throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
2821
+ }
2822
+ if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) {
2823
+ throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
2824
+ }
2825
+ if (typeof cost.accountingComplete !== "boolean") {
2826
+ throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
2827
+ }
2828
+ if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some(
2829
+ (reason) => typeof reason !== "string" || reason.trim().length === 0
2830
+ )) {
2831
+ throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
2832
+ }
2833
+ if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) {
2834
+ throw new Error(
2835
+ `compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`
2836
+ );
2837
+ }
2838
+ }
2839
+
2840
+ // src/campaign/gepa-optimization-method.ts
2841
+ var DEFAULT_MAX_CANDIDATE_CHARS = 2e5;
2842
+ var DEFAULT_TIMEOUT_MS = 30 * 60 * 1e3;
2843
+ var MAX_CALLBACK_BODY_BYTES = 1e6;
2844
+ var MAX_PROCESS_OUTPUT_CHARS = 64e3;
2845
+ var PROCESS_TERMINATION_GRACE_MS = 5e3;
2846
+ function gepaOptimizationMethod(config) {
2847
+ assertConfig(config);
2848
+ const name = config.name ?? defaultMethodName(config.recipe);
2849
+ return {
2850
+ name,
2851
+ async optimize(input) {
2852
+ if (typeof input.baselineSurface !== "string") {
2853
+ throw new Error(`${name}: GEPA bridge requires a string baselineSurface`);
2733
2854
  }
2734
- if (wrapped.length > 0 && reviewed.length === 0) {
2735
- throw new Error(
2736
- `fapoProposer: reviewer blocked every ${decision.level} candidate; refusing to evaluate an unreviewed variant`
2855
+ const started = Date.now();
2856
+ const maxCandidateChars = config.maxCandidateChars ?? DEFAULT_MAX_CANDIDATE_CHARS;
2857
+ const storage = input.runOptions.storage ?? fsCampaignStorage();
2858
+ const runDir = `${input.runDir}/gepa`;
2859
+ storage.ensureDir(runDir);
2860
+ const costLedger = createRunCostLedger({
2861
+ storage,
2862
+ runDir,
2863
+ costCeilingUsd: input.runOptions.costCeiling
2864
+ });
2865
+ const scenarioById = scenarioMap(input.trainScenarios, input.selectionScenarios);
2866
+ const evaluate = createEvaluationFunction({
2867
+ input,
2868
+ config,
2869
+ runDir,
2870
+ costLedger,
2871
+ scenarioById,
2872
+ maxCandidateChars
2873
+ });
2874
+ const callback = await startCallbackServer({
2875
+ token: randomBytes(32).toString("hex"),
2876
+ maxEvaluations: recipeEvaluationLimit(config.recipe),
2877
+ evaluate
2878
+ });
2879
+ try {
2880
+ const outputDir = `${runDir}/external`;
2881
+ await mkdir(outputDir, { recursive: true });
2882
+ const result = await runGepaBridge(
2883
+ {
2884
+ version: 2,
2885
+ callbackUrl: callback.url,
2886
+ callbackToken: callback.token,
2887
+ recipe: config.recipe,
2888
+ objective: config.objective,
2889
+ ...config.background ? { background: config.background } : {},
2890
+ seedCandidate: input.baselineSurface,
2891
+ trainSet: input.trainScenarios.map((scenario) => describeScenario(config, scenario)),
2892
+ selectionSet: input.selectionScenarios.map(
2893
+ (scenario) => describeScenario(config, scenario)
2894
+ ),
2895
+ maxCandidateChars,
2896
+ outputDir
2897
+ },
2898
+ config.runner,
2899
+ config.timeoutMs ?? DEFAULT_TIMEOUT_MS
2737
2900
  );
2901
+ assertBridgeOutput(
2902
+ result,
2903
+ name,
2904
+ maxCandidateChars,
2905
+ config.recipe.kind,
2906
+ recipeEvaluationLimit(config.recipe)
2907
+ );
2908
+ const evaluationCost = costFromLedgerSummary(costLedger.summary());
2909
+ const reportedProposerCost = result.proposerCostUsd ?? 0;
2910
+ return {
2911
+ winnerSurface: result.bestCandidate,
2912
+ cost: {
2913
+ totalCostUsd: evaluationCost.totalCostUsd + reportedProposerCost,
2914
+ accountingComplete: false,
2915
+ incompleteReasons: [
2916
+ ...evaluationCost.incompleteReasons,
2917
+ result.proposerCostAccounting === "reported" ? "GEPA proposer cost is externally reported and has no agent-eval receipt" : "GEPA proposer cost is unavailable"
2918
+ ]
2919
+ },
2920
+ durationMs: Date.now() - started
2921
+ };
2922
+ } finally {
2923
+ await callback.close();
2738
2924
  }
2739
- return reviewed;
2740
- },
2741
- decide({ history }) {
2742
- const last = history[history.length - 1];
2743
- if (last && last.candidates.length === 0) {
2744
- return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
2745
- }
2746
- const state = policyState(history, [], {
2747
- plateauWindow,
2748
- minDistinctStrategies,
2749
- minImprovement
2750
- });
2751
- const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
2752
- return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
2753
2925
  }
2754
2926
  };
2755
2927
  }
2756
- function levelProposers(opts) {
2757
- return {
2758
- ...opts.proposers ?? {},
2759
- ...opts.promptProposer ? { prompt: opts.promptProposer } : {},
2760
- ...opts.parameterProposer ? { parameter: opts.parameterProposer } : {},
2761
- ...opts.structuralProposer ? { structural: opts.structuralProposer } : {}
2928
+ function createEvaluationFunction(args) {
2929
+ const cached = /* @__PURE__ */ new Map();
2930
+ return async ({ candidate, exampleId }) => {
2931
+ if (!args.scenarioById.has(exampleId)) {
2932
+ throw new Error(`GEPA requested unknown train or selection case '${exampleId}'`);
2933
+ }
2934
+ if (!isCandidateText(candidate, args.maxCandidateChars)) {
2935
+ throw new Error("GEPA submitted an invalid candidate");
2936
+ }
2937
+ const scenario = args.scenarioById.get(exampleId);
2938
+ const cacheKey = `${surfaceContentHash(candidate)}:${exampleId}`;
2939
+ const existing = cached.get(cacheKey);
2940
+ if (existing) return existing;
2941
+ const result = scoreOneScenario({
2942
+ input: args.input,
2943
+ candidate,
2944
+ scenario,
2945
+ runDir: args.runDir,
2946
+ costLedger: args.costLedger
2947
+ });
2948
+ cached.set(cacheKey, result);
2949
+ return result;
2762
2950
  };
2763
2951
  }
2764
- function allowedLevels(opts, proposers) {
2765
- const explicit = opts.scope?.allowedLevels;
2766
- const forbidden = new Set(opts.scope?.forbiddenLevels ?? []);
2767
- return FAPO_LEVELS.filter((level) => {
2768
- if (!proposers[level]) return false;
2769
- if (forbidden.has(level)) return false;
2770
- return explicit ? explicit.includes(level) : true;
2952
+ async function scoreOneScenario(args) {
2953
+ const campaign = await runCampaign({
2954
+ ...args.input.runOptions,
2955
+ scenarios: [structuredClone(args.scenario)],
2956
+ dispatch: (scenario, context) => args.input.dispatchWithSurface(args.candidate, scenario, context),
2957
+ judges: [...args.input.judges],
2958
+ runDir: `${args.runDir}/evaluations/${safePathComponent(surfaceContentHash(args.candidate))}/${safePathComponent(args.scenario.id)}`,
2959
+ seed: args.input.seed,
2960
+ costLedger: args.costLedger,
2961
+ costPhase: "gepa.external-evaluation",
2962
+ maxConcurrency: 1
2771
2963
  });
2964
+ const breakdown = campaignBreakdown(campaign);
2965
+ const row = breakdown.scenarios[0];
2966
+ if (!row) throw new Error(`GEPA evaluation produced no score for '${args.scenario.id}'`);
2967
+ return {
2968
+ score: row.composite,
2969
+ info: {
2970
+ scenarioId: row.scenarioId,
2971
+ dimensions: breakdown.dimensions,
2972
+ ...row.notes ? { notes: row.notes } : {}
2973
+ }
2974
+ };
2772
2975
  }
2773
- function policyState(history, findings, opts) {
2774
- const attempts = extractAttempts(history);
2976
+ function scenarioMap(train, selection) {
2977
+ const out = /* @__PURE__ */ new Map();
2978
+ for (const scenario of [...train, ...selection]) {
2979
+ if (out.has(scenario.id)) {
2980
+ throw new Error(
2981
+ `GEPA bridge requires unique train and selection ids; duplicate '${scenario.id}'`
2982
+ );
2983
+ }
2984
+ out.set(scenario.id, scenario);
2985
+ }
2986
+ return out;
2987
+ }
2988
+ function describeScenario(config, scenario) {
2989
+ const data = config.describeScenario ? config.describeScenario(scenario) : { id: scenario.id };
2990
+ assertJsonValue(data, `GEPA scenario '${scenario.id}'`);
2991
+ return { id: scenario.id, data };
2992
+ }
2993
+ async function startCallbackServer(args) {
2994
+ let evaluations = 0;
2995
+ const server = createServer((request, response) => {
2996
+ void handleCallback(request, response, args, () => {
2997
+ evaluations += 1;
2998
+ return evaluations;
2999
+ });
3000
+ });
3001
+ const port = await listen(server);
2775
3002
  return {
2776
- attempts,
2777
- exhausted: {
2778
- prompt: isLevelExhausted("prompt", attempts, opts),
2779
- parameter: isLevelExhausted("parameter", attempts, opts),
2780
- structural: isLevelExhausted("structural", attempts, opts)
2781
- },
2782
- signals: extractFapoAttributionSignals(findings)
3003
+ url: `http://127.0.0.1:${port}/evaluate`,
3004
+ token: args.token,
3005
+ close: () => closeServer(server)
2783
3006
  };
2784
3007
  }
2785
- function extractAttempts(history) {
2786
- const attempts = [];
2787
- for (const generation of history) {
2788
- for (const candidate of generation.candidates) {
2789
- const level = candidateLevel(candidate);
2790
- if (!level) continue;
2791
- attempts.push({
2792
- level,
2793
- composite: candidate.composite,
2794
- strategy: candidateStrategy(candidate)
2795
- });
2796
- }
3008
+ async function handleCallback(request, response, args, nextEvaluation) {
3009
+ try {
3010
+ if (request.method !== "POST" || request.url !== "/evaluate") {
3011
+ sendJson(response, 404, { error: "not found" });
3012
+ return;
3013
+ }
3014
+ if (request.headers.authorization !== `Bearer ${args.token}`) {
3015
+ sendJson(response, 401, { error: "unauthorized" });
3016
+ return;
3017
+ }
3018
+ const body = await readJson(request);
3019
+ if (!isRecord(body) || typeof body.candidate !== "string" || typeof body.exampleId !== "string") {
3020
+ sendJson(response, 400, { error: "candidate and exampleId are required strings" });
3021
+ return;
3022
+ }
3023
+ const count = nextEvaluation();
3024
+ if (count > args.maxEvaluations) {
3025
+ sendJson(response, 429, { error: "evaluation limit reached" });
3026
+ return;
3027
+ }
3028
+ const result = await args.evaluate({ candidate: body.candidate, exampleId: body.exampleId });
3029
+ sendJson(response, 200, result);
3030
+ } catch {
3031
+ sendJson(response, 500, { error: "evaluation failed" });
2797
3032
  }
2798
- return attempts;
2799
- }
2800
- function candidateLevel(candidate) {
2801
- const label = candidate.label ?? "";
2802
- const rationale = candidate.rationale ?? "";
2803
- return parseLevel(label) ?? parseLevel(rationale);
2804
- }
2805
- function parseLevel(text) {
2806
- const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
2807
- return match ? match[1] : null;
2808
- }
2809
- function candidateStrategy(candidate) {
2810
- const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
2811
- return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
2812
3033
  }
2813
- function isLevelExhausted(level, attempts, opts) {
2814
- const own = attempts.filter((attempt) => attempt.level === level);
2815
- if (own.length < opts.plateauWindow) return false;
2816
- const distinctStrategies = new Set(own.map((attempt) => attempt.strategy)).size;
2817
- if (distinctStrategies < opts.minDistinctStrategies) return false;
2818
- let best = Number.NEGATIVE_INFINITY;
2819
- let nonImproving = 0;
2820
- for (const attempt of own) {
2821
- if (attempt.composite > best + opts.minImprovement) {
2822
- best = attempt.composite;
2823
- nonImproving = 0;
2824
- } else {
2825
- nonImproving += 1;
2826
- }
3034
+ async function runGepaBridge(input, runner, timeoutMs) {
3035
+ const dir = await mkdtemp(join2(tmpdir(), "agent-eval-gepa-"));
3036
+ const inputPath = join2(dir, "input.json");
3037
+ const outputPath = join2(dir, "output.json");
3038
+ try {
3039
+ await writeFile(inputPath, `${JSON.stringify(input)}
3040
+ `);
3041
+ const command = runner?.command ?? "python";
3042
+ const args = [
3043
+ ...runner?.args ?? ["-m", "agent_eval_rpc.gepa_bridge"],
3044
+ "--input",
3045
+ inputPath,
3046
+ "--output",
3047
+ outputPath
3048
+ ];
3049
+ await runProcess(command, args, runner?.cwd ?? dir, runner?.env, timeoutMs);
3050
+ const raw = JSON.parse(await readFile(outputPath, "utf8"));
3051
+ if (!isRecord(raw)) throw new Error("GEPA bridge output must be a JSON object");
3052
+ return raw;
3053
+ } finally {
3054
+ await rm(dir, { recursive: true, force: true });
2827
3055
  }
2828
- return nonImproving >= opts.plateauWindow;
2829
3056
  }
2830
- function chooseLevel(args) {
2831
- const { available, state } = args;
2832
- if (available.length === 0) return null;
2833
- const triedPrompt = state.attempts.some((attempt) => attempt.level === "prompt");
2834
- if (args.promptFirst && available.includes("prompt") && !triedPrompt) {
2835
- return {
2836
- level: "prompt",
2837
- reason: "prompt-first policy: no prompt-level variant has been tried yet"
2838
- };
2839
- }
2840
- const supported = available.filter((level) => state.signals.counts[level] > 0);
2841
- if (supported.includes("prompt")) {
2842
- return {
2843
- level: "prompt",
2844
- reason: `attribution has ${state.signals.counts.prompt} prompt-addressable failure(s)`
2845
- };
2846
- }
2847
- const strongest = supported.sort(
2848
- (a, b) => state.signals.counts[b] - state.signals.counts[a] || levelRank(a) - levelRank(b)
2849
- )[0];
2850
- if (strongest) {
2851
- if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
2852
- return {
2853
- level: "parameter",
2854
- reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
2855
- };
2856
- }
2857
- return {
2858
- level: strongest,
2859
- reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
3057
+ function runProcess(command, args, cwd, env, timeoutMs) {
3058
+ return new Promise((resolvePromise, reject) => {
3059
+ const child = spawn(command, args, {
3060
+ cwd,
3061
+ env: { ...process.env, ...env },
3062
+ stdio: ["ignore", "pipe", "pipe"]
3063
+ });
3064
+ let stdout = "";
3065
+ let stderr = "";
3066
+ let settled = false;
3067
+ let timedOut = false;
3068
+ let timeout;
3069
+ let terminationGrace;
3070
+ const finish = (error) => {
3071
+ if (settled) return;
3072
+ settled = true;
3073
+ if (timeout) clearTimeout(timeout);
3074
+ if (terminationGrace) clearTimeout(terminationGrace);
3075
+ if (error) reject(error);
3076
+ else resolvePromise();
2860
3077
  };
2861
- }
2862
- if (state.attempts.length === 0) {
2863
- const first = available.slice().sort((a, b) => levelRank(a) - levelRank(b))[0];
2864
- return { level: first, reason: "no attribution yet; starting at the cheapest allowed level" };
2865
- }
2866
- return null;
2867
- }
2868
- function levelRank(level) {
2869
- return level === "prompt" ? 0 : level === "parameter" ? 1 : 2;
3078
+ timeout = setTimeout(() => {
3079
+ timedOut = true;
3080
+ child.kill("SIGTERM");
3081
+ terminationGrace = setTimeout(() => {
3082
+ child.kill("SIGKILL");
3083
+ finish(new Error(`GEPA bridge exceeded ${timeoutMs}ms`));
3084
+ }, PROCESS_TERMINATION_GRACE_MS);
3085
+ }, timeoutMs);
3086
+ child.stdout.on("data", (chunk) => {
3087
+ stdout = appendProcessOutput(stdout, chunk);
3088
+ });
3089
+ child.stderr.on("data", (chunk) => {
3090
+ stderr = appendProcessOutput(stderr, chunk);
3091
+ });
3092
+ child.on("error", (error) => finish(new Error(`GEPA bridge could not start: ${error.message}`)));
3093
+ child.on("close", (code) => {
3094
+ if (timedOut) {
3095
+ finish(new Error(`GEPA bridge exceeded ${timeoutMs}ms`));
3096
+ return;
3097
+ }
3098
+ if (code === 0) {
3099
+ finish();
3100
+ return;
3101
+ }
3102
+ finish(
3103
+ new Error(
3104
+ `GEPA bridge exited ${String(code)}. stderr=${truncate(stderr)} stdout=${truncate(stdout)}`
3105
+ )
3106
+ );
3107
+ });
3108
+ });
2870
3109
  }
2871
- function normalizeProposal(value) {
2872
- return isProposedCandidate(value) ? value : { surface: value, label: "candidate", rationale: "bare candidate from level proposer" };
3110
+ function appendProcessOutput(current, chunk) {
3111
+ if (current.length >= MAX_PROCESS_OUTPUT_CHARS) return current;
3112
+ return `${current}${chunk.toString()}`.slice(0, MAX_PROCESS_OUTPUT_CHARS);
3113
+ }
3114
+ function readJson(request) {
3115
+ return new Promise((resolvePromise, reject) => {
3116
+ let size = 0;
3117
+ const chunks = [];
3118
+ request.on("data", (chunk) => {
3119
+ size += chunk.length;
3120
+ if (size > MAX_CALLBACK_BODY_BYTES) {
3121
+ reject(new Error("callback body too large"));
3122
+ request.destroy();
3123
+ return;
3124
+ }
3125
+ chunks.push(chunk);
3126
+ });
3127
+ request.on("error", reject);
3128
+ request.on("end", () => {
3129
+ try {
3130
+ resolvePromise(JSON.parse(Buffer.concat(chunks).toString("utf8")));
3131
+ } catch (error) {
3132
+ reject(error);
3133
+ }
3134
+ });
3135
+ });
2873
3136
  }
2874
- function wrapProposal(candidate, level, reason) {
2875
- const label = candidate.label ? `fapo:${level}:${candidate.label}` : `fapo:${level}`;
2876
- return {
2877
- ...candidate,
2878
- label,
2879
- rationale: [
2880
- `fapo:${level} selected by reviewed escalation policy`,
2881
- `Reason: ${reason}`,
2882
- candidate.rationale ? `Level proposer rationale: ${candidate.rationale}` : ""
2883
- ].filter(Boolean).join("\n")
2884
- };
3137
+ function listen(server) {
3138
+ return new Promise((resolvePromise, reject) => {
3139
+ server.once("error", reject);
3140
+ server.listen(0, "127.0.0.1", () => {
3141
+ server.off("error", reject);
3142
+ const address = server.address();
3143
+ if (!address || typeof address === "string") {
3144
+ reject(new Error("GEPA callback did not bind a TCP port"));
3145
+ return;
3146
+ }
3147
+ resolvePromise(address.port);
3148
+ });
3149
+ });
2885
3150
  }
2886
- function renderReviewWarnings(review) {
2887
- const issues = review.issues?.filter((issue) => issue.severity === "warn") ?? [];
2888
- if (issues.length === 0) return "";
2889
- return `Reviewer warnings:
2890
- ${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
3151
+ function closeServer(server) {
3152
+ return new Promise((resolvePromise, reject) => {
3153
+ server.close((error) => error ? reject(error) : resolvePromise());
3154
+ });
2891
3155
  }
2892
- function extractFapoAttributionSignals(findings) {
2893
- const signals = {
2894
- counts: { prompt: 0, parameter: 0, structural: 0 },
2895
- clusters: []
2896
- };
2897
- const seen = /* @__PURE__ */ new WeakSet();
2898
- for (const finding of findings) {
2899
- collectFinding(signals, finding, seen, 0);
3156
+ function assertConfig(config) {
3157
+ if (!config.objective || config.objective.trim() !== config.objective) {
3158
+ throw new Error("gepaOptimizationMethod: objective must be trimmed and non-empty");
3159
+ }
3160
+ assertRecipe(config.recipe);
3161
+ if (config.maxCandidateChars !== void 0 && (!Number.isSafeInteger(config.maxCandidateChars) || config.maxCandidateChars <= 0)) {
3162
+ throw new Error("gepaOptimizationMethod: maxCandidateChars must be a positive safe integer");
3163
+ }
3164
+ if (config.timeoutMs !== void 0 && (!Number.isSafeInteger(config.timeoutMs) || config.timeoutMs <= 0)) {
3165
+ throw new Error("gepaOptimizationMethod: timeoutMs must be a positive safe integer");
2900
3166
  }
2901
- return signals;
2902
3167
  }
2903
- function collectFinding(signals, finding, seen, depth) {
2904
- if (depth > MAX_FINDING_DEPTH) return;
2905
- if (!finding) return;
2906
- if (typeof finding === "string") {
2907
- const level = inferLevelFromText(finding);
2908
- if (level) addCluster(signals, { label: finding, level, count: 1 });
2909
- return;
3168
+ function assertRecipe(recipe) {
3169
+ if (!recipe || typeof recipe !== "object") {
3170
+ throw new Error("gepaOptimizationMethod: recipe is required");
2910
3171
  }
2911
- if (Array.isArray(finding)) {
2912
- if (seen.has(finding)) return;
2913
- seen.add(finding);
2914
- for (const item of finding) collectFinding(signals, item, seen, depth + 1);
3172
+ if (recipe.kind === "engine") {
3173
+ assertEngineRun(recipe.run, "recipe.run");
2915
3174
  return;
2916
3175
  }
2917
- if (typeof finding !== "object") return;
2918
- if (seen.has(finding)) return;
2919
- seen.add(finding);
2920
- const obj = finding;
2921
- collectLevelPartition(signals, obj.level_partition ?? obj.levelPartition, seen, depth + 1);
2922
- collectCounts(signals, obj);
2923
- collectClusters(signals, obj.clusters, true, seen, depth + 1);
2924
- const explicit = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel);
2925
- const text = textFields(obj);
2926
- const inferred = explicit ?? inferLevelFromText(text);
2927
- if (inferred) {
2928
- addCluster(signals, {
2929
- label: text || inferred,
2930
- level: inferred,
2931
- count: numberField(obj.count) ?? 1,
2932
- confidence: confidenceField(obj.confidence),
2933
- suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
2934
- caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
2935
- });
3176
+ if (recipe.kind === "best-of-then-continue") {
3177
+ if (!Array.isArray(recipe.explore) || recipe.explore.length < 2) {
3178
+ throw new Error(
3179
+ "gepaOptimizationMethod: recipe.explore must contain at least two bounded engine runs"
3180
+ );
3181
+ }
3182
+ for (const [index, run] of recipe.explore.entries()) {
3183
+ assertEngineRun(run, `recipe.explore[${index}]`);
3184
+ }
3185
+ assertEngineRun(recipe.continueWith, "recipe.continueWith");
3186
+ return;
2936
3187
  }
3188
+ throw new Error("gepaOptimizationMethod: unsupported recipe");
2937
3189
  }
2938
- function collectLevelPartition(signals, raw, seen, depth) {
2939
- if (!raw || typeof raw !== "object") return;
2940
- if (seen.has(raw)) return;
2941
- seen.add(raw);
2942
- const partition = raw;
2943
- for (const level of FAPO_LEVELS) {
2944
- const bucket = partition[level];
2945
- if (!bucket || typeof bucket !== "object") continue;
2946
- const obj = bucket;
2947
- const count = numberField(obj.count) ?? 0;
2948
- if (count > 0) signals.counts[level] += count;
2949
- collectClusters(signals, obj.clusters, false, seen, depth + 1);
3190
+ function assertEngineRun(run, label) {
3191
+ if (!run || typeof run !== "object") {
3192
+ throw new Error(`gepaOptimizationMethod: ${label} is required`);
2950
3193
  }
3194
+ if (typeof run.engine !== "string" || !run.engine.trim() || run.engine.trim() !== run.engine) {
3195
+ throw new Error(`gepaOptimizationMethod: ${label}.engine must be a trimmed non-empty string`);
3196
+ }
3197
+ if (!Number.isSafeInteger(run.maxEvaluations) || run.maxEvaluations <= 0) {
3198
+ throw new Error(
3199
+ `gepaOptimizationMethod: ${label}.maxEvaluations must be a positive safe integer`
3200
+ );
3201
+ }
3202
+ if (!Number.isFinite(run.maxProposerCostUsd) || run.maxProposerCostUsd <= 0) {
3203
+ throw new Error(
3204
+ `gepaOptimizationMethod: ${label}.maxProposerCostUsd must be a positive finite number`
3205
+ );
3206
+ }
3207
+ assertJsonValue(run.engineConfig ?? {}, `gepaOptimizationMethod: ${label}.engineConfig`);
2951
3208
  }
2952
- function collectCounts(signals, obj) {
2953
- const prompt = numberField(obj.prompt_addressable ?? obj.promptAddressable);
2954
- const structural = numberField(obj.structural_addressable ?? obj.structuralAddressable);
2955
- const tool = numberField(obj.tool_addressable ?? obj.toolAddressable);
2956
- if (prompt) signals.counts.prompt += prompt;
2957
- if (structural) signals.counts.structural += structural;
2958
- if (tool) signals.counts.structural += tool;
2959
- }
2960
- function collectClusters(signals, raw, countClusters, seen, depth) {
2961
- if (!Array.isArray(raw)) return;
2962
- if (seen.has(raw)) return;
2963
- seen.add(raw);
2964
- for (const item of raw) {
2965
- if (countClusters) {
2966
- collectFinding(signals, item, seen, depth + 1);
2967
- continue;
3209
+ function recipeEvaluationLimit(recipe) {
3210
+ const runs = recipe.kind === "engine" ? [recipe.run] : [...recipe.explore, recipe.continueWith];
3211
+ let total = 0;
3212
+ for (const run of runs) {
3213
+ total += run.maxEvaluations;
3214
+ if (!Number.isSafeInteger(total)) {
3215
+ throw new Error("gepaOptimizationMethod: recipe evaluation limit exceeds safe integer range");
2968
3216
  }
2969
- const cluster = parseCluster(item);
2970
- if (cluster) signals.clusters.push(cluster);
2971
3217
  }
3218
+ return total;
2972
3219
  }
2973
- function addCluster(signals, cluster) {
2974
- signals.counts[cluster.level] += Math.max(1, cluster.count);
2975
- signals.clusters.push(cluster);
3220
+ function defaultMethodName(recipe) {
3221
+ if (recipe.kind === "engine") return `gepa:${recipe.run.engine}`;
3222
+ return `gepa:best-of-then-continue:${recipe.continueWith.engine}`;
2976
3223
  }
2977
- function parseCluster(raw) {
2978
- if (!raw || typeof raw !== "object") return null;
2979
- const obj = raw;
2980
- const text = textFields(obj);
2981
- const level = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel) ?? inferLevelFromText(text);
2982
- if (!level) return null;
2983
- return {
2984
- label: text || level,
2985
- level,
2986
- count: numberField(obj.count) ?? 1,
2987
- confidence: confidenceField(obj.confidence),
2988
- suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
2989
- caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
2990
- };
3224
+ function assertBridgeOutput(result, name, maxCandidateChars, recipeKind, maxEvaluations) {
3225
+ if (result.recipeKind !== recipeKind)
3226
+ throw new Error(`${name}: GEPA bridge reported recipe '${String(result.recipeKind)}'`);
3227
+ if (!isCandidateText(result.bestCandidate, maxCandidateChars)) {
3228
+ throw new Error(`${name}: GEPA bridge returned an invalid candidate`);
3229
+ }
3230
+ if (!Number.isFinite(result.bestScore))
3231
+ throw new Error(`${name}: GEPA bridge returned an invalid bestScore`);
3232
+ if (!Number.isSafeInteger(result.totalEvaluations) || result.totalEvaluations < 0 || result.totalEvaluations > maxEvaluations) {
3233
+ throw new Error(`${name}: GEPA bridge returned an invalid totalEvaluations`);
3234
+ }
3235
+ if (result.proposerCostUsd !== void 0 && (!Number.isFinite(result.proposerCostUsd) || result.proposerCostUsd < 0)) {
3236
+ throw new Error(`${name}: GEPA bridge returned an invalid proposerCostUsd`);
3237
+ }
2991
3238
  }
2992
- function parseFindingLevel(raw) {
2993
- if (typeof raw !== "string") return null;
2994
- const lower = raw.toLowerCase();
2995
- if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
2996
- if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
2997
- if (lower === "config" || lower === "params") return "parameter";
2998
- return null;
3239
+ function isCandidateText(value, maxChars) {
3240
+ return typeof value === "string" && value.trim().length > 0 && value.length <= maxChars;
2999
3241
  }
3000
- function inferLevelFromText(text) {
3001
- const lower = text.toLowerCase();
3002
- if (/\b(retrieval_k|temperature|top_p|max_tokens|max_completion_tokens|reasoning_effort|config|parameter)\b/.test(
3003
- lower
3004
- )) {
3005
- return "parameter";
3006
- }
3007
- if (/\b(retriev\w*|search\w*|bm25|hop|evidence|cascade|tool|node|chain|state|structur\w*|topology|import)\b/.test(
3008
- lower
3009
- )) {
3010
- return "structural";
3242
+ function assertJsonValue(value, label, seen = /* @__PURE__ */ new Set()) {
3243
+ if (value === null || typeof value === "string" || typeof value === "boolean") return;
3244
+ if (typeof value === "number" && Number.isFinite(value)) return;
3245
+ if (Array.isArray(value)) {
3246
+ if (seen.has(value)) throw new Error(`${label} must be JSON-serializable`);
3247
+ seen.add(value);
3248
+ for (const item of value) assertJsonValue(item, label, seen);
3249
+ seen.delete(value);
3250
+ return;
3011
3251
  }
3012
- if (/\b(format\w*|verbose|brevity|abstain\w*|reasoning|instruction|prompt|output|answer)\b/.test(
3013
- lower
3014
- )) {
3015
- return "prompt";
3252
+ if (typeof value === "object") {
3253
+ if (seen.has(value)) throw new Error(`${label} must be JSON-serializable`);
3254
+ const prototype = Object.getPrototypeOf(value);
3255
+ if (prototype !== Object.prototype && prototype !== null) {
3256
+ throw new Error(`${label} must be JSON-serializable`);
3257
+ }
3258
+ seen.add(value);
3259
+ for (const item of Object.values(value)) assertJsonValue(item, label, seen);
3260
+ seen.delete(value);
3261
+ return;
3016
3262
  }
3017
- return null;
3263
+ throw new Error(`${label} must be JSON-serializable`);
3018
3264
  }
3019
- function textFields(obj) {
3020
- return [
3021
- obj.label,
3022
- obj.claim,
3023
- obj.recommended_action,
3024
- obj.suggested_fix,
3025
- obj.suggestedFix,
3026
- obj.message,
3027
- obj.text,
3028
- obj.heuristic,
3029
- obj.area
3030
- ].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
3031
- }
3032
- function numberField(value) {
3033
- return typeof value === "number" && Number.isFinite(value) ? value : null;
3265
+ function safePathComponent(value) {
3266
+ return value.replace(/[^a-zA-Z0-9._-]/g, "_");
3034
3267
  }
3035
- function stringField(value) {
3036
- return typeof value === "string" && value.trim() ? value.trim() : void 0;
3268
+ function isRecord(value) {
3269
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3037
3270
  }
3038
- function confidenceField(value) {
3039
- return value === "high" || value === "medium" || value === "low" ? value : void 0;
3271
+ function sendJson(response, status, body) {
3272
+ response.writeHead(status, { "content-type": "application/json; charset=utf-8" });
3273
+ response.end(JSON.stringify(body));
3040
3274
  }
3041
- function stringArrayField(value) {
3042
- return Array.isArray(value) ? value.filter((item) => typeof item === "string") : void 0;
3275
+ function truncate(value, max = 1e3) {
3276
+ const compact2 = value.trim().replace(/\s+/g, " ");
3277
+ return compact2.length <= max ? compact2 : `${compact2.slice(0, max)}\u2026`;
3043
3278
  }
3044
- function parameterSweepProposer(opts) {
3045
- if (opts.candidates.length === 0) {
3046
- throw new Error("parameterSweepProposer: candidates must not be empty");
3047
- }
3048
- return {
3049
- kind: "parameter-sweep",
3050
- async propose(ctx) {
3051
- if (typeof ctx.currentSurface !== "string") {
3052
- throw new Error(
3053
- "parameterSweepProposer: currentSurface must be a JSON string config surface"
3054
- );
3055
- }
3056
- const parse = opts.parse ?? parseJsonObject;
3057
- const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
3058
- const current = parse(ctx.currentSurface);
3059
- const currentCanonical = canonicalJson2(current);
3060
- const tried = triedLabels(ctx.history);
3061
- const out = [];
3062
- for (const candidate of opts.candidates) {
3063
- if (tried.has(candidate.label)) continue;
3064
- const next = applyParameterCandidate(current, candidate);
3065
- const surface = stringify(next);
3066
- if (surface === ctx.currentSurface || canonicalJson2(next) === currentCanonical) continue;
3067
- out.push({ surface, label: candidate.label, rationale: candidate.rationale });
3068
- if (out.length >= ctx.populationSize) break;
3279
+
3280
+ // src/campaign/grounded-reflection.ts
3281
+ function rolloutArgumentDiff(rollouts, opts = {}) {
3282
+ const passThreshold = opts.passThreshold ?? 1;
3283
+ const maxValues = opts.maxValuesPerField ?? 4;
3284
+ const collect = (pass) => {
3285
+ const byField = /* @__PURE__ */ new Map();
3286
+ for (const r of rollouts) {
3287
+ if (pass !== r.score >= passThreshold) continue;
3288
+ for (const c of r.calls) {
3289
+ for (const [k, v] of Object.entries(c.args)) {
3290
+ const set = byField.get(k) ?? /* @__PURE__ */ new Set();
3291
+ set.add(String(v));
3292
+ byField.set(k, set);
3293
+ }
3069
3294
  }
3070
- return out;
3071
3295
  }
3296
+ return byField;
3072
3297
  };
3073
- }
3074
- function parseJsonObject(surface) {
3075
- let parsed;
3076
- try {
3077
- parsed = JSON.parse(surface);
3078
- } catch (error) {
3079
- throw new Error(
3080
- `parameterSweepProposer: currentSurface must be valid JSON object config (${error instanceof Error ? error.message : String(error)})`
3081
- );
3082
- }
3083
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
3084
- throw new Error("parameterSweepProposer: JSON surface must parse to an object");
3085
- }
3086
- return parsed;
3087
- }
3088
- function triedLabels(history) {
3089
- const tried = /* @__PURE__ */ new Set();
3090
- for (const generation of history) {
3091
- for (const candidate of generation.candidates) {
3092
- if (!candidate.label) continue;
3093
- tried.add(candidate.label);
3094
- tried.add(candidateStrategy(candidate));
3095
- }
3298
+ const passing = collect(true);
3299
+ const failing = collect(false);
3300
+ const lines = [];
3301
+ for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
3302
+ const pv = [...passing.get(field) ?? []].slice(0, maxValues);
3303
+ const fv = [...failing.get(field) ?? []].slice(0, maxValues);
3304
+ const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
3305
+ lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
3096
3306
  }
3097
- return tried;
3307
+ const lower = (m) => {
3308
+ const out = /* @__PURE__ */ new Set();
3309
+ for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
3310
+ return out;
3311
+ };
3312
+ return {
3313
+ text: lines.join("\n") || " (no calls observed)",
3314
+ passingValues: lower(passing),
3315
+ failingValues: lower(failing)
3316
+ };
3098
3317
  }
3099
- function applyParameterCandidate(current, candidate) {
3100
- const next = cloneJsonObject(current);
3101
- if (candidate.patch) deepMerge(next, candidate.patch);
3102
- for (const change of candidate.changes ?? []) {
3103
- setPath(
3104
- next,
3105
- typeof change.path === "string" ? change.path.split(".") : [...change.path],
3106
- change.value
3107
- );
3318
+ function classifyUngroundedLiterals(text, diff) {
3319
+ const ungrounded = /* @__PURE__ */ new Set();
3320
+ for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
3321
+ const w = m[1].toLowerCase();
3322
+ if (!diff.passingValues.has(w)) ungrounded.add(w);
3108
3323
  }
3109
- return next;
3110
- }
3111
- function cloneJsonObject(value) {
3112
- return JSON.parse(JSON.stringify(value));
3324
+ const all = [...ungrounded];
3325
+ return {
3326
+ ungrounded: all,
3327
+ harmful: all.filter((w) => diff.failingValues.has(w))
3328
+ };
3113
3329
  }
3114
- function deepMerge(target, patch) {
3115
- for (const [key, value] of Object.entries(patch)) {
3116
- assertSafeJsonKey(key);
3117
- if (isPlainObject(value) && isPlainObject(target[key])) {
3118
- deepMerge(target[key], value);
3119
- } else {
3120
- target[key] = value;
3121
- }
3330
+
3331
+ // src/campaign/labeled-store/fs-adapter.ts
3332
+ import { createHash as createHash4 } from "crypto";
3333
+ import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
3334
+ import { join as join3 } from "path";
3335
+ var LabeledScenarioStoreError = class extends Error {
3336
+ constructor(code, message) {
3337
+ super(message);
3338
+ this.code = code;
3339
+ this.name = "LabeledScenarioStoreError";
3122
3340
  }
3123
- }
3124
- function setPath(target, path, value) {
3125
- if (path.length === 0) throw new Error("parameterSweepProposer: change path must not be empty");
3126
- for (const part of path) assertSafeJsonKey(part);
3127
- let cursor = target;
3128
- for (const part of path.slice(0, -1)) {
3129
- const existing = cursor[part];
3130
- if (!isPlainObject(existing)) {
3131
- cursor[part] = {};
3132
- }
3133
- cursor = cursor[part];
3341
+ code;
3342
+ };
3343
+ var FsLabeledScenarioStore = class {
3344
+ constructor(options) {
3345
+ this.options = options;
3346
+ if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
3347
+ this.now = options.now ?? Date.now;
3134
3348
  }
3135
- cursor[path[path.length - 1]] = value;
3136
- }
3137
- function assertSafeJsonKey(key) {
3138
- if (!key.trim()) throw new Error("parameterSweepProposer: change path contains an empty key");
3139
- if (UNSAFE_JSON_KEYS.has(key)) {
3140
- throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
3349
+ options;
3350
+ now;
3351
+ rateLimits = /* @__PURE__ */ new Map();
3352
+ async observe(write) {
3353
+ this.assertProvenance(write);
3354
+ this.assertRateLimit(write);
3355
+ const record = this.toRecord(write);
3356
+ const path = this.pathForSource(write.source);
3357
+ const line = `${JSON.stringify(record)}
3358
+ `;
3359
+ appendLine(path, line);
3141
3360
  }
3142
- }
3143
- function canonicalJson2(value) {
3144
- if (Array.isArray(value)) return `[${value.map((item) => canonicalJson2(item)).join(",")}]`;
3145
- if (isPlainObject(value)) {
3146
- return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key])}`).join(",")}}`;
3361
+ async sample(args) {
3362
+ if (!args.split) {
3363
+ throw new LabeledScenarioStoreError(
3364
+ "split_required",
3365
+ "sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
3366
+ );
3367
+ }
3368
+ if (!args.capturedBefore) {
3369
+ throw new LabeledScenarioStoreError(
3370
+ "capturedBefore_required",
3371
+ "sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
3372
+ );
3373
+ }
3374
+ const all = [];
3375
+ for (const source of ALL_SOURCES) {
3376
+ if (args.split === "train" && source === "production-trace") {
3377
+ const explicit = sourceFilterContains(args.filter?.source, "production-trace");
3378
+ if (!explicit) continue;
3379
+ }
3380
+ const path = this.pathForSource(source);
3381
+ if (!existsSync2(path)) continue;
3382
+ const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
3383
+ for (const line of lines) {
3384
+ let record;
3385
+ try {
3386
+ record = JSON.parse(line);
3387
+ } catch {
3388
+ continue;
3389
+ }
3390
+ if (!matchesFilter(record, args, source)) continue;
3391
+ all.push(record);
3392
+ }
3393
+ }
3394
+ all.sort((a, b) => {
3395
+ if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
3396
+ return a.recordHash.localeCompare(b.recordHash);
3397
+ });
3398
+ return all.slice(0, args.count);
3147
3399
  }
3148
- return JSON.stringify(value);
3149
- }
3150
- function isPlainObject(value) {
3151
- if (!value || typeof value !== "object" || Array.isArray(value)) return false;
3152
- const proto = Object.getPrototypeOf(value);
3153
- return proto === Object.prototype || proto === null;
3154
- }
3155
-
3156
- // src/campaign/skill-patch.ts
3157
- function applySkillPatch(surface, patch) {
3158
- let lines = surface.split("\n");
3159
- let applied = 0;
3160
- const rejected = [];
3161
- const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
3162
- for (const op of patch.ops) {
3163
- if (op.op === "add") {
3164
- if (typeof op.text !== "string" || op.text.trim() === "") {
3165
- rejected.push({ op, reason: "empty add text" });
3400
+ async size() {
3401
+ const bySource = {};
3402
+ const byTrust = {
3403
+ unverified: 0,
3404
+ "verified-signal": 0,
3405
+ "human-rated": 0
3406
+ };
3407
+ let total = 0;
3408
+ for (const source of ALL_SOURCES) {
3409
+ const path = this.pathForSource(source);
3410
+ if (!existsSync2(path)) {
3411
+ bySource[source] = 0;
3166
3412
  continue;
3167
3413
  }
3168
- const insert = op.text.split("\n");
3169
- if (op.after === void 0 || op.after === "") {
3170
- lines = [...lines, ...insert];
3171
- applied++;
3172
- continue;
3173
- }
3174
- const idx = findLine(op.after);
3175
- if (idx === -1) {
3176
- rejected.push({ op, reason: `add anchor not found: ${truncate(op.after)}` });
3177
- continue;
3178
- }
3179
- lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
3180
- applied++;
3181
- } else if (op.op === "delete") {
3182
- const idx = findLine(op.anchor);
3183
- if (idx === -1) {
3184
- rejected.push({ op, reason: `delete anchor not found: ${truncate(op.anchor)}` });
3185
- continue;
3186
- }
3187
- lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
3188
- applied++;
3189
- } else {
3190
- const idx = findLine(op.anchor);
3191
- if (idx === -1) {
3192
- rejected.push({ op, reason: `replace anchor not found: ${truncate(op.anchor)}` });
3193
- continue;
3194
- }
3195
- if (typeof op.text !== "string") {
3196
- rejected.push({ op, reason: "replace text missing" });
3197
- continue;
3414
+ const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
3415
+ bySource[source] = lines.length;
3416
+ total += lines.length;
3417
+ for (const line of lines) {
3418
+ let trust = "unverified";
3419
+ try {
3420
+ trust = JSON.parse(line).labelTrust ?? "unverified";
3421
+ } catch {
3422
+ }
3423
+ byTrust[trust] += 1;
3198
3424
  }
3199
- lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
3200
- applied++;
3201
3425
  }
3426
+ return { train: total, test: total, bySource, byTrust };
3202
3427
  }
3203
- return { surface: lines.join("\n"), applied, rejected };
3428
+ assertProvenance(write) {
3429
+ if (!write.source) {
3430
+ throw new LabeledScenarioStoreError(
3431
+ "missing_source",
3432
+ "LabeledScenarioWrite requires `source`"
3433
+ );
3434
+ }
3435
+ if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
3436
+ throw new LabeledScenarioStoreError(
3437
+ "missing_source_version",
3438
+ "LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
3439
+ );
3440
+ }
3441
+ if (!write.capturedAt) {
3442
+ throw new LabeledScenarioStoreError(
3443
+ "missing_captured_at",
3444
+ "LabeledScenarioWrite requires `capturedAt` ISO timestamp"
3445
+ );
3446
+ }
3447
+ if (!write.redactionStatus) {
3448
+ throw new LabeledScenarioStoreError(
3449
+ "missing_redaction_status",
3450
+ "LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
3451
+ );
3452
+ }
3453
+ if (!ALL_SOURCES.includes(write.source)) {
3454
+ throw new LabeledScenarioStoreError(
3455
+ "unknown_source",
3456
+ `LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
3457
+ );
3458
+ }
3459
+ }
3460
+ assertRateLimit(write) {
3461
+ const cap = this.options.maxWritesPerMinutePerBucket;
3462
+ if (!cap || !write.rateLimitBucket) return;
3463
+ const now = this.now();
3464
+ const windowMs = 6e4;
3465
+ let state = this.rateLimits.get(write.rateLimitBucket);
3466
+ if (!state || now - state.windowStartMs >= windowMs) {
3467
+ state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
3468
+ this.rateLimits.set(write.rateLimitBucket, state);
3469
+ }
3470
+ if (state.count >= cap) {
3471
+ throw new LabeledScenarioStoreError(
3472
+ "rate_limit_exceeded",
3473
+ `LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
3474
+ );
3475
+ }
3476
+ state.count += 1;
3477
+ }
3478
+ toRecord(write) {
3479
+ const recordHash = sha256(
3480
+ JSON.stringify({
3481
+ id: write.scenario.id,
3482
+ src: write.source,
3483
+ at: write.capturedAt,
3484
+ ver: write.sourceVersionHash
3485
+ })
3486
+ );
3487
+ return {
3488
+ ...write,
3489
+ recordHash,
3490
+ split: "train"
3491
+ };
3492
+ }
3493
+ pathForSource(source) {
3494
+ return join3(this.options.root, `${source}.jsonl`);
3495
+ }
3496
+ };
3497
+ var ALL_SOURCES = [
3498
+ "production-trace",
3499
+ "eval-run",
3500
+ "manual",
3501
+ "red-team",
3502
+ "synthetic"
3503
+ ];
3504
+ function sourceFilterContains(filter, needle) {
3505
+ if (!filter) return false;
3506
+ if (Array.isArray(filter)) return filter.includes(needle);
3507
+ return filter === needle;
3204
3508
  }
3205
- function patchEditCount(patch) {
3206
- return patch.ops.length;
3509
+ function matchesFilter(record, args, source) {
3510
+ if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
3511
+ if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
3512
+ const f = args.filter;
3513
+ if (!f) return true;
3514
+ if (f.kind && record.scenario.kind !== f.kind) return false;
3515
+ if (f.source) {
3516
+ const sources = Array.isArray(f.source) ? f.source : [f.source];
3517
+ if (!sources.includes(source)) return false;
3518
+ }
3519
+ if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
3520
+ const composites = Object.values(record.judgeScores).map((s) => s.composite);
3521
+ const max = composites.length === 0 ? 0 : Math.max(...composites);
3522
+ if (f.minComposite !== void 0 && max < f.minComposite) return false;
3523
+ if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
3524
+ }
3525
+ if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
3526
+ return false;
3527
+ }
3528
+ return true;
3207
3529
  }
3208
- function truncate(s, max = 48) {
3209
- return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
3530
+ function sha256(input) {
3531
+ return createHash4("sha256").update(input).digest("hex").slice(0, 16);
3532
+ }
3533
+ function appendLine(path, line) {
3534
+ if (existsSync2(path)) {
3535
+ const existing = readFileSync2(path, "utf8");
3536
+ writeFileSync(path, existing + line);
3537
+ } else {
3538
+ writeFileSync(path, line);
3539
+ }
3210
3540
  }
3211
3541
 
3212
- // src/campaign/proposers/skill-opt.ts
3213
- var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
3214
- function skillOptProposer(opts) {
3215
- const evidenceK = opts.evidenceK ?? 3;
3216
- const defaultBudget = opts.editBudget ?? 3;
3217
- const directCostLedger = opts.costLedger ?? new CostLedger();
3218
- async function proposePatches(args) {
3219
- const userPrompt = buildPatchPrompt({
3220
- target: opts.target,
3221
- surface: args.surface,
3222
- evidence: args.evidence,
3223
- editBudget: args.editBudget,
3224
- rejectedBuffer: args.rejectedBuffer,
3225
- metaNote: args.metaNote,
3226
- findingsNote: args.findingsNote,
3227
- count: args.count
3228
- });
3229
- const request = {
3230
- model: opts.model,
3231
- messages: [
3232
- { role: "system", content: SKILLOPT_SYSTEM },
3233
- { role: "user", content: userPrompt }
3234
- ],
3235
- jsonMode: true,
3236
- temperature: opts.temperature ?? 0.6,
3237
- maxTokens: opts.maxTokens ?? 4e3
3238
- };
3239
- const paid = await (args.costLedger ?? directCostLedger).runPaidCall({
3240
- channel: "driver",
3241
- phase: args.costPhase ?? "search.proposal",
3242
- actor: "skill-opt.propose",
3243
- model: opts.model,
3244
- maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
3245
- signal: args.signal,
3246
- execute: (signal, callId) => callLlm(request, { ...opts.llm, signal, idempotencyKey: callId }),
3247
- receipt: costReceiptFromLlm,
3248
- receiptFromError: costReceiptFromLlmError
3249
- });
3250
- if (!paid.succeeded) throw paid.error;
3251
- const result = paid.value;
3252
- return parseSkillPatchResponse(result.content, args.count, args.editBudget);
3542
+ // src/campaign/neutralize.ts
3543
+ var FILLER = "#";
3544
+ function neutralizeText(content) {
3545
+ return content.replace(/\S/g, FILLER);
3546
+ }
3547
+
3548
+ // src/campaign/proposers/fapo.ts
3549
+ var FAPO_LEVELS = ["prompt", "parameter", "structural"];
3550
+ var MAX_FINDING_DEPTH = 16;
3551
+ var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
3552
+ function fapoProposer(opts) {
3553
+ const proposers = levelProposers(opts);
3554
+ const allowed = allowedLevels(opts, proposers);
3555
+ const plateauWindow = opts.plateauWindow ?? 3;
3556
+ const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
3557
+ const minImprovement = opts.minImprovement ?? 0;
3558
+ const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
3559
+ if (allowed.length === 0) {
3560
+ throw new Error("fapoProposer: at least one allowed level must have a proposer");
3253
3561
  }
3562
+ if (plateauWindow < 1) throw new Error("fapoProposer: plateauWindow must be >= 1");
3563
+ if (minDistinctStrategies < 1) {
3564
+ throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
3565
+ }
3566
+ if (proposalsPerCycle < 1) throw new Error("fapoProposer: proposalsPerCycle must be >= 1");
3254
3567
  return {
3255
- kind: "skill-opt",
3256
- proposePatches,
3568
+ kind: "fapo",
3257
3569
  async propose(ctx) {
3258
- if (typeof ctx.currentSurface !== "string") {
3570
+ const state = policyState(ctx.history, ctx.findings, {
3571
+ plateauWindow,
3572
+ minDistinctStrategies,
3573
+ minImprovement
3574
+ });
3575
+ const decision = chooseLevel({
3576
+ allowed,
3577
+ available: allowed.filter((level) => !state.exhausted[level]),
3578
+ proposers,
3579
+ state,
3580
+ promptFirst: opts.promptFirst ?? true,
3581
+ parameterBeforeStructural: opts.parameterBeforeStructural ?? true
3582
+ });
3583
+ if (!decision) return [];
3584
+ const proposer = proposers[decision.level];
3585
+ if (!proposer) {
3586
+ throw new Error(`fapoProposer: selected ${decision.level} but no proposer is configured`);
3587
+ }
3588
+ const requested = Math.min(ctx.populationSize, proposalsPerCycle);
3589
+ const raw = await proposer.propose({ ...ctx, populationSize: requested });
3590
+ const wrapped = raw.map((candidate) => normalizeProposal(candidate)).map((candidate) => wrapProposal(candidate, decision.level, decision.reason));
3591
+ const reviewed = [];
3592
+ for (const candidate of wrapped) {
3593
+ const review = opts.reviewCandidate ? await opts.reviewCandidate({
3594
+ level: decision.level,
3595
+ candidate,
3596
+ context: ctx,
3597
+ reason: decision.reason
3598
+ }) : { verdict: "pass" };
3599
+ if (review.verdict === "fail") continue;
3600
+ const warn = review.verdict === "warn" ? renderReviewWarnings(review) : "";
3601
+ reviewed.push(
3602
+ warn ? { ...candidate, rationale: `${candidate.rationale}
3603
+ ${warn}` } : candidate
3604
+ );
3605
+ }
3606
+ if (wrapped.length > 0 && reviewed.length === 0) {
3259
3607
  throw new Error(
3260
- "skillOptProposer: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
3608
+ `fapoProposer: reviewer blocked every ${decision.level} candidate; refusing to evaluate an unreviewed variant`
3261
3609
  );
3262
3610
  }
3263
- const surface = ctx.currentSurface;
3264
- const patches = await proposePatches({
3265
- surface,
3266
- evidence: evidenceFromHistory(ctx, evidenceK),
3267
- editBudget: defaultBudget,
3268
- rejectedBuffer: [],
3269
- findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
3270
- count: ctx.populationSize,
3271
- signal: ctx.signal,
3272
- costLedger: ctx.costLedger ?? directCostLedger,
3273
- costPhase: ctx.costPhase ?? "search.proposal"
3274
- });
3275
- const out = [];
3276
- const seen = /* @__PURE__ */ new Set();
3277
- for (const patch of patches) {
3278
- const { surface: candidate, applied } = applySkillPatch(surface, patch);
3279
- if (applied === 0 || candidate === surface || seen.has(candidate)) continue;
3280
- seen.add(candidate);
3281
- out.push({ surface: candidate, label: patch.label, rationale: patch.rationale });
3282
- if (out.length >= ctx.populationSize) break;
3611
+ return reviewed;
3612
+ },
3613
+ decide({ history }) {
3614
+ const last = history[history.length - 1];
3615
+ if (last && last.candidates.length === 0) {
3616
+ return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
3283
3617
  }
3284
- return out;
3618
+ const state = policyState(history, [], {
3619
+ plateauWindow,
3620
+ minDistinctStrategies,
3621
+ minImprovement
3622
+ });
3623
+ const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
3624
+ return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
3285
3625
  }
3286
3626
  };
3287
3627
  }
3288
- function evidenceFromHistory(ctx, k) {
3289
- const last = ctx.history.at(-1);
3290
- if (!last || last.candidates.length === 0) return { weakScenarios: [], weakDimensions: [] };
3291
- const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
3292
- if (!best) return { weakScenarios: [], weakDimensions: [] };
3293
- const weakScenarios = [...best.scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
3294
- const weakDimensions = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
3295
- return { weakScenarios, weakDimensions };
3628
+ function levelProposers(opts) {
3629
+ return {
3630
+ ...opts.proposers ?? {},
3631
+ ...opts.promptProposer ? { prompt: opts.promptProposer } : {},
3632
+ ...opts.parameterProposer ? { parameter: opts.parameterProposer } : {},
3633
+ ...opts.structuralProposer ? { structural: opts.structuralProposer } : {}
3634
+ };
3296
3635
  }
3297
- function buildPatchPrompt(args) {
3298
- const lines = [
3299
- `Skill document governs: ${args.target}.`,
3300
- "",
3301
- "Current skill document:",
3302
- "```",
3303
- args.surface,
3304
- "```",
3305
- "",
3306
- `Propose ${args.count} candidate patch(es). Each patch is a SMALL bundle of`,
3307
- `at most ${args.editBudget} op(s). Anchors must be verbatim substrings of`,
3308
- "existing lines. Prefer adding a specific missing rule or sharpening a vague",
3309
- "one over deleting; never rewrite the whole document."
3310
- ];
3311
- if (args.evidence.weakScenarios.length > 0) {
3312
- lines.push(
3313
- "",
3314
- "Weakest scenarios (patch to fix these):",
3315
- ...args.evidence.weakScenarios.map((s) => `- ${s.scenarioId} (${s.composite.toFixed(2)})`)
3316
- );
3317
- }
3318
- if (args.evidence.weakDimensions.length > 0) {
3319
- lines.push(
3320
- "",
3321
- "Weakest dimensions (what to improve):",
3322
- ...args.evidence.weakDimensions.map((d) => `- ${d.dimension} (${d.score.toFixed(2)})`)
3323
- );
3324
- }
3325
- if (args.rejectedBuffer.length > 0) {
3326
- lines.push(
3327
- "",
3328
- "Already tried and REJECTED (do not repeat or restate these edits):",
3329
- ...args.rejectedBuffer.map((e) => `- ${e.label}: ${e.rationale} \u2014 ${e.reason}`)
3330
- );
3331
- }
3332
- if (args.findingsNote) {
3333
- lines.push("", args.findingsNote);
3334
- }
3335
- if (args.metaNote) {
3336
- lines.push("", `Strategy note from prior epochs: ${args.metaNote}`);
3337
- }
3338
- return lines.join("\n");
3636
+ function allowedLevels(opts, proposers) {
3637
+ const explicit = opts.scope?.allowedLevels;
3638
+ const forbidden = new Set(opts.scope?.forbiddenLevels ?? []);
3639
+ return FAPO_LEVELS.filter((level) => {
3640
+ if (!proposers[level]) return false;
3641
+ if (forbidden.has(level)) return false;
3642
+ return explicit ? explicit.includes(level) : true;
3643
+ });
3339
3644
  }
3340
- var SkillPatchParseError = class extends Error {
3341
- constructor(message) {
3342
- super(message);
3343
- this.name = "SkillPatchParseError";
3344
- }
3345
- };
3346
- function parseSkillPatchResponse(raw, maxPatches, editBudget) {
3347
- let text = raw.trim();
3348
- if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
3349
- const start = text.indexOf("{");
3350
- const end = text.lastIndexOf("}");
3351
- if (start < 0 || end <= start) {
3352
- throw new SkillPatchParseError(
3353
- `parseSkillPatchResponse: response was not valid JSON (no object found): ${snippet(raw)}`
3354
- );
3355
- }
3356
- let parsed;
3357
- try {
3358
- parsed = JSON.parse(text.slice(start, end + 1));
3359
- } catch (err) {
3360
- throw new SkillPatchParseError(
3361
- `parseSkillPatchResponse: response was not valid JSON (${err instanceof Error ? err.message : String(err)}): ${snippet(raw)}`
3362
- );
3645
+ function policyState(history, findings, opts) {
3646
+ const attempts = extractAttempts(history);
3647
+ return {
3648
+ attempts,
3649
+ exhausted: {
3650
+ prompt: isLevelExhausted("prompt", attempts, opts),
3651
+ parameter: isLevelExhausted("parameter", attempts, opts),
3652
+ structural: isLevelExhausted("structural", attempts, opts)
3653
+ },
3654
+ signals: extractFapoAttributionSignals(findings)
3655
+ };
3656
+ }
3657
+ function extractAttempts(history) {
3658
+ const attempts = [];
3659
+ for (const generation of history) {
3660
+ for (const candidate of generation.candidates) {
3661
+ const level = candidateLevel(candidate);
3662
+ if (!level) continue;
3663
+ attempts.push({
3664
+ level,
3665
+ composite: candidate.composite,
3666
+ strategy: candidateStrategy(candidate)
3667
+ });
3668
+ }
3363
3669
  }
3364
- const rawPatches = Array.isArray(parsed.patches) ? parsed.patches : [];
3365
- const out = [];
3366
- for (const rp of rawPatches) {
3367
- if (typeof rp !== "object" || rp === null) continue;
3368
- const obj = rp;
3369
- const ops = Array.isArray(obj.ops) ? obj.ops.map(normalizeOp).filter(isOp) : [];
3370
- if (ops.length === 0) continue;
3371
- out.push({
3372
- label: typeof obj.label === "string" ? obj.label : "patch",
3373
- rationale: typeof obj.rationale === "string" ? obj.rationale : "",
3374
- ops: ops.slice(0, editBudget)
3375
- });
3376
- if (out.length >= maxPatches) break;
3670
+ return attempts;
3671
+ }
3672
+ function candidateLevel(candidate) {
3673
+ const label = candidate.label ?? "";
3674
+ const rationale = candidate.rationale ?? "";
3675
+ return parseLevel(label) ?? parseLevel(rationale);
3676
+ }
3677
+ function parseLevel(text) {
3678
+ const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
3679
+ return match ? match[1] : null;
3680
+ }
3681
+ function candidateStrategy(candidate) {
3682
+ const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
3683
+ return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
3684
+ }
3685
+ function isLevelExhausted(level, attempts, opts) {
3686
+ const own = attempts.filter((attempt) => attempt.level === level);
3687
+ if (own.length < opts.plateauWindow) return false;
3688
+ const distinctStrategies = new Set(own.map((attempt) => attempt.strategy)).size;
3689
+ if (distinctStrategies < opts.minDistinctStrategies) return false;
3690
+ let best = Number.NEGATIVE_INFINITY;
3691
+ let nonImproving = 0;
3692
+ for (const attempt of own) {
3693
+ if (attempt.composite > best + opts.minImprovement) {
3694
+ best = attempt.composite;
3695
+ nonImproving = 0;
3696
+ } else {
3697
+ nonImproving += 1;
3698
+ }
3377
3699
  }
3378
- return out;
3700
+ return nonImproving >= opts.plateauWindow;
3379
3701
  }
3380
- function normalizeOp(raw) {
3381
- if (typeof raw !== "object" || raw === null) return null;
3382
- const o = raw;
3383
- if (o.op === "add") {
3384
- if (typeof o.text !== "string") return null;
3385
- const op = { op: "add", text: o.text };
3386
- if (typeof o.after === "string") op.after = o.after;
3387
- return op;
3702
+ function chooseLevel(args) {
3703
+ const { available, state } = args;
3704
+ if (available.length === 0) return null;
3705
+ const triedPrompt = state.attempts.some((attempt) => attempt.level === "prompt");
3706
+ if (args.promptFirst && available.includes("prompt") && !triedPrompt) {
3707
+ return {
3708
+ level: "prompt",
3709
+ reason: "prompt-first policy: no prompt-level variant has been tried yet"
3710
+ };
3388
3711
  }
3389
- if (o.op === "delete") {
3390
- if (typeof o.anchor !== "string") return null;
3391
- return { op: "delete", anchor: o.anchor };
3712
+ const supported = available.filter((level) => state.signals.counts[level] > 0);
3713
+ if (supported.includes("prompt")) {
3714
+ return {
3715
+ level: "prompt",
3716
+ reason: `attribution has ${state.signals.counts.prompt} prompt-addressable failure(s)`
3717
+ };
3392
3718
  }
3393
- if (o.op === "replace") {
3394
- if (typeof o.anchor !== "string" || typeof o.text !== "string") return null;
3395
- return { op: "replace", anchor: o.anchor, text: o.text };
3719
+ const strongest = supported.sort(
3720
+ (a, b) => state.signals.counts[b] - state.signals.counts[a] || levelRank(a) - levelRank(b)
3721
+ )[0];
3722
+ if (strongest) {
3723
+ if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
3724
+ return {
3725
+ level: "parameter",
3726
+ reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
3727
+ };
3728
+ }
3729
+ return {
3730
+ level: strongest,
3731
+ reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
3732
+ };
3733
+ }
3734
+ if (state.attempts.length === 0) {
3735
+ const first = available.slice().sort((a, b) => levelRank(a) - levelRank(b))[0];
3736
+ return { level: first, reason: "no attribution yet; starting at the cheapest allowed level" };
3396
3737
  }
3397
3738
  return null;
3398
3739
  }
3399
- function isOp(op) {
3400
- return op !== null;
3740
+ function levelRank(level) {
3741
+ return level === "prompt" ? 0 : level === "parameter" ? 1 : 2;
3401
3742
  }
3402
- function snippet(s, max = 120) {
3403
- const t = s.trim().replace(/\s+/g, " ");
3404
- return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
3743
+ function normalizeProposal(value) {
3744
+ return isProposedCandidate(value) ? value : { surface: value, label: "candidate", rationale: "bare candidate from level proposer" };
3405
3745
  }
3406
-
3407
- // src/campaign/presets/compare-optimization-methods.ts
3408
- import { randomUUID } from "crypto";
3409
- async function compareOptimizationMethods(opts) {
3410
- assertOptimizationMethods(opts.methods);
3411
- assertComparisonPartitions(opts);
3412
- const seed = opts.seed ?? 42;
3413
- const confidence = opts.confidence ?? 0.95;
3414
- assertConfidence(confidence);
3415
- const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
3416
- const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
3417
- const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
3418
- const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
3419
- const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
3420
- assertComparisonControls(opts, seed, resamples, confidence);
3421
- const storage = opts.storage ?? fsCampaignStorage();
3422
- const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
3423
- const testCostPhase = `compareOptimizationMethods:test:${randomUUID()}`;
3424
- const testCostLedger = opts.costLedger ?? createRunCostLedger({
3425
- storage,
3426
- runDir: `${resolvedRunDir}/test/cost`,
3427
- costCeilingUsd: opts.costCeiling
3428
- });
3429
- const scoreOnTest = async (surface, tag) => {
3430
- const campaign = await runCampaign({
3431
- ...opts,
3432
- storage,
3433
- costLedger: testCostLedger,
3434
- costPhase: testCostPhase,
3435
- scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
3436
- dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
3437
- runDir: `${resolvedRunDir}/${tag}`
3438
- });
3439
- const byScenario = {};
3440
- for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
3441
- byScenario[scenarioId] = composite;
3442
- }
3443
- return byScenario;
3746
+ function wrapProposal(candidate, level, reason) {
3747
+ const label = candidate.label ? `fapo:${level}:${candidate.label}` : `fapo:${level}`;
3748
+ return {
3749
+ ...candidate,
3750
+ label,
3751
+ rationale: [
3752
+ `fapo:${level} selected by reviewed escalation policy`,
3753
+ `Reason: ${reason}`,
3754
+ candidate.rationale ? `Level proposer rationale: ${candidate.rationale}` : ""
3755
+ ].filter(Boolean).join("\n")
3444
3756
  };
3445
- const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
3446
- const align = (byScenario, label) => {
3447
- const missing = scenarioIds.filter((id) => !(id in byScenario));
3448
- if (missing.length > 0) {
3449
- throw new Error(
3450
- `compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(
3451
- ", "
3452
- )}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`
3453
- );
3454
- }
3455
- return scenarioIds.map((id) => byScenario[id]);
3757
+ }
3758
+ function renderReviewWarnings(review) {
3759
+ const issues = review.issues?.filter((issue) => issue.severity === "warn") ?? [];
3760
+ if (issues.length === 0) return "";
3761
+ return `Reviewer warnings:
3762
+ ${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
3763
+ }
3764
+ function extractFapoAttributionSignals(findings) {
3765
+ const signals = {
3766
+ counts: { prompt: 0, parameter: 0, structural: 0 },
3767
+ clusters: []
3456
3768
  };
3457
- const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
3458
- const out = await method.optimize(
3459
- createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed)
3460
- );
3461
- assertOptimizationResult(method.name, out);
3462
- const winnerSurface = structuredClone(out.winnerSurface);
3463
- return {
3464
- name: method.name,
3465
- winnerSurface,
3466
- cost: out.cost,
3467
- durationMs: out.durationMs
3468
- };
3469
- });
3470
- const baselineArr = align(await scoreOnTest(opts.baselineSurface, "test/baseline"), "baseline");
3471
- const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(opts.baselineSurface), baselineArr]]);
3472
- const winners = [];
3473
- for (const winner of optimized) {
3474
- const surfaceKey = surfaceContentHash(winner.winnerSurface);
3475
- let arr = testScoresBySurface.get(surfaceKey);
3476
- if (!arr) {
3477
- const byScenario = await scoreOnTest(
3478
- winner.winnerSurface,
3479
- `test/methods/${slug(winner.name)}`
3480
- );
3481
- arr = align(byScenario, `method "${winner.name}"`);
3482
- testScoresBySurface.set(surfaceKey, arr);
3483
- }
3484
- winners.push({
3485
- ...winner,
3486
- arr
3487
- });
3769
+ const seen = /* @__PURE__ */ new WeakSet();
3770
+ for (const finding of findings) {
3771
+ collectFinding(signals, finding, seen, 0);
3488
3772
  }
3489
- const scores = winners.map((w) => {
3490
- const boot = pairedBootstrap(baselineArr, w.arr, {
3491
- seed,
3492
- resamples,
3493
- confidence: intervalConfidence,
3494
- statistic: "mean"
3773
+ return signals;
3774
+ }
3775
+ function collectFinding(signals, finding, seen, depth) {
3776
+ if (depth > MAX_FINDING_DEPTH) return;
3777
+ if (!finding) return;
3778
+ if (typeof finding === "string") {
3779
+ const level = inferLevelFromText(finding);
3780
+ if (level) addCluster(signals, { label: finding, level, count: 1 });
3781
+ return;
3782
+ }
3783
+ if (Array.isArray(finding)) {
3784
+ if (seen.has(finding)) return;
3785
+ seen.add(finding);
3786
+ for (const item of finding) collectFinding(signals, item, seen, depth + 1);
3787
+ return;
3788
+ }
3789
+ if (typeof finding !== "object") return;
3790
+ if (seen.has(finding)) return;
3791
+ seen.add(finding);
3792
+ const obj = finding;
3793
+ collectLevelPartition(signals, obj.level_partition ?? obj.levelPartition, seen, depth + 1);
3794
+ collectCounts(signals, obj);
3795
+ collectClusters(signals, obj.clusters, true, seen, depth + 1);
3796
+ const explicit = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel);
3797
+ const text = textFields(obj);
3798
+ const inferred = explicit ?? inferLevelFromText(text);
3799
+ if (inferred) {
3800
+ addCluster(signals, {
3801
+ label: text || inferred,
3802
+ level: inferred,
3803
+ count: numberField(obj.count) ?? 1,
3804
+ confidence: confidenceField(obj.confidence),
3805
+ suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
3806
+ caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
3495
3807
  });
3496
- const score = {
3497
- name: w.name,
3498
- baselineComposite: mean2(baselineArr),
3499
- winnerComposite: mean2(w.arr),
3500
- lift: boot.mean,
3501
- liftCi: { low: boot.low, high: boot.high },
3502
- optimizationCost: w.cost,
3503
- scenarioScores: scenarioIds.map((scenarioId, index) => ({
3504
- scenarioId,
3505
- baselineComposite: baselineArr[index],
3506
- winnerComposite: w.arr[index],
3507
- lift: w.arr[index] - baselineArr[index]
3508
- })),
3509
- winnerSurface: w.winnerSurface,
3510
- rank: 0
3511
- };
3512
- if (w.durationMs !== void 0) score.durationMs = w.durationMs;
3513
- return score;
3514
- });
3515
- scores.sort((a, b) => b.lift - a.lift);
3516
- for (let start = 0; start < scores.length; ) {
3517
- let end = start + 1;
3518
- while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
3519
- const tied = scores.slice(start, end);
3520
- if (tied.every((score) => score.optimizationCost.accountingComplete)) {
3521
- tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
3522
- scores.splice(start, tied.length, ...tied);
3808
+ }
3809
+ }
3810
+ function collectLevelPartition(signals, raw, seen, depth) {
3811
+ if (!raw || typeof raw !== "object") return;
3812
+ if (seen.has(raw)) return;
3813
+ seen.add(raw);
3814
+ const partition = raw;
3815
+ for (const level of FAPO_LEVELS) {
3816
+ const bucket = partition[level];
3817
+ if (!bucket || typeof bucket !== "object") continue;
3818
+ const obj = bucket;
3819
+ const count = numberField(obj.count) ?? 0;
3820
+ if (count > 0) signals.counts[level] += count;
3821
+ collectClusters(signals, obj.clusters, false, seen, depth + 1);
3822
+ }
3823
+ }
3824
+ function collectCounts(signals, obj) {
3825
+ const prompt = numberField(obj.prompt_addressable ?? obj.promptAddressable);
3826
+ const structural = numberField(obj.structural_addressable ?? obj.structuralAddressable);
3827
+ const tool = numberField(obj.tool_addressable ?? obj.toolAddressable);
3828
+ if (prompt) signals.counts.prompt += prompt;
3829
+ if (structural) signals.counts.structural += structural;
3830
+ if (tool) signals.counts.structural += tool;
3831
+ }
3832
+ function collectClusters(signals, raw, countClusters, seen, depth) {
3833
+ if (!Array.isArray(raw)) return;
3834
+ if (seen.has(raw)) return;
3835
+ seen.add(raw);
3836
+ for (const item of raw) {
3837
+ if (countClusters) {
3838
+ collectFinding(signals, item, seen, depth + 1);
3839
+ continue;
3523
3840
  }
3524
- start = end;
3841
+ const cluster = parseCluster(item);
3842
+ if (cluster) signals.clusters.push(cluster);
3525
3843
  }
3526
- scores.forEach((s, i) => {
3527
- s.rank = i + 1;
3528
- });
3529
- const best = scores[0];
3530
- const byName = new Map(winners.map((w) => [w.name, w]));
3531
- const bestArr = byName.get(best.name).arr;
3532
- const pairwise = scores.slice(1).map((other) => {
3533
- const otherArr = byName.get(other.name).arr;
3534
- const boot = pairedBootstrap(otherArr, bestArr, {
3535
- seed,
3536
- resamples,
3537
- confidence: intervalConfidence,
3538
- statistic: "mean"
3539
- });
3540
- const favored = boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
3541
- return {
3542
- a: best.name,
3543
- b: other.name,
3544
- deltaMean: boot.mean,
3545
- low: boot.low,
3546
- high: boot.high,
3547
- favored
3548
- };
3549
- });
3550
- const optimizationCost = combineCosts(
3551
- scores.map((score) => ({ label: `method '${score.name}'`, cost: score.optimizationCost }))
3552
- );
3553
- const testCost = costFromLedgerSummary(testCostLedger.summary({ phase: testCostPhase }));
3554
- const totalCost = combineCosts([
3555
- { label: "optimization", cost: optimizationCost },
3556
- { label: "final test", cost: testCost }
3557
- ]);
3844
+ }
3845
+ function addCluster(signals, cluster) {
3846
+ signals.counts[cluster.level] += Math.max(1, cluster.count);
3847
+ signals.clusters.push(cluster);
3848
+ }
3849
+ function parseCluster(raw) {
3850
+ if (!raw || typeof raw !== "object") return null;
3851
+ const obj = raw;
3852
+ const text = textFields(obj);
3853
+ const level = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel) ?? inferLevelFromText(text);
3854
+ if (!level) return null;
3558
3855
  return {
3559
- scores,
3560
- best,
3561
- pairwise,
3562
- testScenarioIds: scenarioIds,
3563
- optimizationCost,
3564
- testCost,
3565
- totalCost,
3566
- confidence,
3567
- intervalConfidence,
3568
- comparisonCount,
3569
- seed,
3570
- resamples,
3571
- reps: opts.reps ?? 1
3856
+ label: text || level,
3857
+ level,
3858
+ count: numberField(obj.count) ?? 1,
3859
+ confidence: confidenceField(obj.confidence),
3860
+ suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
3861
+ caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
3572
3862
  };
3573
3863
  }
3574
- function assertOptimizationMethods(methods) {
3575
- if (!Array.isArray(methods) || methods.length === 0) {
3576
- throw new Error("compareOptimizationMethods: no methods to compare");
3864
+ function parseFindingLevel(raw) {
3865
+ if (typeof raw !== "string") return null;
3866
+ const lower = raw.toLowerCase();
3867
+ if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
3868
+ if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
3869
+ if (lower === "config" || lower === "params") return "parameter";
3870
+ return null;
3871
+ }
3872
+ function inferLevelFromText(text) {
3873
+ const lower = text.toLowerCase();
3874
+ if (/\b(retrieval_k|temperature|top_p|max_tokens|max_completion_tokens|reasoning_effort|config|parameter)\b/.test(
3875
+ lower
3876
+ )) {
3877
+ return "parameter";
3577
3878
  }
3578
- const names = /* @__PURE__ */ new Set();
3579
- const pathOwners = /* @__PURE__ */ new Map();
3580
- for (const method of methods) {
3581
- if (!method || typeof method !== "object" || typeof method.optimize !== "function") {
3582
- throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
3583
- }
3584
- if (!method.name || method.name.trim() !== method.name) {
3585
- throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
3586
- }
3587
- if (names.has(method.name)) {
3588
- throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
3589
- }
3590
- names.add(method.name);
3591
- const pathKey = slug(method.name);
3592
- const prior = pathOwners.get(pathKey);
3593
- if (prior) {
3594
- throw new Error(
3595
- `compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`
3596
- );
3879
+ if (/\b(retriev\w*|search\w*|bm25|hop|evidence|cascade|tool|node|chain|state|structur\w*|topology|import)\b/.test(
3880
+ lower
3881
+ )) {
3882
+ return "structural";
3883
+ }
3884
+ if (/\b(format\w*|verbose|brevity|abstain\w*|reasoning|instruction|prompt|output|answer)\b/.test(
3885
+ lower
3886
+ )) {
3887
+ return "prompt";
3888
+ }
3889
+ return null;
3890
+ }
3891
+ function textFields(obj) {
3892
+ return [
3893
+ obj.label,
3894
+ obj.claim,
3895
+ obj.recommended_action,
3896
+ obj.suggested_fix,
3897
+ obj.suggestedFix,
3898
+ obj.message,
3899
+ obj.text,
3900
+ obj.heuristic,
3901
+ obj.area
3902
+ ].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
3903
+ }
3904
+ function numberField(value) {
3905
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
3906
+ }
3907
+ function stringField(value) {
3908
+ return typeof value === "string" && value.trim() ? value.trim() : void 0;
3909
+ }
3910
+ function confidenceField(value) {
3911
+ return value === "high" || value === "medium" || value === "low" ? value : void 0;
3912
+ }
3913
+ function stringArrayField(value) {
3914
+ return Array.isArray(value) ? value.filter((item) => typeof item === "string") : void 0;
3915
+ }
3916
+ function parameterSweepProposer(opts) {
3917
+ if (opts.candidates.length === 0) {
3918
+ throw new Error("parameterSweepProposer: candidates must not be empty");
3919
+ }
3920
+ return {
3921
+ kind: "parameter-sweep",
3922
+ async propose(ctx) {
3923
+ if (typeof ctx.currentSurface !== "string") {
3924
+ throw new Error(
3925
+ "parameterSweepProposer: currentSurface must be a JSON string config surface"
3926
+ );
3927
+ }
3928
+ const parse = opts.parse ?? parseJsonObject;
3929
+ const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
3930
+ const current = parse(ctx.currentSurface);
3931
+ const currentCanonical = canonicalJson2(current);
3932
+ const tried = triedLabels(ctx.history);
3933
+ const out = [];
3934
+ for (const candidate of opts.candidates) {
3935
+ if (tried.has(candidate.label)) continue;
3936
+ const next = applyParameterCandidate(current, candidate);
3937
+ const surface = stringify(next);
3938
+ if (surface === ctx.currentSurface || canonicalJson2(next) === currentCanonical) continue;
3939
+ out.push({ surface, label: candidate.label, rationale: candidate.rationale });
3940
+ if (out.length >= ctx.populationSize) break;
3941
+ }
3942
+ return out;
3597
3943
  }
3598
- pathOwners.set(pathKey, method.name);
3599
- }
3944
+ };
3600
3945
  }
3601
- function assertOptimizationResult(name, result) {
3602
- if (!result || typeof result !== "object") {
3603
- throw new Error(`compareOptimizationMethods: method '${name}' returned no result`);
3604
- }
3946
+ function parseJsonObject(surface) {
3947
+ let parsed;
3605
3948
  try {
3606
- surfaceContentHash(result.winnerSurface);
3607
- } catch (cause) {
3949
+ parsed = JSON.parse(surface);
3950
+ } catch (error) {
3608
3951
  throw new Error(
3609
- `compareOptimizationMethods: method '${name}' returned an invalid winnerSurface`,
3610
- { cause }
3952
+ `parameterSweepProposer: currentSurface must be valid JSON object config (${error instanceof Error ? error.message : String(error)})`
3611
3953
  );
3612
3954
  }
3613
- assertComparisonCost(result.cost, `method '${name}'`);
3614
- if (result.durationMs !== void 0 && (!Number.isFinite(result.durationMs) || result.durationMs < 0)) {
3615
- throw new Error(`compareOptimizationMethods: method '${name}' returned an invalid durationMs`);
3955
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
3956
+ throw new Error("parameterSweepProposer: JSON surface must parse to an object");
3616
3957
  }
3958
+ return parsed;
3617
3959
  }
3618
- function assertComparisonControls(opts, seed, resamples, confidence) {
3619
- if (!opts.judges || opts.judges.length === 0) {
3620
- throw new Error("compareOptimizationMethods: at least one judge is required");
3621
- }
3622
- if (typeof opts.dispatchWithSurface !== "function") {
3623
- throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
3960
+ function triedLabels(history) {
3961
+ const tried = /* @__PURE__ */ new Set();
3962
+ for (const generation of history) {
3963
+ for (const candidate of generation.candidates) {
3964
+ if (!candidate.label) continue;
3965
+ tried.add(candidate.label);
3966
+ tried.add(candidateStrategy(candidate));
3967
+ }
3624
3968
  }
3625
- try {
3626
- surfaceContentHash(opts.baselineSurface);
3627
- } catch (cause) {
3628
- throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
3969
+ return tried;
3970
+ }
3971
+ function applyParameterCandidate(current, candidate) {
3972
+ const next = cloneJsonObject(current);
3973
+ if (candidate.patch) deepMerge(next, candidate.patch);
3974
+ for (const change of candidate.changes ?? []) {
3975
+ setPath(
3976
+ next,
3977
+ typeof change.path === "string" ? change.path.split(".") : [...change.path],
3978
+ change.value
3979
+ );
3629
3980
  }
3630
- const judgeNames = /* @__PURE__ */ new Set();
3631
- for (const judge of opts.judges) {
3632
- if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) {
3633
- throw new Error(
3634
- "compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)"
3635
- );
3981
+ return next;
3982
+ }
3983
+ function cloneJsonObject(value) {
3984
+ return JSON.parse(JSON.stringify(value));
3985
+ }
3986
+ function deepMerge(target, patch) {
3987
+ for (const [key, value] of Object.entries(patch)) {
3988
+ assertSafeJsonKey(key);
3989
+ if (isPlainObject(value) && isPlainObject(target[key])) {
3990
+ deepMerge(target[key], value);
3991
+ } else {
3992
+ target[key] = value;
3636
3993
  }
3637
- if (judgeNames.has(judge.name)) {
3638
- throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
3994
+ }
3995
+ }
3996
+ function setPath(target, path, value) {
3997
+ if (path.length === 0) throw new Error("parameterSweepProposer: change path must not be empty");
3998
+ for (const part of path) assertSafeJsonKey(part);
3999
+ let cursor = target;
4000
+ for (const part of path.slice(0, -1)) {
4001
+ const existing = cursor[part];
4002
+ if (!isPlainObject(existing)) {
4003
+ cursor[part] = {};
3639
4004
  }
3640
- judgeNames.add(judge.name);
3641
- const dimensionKeys = /* @__PURE__ */ new Set();
3642
- for (const dimension of judge.dimensions) {
3643
- if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) {
3644
- throw new Error(
3645
- `compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`
3646
- );
4005
+ cursor = cursor[part];
4006
+ }
4007
+ cursor[path[path.length - 1]] = value;
4008
+ }
4009
+ function assertSafeJsonKey(key) {
4010
+ if (!key.trim()) throw new Error("parameterSweepProposer: change path contains an empty key");
4011
+ if (UNSAFE_JSON_KEYS.has(key)) {
4012
+ throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
4013
+ }
4014
+ }
4015
+ function canonicalJson2(value) {
4016
+ if (Array.isArray(value)) return `[${value.map((item) => canonicalJson2(item)).join(",")}]`;
4017
+ if (isPlainObject(value)) {
4018
+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key])}`).join(",")}}`;
4019
+ }
4020
+ return JSON.stringify(value);
4021
+ }
4022
+ function isPlainObject(value) {
4023
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
4024
+ const proto = Object.getPrototypeOf(value);
4025
+ return proto === Object.prototype || proto === null;
4026
+ }
4027
+
4028
+ // src/campaign/skill-patch.ts
4029
+ function applySkillPatch(surface, patch) {
4030
+ let lines = surface.split("\n");
4031
+ let applied = 0;
4032
+ const rejected = [];
4033
+ const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
4034
+ for (const op of patch.ops) {
4035
+ if (op.op === "add") {
4036
+ if (typeof op.text !== "string" || op.text.trim() === "") {
4037
+ rejected.push({ op, reason: "empty add text" });
4038
+ continue;
3647
4039
  }
3648
- if (dimensionKeys.has(dimension.key)) {
3649
- throw new Error(
3650
- `compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`
3651
- );
4040
+ const insert = op.text.split("\n");
4041
+ if (op.after === void 0 || op.after === "") {
4042
+ lines = [...lines, ...insert];
4043
+ applied++;
4044
+ continue;
3652
4045
  }
3653
- dimensionKeys.add(dimension.key);
4046
+ const idx = findLine(op.after);
4047
+ if (idx === -1) {
4048
+ rejected.push({ op, reason: `add anchor not found: ${truncate2(op.after)}` });
4049
+ continue;
4050
+ }
4051
+ lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
4052
+ applied++;
4053
+ } else if (op.op === "delete") {
4054
+ const idx = findLine(op.anchor);
4055
+ if (idx === -1) {
4056
+ rejected.push({ op, reason: `delete anchor not found: ${truncate2(op.anchor)}` });
4057
+ continue;
4058
+ }
4059
+ lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
4060
+ applied++;
4061
+ } else {
4062
+ const idx = findLine(op.anchor);
4063
+ if (idx === -1) {
4064
+ rejected.push({ op, reason: `replace anchor not found: ${truncate2(op.anchor)}` });
4065
+ continue;
4066
+ }
4067
+ if (typeof op.text !== "string") {
4068
+ rejected.push({ op, reason: "replace text missing" });
4069
+ continue;
4070
+ }
4071
+ lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
4072
+ applied++;
3654
4073
  }
3655
4074
  }
3656
- if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
3657
- throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
3658
- }
3659
- if (!Number.isSafeInteger(seed)) {
3660
- throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
4075
+ return { surface: lines.join("\n"), applied, rejected };
4076
+ }
4077
+ function patchEditCount(patch) {
4078
+ return patch.ops.length;
4079
+ }
4080
+ function truncate2(s, max = 48) {
4081
+ return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
4082
+ }
4083
+
4084
+ // src/campaign/proposers/skill-opt.ts
4085
+ var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
4086
+ function skillOptProposer(opts) {
4087
+ const evidenceK = opts.evidenceK ?? 3;
4088
+ const defaultBudget = opts.editBudget ?? 3;
4089
+ const directCostLedger = opts.costLedger ?? new CostLedger();
4090
+ async function proposePatches(args) {
4091
+ const userPrompt = buildPatchPrompt({
4092
+ target: opts.target,
4093
+ surface: args.surface,
4094
+ evidence: args.evidence,
4095
+ editBudget: args.editBudget,
4096
+ rejectedBuffer: args.rejectedBuffer,
4097
+ metaNote: args.metaNote,
4098
+ findingsNote: args.findingsNote,
4099
+ count: args.count
4100
+ });
4101
+ const request = {
4102
+ model: opts.model,
4103
+ messages: [
4104
+ { role: "system", content: SKILLOPT_SYSTEM },
4105
+ { role: "user", content: userPrompt }
4106
+ ],
4107
+ jsonMode: true,
4108
+ temperature: opts.temperature ?? 0.6,
4109
+ maxTokens: opts.maxTokens ?? 4e3
4110
+ };
4111
+ const paid = await (args.costLedger ?? directCostLedger).runPaidCall({
4112
+ channel: "driver",
4113
+ phase: args.costPhase ?? "search.proposal",
4114
+ actor: "skill-opt.propose",
4115
+ model: opts.model,
4116
+ maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
4117
+ signal: args.signal,
4118
+ execute: (signal, callId) => callLlm(request, { ...opts.llm, signal, idempotencyKey: callId }),
4119
+ receipt: costReceiptFromLlm,
4120
+ receiptFromError: costReceiptFromLlmError
4121
+ });
4122
+ if (!paid.succeeded) throw paid.error;
4123
+ const result = paid.value;
4124
+ return parseSkillPatchResponse(result.content, args.count, args.editBudget);
3661
4125
  }
3662
- if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) {
3663
- throw new Error(
3664
- `compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`
4126
+ return {
4127
+ kind: "skill-opt",
4128
+ proposePatches,
4129
+ async propose(ctx) {
4130
+ if (typeof ctx.currentSurface !== "string") {
4131
+ throw new Error(
4132
+ "skillOptProposer: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
4133
+ );
4134
+ }
4135
+ const surface = ctx.currentSurface;
4136
+ const patches = await proposePatches({
4137
+ surface,
4138
+ evidence: evidenceFromHistory(ctx, evidenceK),
4139
+ editBudget: defaultBudget,
4140
+ rejectedBuffer: [],
4141
+ findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
4142
+ count: ctx.populationSize,
4143
+ signal: ctx.signal,
4144
+ costLedger: ctx.costLedger ?? directCostLedger,
4145
+ costPhase: ctx.costPhase ?? "search.proposal"
4146
+ });
4147
+ const out = [];
4148
+ const seen = /* @__PURE__ */ new Set();
4149
+ for (const patch of patches) {
4150
+ const { surface: candidate, applied } = applySkillPatch(surface, patch);
4151
+ if (applied === 0 || candidate === surface || seen.has(candidate)) continue;
4152
+ seen.add(candidate);
4153
+ out.push({ surface: candidate, label: patch.label, rationale: patch.rationale });
4154
+ if (out.length >= ctx.populationSize) break;
4155
+ }
4156
+ return out;
4157
+ }
4158
+ };
4159
+ }
4160
+ function evidenceFromHistory(ctx, k) {
4161
+ const last = ctx.history.at(-1);
4162
+ if (!last || last.candidates.length === 0) return { weakScenarios: [], weakDimensions: [] };
4163
+ const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
4164
+ if (!best) return { weakScenarios: [], weakDimensions: [] };
4165
+ const weakScenarios = [...best.scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
4166
+ const weakDimensions = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
4167
+ return { weakScenarios, weakDimensions };
4168
+ }
4169
+ function buildPatchPrompt(args) {
4170
+ const lines = [
4171
+ `Skill document governs: ${args.target}.`,
4172
+ "",
4173
+ "Current skill document:",
4174
+ "```",
4175
+ args.surface,
4176
+ "```",
4177
+ "",
4178
+ `Propose ${args.count} candidate patch(es). Each patch is a SMALL bundle of`,
4179
+ `at most ${args.editBudget} op(s). Anchors must be verbatim substrings of`,
4180
+ "existing lines. Prefer adding a specific missing rule or sharpening a vague",
4181
+ "one over deleting; never rewrite the whole document."
4182
+ ];
4183
+ if (args.evidence.weakScenarios.length > 0) {
4184
+ lines.push(
4185
+ "",
4186
+ "Weakest scenarios (patch to fix these):",
4187
+ ...args.evidence.weakScenarios.map((s) => `- ${s.scenarioId} (${s.composite.toFixed(2)})`)
3665
4188
  );
3666
4189
  }
3667
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
3668
- throw new Error(
3669
- `compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
4190
+ if (args.evidence.weakDimensions.length > 0) {
4191
+ lines.push(
4192
+ "",
4193
+ "Weakest dimensions (what to improve):",
4194
+ ...args.evidence.weakDimensions.map((d) => `- ${d.dimension} (${d.score.toFixed(2)})`)
3670
4195
  );
3671
4196
  }
3672
- const minimumResamples = minimumBootstrapResamples(
3673
- confidence,
3674
- opts.methods.length * (opts.methods.length + 1) / 2
3675
- );
3676
- if (resamples < minimumResamples) {
3677
- throw new Error(
3678
- `compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`
4197
+ if (args.rejectedBuffer.length > 0) {
4198
+ lines.push(
4199
+ "",
4200
+ "Already tried and REJECTED (do not repeat or restate these edits):",
4201
+ ...args.rejectedBuffer.map((e) => `- ${e.label}: ${e.rationale} \u2014 ${e.reason}`)
3679
4202
  );
3680
4203
  }
3681
- if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) {
3682
- throw new Error(
3683
- "compareOptimizationMethods: optimizationConcurrency must be a positive safe integer"
3684
- );
4204
+ if (args.findingsNote) {
4205
+ lines.push("", args.findingsNote);
3685
4206
  }
3686
- if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) {
3687
- throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
4207
+ if (args.metaNote) {
4208
+ lines.push("", `Strategy note from prior epochs: ${args.metaNote}`);
3688
4209
  }
3689
- if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) {
3690
- throw new Error(
3691
- "compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647"
3692
- );
4210
+ return lines.join("\n");
4211
+ }
4212
+ var SkillPatchParseError = class extends Error {
4213
+ constructor(message) {
4214
+ super(message);
4215
+ this.name = "SkillPatchParseError";
3693
4216
  }
3694
- if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) {
3695
- throw new Error(
3696
- "compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0"
4217
+ };
4218
+ function parseSkillPatchResponse(raw, maxPatches, editBudget) {
4219
+ let text = raw.trim();
4220
+ if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
4221
+ const start = text.indexOf("{");
4222
+ const end = text.lastIndexOf("}");
4223
+ if (start < 0 || end <= start) {
4224
+ throw new SkillPatchParseError(
4225
+ `parseSkillPatchResponse: response was not valid JSON (no object found): ${snippet(raw)}`
3697
4226
  );
3698
4227
  }
3699
- if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) {
3700
- throw new Error(
3701
- "compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling"
4228
+ let parsed;
4229
+ try {
4230
+ parsed = JSON.parse(text.slice(start, end + 1));
4231
+ } catch (err) {
4232
+ throw new SkillPatchParseError(
4233
+ `parseSkillPatchResponse: response was not valid JSON (${err instanceof Error ? err.message : String(err)}): ${snippet(raw)}`
3702
4234
  );
3703
4235
  }
4236
+ const rawPatches = Array.isArray(parsed.patches) ? parsed.patches : [];
4237
+ const out = [];
4238
+ for (const rp of rawPatches) {
4239
+ if (typeof rp !== "object" || rp === null) continue;
4240
+ const obj = rp;
4241
+ const ops = Array.isArray(obj.ops) ? obj.ops.map(normalizeOp).filter(isOp) : [];
4242
+ if (ops.length === 0) continue;
4243
+ out.push({
4244
+ label: typeof obj.label === "string" ? obj.label : "patch",
4245
+ rationale: typeof obj.rationale === "string" ? obj.rationale : "",
4246
+ ops: ops.slice(0, editBudget)
4247
+ });
4248
+ if (out.length >= maxPatches) break;
4249
+ }
4250
+ return out;
3704
4251
  }
3705
- function assertComparisonPartitions(opts) {
3706
- const legacy = opts;
3707
- if (legacy.holdoutScenarios !== void 0) {
3708
- throw new Error(
3709
- "compareOptimizationMethods: holdoutScenarios is ambiguous and no longer accepted. Provide disjoint trainScenarios, selectionScenarios, and testScenarios; selection may be reused adaptively, test must remain untouched."
3710
- );
4252
+ function normalizeOp(raw) {
4253
+ if (typeof raw !== "object" || raw === null) return null;
4254
+ const o = raw;
4255
+ if (o.op === "add") {
4256
+ if (typeof o.text !== "string") return null;
4257
+ const op = { op: "add", text: o.text };
4258
+ if (typeof o.after === "string") op.after = o.after;
4259
+ return op;
3711
4260
  }
3712
- const partitions = [
3713
- { name: "trainScenarios", scenarios: opts.trainScenarios },
3714
- { name: "selectionScenarios", scenarios: opts.selectionScenarios },
3715
- { name: "testScenarios", scenarios: opts.testScenarios }
3716
- ];
3717
- const owner = /* @__PURE__ */ new Map();
3718
- for (const partition of partitions) {
3719
- if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) {
3720
- throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
3721
- }
3722
- if (partition.name === "testScenarios" && partition.scenarios.length < 2) {
3723
- throw new Error(
3724
- "compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty"
3725
- );
3726
- }
3727
- const local = /* @__PURE__ */ new Set();
3728
- const duplicates = /* @__PURE__ */ new Set();
3729
- const overlaps = /* @__PURE__ */ new Map();
3730
- for (const scenario of partition.scenarios) {
3731
- if (local.has(scenario.id)) duplicates.add(scenario.id);
3732
- local.add(scenario.id);
3733
- const prior = owner.get(scenario.id);
3734
- if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
3735
- }
3736
- if (duplicates.size > 0) {
3737
- throw new Error(
3738
- `compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[
3739
- ...duplicates
3740
- ].join(", ")}]`
3741
- );
3742
- }
3743
- if (overlaps.size > 0) {
3744
- const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} \u2229 ${partition.name})`).join(", ");
3745
- throw new Error(
3746
- `compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`
3747
- );
3748
- }
3749
- assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
3750
- for (const id of local) owner.set(id, partition.name);
4261
+ if (o.op === "delete") {
4262
+ if (typeof o.anchor !== "string") return null;
4263
+ return { op: "delete", anchor: o.anchor };
3751
4264
  }
3752
- }
3753
- function mean2(xs) {
3754
- return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
3755
- }
3756
- function slug(name) {
3757
- return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
3758
- }
3759
- function assertConfidence(confidence) {
3760
- if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
3761
- throw new Error(
3762
- `compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
3763
- );
4265
+ if (o.op === "replace") {
4266
+ if (typeof o.anchor !== "string" || typeof o.text !== "string") return null;
4267
+ return { op: "replace", anchor: o.anchor, text: o.text };
3764
4268
  }
4269
+ return null;
3765
4270
  }
3766
- function minimumBootstrapResamples(confidence, comparisonCount) {
3767
- const exact = 2 * comparisonCount / (1 - confidence);
3768
- return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
3769
- }
3770
- function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed) {
3771
- const cloneScenarios = (scenarios) => Object.freeze(scenarios.map((scenario) => structuredClone(scenario)));
3772
- const judges = opts.judges.map(
3773
- (judge) => Object.freeze({
3774
- ...judge,
3775
- dimensions: Object.freeze(
3776
- judge.dimensions.map((dimension) => Object.freeze({ ...dimension }))
3777
- )
3778
- })
3779
- );
3780
- return Object.freeze({
3781
- baselineSurface: structuredClone(opts.baselineSurface),
3782
- trainScenarios: cloneScenarios(opts.trainScenarios),
3783
- selectionScenarios: cloneScenarios(opts.selectionScenarios),
3784
- dispatchWithSurface: opts.dispatchWithSurface,
3785
- judges: Object.freeze(judges),
3786
- runDir: `${resolvedRunDir}/optimization/${slug(methodName)}`,
3787
- seed,
3788
- runOptions: Object.freeze({ ...opts.optimizationRunOptions ?? {} })
3789
- });
3790
- }
3791
- function costFromLedgerSummary(summary) {
3792
- const cost = {
3793
- totalCostUsd: summary.totalCostUsd,
3794
- accountingComplete: summary.accountingComplete,
3795
- incompleteReasons: [...summary.incompleteReasons]
3796
- };
3797
- assertComparisonCost(cost, "cost ledger");
3798
- return cost;
3799
- }
3800
- function combineCosts(entries) {
3801
- return {
3802
- totalCostUsd: entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0),
3803
- accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
3804
- incompleteReasons: entries.flatMap(
3805
- (entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`)
3806
- )
3807
- };
4271
+ function isOp(op) {
4272
+ return op !== null;
3808
4273
  }
3809
- function assertComparisonCost(cost, label) {
3810
- if (!cost || typeof cost !== "object") {
3811
- throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
3812
- }
3813
- if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) {
3814
- throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
3815
- }
3816
- if (typeof cost.accountingComplete !== "boolean") {
3817
- throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
3818
- }
3819
- if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some(
3820
- (reason) => typeof reason !== "string" || reason.trim().length === 0
3821
- )) {
3822
- throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
3823
- }
3824
- if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) {
3825
- throw new Error(
3826
- `compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`
3827
- );
3828
- }
4274
+ function snippet(s, max = 120) {
4275
+ const t = s.trim().replace(/\s+/g, " ");
4276
+ return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
3829
4277
  }
3830
4278
 
3831
4279
  // src/campaign/presets/run-skill-opt.ts
@@ -4627,14 +5075,14 @@ function scoreboardSummary(rows) {
4627
5075
  function escapeCell(s) {
4628
5076
  return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
4629
5077
  }
4630
- function truncate2(s, max) {
5078
+ function truncate3(s, max) {
4631
5079
  return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
4632
5080
  }
4633
5081
  function renderScoreboardMarkdown(rows, opts = {}) {
4634
5082
  const maxEv = opts.maxEvidenceChars ?? 160;
4635
5083
  const sum = scoreboardSummary(rows);
4636
5084
  const pct = (n) => `${Math.round(n * 100)}%`;
4637
- const ev = (e) => escapeCell(truncate2(e.join("; "), maxEv)) || "\u2014";
5085
+ const ev = (e) => escapeCell(truncate3(e.join("; "), maxEv)) || "\u2014";
4638
5086
  const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
4639
5087
  if (opts.meta) {
4640
5088
  for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
@@ -4872,7 +5320,7 @@ async function runLineageLoop(opts) {
4872
5320
 
4873
5321
  // src/campaign/presets/run-profile-matrix.ts
4874
5322
  import { createHash as createHash6 } from "crypto";
4875
- import { join as join3 } from "path";
5323
+ import { join as join4 } from "path";
4876
5324
 
4877
5325
  // src/agent-profile.ts
4878
5326
  import { createHash as createHash5 } from "crypto";
@@ -5137,7 +5585,7 @@ async function runProfileMatrix(opts) {
5137
5585
  captureSource: opts.captureSource,
5138
5586
  storage: opts.storage,
5139
5587
  now: opts.now,
5140
- runDir: join3(opts.runDir, sanitize(profileId))
5588
+ runDir: join4(opts.runDir, sanitize(profileId))
5141
5589
  });
5142
5590
  const axis = harnessAxisOf(profile);
5143
5591
  const buildCellIdentity = (cellModel) => buildAgentProfileCell({
@@ -5420,8 +5868,8 @@ import { promisify } from "util";
5420
5868
 
5421
5869
  // src/campaign/proposers/analysis-edit.ts
5422
5870
  import { mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
5423
- import { tmpdir } from "os";
5424
- import { join as join4 } from "path";
5871
+ import { tmpdir as tmpdir2 } from "os";
5872
+ import { join as join5 } from "path";
5425
5873
  var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
5426
5874
  function surfaceToPromptText(surface) {
5427
5875
  return typeof surface === "string" ? surface : JSON.stringify(surface);
@@ -5436,8 +5884,8 @@ function analysisEditProposer(opts) {
5436
5884
  const phase = ctx.costPhase ?? "search.proposal";
5437
5885
  const traces = await opts.resolveTraces(ctx) ?? "";
5438
5886
  if (!traces.trim()) throw new Error(opts.noTracesError);
5439
- const dir = mkdtempSync(join4(tmpdir(), `${opts.kind}-proposer-`));
5440
- const tracePath = join4(dir, "traces.jsonl");
5887
+ const dir = mkdtempSync(join5(tmpdir2(), `${opts.kind}-proposer-`));
5888
+ const tracePath = join5(dir, "traces.jsonl");
5441
5889
  writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
5442
5890
  `);
5443
5891
  let report;
@@ -8064,8 +8512,8 @@ import {
8064
8512
  realpathSync,
8065
8513
  rmSync
8066
8514
  } from "fs";
8067
- import { devNull, tmpdir as tmpdir2 } from "os";
8068
- import { basename, dirname as dirname2, isAbsolute as isAbsolute2, join as join5, relative as relative2, resolve as resolve3, sep } from "path";
8515
+ import { devNull, tmpdir as tmpdir3 } from "os";
8516
+ import { basename, dirname as dirname2, isAbsolute as isAbsolute2, join as join6, relative as relative2, resolve as resolve3, sep } from "path";
8069
8517
  var MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024;
8070
8518
  var FILE_HASH_CHUNK_BYTES = 1024 * 1024;
8071
8519
  var GIT_REPOSITORY_ENV = /* @__PURE__ */ new Set([
@@ -8194,9 +8642,9 @@ var CANONICAL_GIT_ENV = {
8194
8642
  GIT_CONFIG_SYSTEM: devNull
8195
8643
  };
8196
8644
  function patchBytes(git, cwd, baseCommit, candidateCommit) {
8197
- const scratch = mkdtempSync2(join5(tmpdir2(), "agent-eval-patch-"));
8198
- const bareRepo = join5(scratch, "repo.git");
8199
- const emptyTemplate = join5(scratch, "empty-template");
8645
+ const scratch = mkdtempSync2(join6(tmpdir3(), "agent-eval-patch-"));
8646
+ const bareRepo = join6(scratch, "repo.git");
8647
+ const emptyTemplate = join6(scratch, "empty-template");
8200
8648
  mkdirSync2(emptyTemplate);
8201
8649
  try {
8202
8650
  const objectFormat = gitObjectHashAlgorithm(candidateCommit);
@@ -8232,7 +8680,7 @@ function resolveCommit(git, cwd, ref) {
8232
8680
  }
8233
8681
  function unresolvedWorktreePath(surface, worktreeDir) {
8234
8682
  if (isAbsolute2(surface.worktreeRef)) return surface.worktreeRef;
8235
- if (worktreeDir) return join5(worktreeDir, basename(surface.worktreeRef));
8683
+ if (worktreeDir) return join6(worktreeDir, basename(surface.worktreeRef));
8236
8684
  return surface.worktreeRef;
8237
8685
  }
8238
8686
  function displayGitPath(path) {
@@ -8308,7 +8756,7 @@ function assertSafeRelativePath(root, path) {
8308
8756
  if (segment.length === 0 || segment === "." || segment === "..") {
8309
8757
  throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
8310
8758
  }
8311
- parent = join5(parent, segment);
8759
+ parent = join6(parent, segment);
8312
8760
  const stat = lstatSync(parent);
8313
8761
  if (!stat.isDirectory() || stat.isSymbolicLink()) {
8314
8762
  throw new WorktreeAdapterError(
@@ -8568,13 +9016,13 @@ function slug2(label) {
8568
9016
  }
8569
9017
  function gitWorktreeAdapter(opts) {
8570
9018
  const git = opts.git ?? defaultGit;
8571
- const worktreeDir = opts.worktreeDir ?? join5(opts.repoRoot, ".worktrees");
9019
+ const worktreeDir = opts.worktreeDir ?? join6(opts.repoRoot, ".worktrees");
8572
9020
  const branchPrefix = opts.branchPrefix ?? "improve";
8573
9021
  return {
8574
9022
  async create({ baseRef, label }) {
8575
9023
  const id = `${slug2(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
8576
9024
  const branch = `${branchPrefix}/${id}`;
8577
- const path = join5(worktreeDir, id);
9025
+ const path = join6(worktreeDir, id);
8578
9026
  const baseCommit = resolveCommit(git, opts.repoRoot, baseRef);
8579
9027
  const baseTree = gitText(
8580
9028
  git,
@@ -8683,6 +9131,9 @@ export {
8683
9131
  neutralizationGate,
8684
9132
  sequentialPairedGate,
8685
9133
  sequentialDecide,
9134
+ compareOptimizationMethods,
9135
+ costFromLedgerSummary,
9136
+ gepaOptimizationMethod,
8686
9137
  rolloutArgumentDiff,
8687
9138
  classifyUngroundedLiterals,
8688
9139
  LabeledScenarioStoreError,
@@ -8696,8 +9147,6 @@ export {
8696
9147
  skillOptProposer,
8697
9148
  SkillPatchParseError,
8698
9149
  parseSkillPatchResponse,
8699
- compareOptimizationMethods,
8700
- costFromLedgerSummary,
8701
9150
  runSkillOpt,
8702
9151
  gepaReflectionMethod,
8703
9152
  gepaParetoMethod,
@@ -8736,4 +9185,4 @@ export {
8736
9185
  verifyCodeSurface,
8737
9186
  resolveWorktreePath
8738
9187
  };
8739
- //# sourceMappingURL=chunk-LT4J7ULK.js.map
9188
+ //# sourceMappingURL=chunk-VPDOSN3L.js.map