@tangle-network/agent-eval 0.123.5 → 0.123.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +214 -147
- package/dist/campaign/index.js +4 -2
- package/dist/{chunk-QBRSJK47.js → chunk-J3LHTAAB.js} +47 -47
- package/dist/chunk-J3LHTAAB.js.map +1 -0
- package/dist/{chunk-SUN7QLPB.js → chunk-KKPPFIDS.js} +87 -87
- package/dist/{chunk-SUN7QLPB.js.map → chunk-KKPPFIDS.js.map} +1 -1
- package/dist/{chunk-LT4J7ULK.js → chunk-VPDOSN3L.js} +1731 -1282
- package/dist/chunk-VPDOSN3L.js.map +1 -0
- package/dist/contract/index.js +1 -1
- package/dist/index.d.ts +115 -1
- package/dist/index.js +120 -3
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +1 -1
- package/docs/campaign-proposers.md +66 -0
- package/package.json +1 -1
- package/dist/chunk-LT4J7ULK.js.map +0 -1
- package/dist/chunk-QBRSJK47.js.map +0 -1
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
runImprovementLoop,
|
|
16
16
|
surfaceContentHash,
|
|
17
17
|
surfaceHash
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-KKPPFIDS.js";
|
|
19
19
|
import {
|
|
20
20
|
SearchLedgerConflictError,
|
|
21
21
|
SearchLedgerError,
|
|
@@ -2405,1427 +2405,1875 @@ function sequentialDecide(options = {}) {
|
|
|
2405
2405
|
return decide;
|
|
2406
2406
|
}
|
|
2407
2407
|
|
|
2408
|
-
// src/campaign/
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2408
|
+
// src/campaign/gepa-optimization-method.ts
|
|
2409
|
+
import { spawn } from "child_process";
|
|
2410
|
+
import { randomBytes } from "crypto";
|
|
2411
|
+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises";
|
|
2412
|
+
import { createServer } from "http";
|
|
2413
|
+
import { tmpdir } from "os";
|
|
2414
|
+
import { join as join2 } from "path";
|
|
2415
|
+
|
|
2416
|
+
// src/campaign/presets/compare-optimization-methods.ts
|
|
2417
|
+
import { randomUUID } from "crypto";
|
|
2418
|
+
async function compareOptimizationMethods(opts) {
|
|
2419
|
+
assertOptimizationMethods(opts.methods);
|
|
2420
|
+
assertComparisonPartitions(opts);
|
|
2421
|
+
const seed = opts.seed ?? 42;
|
|
2422
|
+
const confidence = opts.confidence ?? 0.95;
|
|
2423
|
+
assertConfidence(confidence);
|
|
2424
|
+
const optimizationConcurrency = opts.optimizationConcurrency ?? 1;
|
|
2425
|
+
const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
|
|
2426
|
+
const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
|
|
2427
|
+
const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
|
|
2428
|
+
const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
|
|
2429
|
+
assertComparisonControls(opts, seed, resamples, confidence);
|
|
2430
|
+
const storage = opts.storage ?? fsCampaignStorage();
|
|
2431
|
+
const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
|
|
2432
|
+
const testCostPhase = `compareOptimizationMethods:test:${randomUUID()}`;
|
|
2433
|
+
const testCostLedger = opts.costLedger ?? createRunCostLedger({
|
|
2434
|
+
storage,
|
|
2435
|
+
runDir: `${resolvedRunDir}/test/cost`,
|
|
2436
|
+
costCeilingUsd: opts.costCeiling
|
|
2437
|
+
});
|
|
2438
|
+
const scoreOnTest = async (surface, tag) => {
|
|
2439
|
+
const campaign = await runCampaign({
|
|
2440
|
+
...opts,
|
|
2441
|
+
storage,
|
|
2442
|
+
costLedger: testCostLedger,
|
|
2443
|
+
costPhase: testCostPhase,
|
|
2444
|
+
scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
|
|
2445
|
+
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
2446
|
+
runDir: `${resolvedRunDir}/${tag}`
|
|
2447
|
+
});
|
|
2448
|
+
const byScenario = {};
|
|
2449
|
+
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
|
|
2450
|
+
byScenario[scenarioId] = composite;
|
|
2423
2451
|
}
|
|
2424
|
-
return
|
|
2425
|
-
};
|
|
2426
|
-
const passing = collect(true);
|
|
2427
|
-
const failing = collect(false);
|
|
2428
|
-
const lines = [];
|
|
2429
|
-
for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
|
|
2430
|
-
const pv = [...passing.get(field) ?? []].slice(0, maxValues);
|
|
2431
|
-
const fv = [...failing.get(field) ?? []].slice(0, maxValues);
|
|
2432
|
-
const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
|
|
2433
|
-
lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
|
|
2434
|
-
}
|
|
2435
|
-
const lower = (m) => {
|
|
2436
|
-
const out = /* @__PURE__ */ new Set();
|
|
2437
|
-
for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
|
|
2438
|
-
return out;
|
|
2439
|
-
};
|
|
2440
|
-
return {
|
|
2441
|
-
text: lines.join("\n") || " (no calls observed)",
|
|
2442
|
-
passingValues: lower(passing),
|
|
2443
|
-
failingValues: lower(failing)
|
|
2444
|
-
};
|
|
2445
|
-
}
|
|
2446
|
-
function classifyUngroundedLiterals(text, diff) {
|
|
2447
|
-
const ungrounded = /* @__PURE__ */ new Set();
|
|
2448
|
-
for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
|
|
2449
|
-
const w = m[1].toLowerCase();
|
|
2450
|
-
if (!diff.passingValues.has(w)) ungrounded.add(w);
|
|
2451
|
-
}
|
|
2452
|
-
const all = [...ungrounded];
|
|
2453
|
-
return {
|
|
2454
|
-
ungrounded: all,
|
|
2455
|
-
harmful: all.filter((w) => diff.failingValues.has(w))
|
|
2452
|
+
return byScenario;
|
|
2456
2453
|
};
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
super(message);
|
|
2466
|
-
this.code = code;
|
|
2467
|
-
this.name = "LabeledScenarioStoreError";
|
|
2468
|
-
}
|
|
2469
|
-
code;
|
|
2470
|
-
};
|
|
2471
|
-
var FsLabeledScenarioStore = class {
|
|
2472
|
-
constructor(options) {
|
|
2473
|
-
this.options = options;
|
|
2474
|
-
if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
|
|
2475
|
-
this.now = options.now ?? Date.now;
|
|
2476
|
-
}
|
|
2477
|
-
options;
|
|
2478
|
-
now;
|
|
2479
|
-
rateLimits = /* @__PURE__ */ new Map();
|
|
2480
|
-
async observe(write) {
|
|
2481
|
-
this.assertProvenance(write);
|
|
2482
|
-
this.assertRateLimit(write);
|
|
2483
|
-
const record = this.toRecord(write);
|
|
2484
|
-
const path = this.pathForSource(write.source);
|
|
2485
|
-
const line = `${JSON.stringify(record)}
|
|
2486
|
-
`;
|
|
2487
|
-
appendLine(path, line);
|
|
2488
|
-
}
|
|
2489
|
-
async sample(args) {
|
|
2490
|
-
if (!args.split) {
|
|
2491
|
-
throw new LabeledScenarioStoreError(
|
|
2492
|
-
"split_required",
|
|
2493
|
-
"sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
|
|
2454
|
+
const scenarioIds = opts.testScenarios.map((s) => s.id).sort();
|
|
2455
|
+
const align = (byScenario, label) => {
|
|
2456
|
+
const missing = scenarioIds.filter((id) => !(id in byScenario));
|
|
2457
|
+
if (missing.length > 0) {
|
|
2458
|
+
throw new Error(
|
|
2459
|
+
`compareOptimizationMethods: ${label} produced no test score for scenario(s) [${missing.join(
|
|
2460
|
+
", "
|
|
2461
|
+
)}]. A cell failed or its judges returned nothing. Fix the dispatch or judge; the comparison will not replace missing scores with zero.`
|
|
2494
2462
|
);
|
|
2495
2463
|
}
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2464
|
+
return scenarioIds.map((id) => byScenario[id]);
|
|
2465
|
+
};
|
|
2466
|
+
const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
|
|
2467
|
+
const out = await method.optimize(
|
|
2468
|
+
createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed)
|
|
2469
|
+
);
|
|
2470
|
+
assertOptimizationResult(method.name, out);
|
|
2471
|
+
const winnerSurface = structuredClone(out.winnerSurface);
|
|
2472
|
+
return {
|
|
2473
|
+
name: method.name,
|
|
2474
|
+
winnerSurface,
|
|
2475
|
+
cost: out.cost,
|
|
2476
|
+
durationMs: out.durationMs
|
|
2477
|
+
};
|
|
2478
|
+
});
|
|
2479
|
+
const baselineArr = align(await scoreOnTest(opts.baselineSurface, "test/baseline"), "baseline");
|
|
2480
|
+
const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(opts.baselineSurface), baselineArr]]);
|
|
2481
|
+
const winners = [];
|
|
2482
|
+
for (const winner of optimized) {
|
|
2483
|
+
const surfaceKey = surfaceContentHash(winner.winnerSurface);
|
|
2484
|
+
let arr = testScoresBySurface.get(surfaceKey);
|
|
2485
|
+
if (!arr) {
|
|
2486
|
+
const byScenario = await scoreOnTest(
|
|
2487
|
+
winner.winnerSurface,
|
|
2488
|
+
`test/methods/${slug(winner.name)}`
|
|
2500
2489
|
);
|
|
2490
|
+
arr = align(byScenario, `method "${winner.name}"`);
|
|
2491
|
+
testScoresBySurface.set(surfaceKey, arr);
|
|
2501
2492
|
}
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
const explicit = sourceFilterContains(args.filter?.source, "production-trace");
|
|
2506
|
-
if (!explicit) continue;
|
|
2507
|
-
}
|
|
2508
|
-
const path = this.pathForSource(source);
|
|
2509
|
-
if (!existsSync2(path)) continue;
|
|
2510
|
-
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
2511
|
-
for (const line of lines) {
|
|
2512
|
-
let record;
|
|
2513
|
-
try {
|
|
2514
|
-
record = JSON.parse(line);
|
|
2515
|
-
} catch {
|
|
2516
|
-
continue;
|
|
2517
|
-
}
|
|
2518
|
-
if (!matchesFilter(record, args, source)) continue;
|
|
2519
|
-
all.push(record);
|
|
2520
|
-
}
|
|
2521
|
-
}
|
|
2522
|
-
all.sort((a, b) => {
|
|
2523
|
-
if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
|
|
2524
|
-
return a.recordHash.localeCompare(b.recordHash);
|
|
2493
|
+
winners.push({
|
|
2494
|
+
...winner,
|
|
2495
|
+
arr
|
|
2525
2496
|
});
|
|
2526
|
-
return all.slice(0, args.count);
|
|
2527
2497
|
}
|
|
2528
|
-
|
|
2529
|
-
const
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
"
|
|
2498
|
+
const scores = winners.map((w) => {
|
|
2499
|
+
const boot = pairedBootstrap(baselineArr, w.arr, {
|
|
2500
|
+
seed,
|
|
2501
|
+
resamples,
|
|
2502
|
+
confidence: intervalConfidence,
|
|
2503
|
+
statistic: "mean"
|
|
2504
|
+
});
|
|
2505
|
+
const score = {
|
|
2506
|
+
name: w.name,
|
|
2507
|
+
baselineComposite: mean2(baselineArr),
|
|
2508
|
+
winnerComposite: mean2(w.arr),
|
|
2509
|
+
lift: boot.mean,
|
|
2510
|
+
liftCi: { low: boot.low, high: boot.high },
|
|
2511
|
+
optimizationCost: w.cost,
|
|
2512
|
+
scenarioScores: scenarioIds.map((scenarioId, index) => ({
|
|
2513
|
+
scenarioId,
|
|
2514
|
+
baselineComposite: baselineArr[index],
|
|
2515
|
+
winnerComposite: w.arr[index],
|
|
2516
|
+
lift: w.arr[index] - baselineArr[index]
|
|
2517
|
+
})),
|
|
2518
|
+
winnerSurface: w.winnerSurface,
|
|
2519
|
+
rank: 0
|
|
2534
2520
|
};
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
let trust = "unverified";
|
|
2547
|
-
try {
|
|
2548
|
-
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
2549
|
-
} catch {
|
|
2550
|
-
}
|
|
2551
|
-
byTrust[trust] += 1;
|
|
2552
|
-
}
|
|
2521
|
+
if (w.durationMs !== void 0) score.durationMs = w.durationMs;
|
|
2522
|
+
return score;
|
|
2523
|
+
});
|
|
2524
|
+
scores.sort((a, b) => b.lift - a.lift);
|
|
2525
|
+
for (let start = 0; start < scores.length; ) {
|
|
2526
|
+
let end = start + 1;
|
|
2527
|
+
while (end < scores.length && scores[end].lift === scores[start].lift) end += 1;
|
|
2528
|
+
const tied = scores.slice(start, end);
|
|
2529
|
+
if (tied.every((score) => score.optimizationCost.accountingComplete)) {
|
|
2530
|
+
tied.sort((a, b) => a.optimizationCost.totalCostUsd - b.optimizationCost.totalCostUsd);
|
|
2531
|
+
scores.splice(start, tied.length, ...tied);
|
|
2553
2532
|
}
|
|
2554
|
-
|
|
2533
|
+
start = end;
|
|
2555
2534
|
}
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2535
|
+
scores.forEach((s, i) => {
|
|
2536
|
+
s.rank = i + 1;
|
|
2537
|
+
});
|
|
2538
|
+
const best = scores[0];
|
|
2539
|
+
const byName = new Map(winners.map((w) => [w.name, w]));
|
|
2540
|
+
const bestArr = byName.get(best.name).arr;
|
|
2541
|
+
const pairwise = scores.slice(1).map((other) => {
|
|
2542
|
+
const otherArr = byName.get(other.name).arr;
|
|
2543
|
+
const boot = pairedBootstrap(otherArr, bestArr, {
|
|
2544
|
+
seed,
|
|
2545
|
+
resamples,
|
|
2546
|
+
confidence: intervalConfidence,
|
|
2547
|
+
statistic: "mean"
|
|
2548
|
+
});
|
|
2549
|
+
const favored = boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
|
|
2550
|
+
return {
|
|
2551
|
+
a: best.name,
|
|
2552
|
+
b: other.name,
|
|
2553
|
+
deltaMean: boot.mean,
|
|
2554
|
+
low: boot.low,
|
|
2555
|
+
high: boot.high,
|
|
2556
|
+
favored
|
|
2557
|
+
};
|
|
2558
|
+
});
|
|
2559
|
+
const optimizationCost = combineCosts(
|
|
2560
|
+
scores.map((score) => ({ label: `method '${score.name}'`, cost: score.optimizationCost }))
|
|
2561
|
+
);
|
|
2562
|
+
const testCost = costFromLedgerSummary(testCostLedger.summary({ phase: testCostPhase }));
|
|
2563
|
+
const totalCost = combineCosts([
|
|
2564
|
+
{ label: "optimization", cost: optimizationCost },
|
|
2565
|
+
{ label: "final test", cost: testCost }
|
|
2566
|
+
]);
|
|
2567
|
+
return {
|
|
2568
|
+
scores,
|
|
2569
|
+
best,
|
|
2570
|
+
pairwise,
|
|
2571
|
+
testScenarioIds: scenarioIds,
|
|
2572
|
+
optimizationCost,
|
|
2573
|
+
testCost,
|
|
2574
|
+
totalCost,
|
|
2575
|
+
confidence,
|
|
2576
|
+
intervalConfidence,
|
|
2577
|
+
comparisonCount,
|
|
2578
|
+
seed,
|
|
2579
|
+
resamples,
|
|
2580
|
+
reps: opts.reps ?? 1
|
|
2581
|
+
};
|
|
2582
|
+
}
|
|
2583
|
+
function assertOptimizationMethods(methods) {
|
|
2584
|
+
if (!Array.isArray(methods) || methods.length === 0) {
|
|
2585
|
+
throw new Error("compareOptimizationMethods: no methods to compare");
|
|
2586
|
+
}
|
|
2587
|
+
const names = /* @__PURE__ */ new Set();
|
|
2588
|
+
const pathOwners = /* @__PURE__ */ new Map();
|
|
2589
|
+
for (const method of methods) {
|
|
2590
|
+
if (!method || typeof method !== "object" || typeof method.optimize !== "function") {
|
|
2591
|
+
throw new Error("compareOptimizationMethods: every method must provide optimize(input)");
|
|
2568
2592
|
}
|
|
2569
|
-
if (!
|
|
2570
|
-
throw new
|
|
2571
|
-
"missing_captured_at",
|
|
2572
|
-
"LabeledScenarioWrite requires `capturedAt` ISO timestamp"
|
|
2573
|
-
);
|
|
2593
|
+
if (!method.name || method.name.trim() !== method.name) {
|
|
2594
|
+
throw new Error("compareOptimizationMethods: method names must be trimmed and non-empty");
|
|
2574
2595
|
}
|
|
2575
|
-
if (
|
|
2576
|
-
throw new
|
|
2577
|
-
"missing_redaction_status",
|
|
2578
|
-
"LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
|
|
2579
|
-
);
|
|
2596
|
+
if (names.has(method.name)) {
|
|
2597
|
+
throw new Error(`compareOptimizationMethods: duplicate method name '${method.name}'`);
|
|
2580
2598
|
}
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2599
|
+
names.add(method.name);
|
|
2600
|
+
const pathKey = slug(method.name);
|
|
2601
|
+
const prior = pathOwners.get(pathKey);
|
|
2602
|
+
if (prior) {
|
|
2603
|
+
throw new Error(
|
|
2604
|
+
`compareOptimizationMethods: method names '${prior}' and '${method.name}' map to the same run path '${pathKey}'`
|
|
2585
2605
|
);
|
|
2586
2606
|
}
|
|
2607
|
+
pathOwners.set(pathKey, method.name);
|
|
2587
2608
|
}
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
const windowMs = 6e4;
|
|
2593
|
-
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
2594
|
-
if (!state || now - state.windowStartMs >= windowMs) {
|
|
2595
|
-
state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
|
|
2596
|
-
this.rateLimits.set(write.rateLimitBucket, state);
|
|
2597
|
-
}
|
|
2598
|
-
if (state.count >= cap) {
|
|
2599
|
-
throw new LabeledScenarioStoreError(
|
|
2600
|
-
"rate_limit_exceeded",
|
|
2601
|
-
`LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
|
|
2602
|
-
);
|
|
2603
|
-
}
|
|
2604
|
-
state.count += 1;
|
|
2609
|
+
}
|
|
2610
|
+
function assertOptimizationResult(name, result) {
|
|
2611
|
+
if (!result || typeof result !== "object") {
|
|
2612
|
+
throw new Error(`compareOptimizationMethods: method '${name}' returned no result`);
|
|
2605
2613
|
}
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
ver: write.sourceVersionHash
|
|
2613
|
-
})
|
|
2614
|
+
try {
|
|
2615
|
+
surfaceContentHash(result.winnerSurface);
|
|
2616
|
+
} catch (cause) {
|
|
2617
|
+
throw new Error(
|
|
2618
|
+
`compareOptimizationMethods: method '${name}' returned an invalid winnerSurface`,
|
|
2619
|
+
{ cause }
|
|
2614
2620
|
);
|
|
2615
|
-
return {
|
|
2616
|
-
...write,
|
|
2617
|
-
recordHash,
|
|
2618
|
-
split: "train"
|
|
2619
|
-
};
|
|
2620
2621
|
}
|
|
2621
|
-
|
|
2622
|
-
|
|
2622
|
+
assertComparisonCost(result.cost, `method '${name}'`);
|
|
2623
|
+
if (result.durationMs !== void 0 && (!Number.isFinite(result.durationMs) || result.durationMs < 0)) {
|
|
2624
|
+
throw new Error(`compareOptimizationMethods: method '${name}' returned an invalid durationMs`);
|
|
2623
2625
|
}
|
|
2624
|
-
};
|
|
2625
|
-
var ALL_SOURCES = [
|
|
2626
|
-
"production-trace",
|
|
2627
|
-
"eval-run",
|
|
2628
|
-
"manual",
|
|
2629
|
-
"red-team",
|
|
2630
|
-
"synthetic"
|
|
2631
|
-
];
|
|
2632
|
-
function sourceFilterContains(filter, needle) {
|
|
2633
|
-
if (!filter) return false;
|
|
2634
|
-
if (Array.isArray(filter)) return filter.includes(needle);
|
|
2635
|
-
return filter === needle;
|
|
2636
2626
|
}
|
|
2637
|
-
function
|
|
2638
|
-
if (
|
|
2639
|
-
|
|
2640
|
-
const f = args.filter;
|
|
2641
|
-
if (!f) return true;
|
|
2642
|
-
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
2643
|
-
if (f.source) {
|
|
2644
|
-
const sources = Array.isArray(f.source) ? f.source : [f.source];
|
|
2645
|
-
if (!sources.includes(source)) return false;
|
|
2627
|
+
function assertComparisonControls(opts, seed, resamples, confidence) {
|
|
2628
|
+
if (!opts.judges || opts.judges.length === 0) {
|
|
2629
|
+
throw new Error("compareOptimizationMethods: at least one judge is required");
|
|
2646
2630
|
}
|
|
2647
|
-
if (
|
|
2648
|
-
|
|
2649
|
-
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
2650
|
-
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
2651
|
-
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
2631
|
+
if (typeof opts.dispatchWithSurface !== "function") {
|
|
2632
|
+
throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
|
|
2652
2633
|
}
|
|
2653
|
-
|
|
2654
|
-
|
|
2634
|
+
try {
|
|
2635
|
+
surfaceContentHash(opts.baselineSurface);
|
|
2636
|
+
} catch (cause) {
|
|
2637
|
+
throw new Error("compareOptimizationMethods: baselineSurface is invalid", { cause });
|
|
2655
2638
|
}
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
function
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2639
|
+
const judgeNames = /* @__PURE__ */ new Set();
|
|
2640
|
+
for (const judge of opts.judges) {
|
|
2641
|
+
if (!judge || typeof judge !== "object" || typeof judge.name !== "string" || judge.name.trim().length === 0 || judge.name.trim() !== judge.name || typeof judge.score !== "function" || !Array.isArray(judge.dimensions) || judge.dimensions.length === 0) {
|
|
2642
|
+
throw new Error(
|
|
2643
|
+
"compareOptimizationMethods: every judge needs a trimmed name, at least one dimension, and score(input)"
|
|
2644
|
+
);
|
|
2645
|
+
}
|
|
2646
|
+
if (judgeNames.has(judge.name)) {
|
|
2647
|
+
throw new Error(`compareOptimizationMethods: duplicate judge name '${judge.name}'`);
|
|
2648
|
+
}
|
|
2649
|
+
judgeNames.add(judge.name);
|
|
2650
|
+
const dimensionKeys = /* @__PURE__ */ new Set();
|
|
2651
|
+
for (const dimension of judge.dimensions) {
|
|
2652
|
+
if (!dimension || typeof dimension.key !== "string" || dimension.key.trim().length === 0 || dimension.key.trim() !== dimension.key || typeof dimension.description !== "string" || dimension.description.trim().length === 0) {
|
|
2653
|
+
throw new Error(
|
|
2654
|
+
`compareOptimizationMethods: judge '${judge.name}' has an invalid dimension`
|
|
2655
|
+
);
|
|
2656
|
+
}
|
|
2657
|
+
if (dimensionKeys.has(dimension.key)) {
|
|
2658
|
+
throw new Error(
|
|
2659
|
+
`compareOptimizationMethods: judge '${judge.name}' has duplicate dimension '${dimension.key}'`
|
|
2660
|
+
);
|
|
2661
|
+
}
|
|
2662
|
+
dimensionKeys.add(dimension.key);
|
|
2663
|
+
}
|
|
2667
2664
|
}
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
// src/campaign/neutralize.ts
|
|
2671
|
-
var FILLER = "#";
|
|
2672
|
-
function neutralizeText(content) {
|
|
2673
|
-
return content.replace(/\S/g, FILLER);
|
|
2674
|
-
}
|
|
2675
|
-
|
|
2676
|
-
// src/campaign/proposers/fapo.ts
|
|
2677
|
-
var FAPO_LEVELS = ["prompt", "parameter", "structural"];
|
|
2678
|
-
var MAX_FINDING_DEPTH = 16;
|
|
2679
|
-
var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
2680
|
-
function fapoProposer(opts) {
|
|
2681
|
-
const proposers = levelProposers(opts);
|
|
2682
|
-
const allowed = allowedLevels(opts, proposers);
|
|
2683
|
-
const plateauWindow = opts.plateauWindow ?? 3;
|
|
2684
|
-
const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
|
|
2685
|
-
const minImprovement = opts.minImprovement ?? 0;
|
|
2686
|
-
const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
|
|
2687
|
-
if (allowed.length === 0) {
|
|
2688
|
-
throw new Error("fapoProposer: at least one allowed level must have a proposer");
|
|
2665
|
+
if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
|
|
2666
|
+
throw new Error("compareOptimizationMethods: runDir must be a non-empty string");
|
|
2689
2667
|
}
|
|
2690
|
-
if (
|
|
2691
|
-
|
|
2692
|
-
throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
|
|
2668
|
+
if (!Number.isSafeInteger(seed)) {
|
|
2669
|
+
throw new Error(`compareOptimizationMethods: seed must be a safe integer, got ${String(seed)}`);
|
|
2693
2670
|
}
|
|
2694
|
-
if (
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
}
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2671
|
+
if (!Number.isSafeInteger(resamples) || resamples <= 0 || resamples > 1e6) {
|
|
2672
|
+
throw new Error(
|
|
2673
|
+
`compareOptimizationMethods: resamples must be a positive safe integer no greater than 1000000, got ${String(resamples)}`
|
|
2674
|
+
);
|
|
2675
|
+
}
|
|
2676
|
+
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
|
|
2677
|
+
throw new Error(
|
|
2678
|
+
`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
|
|
2679
|
+
);
|
|
2680
|
+
}
|
|
2681
|
+
const minimumResamples = minimumBootstrapResamples(
|
|
2682
|
+
confidence,
|
|
2683
|
+
opts.methods.length * (opts.methods.length + 1) / 2
|
|
2684
|
+
);
|
|
2685
|
+
if (resamples < minimumResamples) {
|
|
2686
|
+
throw new Error(
|
|
2687
|
+
`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`
|
|
2688
|
+
);
|
|
2689
|
+
}
|
|
2690
|
+
if (opts.optimizationConcurrency !== void 0 && (!Number.isSafeInteger(opts.optimizationConcurrency) || opts.optimizationConcurrency <= 0)) {
|
|
2691
|
+
throw new Error(
|
|
2692
|
+
"compareOptimizationMethods: optimizationConcurrency must be a positive safe integer"
|
|
2693
|
+
);
|
|
2694
|
+
}
|
|
2695
|
+
if (opts.maxConcurrency !== void 0 && (!Number.isSafeInteger(opts.maxConcurrency) || opts.maxConcurrency <= 0)) {
|
|
2696
|
+
throw new Error("compareOptimizationMethods: maxConcurrency must be a positive safe integer");
|
|
2697
|
+
}
|
|
2698
|
+
if (opts.dispatchTimeoutMs !== void 0 && (!Number.isSafeInteger(opts.dispatchTimeoutMs) || opts.dispatchTimeoutMs < 0 || opts.dispatchTimeoutMs > 2147483647)) {
|
|
2699
|
+
throw new Error(
|
|
2700
|
+
"compareOptimizationMethods: dispatchTimeoutMs must be a non-negative safe integer no greater than 2147483647"
|
|
2701
|
+
);
|
|
2702
|
+
}
|
|
2703
|
+
if (opts.costCeiling !== void 0 && (!Number.isFinite(opts.costCeiling) || opts.costCeiling < 0)) {
|
|
2704
|
+
throw new Error(
|
|
2705
|
+
"compareOptimizationMethods: costCeiling must be a finite number greater than or equal to 0"
|
|
2706
|
+
);
|
|
2707
|
+
}
|
|
2708
|
+
if (opts.costCeiling !== void 0 && opts.costLedger !== void 0 && opts.costLedger.costCeilingUsd !== opts.costCeiling) {
|
|
2709
|
+
throw new Error(
|
|
2710
|
+
"compareOptimizationMethods: costCeiling must match the shared CostLedger ceiling"
|
|
2711
|
+
);
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
function assertComparisonPartitions(opts) {
|
|
2715
|
+
const legacy = opts;
|
|
2716
|
+
if (legacy.holdoutScenarios !== void 0) {
|
|
2717
|
+
throw new Error(
|
|
2718
|
+
"compareOptimizationMethods: holdoutScenarios is ambiguous and no longer accepted. Provide disjoint trainScenarios, selectionScenarios, and testScenarios; selection may be reused adaptively, test must remain untouched."
|
|
2719
|
+
);
|
|
2720
|
+
}
|
|
2721
|
+
const partitions = [
|
|
2722
|
+
{ name: "trainScenarios", scenarios: opts.trainScenarios },
|
|
2723
|
+
{ name: "selectionScenarios", scenarios: opts.selectionScenarios },
|
|
2724
|
+
{ name: "testScenarios", scenarios: opts.testScenarios }
|
|
2725
|
+
];
|
|
2726
|
+
const owner = /* @__PURE__ */ new Map();
|
|
2727
|
+
for (const partition of partitions) {
|
|
2728
|
+
if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) {
|
|
2729
|
+
throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
|
|
2730
|
+
}
|
|
2731
|
+
if (partition.name === "testScenarios" && partition.scenarios.length < 2) {
|
|
2732
|
+
throw new Error(
|
|
2733
|
+
"compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty"
|
|
2734
|
+
);
|
|
2735
|
+
}
|
|
2736
|
+
const local = /* @__PURE__ */ new Set();
|
|
2737
|
+
const duplicates = /* @__PURE__ */ new Set();
|
|
2738
|
+
const overlaps = /* @__PURE__ */ new Map();
|
|
2739
|
+
for (const scenario of partition.scenarios) {
|
|
2740
|
+
if (local.has(scenario.id)) duplicates.add(scenario.id);
|
|
2741
|
+
local.add(scenario.id);
|
|
2742
|
+
const prior = owner.get(scenario.id);
|
|
2743
|
+
if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
|
|
2744
|
+
}
|
|
2745
|
+
if (duplicates.size > 0) {
|
|
2746
|
+
throw new Error(
|
|
2747
|
+
`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[
|
|
2748
|
+
...duplicates
|
|
2749
|
+
].join(", ")}]`
|
|
2750
|
+
);
|
|
2751
|
+
}
|
|
2752
|
+
if (overlaps.size > 0) {
|
|
2753
|
+
const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} \u2229 ${partition.name})`).join(", ");
|
|
2754
|
+
throw new Error(
|
|
2755
|
+
`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`
|
|
2756
|
+
);
|
|
2757
|
+
}
|
|
2758
|
+
assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
|
|
2759
|
+
for (const id of local) owner.set(id, partition.name);
|
|
2760
|
+
}
|
|
2761
|
+
}
|
|
2762
|
+
function mean2(xs) {
|
|
2763
|
+
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
2764
|
+
}
|
|
2765
|
+
function slug(name) {
|
|
2766
|
+
return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
|
|
2767
|
+
}
|
|
2768
|
+
function assertConfidence(confidence) {
|
|
2769
|
+
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
|
|
2770
|
+
throw new Error(
|
|
2771
|
+
`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
|
|
2772
|
+
);
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
function minimumBootstrapResamples(confidence, comparisonCount) {
|
|
2776
|
+
const exact = 2 * comparisonCount / (1 - confidence);
|
|
2777
|
+
return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
|
|
2778
|
+
}
|
|
2779
|
+
function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed) {
|
|
2780
|
+
const cloneScenarios = (scenarios) => Object.freeze(scenarios.map((scenario) => structuredClone(scenario)));
|
|
2781
|
+
const judges = opts.judges.map(
|
|
2782
|
+
(judge) => Object.freeze({
|
|
2783
|
+
...judge,
|
|
2784
|
+
dimensions: Object.freeze(
|
|
2785
|
+
judge.dimensions.map((dimension) => Object.freeze({ ...dimension }))
|
|
2786
|
+
)
|
|
2787
|
+
})
|
|
2788
|
+
);
|
|
2789
|
+
return Object.freeze({
|
|
2790
|
+
baselineSurface: structuredClone(opts.baselineSurface),
|
|
2791
|
+
trainScenarios: cloneScenarios(opts.trainScenarios),
|
|
2792
|
+
selectionScenarios: cloneScenarios(opts.selectionScenarios),
|
|
2793
|
+
dispatchWithSurface: opts.dispatchWithSurface,
|
|
2794
|
+
judges: Object.freeze(judges),
|
|
2795
|
+
runDir: `${resolvedRunDir}/optimization/${slug(methodName)}`,
|
|
2796
|
+
seed,
|
|
2797
|
+
runOptions: Object.freeze({ ...opts.optimizationRunOptions ?? {} })
|
|
2798
|
+
});
|
|
2799
|
+
}
|
|
2800
|
+
function costFromLedgerSummary(summary) {
|
|
2801
|
+
const cost = {
|
|
2802
|
+
totalCostUsd: summary.totalCostUsd,
|
|
2803
|
+
accountingComplete: summary.accountingComplete,
|
|
2804
|
+
incompleteReasons: [...summary.incompleteReasons]
|
|
2805
|
+
};
|
|
2806
|
+
assertComparisonCost(cost, "cost ledger");
|
|
2807
|
+
return cost;
|
|
2808
|
+
}
|
|
2809
|
+
function combineCosts(entries) {
|
|
2810
|
+
return {
|
|
2811
|
+
totalCostUsd: entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0),
|
|
2812
|
+
accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
|
|
2813
|
+
incompleteReasons: entries.flatMap(
|
|
2814
|
+
(entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`)
|
|
2815
|
+
)
|
|
2816
|
+
};
|
|
2817
|
+
}
|
|
2818
|
+
function assertComparisonCost(cost, label) {
|
|
2819
|
+
if (!cost || typeof cost !== "object") {
|
|
2820
|
+
throw new Error(`compareOptimizationMethods: ${label} returned no cost`);
|
|
2821
|
+
}
|
|
2822
|
+
if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) {
|
|
2823
|
+
throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
|
|
2824
|
+
}
|
|
2825
|
+
if (typeof cost.accountingComplete !== "boolean") {
|
|
2826
|
+
throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
|
|
2827
|
+
}
|
|
2828
|
+
if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some(
|
|
2829
|
+
(reason) => typeof reason !== "string" || reason.trim().length === 0
|
|
2830
|
+
)) {
|
|
2831
|
+
throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
|
|
2832
|
+
}
|
|
2833
|
+
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) {
|
|
2834
|
+
throw new Error(
|
|
2835
|
+
`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`
|
|
2836
|
+
);
|
|
2837
|
+
}
|
|
2838
|
+
}
|
|
2839
|
+
|
|
2840
|
+
// src/campaign/gepa-optimization-method.ts
|
|
2841
|
+
var DEFAULT_MAX_CANDIDATE_CHARS = 2e5;
|
|
2842
|
+
var DEFAULT_TIMEOUT_MS = 30 * 60 * 1e3;
|
|
2843
|
+
var MAX_CALLBACK_BODY_BYTES = 1e6;
|
|
2844
|
+
var MAX_PROCESS_OUTPUT_CHARS = 64e3;
|
|
2845
|
+
var PROCESS_TERMINATION_GRACE_MS = 5e3;
|
|
2846
|
+
function gepaOptimizationMethod(config) {
|
|
2847
|
+
assertConfig(config);
|
|
2848
|
+
const name = config.name ?? defaultMethodName(config.recipe);
|
|
2849
|
+
return {
|
|
2850
|
+
name,
|
|
2851
|
+
async optimize(input) {
|
|
2852
|
+
if (typeof input.baselineSurface !== "string") {
|
|
2853
|
+
throw new Error(`${name}: GEPA bridge requires a string baselineSurface`);
|
|
2733
2854
|
}
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2855
|
+
const started = Date.now();
|
|
2856
|
+
const maxCandidateChars = config.maxCandidateChars ?? DEFAULT_MAX_CANDIDATE_CHARS;
|
|
2857
|
+
const storage = input.runOptions.storage ?? fsCampaignStorage();
|
|
2858
|
+
const runDir = `${input.runDir}/gepa`;
|
|
2859
|
+
storage.ensureDir(runDir);
|
|
2860
|
+
const costLedger = createRunCostLedger({
|
|
2861
|
+
storage,
|
|
2862
|
+
runDir,
|
|
2863
|
+
costCeilingUsd: input.runOptions.costCeiling
|
|
2864
|
+
});
|
|
2865
|
+
const scenarioById = scenarioMap(input.trainScenarios, input.selectionScenarios);
|
|
2866
|
+
const evaluate = createEvaluationFunction({
|
|
2867
|
+
input,
|
|
2868
|
+
config,
|
|
2869
|
+
runDir,
|
|
2870
|
+
costLedger,
|
|
2871
|
+
scenarioById,
|
|
2872
|
+
maxCandidateChars
|
|
2873
|
+
});
|
|
2874
|
+
const callback = await startCallbackServer({
|
|
2875
|
+
token: randomBytes(32).toString("hex"),
|
|
2876
|
+
maxEvaluations: recipeEvaluationLimit(config.recipe),
|
|
2877
|
+
evaluate
|
|
2878
|
+
});
|
|
2879
|
+
try {
|
|
2880
|
+
const outputDir = `${runDir}/external`;
|
|
2881
|
+
await mkdir(outputDir, { recursive: true });
|
|
2882
|
+
const result = await runGepaBridge(
|
|
2883
|
+
{
|
|
2884
|
+
version: 2,
|
|
2885
|
+
callbackUrl: callback.url,
|
|
2886
|
+
callbackToken: callback.token,
|
|
2887
|
+
recipe: config.recipe,
|
|
2888
|
+
objective: config.objective,
|
|
2889
|
+
...config.background ? { background: config.background } : {},
|
|
2890
|
+
seedCandidate: input.baselineSurface,
|
|
2891
|
+
trainSet: input.trainScenarios.map((scenario) => describeScenario(config, scenario)),
|
|
2892
|
+
selectionSet: input.selectionScenarios.map(
|
|
2893
|
+
(scenario) => describeScenario(config, scenario)
|
|
2894
|
+
),
|
|
2895
|
+
maxCandidateChars,
|
|
2896
|
+
outputDir
|
|
2897
|
+
},
|
|
2898
|
+
config.runner,
|
|
2899
|
+
config.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
|
2737
2900
|
);
|
|
2901
|
+
assertBridgeOutput(
|
|
2902
|
+
result,
|
|
2903
|
+
name,
|
|
2904
|
+
maxCandidateChars,
|
|
2905
|
+
config.recipe.kind,
|
|
2906
|
+
recipeEvaluationLimit(config.recipe)
|
|
2907
|
+
);
|
|
2908
|
+
const evaluationCost = costFromLedgerSummary(costLedger.summary());
|
|
2909
|
+
const reportedProposerCost = result.proposerCostUsd ?? 0;
|
|
2910
|
+
return {
|
|
2911
|
+
winnerSurface: result.bestCandidate,
|
|
2912
|
+
cost: {
|
|
2913
|
+
totalCostUsd: evaluationCost.totalCostUsd + reportedProposerCost,
|
|
2914
|
+
accountingComplete: false,
|
|
2915
|
+
incompleteReasons: [
|
|
2916
|
+
...evaluationCost.incompleteReasons,
|
|
2917
|
+
result.proposerCostAccounting === "reported" ? "GEPA proposer cost is externally reported and has no agent-eval receipt" : "GEPA proposer cost is unavailable"
|
|
2918
|
+
]
|
|
2919
|
+
},
|
|
2920
|
+
durationMs: Date.now() - started
|
|
2921
|
+
};
|
|
2922
|
+
} finally {
|
|
2923
|
+
await callback.close();
|
|
2738
2924
|
}
|
|
2739
|
-
return reviewed;
|
|
2740
|
-
},
|
|
2741
|
-
decide({ history }) {
|
|
2742
|
-
const last = history[history.length - 1];
|
|
2743
|
-
if (last && last.candidates.length === 0) {
|
|
2744
|
-
return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
|
|
2745
|
-
}
|
|
2746
|
-
const state = policyState(history, [], {
|
|
2747
|
-
plateauWindow,
|
|
2748
|
-
minDistinctStrategies,
|
|
2749
|
-
minImprovement
|
|
2750
|
-
});
|
|
2751
|
-
const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
|
|
2752
|
-
return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
|
|
2753
2925
|
}
|
|
2754
2926
|
};
|
|
2755
2927
|
}
|
|
2756
|
-
function
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
|
|
2928
|
+
function createEvaluationFunction(args) {
|
|
2929
|
+
const cached = /* @__PURE__ */ new Map();
|
|
2930
|
+
return async ({ candidate, exampleId }) => {
|
|
2931
|
+
if (!args.scenarioById.has(exampleId)) {
|
|
2932
|
+
throw new Error(`GEPA requested unknown train or selection case '${exampleId}'`);
|
|
2933
|
+
}
|
|
2934
|
+
if (!isCandidateText(candidate, args.maxCandidateChars)) {
|
|
2935
|
+
throw new Error("GEPA submitted an invalid candidate");
|
|
2936
|
+
}
|
|
2937
|
+
const scenario = args.scenarioById.get(exampleId);
|
|
2938
|
+
const cacheKey = `${surfaceContentHash(candidate)}:${exampleId}`;
|
|
2939
|
+
const existing = cached.get(cacheKey);
|
|
2940
|
+
if (existing) return existing;
|
|
2941
|
+
const result = scoreOneScenario({
|
|
2942
|
+
input: args.input,
|
|
2943
|
+
candidate,
|
|
2944
|
+
scenario,
|
|
2945
|
+
runDir: args.runDir,
|
|
2946
|
+
costLedger: args.costLedger
|
|
2947
|
+
});
|
|
2948
|
+
cached.set(cacheKey, result);
|
|
2949
|
+
return result;
|
|
2762
2950
|
};
|
|
2763
2951
|
}
|
|
2764
|
-
function
|
|
2765
|
-
const
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2952
|
+
async function scoreOneScenario(args) {
|
|
2953
|
+
const campaign = await runCampaign({
|
|
2954
|
+
...args.input.runOptions,
|
|
2955
|
+
scenarios: [structuredClone(args.scenario)],
|
|
2956
|
+
dispatch: (scenario, context) => args.input.dispatchWithSurface(args.candidate, scenario, context),
|
|
2957
|
+
judges: [...args.input.judges],
|
|
2958
|
+
runDir: `${args.runDir}/evaluations/${safePathComponent(surfaceContentHash(args.candidate))}/${safePathComponent(args.scenario.id)}`,
|
|
2959
|
+
seed: args.input.seed,
|
|
2960
|
+
costLedger: args.costLedger,
|
|
2961
|
+
costPhase: "gepa.external-evaluation",
|
|
2962
|
+
maxConcurrency: 1
|
|
2771
2963
|
});
|
|
2964
|
+
const breakdown = campaignBreakdown(campaign);
|
|
2965
|
+
const row = breakdown.scenarios[0];
|
|
2966
|
+
if (!row) throw new Error(`GEPA evaluation produced no score for '${args.scenario.id}'`);
|
|
2967
|
+
return {
|
|
2968
|
+
score: row.composite,
|
|
2969
|
+
info: {
|
|
2970
|
+
scenarioId: row.scenarioId,
|
|
2971
|
+
dimensions: breakdown.dimensions,
|
|
2972
|
+
...row.notes ? { notes: row.notes } : {}
|
|
2973
|
+
}
|
|
2974
|
+
};
|
|
2772
2975
|
}
|
|
2773
|
-
function
|
|
2774
|
-
const
|
|
2976
|
+
function scenarioMap(train, selection) {
|
|
2977
|
+
const out = /* @__PURE__ */ new Map();
|
|
2978
|
+
for (const scenario of [...train, ...selection]) {
|
|
2979
|
+
if (out.has(scenario.id)) {
|
|
2980
|
+
throw new Error(
|
|
2981
|
+
`GEPA bridge requires unique train and selection ids; duplicate '${scenario.id}'`
|
|
2982
|
+
);
|
|
2983
|
+
}
|
|
2984
|
+
out.set(scenario.id, scenario);
|
|
2985
|
+
}
|
|
2986
|
+
return out;
|
|
2987
|
+
}
|
|
2988
|
+
function describeScenario(config, scenario) {
|
|
2989
|
+
const data = config.describeScenario ? config.describeScenario(scenario) : { id: scenario.id };
|
|
2990
|
+
assertJsonValue(data, `GEPA scenario '${scenario.id}'`);
|
|
2991
|
+
return { id: scenario.id, data };
|
|
2992
|
+
}
|
|
2993
|
+
async function startCallbackServer(args) {
|
|
2994
|
+
let evaluations = 0;
|
|
2995
|
+
const server = createServer((request, response) => {
|
|
2996
|
+
void handleCallback(request, response, args, () => {
|
|
2997
|
+
evaluations += 1;
|
|
2998
|
+
return evaluations;
|
|
2999
|
+
});
|
|
3000
|
+
});
|
|
3001
|
+
const port = await listen(server);
|
|
2775
3002
|
return {
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
parameter: isLevelExhausted("parameter", attempts, opts),
|
|
2780
|
-
structural: isLevelExhausted("structural", attempts, opts)
|
|
2781
|
-
},
|
|
2782
|
-
signals: extractFapoAttributionSignals(findings)
|
|
3003
|
+
url: `http://127.0.0.1:${port}/evaluate`,
|
|
3004
|
+
token: args.token,
|
|
3005
|
+
close: () => closeServer(server)
|
|
2783
3006
|
};
|
|
2784
3007
|
}
|
|
2785
|
-
function
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
3008
|
+
async function handleCallback(request, response, args, nextEvaluation) {
|
|
3009
|
+
try {
|
|
3010
|
+
if (request.method !== "POST" || request.url !== "/evaluate") {
|
|
3011
|
+
sendJson(response, 404, { error: "not found" });
|
|
3012
|
+
return;
|
|
3013
|
+
}
|
|
3014
|
+
if (request.headers.authorization !== `Bearer ${args.token}`) {
|
|
3015
|
+
sendJson(response, 401, { error: "unauthorized" });
|
|
3016
|
+
return;
|
|
3017
|
+
}
|
|
3018
|
+
const body = await readJson(request);
|
|
3019
|
+
if (!isRecord(body) || typeof body.candidate !== "string" || typeof body.exampleId !== "string") {
|
|
3020
|
+
sendJson(response, 400, { error: "candidate and exampleId are required strings" });
|
|
3021
|
+
return;
|
|
3022
|
+
}
|
|
3023
|
+
const count = nextEvaluation();
|
|
3024
|
+
if (count > args.maxEvaluations) {
|
|
3025
|
+
sendJson(response, 429, { error: "evaluation limit reached" });
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
const result = await args.evaluate({ candidate: body.candidate, exampleId: body.exampleId });
|
|
3029
|
+
sendJson(response, 200, result);
|
|
3030
|
+
} catch {
|
|
3031
|
+
sendJson(response, 500, { error: "evaluation failed" });
|
|
2797
3032
|
}
|
|
2798
|
-
return attempts;
|
|
2799
|
-
}
|
|
2800
|
-
function candidateLevel(candidate) {
|
|
2801
|
-
const label = candidate.label ?? "";
|
|
2802
|
-
const rationale = candidate.rationale ?? "";
|
|
2803
|
-
return parseLevel(label) ?? parseLevel(rationale);
|
|
2804
|
-
}
|
|
2805
|
-
function parseLevel(text) {
|
|
2806
|
-
const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
|
|
2807
|
-
return match ? match[1] : null;
|
|
2808
|
-
}
|
|
2809
|
-
function candidateStrategy(candidate) {
|
|
2810
|
-
const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
|
|
2811
|
-
return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
|
|
2812
3033
|
}
|
|
2813
|
-
function
|
|
2814
|
-
const
|
|
2815
|
-
|
|
2816
|
-
const
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
3034
|
+
async function runGepaBridge(input, runner, timeoutMs) {
|
|
3035
|
+
const dir = await mkdtemp(join2(tmpdir(), "agent-eval-gepa-"));
|
|
3036
|
+
const inputPath = join2(dir, "input.json");
|
|
3037
|
+
const outputPath = join2(dir, "output.json");
|
|
3038
|
+
try {
|
|
3039
|
+
await writeFile(inputPath, `${JSON.stringify(input)}
|
|
3040
|
+
`);
|
|
3041
|
+
const command = runner?.command ?? "python";
|
|
3042
|
+
const args = [
|
|
3043
|
+
...runner?.args ?? ["-m", "agent_eval_rpc.gepa_bridge"],
|
|
3044
|
+
"--input",
|
|
3045
|
+
inputPath,
|
|
3046
|
+
"--output",
|
|
3047
|
+
outputPath
|
|
3048
|
+
];
|
|
3049
|
+
await runProcess(command, args, runner?.cwd ?? dir, runner?.env, timeoutMs);
|
|
3050
|
+
const raw = JSON.parse(await readFile(outputPath, "utf8"));
|
|
3051
|
+
if (!isRecord(raw)) throw new Error("GEPA bridge output must be a JSON object");
|
|
3052
|
+
return raw;
|
|
3053
|
+
} finally {
|
|
3054
|
+
await rm(dir, { recursive: true, force: true });
|
|
2827
3055
|
}
|
|
2828
|
-
return nonImproving >= opts.plateauWindow;
|
|
2829
3056
|
}
|
|
2830
|
-
function
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
if (strongest) {
|
|
2851
|
-
if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
|
|
2852
|
-
return {
|
|
2853
|
-
level: "parameter",
|
|
2854
|
-
reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
|
|
2855
|
-
};
|
|
2856
|
-
}
|
|
2857
|
-
return {
|
|
2858
|
-
level: strongest,
|
|
2859
|
-
reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
|
|
3057
|
+
function runProcess(command, args, cwd, env, timeoutMs) {
|
|
3058
|
+
return new Promise((resolvePromise, reject) => {
|
|
3059
|
+
const child = spawn(command, args, {
|
|
3060
|
+
cwd,
|
|
3061
|
+
env: { ...process.env, ...env },
|
|
3062
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
3063
|
+
});
|
|
3064
|
+
let stdout = "";
|
|
3065
|
+
let stderr = "";
|
|
3066
|
+
let settled = false;
|
|
3067
|
+
let timedOut = false;
|
|
3068
|
+
let timeout;
|
|
3069
|
+
let terminationGrace;
|
|
3070
|
+
const finish = (error) => {
|
|
3071
|
+
if (settled) return;
|
|
3072
|
+
settled = true;
|
|
3073
|
+
if (timeout) clearTimeout(timeout);
|
|
3074
|
+
if (terminationGrace) clearTimeout(terminationGrace);
|
|
3075
|
+
if (error) reject(error);
|
|
3076
|
+
else resolvePromise();
|
|
2860
3077
|
};
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
}
|
|
2868
|
-
|
|
2869
|
-
|
|
3078
|
+
timeout = setTimeout(() => {
|
|
3079
|
+
timedOut = true;
|
|
3080
|
+
child.kill("SIGTERM");
|
|
3081
|
+
terminationGrace = setTimeout(() => {
|
|
3082
|
+
child.kill("SIGKILL");
|
|
3083
|
+
finish(new Error(`GEPA bridge exceeded ${timeoutMs}ms`));
|
|
3084
|
+
}, PROCESS_TERMINATION_GRACE_MS);
|
|
3085
|
+
}, timeoutMs);
|
|
3086
|
+
child.stdout.on("data", (chunk) => {
|
|
3087
|
+
stdout = appendProcessOutput(stdout, chunk);
|
|
3088
|
+
});
|
|
3089
|
+
child.stderr.on("data", (chunk) => {
|
|
3090
|
+
stderr = appendProcessOutput(stderr, chunk);
|
|
3091
|
+
});
|
|
3092
|
+
child.on("error", (error) => finish(new Error(`GEPA bridge could not start: ${error.message}`)));
|
|
3093
|
+
child.on("close", (code) => {
|
|
3094
|
+
if (timedOut) {
|
|
3095
|
+
finish(new Error(`GEPA bridge exceeded ${timeoutMs}ms`));
|
|
3096
|
+
return;
|
|
3097
|
+
}
|
|
3098
|
+
if (code === 0) {
|
|
3099
|
+
finish();
|
|
3100
|
+
return;
|
|
3101
|
+
}
|
|
3102
|
+
finish(
|
|
3103
|
+
new Error(
|
|
3104
|
+
`GEPA bridge exited ${String(code)}. stderr=${truncate(stderr)} stdout=${truncate(stdout)}`
|
|
3105
|
+
)
|
|
3106
|
+
);
|
|
3107
|
+
});
|
|
3108
|
+
});
|
|
2870
3109
|
}
|
|
2871
|
-
function
|
|
2872
|
-
|
|
3110
|
+
function appendProcessOutput(current, chunk) {
|
|
3111
|
+
if (current.length >= MAX_PROCESS_OUTPUT_CHARS) return current;
|
|
3112
|
+
return `${current}${chunk.toString()}`.slice(0, MAX_PROCESS_OUTPUT_CHARS);
|
|
3113
|
+
}
|
|
3114
|
+
function readJson(request) {
|
|
3115
|
+
return new Promise((resolvePromise, reject) => {
|
|
3116
|
+
let size = 0;
|
|
3117
|
+
const chunks = [];
|
|
3118
|
+
request.on("data", (chunk) => {
|
|
3119
|
+
size += chunk.length;
|
|
3120
|
+
if (size > MAX_CALLBACK_BODY_BYTES) {
|
|
3121
|
+
reject(new Error("callback body too large"));
|
|
3122
|
+
request.destroy();
|
|
3123
|
+
return;
|
|
3124
|
+
}
|
|
3125
|
+
chunks.push(chunk);
|
|
3126
|
+
});
|
|
3127
|
+
request.on("error", reject);
|
|
3128
|
+
request.on("end", () => {
|
|
3129
|
+
try {
|
|
3130
|
+
resolvePromise(JSON.parse(Buffer.concat(chunks).toString("utf8")));
|
|
3131
|
+
} catch (error) {
|
|
3132
|
+
reject(error);
|
|
3133
|
+
}
|
|
3134
|
+
});
|
|
3135
|
+
});
|
|
2873
3136
|
}
|
|
2874
|
-
function
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
3137
|
+
function listen(server) {
|
|
3138
|
+
return new Promise((resolvePromise, reject) => {
|
|
3139
|
+
server.once("error", reject);
|
|
3140
|
+
server.listen(0, "127.0.0.1", () => {
|
|
3141
|
+
server.off("error", reject);
|
|
3142
|
+
const address = server.address();
|
|
3143
|
+
if (!address || typeof address === "string") {
|
|
3144
|
+
reject(new Error("GEPA callback did not bind a TCP port"));
|
|
3145
|
+
return;
|
|
3146
|
+
}
|
|
3147
|
+
resolvePromise(address.port);
|
|
3148
|
+
});
|
|
3149
|
+
});
|
|
2885
3150
|
}
|
|
2886
|
-
function
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
|
|
3151
|
+
function closeServer(server) {
|
|
3152
|
+
return new Promise((resolvePromise, reject) => {
|
|
3153
|
+
server.close((error) => error ? reject(error) : resolvePromise());
|
|
3154
|
+
});
|
|
2891
3155
|
}
|
|
2892
|
-
function
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
3156
|
+
function assertConfig(config) {
|
|
3157
|
+
if (!config.objective || config.objective.trim() !== config.objective) {
|
|
3158
|
+
throw new Error("gepaOptimizationMethod: objective must be trimmed and non-empty");
|
|
3159
|
+
}
|
|
3160
|
+
assertRecipe(config.recipe);
|
|
3161
|
+
if (config.maxCandidateChars !== void 0 && (!Number.isSafeInteger(config.maxCandidateChars) || config.maxCandidateChars <= 0)) {
|
|
3162
|
+
throw new Error("gepaOptimizationMethod: maxCandidateChars must be a positive safe integer");
|
|
3163
|
+
}
|
|
3164
|
+
if (config.timeoutMs !== void 0 && (!Number.isSafeInteger(config.timeoutMs) || config.timeoutMs <= 0)) {
|
|
3165
|
+
throw new Error("gepaOptimizationMethod: timeoutMs must be a positive safe integer");
|
|
2900
3166
|
}
|
|
2901
|
-
return signals;
|
|
2902
3167
|
}
|
|
2903
|
-
function
|
|
2904
|
-
if (
|
|
2905
|
-
|
|
2906
|
-
if (typeof finding === "string") {
|
|
2907
|
-
const level = inferLevelFromText(finding);
|
|
2908
|
-
if (level) addCluster(signals, { label: finding, level, count: 1 });
|
|
2909
|
-
return;
|
|
3168
|
+
function assertRecipe(recipe) {
|
|
3169
|
+
if (!recipe || typeof recipe !== "object") {
|
|
3170
|
+
throw new Error("gepaOptimizationMethod: recipe is required");
|
|
2910
3171
|
}
|
|
2911
|
-
if (
|
|
2912
|
-
|
|
2913
|
-
seen.add(finding);
|
|
2914
|
-
for (const item of finding) collectFinding(signals, item, seen, depth + 1);
|
|
3172
|
+
if (recipe.kind === "engine") {
|
|
3173
|
+
assertEngineRun(recipe.run, "recipe.run");
|
|
2915
3174
|
return;
|
|
2916
3175
|
}
|
|
2917
|
-
if (
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
addCluster(signals, {
|
|
2929
|
-
label: text || inferred,
|
|
2930
|
-
level: inferred,
|
|
2931
|
-
count: numberField(obj.count) ?? 1,
|
|
2932
|
-
confidence: confidenceField(obj.confidence),
|
|
2933
|
-
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
2934
|
-
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
2935
|
-
});
|
|
3176
|
+
if (recipe.kind === "best-of-then-continue") {
|
|
3177
|
+
if (!Array.isArray(recipe.explore) || recipe.explore.length < 2) {
|
|
3178
|
+
throw new Error(
|
|
3179
|
+
"gepaOptimizationMethod: recipe.explore must contain at least two bounded engine runs"
|
|
3180
|
+
);
|
|
3181
|
+
}
|
|
3182
|
+
for (const [index, run] of recipe.explore.entries()) {
|
|
3183
|
+
assertEngineRun(run, `recipe.explore[${index}]`);
|
|
3184
|
+
}
|
|
3185
|
+
assertEngineRun(recipe.continueWith, "recipe.continueWith");
|
|
3186
|
+
return;
|
|
2936
3187
|
}
|
|
3188
|
+
throw new Error("gepaOptimizationMethod: unsupported recipe");
|
|
2937
3189
|
}
|
|
2938
|
-
function
|
|
2939
|
-
if (!
|
|
2940
|
-
|
|
2941
|
-
seen.add(raw);
|
|
2942
|
-
const partition = raw;
|
|
2943
|
-
for (const level of FAPO_LEVELS) {
|
|
2944
|
-
const bucket = partition[level];
|
|
2945
|
-
if (!bucket || typeof bucket !== "object") continue;
|
|
2946
|
-
const obj = bucket;
|
|
2947
|
-
const count = numberField(obj.count) ?? 0;
|
|
2948
|
-
if (count > 0) signals.counts[level] += count;
|
|
2949
|
-
collectClusters(signals, obj.clusters, false, seen, depth + 1);
|
|
3190
|
+
function assertEngineRun(run, label) {
|
|
3191
|
+
if (!run || typeof run !== "object") {
|
|
3192
|
+
throw new Error(`gepaOptimizationMethod: ${label} is required`);
|
|
2950
3193
|
}
|
|
3194
|
+
if (typeof run.engine !== "string" || !run.engine.trim() || run.engine.trim() !== run.engine) {
|
|
3195
|
+
throw new Error(`gepaOptimizationMethod: ${label}.engine must be a trimmed non-empty string`);
|
|
3196
|
+
}
|
|
3197
|
+
if (!Number.isSafeInteger(run.maxEvaluations) || run.maxEvaluations <= 0) {
|
|
3198
|
+
throw new Error(
|
|
3199
|
+
`gepaOptimizationMethod: ${label}.maxEvaluations must be a positive safe integer`
|
|
3200
|
+
);
|
|
3201
|
+
}
|
|
3202
|
+
if (!Number.isFinite(run.maxProposerCostUsd) || run.maxProposerCostUsd <= 0) {
|
|
3203
|
+
throw new Error(
|
|
3204
|
+
`gepaOptimizationMethod: ${label}.maxProposerCostUsd must be a positive finite number`
|
|
3205
|
+
);
|
|
3206
|
+
}
|
|
3207
|
+
assertJsonValue(run.engineConfig ?? {}, `gepaOptimizationMethod: ${label}.engineConfig`);
|
|
2951
3208
|
}
|
|
2952
|
-
function
|
|
2953
|
-
const
|
|
2954
|
-
|
|
2955
|
-
const
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
}
|
|
2960
|
-
function collectClusters(signals, raw, countClusters, seen, depth) {
|
|
2961
|
-
if (!Array.isArray(raw)) return;
|
|
2962
|
-
if (seen.has(raw)) return;
|
|
2963
|
-
seen.add(raw);
|
|
2964
|
-
for (const item of raw) {
|
|
2965
|
-
if (countClusters) {
|
|
2966
|
-
collectFinding(signals, item, seen, depth + 1);
|
|
2967
|
-
continue;
|
|
3209
|
+
function recipeEvaluationLimit(recipe) {
|
|
3210
|
+
const runs = recipe.kind === "engine" ? [recipe.run] : [...recipe.explore, recipe.continueWith];
|
|
3211
|
+
let total = 0;
|
|
3212
|
+
for (const run of runs) {
|
|
3213
|
+
total += run.maxEvaluations;
|
|
3214
|
+
if (!Number.isSafeInteger(total)) {
|
|
3215
|
+
throw new Error("gepaOptimizationMethod: recipe evaluation limit exceeds safe integer range");
|
|
2968
3216
|
}
|
|
2969
|
-
const cluster = parseCluster(item);
|
|
2970
|
-
if (cluster) signals.clusters.push(cluster);
|
|
2971
3217
|
}
|
|
3218
|
+
return total;
|
|
2972
3219
|
}
|
|
2973
|
-
function
|
|
2974
|
-
|
|
2975
|
-
|
|
3220
|
+
function defaultMethodName(recipe) {
|
|
3221
|
+
if (recipe.kind === "engine") return `gepa:${recipe.run.engine}`;
|
|
3222
|
+
return `gepa:best-of-then-continue:${recipe.continueWith.engine}`;
|
|
2976
3223
|
}
|
|
2977
|
-
function
|
|
2978
|
-
if (
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
}
|
|
3224
|
+
function assertBridgeOutput(result, name, maxCandidateChars, recipeKind, maxEvaluations) {
|
|
3225
|
+
if (result.recipeKind !== recipeKind)
|
|
3226
|
+
throw new Error(`${name}: GEPA bridge reported recipe '${String(result.recipeKind)}'`);
|
|
3227
|
+
if (!isCandidateText(result.bestCandidate, maxCandidateChars)) {
|
|
3228
|
+
throw new Error(`${name}: GEPA bridge returned an invalid candidate`);
|
|
3229
|
+
}
|
|
3230
|
+
if (!Number.isFinite(result.bestScore))
|
|
3231
|
+
throw new Error(`${name}: GEPA bridge returned an invalid bestScore`);
|
|
3232
|
+
if (!Number.isSafeInteger(result.totalEvaluations) || result.totalEvaluations < 0 || result.totalEvaluations > maxEvaluations) {
|
|
3233
|
+
throw new Error(`${name}: GEPA bridge returned an invalid totalEvaluations`);
|
|
3234
|
+
}
|
|
3235
|
+
if (result.proposerCostUsd !== void 0 && (!Number.isFinite(result.proposerCostUsd) || result.proposerCostUsd < 0)) {
|
|
3236
|
+
throw new Error(`${name}: GEPA bridge returned an invalid proposerCostUsd`);
|
|
3237
|
+
}
|
|
2991
3238
|
}
|
|
2992
|
-
function
|
|
2993
|
-
|
|
2994
|
-
const lower = raw.toLowerCase();
|
|
2995
|
-
if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
|
|
2996
|
-
if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
|
|
2997
|
-
if (lower === "config" || lower === "params") return "parameter";
|
|
2998
|
-
return null;
|
|
3239
|
+
function isCandidateText(value, maxChars) {
|
|
3240
|
+
return typeof value === "string" && value.trim().length > 0 && value.length <= maxChars;
|
|
2999
3241
|
}
|
|
3000
|
-
function
|
|
3001
|
-
|
|
3002
|
-
if (
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
)) {
|
|
3010
|
-
return "structural";
|
|
3242
|
+
function assertJsonValue(value, label, seen = /* @__PURE__ */ new Set()) {
|
|
3243
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return;
|
|
3244
|
+
if (typeof value === "number" && Number.isFinite(value)) return;
|
|
3245
|
+
if (Array.isArray(value)) {
|
|
3246
|
+
if (seen.has(value)) throw new Error(`${label} must be JSON-serializable`);
|
|
3247
|
+
seen.add(value);
|
|
3248
|
+
for (const item of value) assertJsonValue(item, label, seen);
|
|
3249
|
+
seen.delete(value);
|
|
3250
|
+
return;
|
|
3011
3251
|
}
|
|
3012
|
-
if (
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3252
|
+
if (typeof value === "object") {
|
|
3253
|
+
if (seen.has(value)) throw new Error(`${label} must be JSON-serializable`);
|
|
3254
|
+
const prototype = Object.getPrototypeOf(value);
|
|
3255
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
3256
|
+
throw new Error(`${label} must be JSON-serializable`);
|
|
3257
|
+
}
|
|
3258
|
+
seen.add(value);
|
|
3259
|
+
for (const item of Object.values(value)) assertJsonValue(item, label, seen);
|
|
3260
|
+
seen.delete(value);
|
|
3261
|
+
return;
|
|
3016
3262
|
}
|
|
3017
|
-
|
|
3263
|
+
throw new Error(`${label} must be JSON-serializable`);
|
|
3018
3264
|
}
|
|
3019
|
-
function
|
|
3020
|
-
return [
|
|
3021
|
-
obj.label,
|
|
3022
|
-
obj.claim,
|
|
3023
|
-
obj.recommended_action,
|
|
3024
|
-
obj.suggested_fix,
|
|
3025
|
-
obj.suggestedFix,
|
|
3026
|
-
obj.message,
|
|
3027
|
-
obj.text,
|
|
3028
|
-
obj.heuristic,
|
|
3029
|
-
obj.area
|
|
3030
|
-
].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
|
|
3031
|
-
}
|
|
3032
|
-
function numberField(value) {
|
|
3033
|
-
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
3265
|
+
function safePathComponent(value) {
|
|
3266
|
+
return value.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
3034
3267
|
}
|
|
3035
|
-
function
|
|
3036
|
-
return typeof value === "
|
|
3268
|
+
function isRecord(value) {
|
|
3269
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3037
3270
|
}
|
|
3038
|
-
function
|
|
3039
|
-
|
|
3271
|
+
function sendJson(response, status, body) {
|
|
3272
|
+
response.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
|
3273
|
+
response.end(JSON.stringify(body));
|
|
3040
3274
|
}
|
|
3041
|
-
function
|
|
3042
|
-
|
|
3275
|
+
function truncate(value, max = 1e3) {
|
|
3276
|
+
const compact2 = value.trim().replace(/\s+/g, " ");
|
|
3277
|
+
return compact2.length <= max ? compact2 : `${compact2.slice(0, max)}\u2026`;
|
|
3043
3278
|
}
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
)
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
const currentCanonical = canonicalJson2(current);
|
|
3060
|
-
const tried = triedLabels(ctx.history);
|
|
3061
|
-
const out = [];
|
|
3062
|
-
for (const candidate of opts.candidates) {
|
|
3063
|
-
if (tried.has(candidate.label)) continue;
|
|
3064
|
-
const next = applyParameterCandidate(current, candidate);
|
|
3065
|
-
const surface = stringify(next);
|
|
3066
|
-
if (surface === ctx.currentSurface || canonicalJson2(next) === currentCanonical) continue;
|
|
3067
|
-
out.push({ surface, label: candidate.label, rationale: candidate.rationale });
|
|
3068
|
-
if (out.length >= ctx.populationSize) break;
|
|
3279
|
+
|
|
3280
|
+
// src/campaign/grounded-reflection.ts
|
|
3281
|
+
function rolloutArgumentDiff(rollouts, opts = {}) {
|
|
3282
|
+
const passThreshold = opts.passThreshold ?? 1;
|
|
3283
|
+
const maxValues = opts.maxValuesPerField ?? 4;
|
|
3284
|
+
const collect = (pass) => {
|
|
3285
|
+
const byField = /* @__PURE__ */ new Map();
|
|
3286
|
+
for (const r of rollouts) {
|
|
3287
|
+
if (pass !== r.score >= passThreshold) continue;
|
|
3288
|
+
for (const c of r.calls) {
|
|
3289
|
+
for (const [k, v] of Object.entries(c.args)) {
|
|
3290
|
+
const set = byField.get(k) ?? /* @__PURE__ */ new Set();
|
|
3291
|
+
set.add(String(v));
|
|
3292
|
+
byField.set(k, set);
|
|
3293
|
+
}
|
|
3069
3294
|
}
|
|
3070
|
-
return out;
|
|
3071
3295
|
}
|
|
3296
|
+
return byField;
|
|
3072
3297
|
};
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
);
|
|
3082
|
-
}
|
|
3083
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3084
|
-
throw new Error("parameterSweepProposer: JSON surface must parse to an object");
|
|
3085
|
-
}
|
|
3086
|
-
return parsed;
|
|
3087
|
-
}
|
|
3088
|
-
function triedLabels(history) {
|
|
3089
|
-
const tried = /* @__PURE__ */ new Set();
|
|
3090
|
-
for (const generation of history) {
|
|
3091
|
-
for (const candidate of generation.candidates) {
|
|
3092
|
-
if (!candidate.label) continue;
|
|
3093
|
-
tried.add(candidate.label);
|
|
3094
|
-
tried.add(candidateStrategy(candidate));
|
|
3095
|
-
}
|
|
3298
|
+
const passing = collect(true);
|
|
3299
|
+
const failing = collect(false);
|
|
3300
|
+
const lines = [];
|
|
3301
|
+
for (const field of /* @__PURE__ */ new Set([...passing.keys(), ...failing.keys()])) {
|
|
3302
|
+
const pv = [...passing.get(field) ?? []].slice(0, maxValues);
|
|
3303
|
+
const fv = [...failing.get(field) ?? []].slice(0, maxValues);
|
|
3304
|
+
const render = (vals) => vals.length ? JSON.stringify(vals) : "NOT SET (omitted)";
|
|
3305
|
+
lines.push(` ${field}: passing runs -> ${render(pv)} | failing runs -> ${render(fv)}`);
|
|
3096
3306
|
}
|
|
3097
|
-
|
|
3307
|
+
const lower = (m) => {
|
|
3308
|
+
const out = /* @__PURE__ */ new Set();
|
|
3309
|
+
for (const vals of m.values()) for (const v of vals) out.add(v.toLowerCase());
|
|
3310
|
+
return out;
|
|
3311
|
+
};
|
|
3312
|
+
return {
|
|
3313
|
+
text: lines.join("\n") || " (no calls observed)",
|
|
3314
|
+
passingValues: lower(passing),
|
|
3315
|
+
failingValues: lower(failing)
|
|
3316
|
+
};
|
|
3098
3317
|
}
|
|
3099
|
-
function
|
|
3100
|
-
const
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
next,
|
|
3105
|
-
typeof change.path === "string" ? change.path.split(".") : [...change.path],
|
|
3106
|
-
change.value
|
|
3107
|
-
);
|
|
3318
|
+
function classifyUngroundedLiterals(text, diff) {
|
|
3319
|
+
const ungrounded = /* @__PURE__ */ new Set();
|
|
3320
|
+
for (const m of text.matchAll(/'([a-z][a-z_-]{1,19})'/gi)) {
|
|
3321
|
+
const w = m[1].toLowerCase();
|
|
3322
|
+
if (!diff.passingValues.has(w)) ungrounded.add(w);
|
|
3108
3323
|
}
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3324
|
+
const all = [...ungrounded];
|
|
3325
|
+
return {
|
|
3326
|
+
ungrounded: all,
|
|
3327
|
+
harmful: all.filter((w) => diff.failingValues.has(w))
|
|
3328
|
+
};
|
|
3113
3329
|
}
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3330
|
+
|
|
3331
|
+
// src/campaign/labeled-store/fs-adapter.ts
|
|
3332
|
+
import { createHash as createHash4 } from "crypto";
|
|
3333
|
+
import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
3334
|
+
import { join as join3 } from "path";
|
|
3335
|
+
var LabeledScenarioStoreError = class extends Error {
|
|
3336
|
+
constructor(code, message) {
|
|
3337
|
+
super(message);
|
|
3338
|
+
this.code = code;
|
|
3339
|
+
this.name = "LabeledScenarioStoreError";
|
|
3122
3340
|
}
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
if (!isPlainObject(existing)) {
|
|
3131
|
-
cursor[part] = {};
|
|
3132
|
-
}
|
|
3133
|
-
cursor = cursor[part];
|
|
3341
|
+
code;
|
|
3342
|
+
};
|
|
3343
|
+
var FsLabeledScenarioStore = class {
|
|
3344
|
+
constructor(options) {
|
|
3345
|
+
this.options = options;
|
|
3346
|
+
if (!existsSync2(options.root)) mkdirSync(options.root, { recursive: true });
|
|
3347
|
+
this.now = options.now ?? Date.now;
|
|
3134
3348
|
}
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3349
|
+
options;
|
|
3350
|
+
now;
|
|
3351
|
+
rateLimits = /* @__PURE__ */ new Map();
|
|
3352
|
+
async observe(write) {
|
|
3353
|
+
this.assertProvenance(write);
|
|
3354
|
+
this.assertRateLimit(write);
|
|
3355
|
+
const record = this.toRecord(write);
|
|
3356
|
+
const path = this.pathForSource(write.source);
|
|
3357
|
+
const line = `${JSON.stringify(record)}
|
|
3358
|
+
`;
|
|
3359
|
+
appendLine(path, line);
|
|
3141
3360
|
}
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3361
|
+
async sample(args) {
|
|
3362
|
+
if (!args.split) {
|
|
3363
|
+
throw new LabeledScenarioStoreError(
|
|
3364
|
+
"split_required",
|
|
3365
|
+
"sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
|
|
3366
|
+
);
|
|
3367
|
+
}
|
|
3368
|
+
if (!args.capturedBefore) {
|
|
3369
|
+
throw new LabeledScenarioStoreError(
|
|
3370
|
+
"capturedBefore_required",
|
|
3371
|
+
"sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
|
|
3372
|
+
);
|
|
3373
|
+
}
|
|
3374
|
+
const all = [];
|
|
3375
|
+
for (const source of ALL_SOURCES) {
|
|
3376
|
+
if (args.split === "train" && source === "production-trace") {
|
|
3377
|
+
const explicit = sourceFilterContains(args.filter?.source, "production-trace");
|
|
3378
|
+
if (!explicit) continue;
|
|
3379
|
+
}
|
|
3380
|
+
const path = this.pathForSource(source);
|
|
3381
|
+
if (!existsSync2(path)) continue;
|
|
3382
|
+
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
3383
|
+
for (const line of lines) {
|
|
3384
|
+
let record;
|
|
3385
|
+
try {
|
|
3386
|
+
record = JSON.parse(line);
|
|
3387
|
+
} catch {
|
|
3388
|
+
continue;
|
|
3389
|
+
}
|
|
3390
|
+
if (!matchesFilter(record, args, source)) continue;
|
|
3391
|
+
all.push(record);
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
all.sort((a, b) => {
|
|
3395
|
+
if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
|
|
3396
|
+
return a.recordHash.localeCompare(b.recordHash);
|
|
3397
|
+
});
|
|
3398
|
+
return all.slice(0, args.count);
|
|
3147
3399
|
}
|
|
3148
|
-
|
|
3149
|
-
}
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
}
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
const rejected = [];
|
|
3161
|
-
const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
|
|
3162
|
-
for (const op of patch.ops) {
|
|
3163
|
-
if (op.op === "add") {
|
|
3164
|
-
if (typeof op.text !== "string" || op.text.trim() === "") {
|
|
3165
|
-
rejected.push({ op, reason: "empty add text" });
|
|
3400
|
+
async size() {
|
|
3401
|
+
const bySource = {};
|
|
3402
|
+
const byTrust = {
|
|
3403
|
+
unverified: 0,
|
|
3404
|
+
"verified-signal": 0,
|
|
3405
|
+
"human-rated": 0
|
|
3406
|
+
};
|
|
3407
|
+
let total = 0;
|
|
3408
|
+
for (const source of ALL_SOURCES) {
|
|
3409
|
+
const path = this.pathForSource(source);
|
|
3410
|
+
if (!existsSync2(path)) {
|
|
3411
|
+
bySource[source] = 0;
|
|
3166
3412
|
continue;
|
|
3167
3413
|
}
|
|
3168
|
-
const
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
}
|
|
3179
|
-
lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
|
|
3180
|
-
applied++;
|
|
3181
|
-
} else if (op.op === "delete") {
|
|
3182
|
-
const idx = findLine(op.anchor);
|
|
3183
|
-
if (idx === -1) {
|
|
3184
|
-
rejected.push({ op, reason: `delete anchor not found: ${truncate(op.anchor)}` });
|
|
3185
|
-
continue;
|
|
3186
|
-
}
|
|
3187
|
-
lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
|
|
3188
|
-
applied++;
|
|
3189
|
-
} else {
|
|
3190
|
-
const idx = findLine(op.anchor);
|
|
3191
|
-
if (idx === -1) {
|
|
3192
|
-
rejected.push({ op, reason: `replace anchor not found: ${truncate(op.anchor)}` });
|
|
3193
|
-
continue;
|
|
3194
|
-
}
|
|
3195
|
-
if (typeof op.text !== "string") {
|
|
3196
|
-
rejected.push({ op, reason: "replace text missing" });
|
|
3197
|
-
continue;
|
|
3414
|
+
const lines = readFileSync2(path, "utf8").split("\n").filter(Boolean);
|
|
3415
|
+
bySource[source] = lines.length;
|
|
3416
|
+
total += lines.length;
|
|
3417
|
+
for (const line of lines) {
|
|
3418
|
+
let trust = "unverified";
|
|
3419
|
+
try {
|
|
3420
|
+
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
3421
|
+
} catch {
|
|
3422
|
+
}
|
|
3423
|
+
byTrust[trust] += 1;
|
|
3198
3424
|
}
|
|
3199
|
-
lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
|
|
3200
|
-
applied++;
|
|
3201
3425
|
}
|
|
3426
|
+
return { train: total, test: total, bySource, byTrust };
|
|
3202
3427
|
}
|
|
3203
|
-
|
|
3428
|
+
assertProvenance(write) {
|
|
3429
|
+
if (!write.source) {
|
|
3430
|
+
throw new LabeledScenarioStoreError(
|
|
3431
|
+
"missing_source",
|
|
3432
|
+
"LabeledScenarioWrite requires `source`"
|
|
3433
|
+
);
|
|
3434
|
+
}
|
|
3435
|
+
if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
|
|
3436
|
+
throw new LabeledScenarioStoreError(
|
|
3437
|
+
"missing_source_version",
|
|
3438
|
+
"LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
|
|
3439
|
+
);
|
|
3440
|
+
}
|
|
3441
|
+
if (!write.capturedAt) {
|
|
3442
|
+
throw new LabeledScenarioStoreError(
|
|
3443
|
+
"missing_captured_at",
|
|
3444
|
+
"LabeledScenarioWrite requires `capturedAt` ISO timestamp"
|
|
3445
|
+
);
|
|
3446
|
+
}
|
|
3447
|
+
if (!write.redactionStatus) {
|
|
3448
|
+
throw new LabeledScenarioStoreError(
|
|
3449
|
+
"missing_redaction_status",
|
|
3450
|
+
"LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
|
|
3451
|
+
);
|
|
3452
|
+
}
|
|
3453
|
+
if (!ALL_SOURCES.includes(write.source)) {
|
|
3454
|
+
throw new LabeledScenarioStoreError(
|
|
3455
|
+
"unknown_source",
|
|
3456
|
+
`LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
|
|
3457
|
+
);
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
assertRateLimit(write) {
|
|
3461
|
+
const cap = this.options.maxWritesPerMinutePerBucket;
|
|
3462
|
+
if (!cap || !write.rateLimitBucket) return;
|
|
3463
|
+
const now = this.now();
|
|
3464
|
+
const windowMs = 6e4;
|
|
3465
|
+
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
3466
|
+
if (!state || now - state.windowStartMs >= windowMs) {
|
|
3467
|
+
state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
|
|
3468
|
+
this.rateLimits.set(write.rateLimitBucket, state);
|
|
3469
|
+
}
|
|
3470
|
+
if (state.count >= cap) {
|
|
3471
|
+
throw new LabeledScenarioStoreError(
|
|
3472
|
+
"rate_limit_exceeded",
|
|
3473
|
+
`LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
|
|
3474
|
+
);
|
|
3475
|
+
}
|
|
3476
|
+
state.count += 1;
|
|
3477
|
+
}
|
|
3478
|
+
toRecord(write) {
|
|
3479
|
+
const recordHash = sha256(
|
|
3480
|
+
JSON.stringify({
|
|
3481
|
+
id: write.scenario.id,
|
|
3482
|
+
src: write.source,
|
|
3483
|
+
at: write.capturedAt,
|
|
3484
|
+
ver: write.sourceVersionHash
|
|
3485
|
+
})
|
|
3486
|
+
);
|
|
3487
|
+
return {
|
|
3488
|
+
...write,
|
|
3489
|
+
recordHash,
|
|
3490
|
+
split: "train"
|
|
3491
|
+
};
|
|
3492
|
+
}
|
|
3493
|
+
pathForSource(source) {
|
|
3494
|
+
return join3(this.options.root, `${source}.jsonl`);
|
|
3495
|
+
}
|
|
3496
|
+
};
|
|
3497
|
+
var ALL_SOURCES = [
|
|
3498
|
+
"production-trace",
|
|
3499
|
+
"eval-run",
|
|
3500
|
+
"manual",
|
|
3501
|
+
"red-team",
|
|
3502
|
+
"synthetic"
|
|
3503
|
+
];
|
|
3504
|
+
function sourceFilterContains(filter, needle) {
|
|
3505
|
+
if (!filter) return false;
|
|
3506
|
+
if (Array.isArray(filter)) return filter.includes(needle);
|
|
3507
|
+
return filter === needle;
|
|
3204
3508
|
}
|
|
3205
|
-
function
|
|
3206
|
-
|
|
3509
|
+
function matchesFilter(record, args, source) {
|
|
3510
|
+
if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
|
|
3511
|
+
if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
|
|
3512
|
+
const f = args.filter;
|
|
3513
|
+
if (!f) return true;
|
|
3514
|
+
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
3515
|
+
if (f.source) {
|
|
3516
|
+
const sources = Array.isArray(f.source) ? f.source : [f.source];
|
|
3517
|
+
if (!sources.includes(source)) return false;
|
|
3518
|
+
}
|
|
3519
|
+
if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
|
|
3520
|
+
const composites = Object.values(record.judgeScores).map((s) => s.composite);
|
|
3521
|
+
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
3522
|
+
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
3523
|
+
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
3524
|
+
}
|
|
3525
|
+
if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
|
|
3526
|
+
return false;
|
|
3527
|
+
}
|
|
3528
|
+
return true;
|
|
3207
3529
|
}
|
|
3208
|
-
function
|
|
3209
|
-
return
|
|
3530
|
+
function sha256(input) {
|
|
3531
|
+
return createHash4("sha256").update(input).digest("hex").slice(0, 16);
|
|
3532
|
+
}
|
|
3533
|
+
function appendLine(path, line) {
|
|
3534
|
+
if (existsSync2(path)) {
|
|
3535
|
+
const existing = readFileSync2(path, "utf8");
|
|
3536
|
+
writeFileSync(path, existing + line);
|
|
3537
|
+
} else {
|
|
3538
|
+
writeFileSync(path, line);
|
|
3539
|
+
}
|
|
3210
3540
|
}
|
|
3211
3541
|
|
|
3212
|
-
// src/campaign/
|
|
3213
|
-
var
|
|
3214
|
-
function
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
messages: [
|
|
3232
|
-
{ role: "system", content: SKILLOPT_SYSTEM },
|
|
3233
|
-
{ role: "user", content: userPrompt }
|
|
3234
|
-
],
|
|
3235
|
-
jsonMode: true,
|
|
3236
|
-
temperature: opts.temperature ?? 0.6,
|
|
3237
|
-
maxTokens: opts.maxTokens ?? 4e3
|
|
3238
|
-
};
|
|
3239
|
-
const paid = await (args.costLedger ?? directCostLedger).runPaidCall({
|
|
3240
|
-
channel: "driver",
|
|
3241
|
-
phase: args.costPhase ?? "search.proposal",
|
|
3242
|
-
actor: "skill-opt.propose",
|
|
3243
|
-
model: opts.model,
|
|
3244
|
-
maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
|
|
3245
|
-
signal: args.signal,
|
|
3246
|
-
execute: (signal, callId) => callLlm(request, { ...opts.llm, signal, idempotencyKey: callId }),
|
|
3247
|
-
receipt: costReceiptFromLlm,
|
|
3248
|
-
receiptFromError: costReceiptFromLlmError
|
|
3249
|
-
});
|
|
3250
|
-
if (!paid.succeeded) throw paid.error;
|
|
3251
|
-
const result = paid.value;
|
|
3252
|
-
return parseSkillPatchResponse(result.content, args.count, args.editBudget);
|
|
3542
|
+
// src/campaign/neutralize.ts
|
|
3543
|
+
var FILLER = "#";
|
|
3544
|
+
function neutralizeText(content) {
|
|
3545
|
+
return content.replace(/\S/g, FILLER);
|
|
3546
|
+
}
|
|
3547
|
+
|
|
3548
|
+
// src/campaign/proposers/fapo.ts
|
|
3549
|
+
var FAPO_LEVELS = ["prompt", "parameter", "structural"];
|
|
3550
|
+
var MAX_FINDING_DEPTH = 16;
|
|
3551
|
+
var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
3552
|
+
function fapoProposer(opts) {
|
|
3553
|
+
const proposers = levelProposers(opts);
|
|
3554
|
+
const allowed = allowedLevels(opts, proposers);
|
|
3555
|
+
const plateauWindow = opts.plateauWindow ?? 3;
|
|
3556
|
+
const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
|
|
3557
|
+
const minImprovement = opts.minImprovement ?? 0;
|
|
3558
|
+
const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
|
|
3559
|
+
if (allowed.length === 0) {
|
|
3560
|
+
throw new Error("fapoProposer: at least one allowed level must have a proposer");
|
|
3253
3561
|
}
|
|
3562
|
+
if (plateauWindow < 1) throw new Error("fapoProposer: plateauWindow must be >= 1");
|
|
3563
|
+
if (minDistinctStrategies < 1) {
|
|
3564
|
+
throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
|
|
3565
|
+
}
|
|
3566
|
+
if (proposalsPerCycle < 1) throw new Error("fapoProposer: proposalsPerCycle must be >= 1");
|
|
3254
3567
|
return {
|
|
3255
|
-
kind: "
|
|
3256
|
-
proposePatches,
|
|
3568
|
+
kind: "fapo",
|
|
3257
3569
|
async propose(ctx) {
|
|
3258
|
-
|
|
3570
|
+
const state = policyState(ctx.history, ctx.findings, {
|
|
3571
|
+
plateauWindow,
|
|
3572
|
+
minDistinctStrategies,
|
|
3573
|
+
minImprovement
|
|
3574
|
+
});
|
|
3575
|
+
const decision = chooseLevel({
|
|
3576
|
+
allowed,
|
|
3577
|
+
available: allowed.filter((level) => !state.exhausted[level]),
|
|
3578
|
+
proposers,
|
|
3579
|
+
state,
|
|
3580
|
+
promptFirst: opts.promptFirst ?? true,
|
|
3581
|
+
parameterBeforeStructural: opts.parameterBeforeStructural ?? true
|
|
3582
|
+
});
|
|
3583
|
+
if (!decision) return [];
|
|
3584
|
+
const proposer = proposers[decision.level];
|
|
3585
|
+
if (!proposer) {
|
|
3586
|
+
throw new Error(`fapoProposer: selected ${decision.level} but no proposer is configured`);
|
|
3587
|
+
}
|
|
3588
|
+
const requested = Math.min(ctx.populationSize, proposalsPerCycle);
|
|
3589
|
+
const raw = await proposer.propose({ ...ctx, populationSize: requested });
|
|
3590
|
+
const wrapped = raw.map((candidate) => normalizeProposal(candidate)).map((candidate) => wrapProposal(candidate, decision.level, decision.reason));
|
|
3591
|
+
const reviewed = [];
|
|
3592
|
+
for (const candidate of wrapped) {
|
|
3593
|
+
const review = opts.reviewCandidate ? await opts.reviewCandidate({
|
|
3594
|
+
level: decision.level,
|
|
3595
|
+
candidate,
|
|
3596
|
+
context: ctx,
|
|
3597
|
+
reason: decision.reason
|
|
3598
|
+
}) : { verdict: "pass" };
|
|
3599
|
+
if (review.verdict === "fail") continue;
|
|
3600
|
+
const warn = review.verdict === "warn" ? renderReviewWarnings(review) : "";
|
|
3601
|
+
reviewed.push(
|
|
3602
|
+
warn ? { ...candidate, rationale: `${candidate.rationale}
|
|
3603
|
+
${warn}` } : candidate
|
|
3604
|
+
);
|
|
3605
|
+
}
|
|
3606
|
+
if (wrapped.length > 0 && reviewed.length === 0) {
|
|
3259
3607
|
throw new Error(
|
|
3260
|
-
|
|
3608
|
+
`fapoProposer: reviewer blocked every ${decision.level} candidate; refusing to evaluate an unreviewed variant`
|
|
3261
3609
|
);
|
|
3262
3610
|
}
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
|
|
3270
|
-
count: ctx.populationSize,
|
|
3271
|
-
signal: ctx.signal,
|
|
3272
|
-
costLedger: ctx.costLedger ?? directCostLedger,
|
|
3273
|
-
costPhase: ctx.costPhase ?? "search.proposal"
|
|
3274
|
-
});
|
|
3275
|
-
const out = [];
|
|
3276
|
-
const seen = /* @__PURE__ */ new Set();
|
|
3277
|
-
for (const patch of patches) {
|
|
3278
|
-
const { surface: candidate, applied } = applySkillPatch(surface, patch);
|
|
3279
|
-
if (applied === 0 || candidate === surface || seen.has(candidate)) continue;
|
|
3280
|
-
seen.add(candidate);
|
|
3281
|
-
out.push({ surface: candidate, label: patch.label, rationale: patch.rationale });
|
|
3282
|
-
if (out.length >= ctx.populationSize) break;
|
|
3611
|
+
return reviewed;
|
|
3612
|
+
},
|
|
3613
|
+
decide({ history }) {
|
|
3614
|
+
const last = history[history.length - 1];
|
|
3615
|
+
if (last && last.candidates.length === 0) {
|
|
3616
|
+
return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
|
|
3283
3617
|
}
|
|
3284
|
-
|
|
3618
|
+
const state = policyState(history, [], {
|
|
3619
|
+
plateauWindow,
|
|
3620
|
+
minDistinctStrategies,
|
|
3621
|
+
minImprovement
|
|
3622
|
+
});
|
|
3623
|
+
const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
|
|
3624
|
+
return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
|
|
3285
3625
|
}
|
|
3286
3626
|
};
|
|
3287
3627
|
}
|
|
3288
|
-
function
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
return { weakScenarios, weakDimensions };
|
|
3628
|
+
function levelProposers(opts) {
|
|
3629
|
+
return {
|
|
3630
|
+
...opts.proposers ?? {},
|
|
3631
|
+
...opts.promptProposer ? { prompt: opts.promptProposer } : {},
|
|
3632
|
+
...opts.parameterProposer ? { parameter: opts.parameterProposer } : {},
|
|
3633
|
+
...opts.structuralProposer ? { structural: opts.structuralProposer } : {}
|
|
3634
|
+
};
|
|
3296
3635
|
}
|
|
3297
|
-
function
|
|
3298
|
-
const
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
"",
|
|
3306
|
-
`Propose ${args.count} candidate patch(es). Each patch is a SMALL bundle of`,
|
|
3307
|
-
`at most ${args.editBudget} op(s). Anchors must be verbatim substrings of`,
|
|
3308
|
-
"existing lines. Prefer adding a specific missing rule or sharpening a vague",
|
|
3309
|
-
"one over deleting; never rewrite the whole document."
|
|
3310
|
-
];
|
|
3311
|
-
if (args.evidence.weakScenarios.length > 0) {
|
|
3312
|
-
lines.push(
|
|
3313
|
-
"",
|
|
3314
|
-
"Weakest scenarios (patch to fix these):",
|
|
3315
|
-
...args.evidence.weakScenarios.map((s) => `- ${s.scenarioId} (${s.composite.toFixed(2)})`)
|
|
3316
|
-
);
|
|
3317
|
-
}
|
|
3318
|
-
if (args.evidence.weakDimensions.length > 0) {
|
|
3319
|
-
lines.push(
|
|
3320
|
-
"",
|
|
3321
|
-
"Weakest dimensions (what to improve):",
|
|
3322
|
-
...args.evidence.weakDimensions.map((d) => `- ${d.dimension} (${d.score.toFixed(2)})`)
|
|
3323
|
-
);
|
|
3324
|
-
}
|
|
3325
|
-
if (args.rejectedBuffer.length > 0) {
|
|
3326
|
-
lines.push(
|
|
3327
|
-
"",
|
|
3328
|
-
"Already tried and REJECTED (do not repeat or restate these edits):",
|
|
3329
|
-
...args.rejectedBuffer.map((e) => `- ${e.label}: ${e.rationale} \u2014 ${e.reason}`)
|
|
3330
|
-
);
|
|
3331
|
-
}
|
|
3332
|
-
if (args.findingsNote) {
|
|
3333
|
-
lines.push("", args.findingsNote);
|
|
3334
|
-
}
|
|
3335
|
-
if (args.metaNote) {
|
|
3336
|
-
lines.push("", `Strategy note from prior epochs: ${args.metaNote}`);
|
|
3337
|
-
}
|
|
3338
|
-
return lines.join("\n");
|
|
3636
|
+
function allowedLevels(opts, proposers) {
|
|
3637
|
+
const explicit = opts.scope?.allowedLevels;
|
|
3638
|
+
const forbidden = new Set(opts.scope?.forbiddenLevels ?? []);
|
|
3639
|
+
return FAPO_LEVELS.filter((level) => {
|
|
3640
|
+
if (!proposers[level]) return false;
|
|
3641
|
+
if (forbidden.has(level)) return false;
|
|
3642
|
+
return explicit ? explicit.includes(level) : true;
|
|
3643
|
+
});
|
|
3339
3644
|
}
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3645
|
+
function policyState(history, findings, opts) {
|
|
3646
|
+
const attempts = extractAttempts(history);
|
|
3647
|
+
return {
|
|
3648
|
+
attempts,
|
|
3649
|
+
exhausted: {
|
|
3650
|
+
prompt: isLevelExhausted("prompt", attempts, opts),
|
|
3651
|
+
parameter: isLevelExhausted("parameter", attempts, opts),
|
|
3652
|
+
structural: isLevelExhausted("structural", attempts, opts)
|
|
3653
|
+
},
|
|
3654
|
+
signals: extractFapoAttributionSignals(findings)
|
|
3655
|
+
};
|
|
3656
|
+
}
|
|
3657
|
+
function extractAttempts(history) {
|
|
3658
|
+
const attempts = [];
|
|
3659
|
+
for (const generation of history) {
|
|
3660
|
+
for (const candidate of generation.candidates) {
|
|
3661
|
+
const level = candidateLevel(candidate);
|
|
3662
|
+
if (!level) continue;
|
|
3663
|
+
attempts.push({
|
|
3664
|
+
level,
|
|
3665
|
+
composite: candidate.composite,
|
|
3666
|
+
strategy: candidateStrategy(candidate)
|
|
3667
|
+
});
|
|
3668
|
+
}
|
|
3363
3669
|
}
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3670
|
+
return attempts;
|
|
3671
|
+
}
|
|
3672
|
+
function candidateLevel(candidate) {
|
|
3673
|
+
const label = candidate.label ?? "";
|
|
3674
|
+
const rationale = candidate.rationale ?? "";
|
|
3675
|
+
return parseLevel(label) ?? parseLevel(rationale);
|
|
3676
|
+
}
|
|
3677
|
+
function parseLevel(text) {
|
|
3678
|
+
const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
|
|
3679
|
+
return match ? match[1] : null;
|
|
3680
|
+
}
|
|
3681
|
+
function candidateStrategy(candidate) {
|
|
3682
|
+
const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
|
|
3683
|
+
return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
|
|
3684
|
+
}
|
|
3685
|
+
function isLevelExhausted(level, attempts, opts) {
|
|
3686
|
+
const own = attempts.filter((attempt) => attempt.level === level);
|
|
3687
|
+
if (own.length < opts.plateauWindow) return false;
|
|
3688
|
+
const distinctStrategies = new Set(own.map((attempt) => attempt.strategy)).size;
|
|
3689
|
+
if (distinctStrategies < opts.minDistinctStrategies) return false;
|
|
3690
|
+
let best = Number.NEGATIVE_INFINITY;
|
|
3691
|
+
let nonImproving = 0;
|
|
3692
|
+
for (const attempt of own) {
|
|
3693
|
+
if (attempt.composite > best + opts.minImprovement) {
|
|
3694
|
+
best = attempt.composite;
|
|
3695
|
+
nonImproving = 0;
|
|
3696
|
+
} else {
|
|
3697
|
+
nonImproving += 1;
|
|
3698
|
+
}
|
|
3377
3699
|
}
|
|
3378
|
-
return
|
|
3700
|
+
return nonImproving >= opts.plateauWindow;
|
|
3379
3701
|
}
|
|
3380
|
-
function
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3702
|
+
function chooseLevel(args) {
|
|
3703
|
+
const { available, state } = args;
|
|
3704
|
+
if (available.length === 0) return null;
|
|
3705
|
+
const triedPrompt = state.attempts.some((attempt) => attempt.level === "prompt");
|
|
3706
|
+
if (args.promptFirst && available.includes("prompt") && !triedPrompt) {
|
|
3707
|
+
return {
|
|
3708
|
+
level: "prompt",
|
|
3709
|
+
reason: "prompt-first policy: no prompt-level variant has been tried yet"
|
|
3710
|
+
};
|
|
3388
3711
|
}
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
return {
|
|
3712
|
+
const supported = available.filter((level) => state.signals.counts[level] > 0);
|
|
3713
|
+
if (supported.includes("prompt")) {
|
|
3714
|
+
return {
|
|
3715
|
+
level: "prompt",
|
|
3716
|
+
reason: `attribution has ${state.signals.counts.prompt} prompt-addressable failure(s)`
|
|
3717
|
+
};
|
|
3392
3718
|
}
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
|
|
3719
|
+
const strongest = supported.sort(
|
|
3720
|
+
(a, b) => state.signals.counts[b] - state.signals.counts[a] || levelRank(a) - levelRank(b)
|
|
3721
|
+
)[0];
|
|
3722
|
+
if (strongest) {
|
|
3723
|
+
if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
|
|
3724
|
+
return {
|
|
3725
|
+
level: "parameter",
|
|
3726
|
+
reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
|
|
3727
|
+
};
|
|
3728
|
+
}
|
|
3729
|
+
return {
|
|
3730
|
+
level: strongest,
|
|
3731
|
+
reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
|
|
3732
|
+
};
|
|
3733
|
+
}
|
|
3734
|
+
if (state.attempts.length === 0) {
|
|
3735
|
+
const first = available.slice().sort((a, b) => levelRank(a) - levelRank(b))[0];
|
|
3736
|
+
return { level: first, reason: "no attribution yet; starting at the cheapest allowed level" };
|
|
3396
3737
|
}
|
|
3397
3738
|
return null;
|
|
3398
3739
|
}
|
|
3399
|
-
function
|
|
3400
|
-
return
|
|
3740
|
+
function levelRank(level) {
|
|
3741
|
+
return level === "prompt" ? 0 : level === "parameter" ? 1 : 2;
|
|
3401
3742
|
}
|
|
3402
|
-
function
|
|
3403
|
-
|
|
3404
|
-
return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
|
|
3743
|
+
function normalizeProposal(value) {
|
|
3744
|
+
return isProposedCandidate(value) ? value : { surface: value, label: "candidate", rationale: "bare candidate from level proposer" };
|
|
3405
3745
|
}
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
const comparisonCount = opts.methods.length * (opts.methods.length + 1) / 2;
|
|
3417
|
-
const intervalConfidence = 1 - (1 - confidence) / comparisonCount;
|
|
3418
|
-
const minimumResamples = minimumBootstrapResamples(confidence, comparisonCount);
|
|
3419
|
-
const resamples = opts.resamples ?? Math.max(2e3, minimumResamples);
|
|
3420
|
-
assertComparisonControls(opts, seed, resamples, confidence);
|
|
3421
|
-
const storage = opts.storage ?? fsCampaignStorage();
|
|
3422
|
-
const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
|
|
3423
|
-
const testCostPhase = `compareOptimizationMethods:test:${randomUUID()}`;
|
|
3424
|
-
const testCostLedger = opts.costLedger ?? createRunCostLedger({
|
|
3425
|
-
storage,
|
|
3426
|
-
runDir: `${resolvedRunDir}/test/cost`,
|
|
3427
|
-
costCeilingUsd: opts.costCeiling
|
|
3428
|
-
});
|
|
3429
|
-
const scoreOnTest = async (surface, tag) => {
|
|
3430
|
-
const campaign = await runCampaign({
|
|
3431
|
-
...opts,
|
|
3432
|
-
storage,
|
|
3433
|
-
costLedger: testCostLedger,
|
|
3434
|
-
costPhase: testCostPhase,
|
|
3435
|
-
scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
|
|
3436
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
3437
|
-
runDir: `${resolvedRunDir}/${tag}`
|
|
3438
|
-
});
|
|
3439
|
-
const byScenario = {};
|
|
3440
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
|
|
3441
|
-
byScenario[scenarioId] = composite;
|
|
3442
|
-
}
|
|
3443
|
-
return byScenario;
|
|
3746
|
+
function wrapProposal(candidate, level, reason) {
|
|
3747
|
+
const label = candidate.label ? `fapo:${level}:${candidate.label}` : `fapo:${level}`;
|
|
3748
|
+
return {
|
|
3749
|
+
...candidate,
|
|
3750
|
+
label,
|
|
3751
|
+
rationale: [
|
|
3752
|
+
`fapo:${level} selected by reviewed escalation policy`,
|
|
3753
|
+
`Reason: ${reason}`,
|
|
3754
|
+
candidate.rationale ? `Level proposer rationale: ${candidate.rationale}` : ""
|
|
3755
|
+
].filter(Boolean).join("\n")
|
|
3444
3756
|
};
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
}
|
|
3455
|
-
|
|
3757
|
+
}
|
|
3758
|
+
function renderReviewWarnings(review) {
|
|
3759
|
+
const issues = review.issues?.filter((issue) => issue.severity === "warn") ?? [];
|
|
3760
|
+
if (issues.length === 0) return "";
|
|
3761
|
+
return `Reviewer warnings:
|
|
3762
|
+
${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
|
|
3763
|
+
}
|
|
3764
|
+
function extractFapoAttributionSignals(findings) {
|
|
3765
|
+
const signals = {
|
|
3766
|
+
counts: { prompt: 0, parameter: 0, structural: 0 },
|
|
3767
|
+
clusters: []
|
|
3456
3768
|
};
|
|
3457
|
-
const
|
|
3458
|
-
|
|
3459
|
-
|
|
3460
|
-
);
|
|
3461
|
-
assertOptimizationResult(method.name, out);
|
|
3462
|
-
const winnerSurface = structuredClone(out.winnerSurface);
|
|
3463
|
-
return {
|
|
3464
|
-
name: method.name,
|
|
3465
|
-
winnerSurface,
|
|
3466
|
-
cost: out.cost,
|
|
3467
|
-
durationMs: out.durationMs
|
|
3468
|
-
};
|
|
3469
|
-
});
|
|
3470
|
-
const baselineArr = align(await scoreOnTest(opts.baselineSurface, "test/baseline"), "baseline");
|
|
3471
|
-
const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(opts.baselineSurface), baselineArr]]);
|
|
3472
|
-
const winners = [];
|
|
3473
|
-
for (const winner of optimized) {
|
|
3474
|
-
const surfaceKey = surfaceContentHash(winner.winnerSurface);
|
|
3475
|
-
let arr = testScoresBySurface.get(surfaceKey);
|
|
3476
|
-
if (!arr) {
|
|
3477
|
-
const byScenario = await scoreOnTest(
|
|
3478
|
-
winner.winnerSurface,
|
|
3479
|
-
`test/methods/${slug(winner.name)}`
|
|
3480
|
-
);
|
|
3481
|
-
arr = align(byScenario, `method "${winner.name}"`);
|
|
3482
|
-
testScoresBySurface.set(surfaceKey, arr);
|
|
3483
|
-
}
|
|
3484
|
-
winners.push({
|
|
3485
|
-
...winner,
|
|
3486
|
-
arr
|
|
3487
|
-
});
|
|
3769
|
+
const seen = /* @__PURE__ */ new WeakSet();
|
|
3770
|
+
for (const finding of findings) {
|
|
3771
|
+
collectFinding(signals, finding, seen, 0);
|
|
3488
3772
|
}
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3773
|
+
return signals;
|
|
3774
|
+
}
|
|
3775
|
+
function collectFinding(signals, finding, seen, depth) {
|
|
3776
|
+
if (depth > MAX_FINDING_DEPTH) return;
|
|
3777
|
+
if (!finding) return;
|
|
3778
|
+
if (typeof finding === "string") {
|
|
3779
|
+
const level = inferLevelFromText(finding);
|
|
3780
|
+
if (level) addCluster(signals, { label: finding, level, count: 1 });
|
|
3781
|
+
return;
|
|
3782
|
+
}
|
|
3783
|
+
if (Array.isArray(finding)) {
|
|
3784
|
+
if (seen.has(finding)) return;
|
|
3785
|
+
seen.add(finding);
|
|
3786
|
+
for (const item of finding) collectFinding(signals, item, seen, depth + 1);
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3789
|
+
if (typeof finding !== "object") return;
|
|
3790
|
+
if (seen.has(finding)) return;
|
|
3791
|
+
seen.add(finding);
|
|
3792
|
+
const obj = finding;
|
|
3793
|
+
collectLevelPartition(signals, obj.level_partition ?? obj.levelPartition, seen, depth + 1);
|
|
3794
|
+
collectCounts(signals, obj);
|
|
3795
|
+
collectClusters(signals, obj.clusters, true, seen, depth + 1);
|
|
3796
|
+
const explicit = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel);
|
|
3797
|
+
const text = textFields(obj);
|
|
3798
|
+
const inferred = explicit ?? inferLevelFromText(text);
|
|
3799
|
+
if (inferred) {
|
|
3800
|
+
addCluster(signals, {
|
|
3801
|
+
label: text || inferred,
|
|
3802
|
+
level: inferred,
|
|
3803
|
+
count: numberField(obj.count) ?? 1,
|
|
3804
|
+
confidence: confidenceField(obj.confidence),
|
|
3805
|
+
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
3806
|
+
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
3495
3807
|
});
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3808
|
+
}
|
|
3809
|
+
}
|
|
3810
|
+
function collectLevelPartition(signals, raw, seen, depth) {
|
|
3811
|
+
if (!raw || typeof raw !== "object") return;
|
|
3812
|
+
if (seen.has(raw)) return;
|
|
3813
|
+
seen.add(raw);
|
|
3814
|
+
const partition = raw;
|
|
3815
|
+
for (const level of FAPO_LEVELS) {
|
|
3816
|
+
const bucket = partition[level];
|
|
3817
|
+
if (!bucket || typeof bucket !== "object") continue;
|
|
3818
|
+
const obj = bucket;
|
|
3819
|
+
const count = numberField(obj.count) ?? 0;
|
|
3820
|
+
if (count > 0) signals.counts[level] += count;
|
|
3821
|
+
collectClusters(signals, obj.clusters, false, seen, depth + 1);
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
function collectCounts(signals, obj) {
|
|
3825
|
+
const prompt = numberField(obj.prompt_addressable ?? obj.promptAddressable);
|
|
3826
|
+
const structural = numberField(obj.structural_addressable ?? obj.structuralAddressable);
|
|
3827
|
+
const tool = numberField(obj.tool_addressable ?? obj.toolAddressable);
|
|
3828
|
+
if (prompt) signals.counts.prompt += prompt;
|
|
3829
|
+
if (structural) signals.counts.structural += structural;
|
|
3830
|
+
if (tool) signals.counts.structural += tool;
|
|
3831
|
+
}
|
|
3832
|
+
function collectClusters(signals, raw, countClusters, seen, depth) {
|
|
3833
|
+
if (!Array.isArray(raw)) return;
|
|
3834
|
+
if (seen.has(raw)) return;
|
|
3835
|
+
seen.add(raw);
|
|
3836
|
+
for (const item of raw) {
|
|
3837
|
+
if (countClusters) {
|
|
3838
|
+
collectFinding(signals, item, seen, depth + 1);
|
|
3839
|
+
continue;
|
|
3523
3840
|
}
|
|
3524
|
-
|
|
3841
|
+
const cluster = parseCluster(item);
|
|
3842
|
+
if (cluster) signals.clusters.push(cluster);
|
|
3525
3843
|
}
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
confidence: intervalConfidence,
|
|
3538
|
-
statistic: "mean"
|
|
3539
|
-
});
|
|
3540
|
-
const favored = boot.low > 0 ? best.name : boot.high < 0 ? other.name : "tie";
|
|
3541
|
-
return {
|
|
3542
|
-
a: best.name,
|
|
3543
|
-
b: other.name,
|
|
3544
|
-
deltaMean: boot.mean,
|
|
3545
|
-
low: boot.low,
|
|
3546
|
-
high: boot.high,
|
|
3547
|
-
favored
|
|
3548
|
-
};
|
|
3549
|
-
});
|
|
3550
|
-
const optimizationCost = combineCosts(
|
|
3551
|
-
scores.map((score) => ({ label: `method '${score.name}'`, cost: score.optimizationCost }))
|
|
3552
|
-
);
|
|
3553
|
-
const testCost = costFromLedgerSummary(testCostLedger.summary({ phase: testCostPhase }));
|
|
3554
|
-
const totalCost = combineCosts([
|
|
3555
|
-
{ label: "optimization", cost: optimizationCost },
|
|
3556
|
-
{ label: "final test", cost: testCost }
|
|
3557
|
-
]);
|
|
3844
|
+
}
|
|
3845
|
+
function addCluster(signals, cluster) {
|
|
3846
|
+
signals.counts[cluster.level] += Math.max(1, cluster.count);
|
|
3847
|
+
signals.clusters.push(cluster);
|
|
3848
|
+
}
|
|
3849
|
+
function parseCluster(raw) {
|
|
3850
|
+
if (!raw || typeof raw !== "object") return null;
|
|
3851
|
+
const obj = raw;
|
|
3852
|
+
const text = textFields(obj);
|
|
3853
|
+
const level = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel) ?? inferLevelFromText(text);
|
|
3854
|
+
if (!level) return null;
|
|
3558
3855
|
return {
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
totalCost,
|
|
3566
|
-
confidence,
|
|
3567
|
-
intervalConfidence,
|
|
3568
|
-
comparisonCount,
|
|
3569
|
-
seed,
|
|
3570
|
-
resamples,
|
|
3571
|
-
reps: opts.reps ?? 1
|
|
3856
|
+
label: text || level,
|
|
3857
|
+
level,
|
|
3858
|
+
count: numberField(obj.count) ?? 1,
|
|
3859
|
+
confidence: confidenceField(obj.confidence),
|
|
3860
|
+
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
3861
|
+
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
3572
3862
|
};
|
|
3573
3863
|
}
|
|
3574
|
-
function
|
|
3575
|
-
if (
|
|
3576
|
-
|
|
3864
|
+
function parseFindingLevel(raw) {
|
|
3865
|
+
if (typeof raw !== "string") return null;
|
|
3866
|
+
const lower = raw.toLowerCase();
|
|
3867
|
+
if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
|
|
3868
|
+
if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
|
|
3869
|
+
if (lower === "config" || lower === "params") return "parameter";
|
|
3870
|
+
return null;
|
|
3871
|
+
}
|
|
3872
|
+
function inferLevelFromText(text) {
|
|
3873
|
+
const lower = text.toLowerCase();
|
|
3874
|
+
if (/\b(retrieval_k|temperature|top_p|max_tokens|max_completion_tokens|reasoning_effort|config|parameter)\b/.test(
|
|
3875
|
+
lower
|
|
3876
|
+
)) {
|
|
3877
|
+
return "parameter";
|
|
3577
3878
|
}
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3879
|
+
if (/\b(retriev\w*|search\w*|bm25|hop|evidence|cascade|tool|node|chain|state|structur\w*|topology|import)\b/.test(
|
|
3880
|
+
lower
|
|
3881
|
+
)) {
|
|
3882
|
+
return "structural";
|
|
3883
|
+
}
|
|
3884
|
+
if (/\b(format\w*|verbose|brevity|abstain\w*|reasoning|instruction|prompt|output|answer)\b/.test(
|
|
3885
|
+
lower
|
|
3886
|
+
)) {
|
|
3887
|
+
return "prompt";
|
|
3888
|
+
}
|
|
3889
|
+
return null;
|
|
3890
|
+
}
|
|
3891
|
+
function textFields(obj) {
|
|
3892
|
+
return [
|
|
3893
|
+
obj.label,
|
|
3894
|
+
obj.claim,
|
|
3895
|
+
obj.recommended_action,
|
|
3896
|
+
obj.suggested_fix,
|
|
3897
|
+
obj.suggestedFix,
|
|
3898
|
+
obj.message,
|
|
3899
|
+
obj.text,
|
|
3900
|
+
obj.heuristic,
|
|
3901
|
+
obj.area
|
|
3902
|
+
].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
|
|
3903
|
+
}
|
|
3904
|
+
function numberField(value) {
|
|
3905
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
3906
|
+
}
|
|
3907
|
+
function stringField(value) {
|
|
3908
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
3909
|
+
}
|
|
3910
|
+
function confidenceField(value) {
|
|
3911
|
+
return value === "high" || value === "medium" || value === "low" ? value : void 0;
|
|
3912
|
+
}
|
|
3913
|
+
function stringArrayField(value) {
|
|
3914
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : void 0;
|
|
3915
|
+
}
|
|
3916
|
+
function parameterSweepProposer(opts) {
|
|
3917
|
+
if (opts.candidates.length === 0) {
|
|
3918
|
+
throw new Error("parameterSweepProposer: candidates must not be empty");
|
|
3919
|
+
}
|
|
3920
|
+
return {
|
|
3921
|
+
kind: "parameter-sweep",
|
|
3922
|
+
async propose(ctx) {
|
|
3923
|
+
if (typeof ctx.currentSurface !== "string") {
|
|
3924
|
+
throw new Error(
|
|
3925
|
+
"parameterSweepProposer: currentSurface must be a JSON string config surface"
|
|
3926
|
+
);
|
|
3927
|
+
}
|
|
3928
|
+
const parse = opts.parse ?? parseJsonObject;
|
|
3929
|
+
const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
|
|
3930
|
+
const current = parse(ctx.currentSurface);
|
|
3931
|
+
const currentCanonical = canonicalJson2(current);
|
|
3932
|
+
const tried = triedLabels(ctx.history);
|
|
3933
|
+
const out = [];
|
|
3934
|
+
for (const candidate of opts.candidates) {
|
|
3935
|
+
if (tried.has(candidate.label)) continue;
|
|
3936
|
+
const next = applyParameterCandidate(current, candidate);
|
|
3937
|
+
const surface = stringify(next);
|
|
3938
|
+
if (surface === ctx.currentSurface || canonicalJson2(next) === currentCanonical) continue;
|
|
3939
|
+
out.push({ surface, label: candidate.label, rationale: candidate.rationale });
|
|
3940
|
+
if (out.length >= ctx.populationSize) break;
|
|
3941
|
+
}
|
|
3942
|
+
return out;
|
|
3597
3943
|
}
|
|
3598
|
-
|
|
3599
|
-
}
|
|
3944
|
+
};
|
|
3600
3945
|
}
|
|
3601
|
-
function
|
|
3602
|
-
|
|
3603
|
-
throw new Error(`compareOptimizationMethods: method '${name}' returned no result`);
|
|
3604
|
-
}
|
|
3946
|
+
function parseJsonObject(surface) {
|
|
3947
|
+
let parsed;
|
|
3605
3948
|
try {
|
|
3606
|
-
|
|
3607
|
-
} catch (
|
|
3949
|
+
parsed = JSON.parse(surface);
|
|
3950
|
+
} catch (error) {
|
|
3608
3951
|
throw new Error(
|
|
3609
|
-
`
|
|
3610
|
-
{ cause }
|
|
3952
|
+
`parameterSweepProposer: currentSurface must be valid JSON object config (${error instanceof Error ? error.message : String(error)})`
|
|
3611
3953
|
);
|
|
3612
3954
|
}
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
throw new Error(`compareOptimizationMethods: method '${name}' returned an invalid durationMs`);
|
|
3955
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3956
|
+
throw new Error("parameterSweepProposer: JSON surface must parse to an object");
|
|
3616
3957
|
}
|
|
3958
|
+
return parsed;
|
|
3617
3959
|
}
|
|
3618
|
-
function
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3960
|
+
function triedLabels(history) {
|
|
3961
|
+
const tried = /* @__PURE__ */ new Set();
|
|
3962
|
+
for (const generation of history) {
|
|
3963
|
+
for (const candidate of generation.candidates) {
|
|
3964
|
+
if (!candidate.label) continue;
|
|
3965
|
+
tried.add(candidate.label);
|
|
3966
|
+
tried.add(candidateStrategy(candidate));
|
|
3967
|
+
}
|
|
3624
3968
|
}
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3969
|
+
return tried;
|
|
3970
|
+
}
|
|
3971
|
+
function applyParameterCandidate(current, candidate) {
|
|
3972
|
+
const next = cloneJsonObject(current);
|
|
3973
|
+
if (candidate.patch) deepMerge(next, candidate.patch);
|
|
3974
|
+
for (const change of candidate.changes ?? []) {
|
|
3975
|
+
setPath(
|
|
3976
|
+
next,
|
|
3977
|
+
typeof change.path === "string" ? change.path.split(".") : [...change.path],
|
|
3978
|
+
change.value
|
|
3979
|
+
);
|
|
3629
3980
|
}
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3981
|
+
return next;
|
|
3982
|
+
}
|
|
3983
|
+
function cloneJsonObject(value) {
|
|
3984
|
+
return JSON.parse(JSON.stringify(value));
|
|
3985
|
+
}
|
|
3986
|
+
function deepMerge(target, patch) {
|
|
3987
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
3988
|
+
assertSafeJsonKey(key);
|
|
3989
|
+
if (isPlainObject(value) && isPlainObject(target[key])) {
|
|
3990
|
+
deepMerge(target[key], value);
|
|
3991
|
+
} else {
|
|
3992
|
+
target[key] = value;
|
|
3636
3993
|
}
|
|
3637
|
-
|
|
3638
|
-
|
|
3994
|
+
}
|
|
3995
|
+
}
|
|
3996
|
+
function setPath(target, path, value) {
|
|
3997
|
+
if (path.length === 0) throw new Error("parameterSweepProposer: change path must not be empty");
|
|
3998
|
+
for (const part of path) assertSafeJsonKey(part);
|
|
3999
|
+
let cursor = target;
|
|
4000
|
+
for (const part of path.slice(0, -1)) {
|
|
4001
|
+
const existing = cursor[part];
|
|
4002
|
+
if (!isPlainObject(existing)) {
|
|
4003
|
+
cursor[part] = {};
|
|
3639
4004
|
}
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
4005
|
+
cursor = cursor[part];
|
|
4006
|
+
}
|
|
4007
|
+
cursor[path[path.length - 1]] = value;
|
|
4008
|
+
}
|
|
4009
|
+
function assertSafeJsonKey(key) {
|
|
4010
|
+
if (!key.trim()) throw new Error("parameterSweepProposer: change path contains an empty key");
|
|
4011
|
+
if (UNSAFE_JSON_KEYS.has(key)) {
|
|
4012
|
+
throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
|
|
4013
|
+
}
|
|
4014
|
+
}
|
|
4015
|
+
function canonicalJson2(value) {
|
|
4016
|
+
if (Array.isArray(value)) return `[${value.map((item) => canonicalJson2(item)).join(",")}]`;
|
|
4017
|
+
if (isPlainObject(value)) {
|
|
4018
|
+
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key])}`).join(",")}}`;
|
|
4019
|
+
}
|
|
4020
|
+
return JSON.stringify(value);
|
|
4021
|
+
}
|
|
4022
|
+
function isPlainObject(value) {
|
|
4023
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
4024
|
+
const proto = Object.getPrototypeOf(value);
|
|
4025
|
+
return proto === Object.prototype || proto === null;
|
|
4026
|
+
}
|
|
4027
|
+
|
|
4028
|
+
// src/campaign/skill-patch.ts
|
|
4029
|
+
function applySkillPatch(surface, patch) {
|
|
4030
|
+
let lines = surface.split("\n");
|
|
4031
|
+
let applied = 0;
|
|
4032
|
+
const rejected = [];
|
|
4033
|
+
const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
|
|
4034
|
+
for (const op of patch.ops) {
|
|
4035
|
+
if (op.op === "add") {
|
|
4036
|
+
if (typeof op.text !== "string" || op.text.trim() === "") {
|
|
4037
|
+
rejected.push({ op, reason: "empty add text" });
|
|
4038
|
+
continue;
|
|
3647
4039
|
}
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
4040
|
+
const insert = op.text.split("\n");
|
|
4041
|
+
if (op.after === void 0 || op.after === "") {
|
|
4042
|
+
lines = [...lines, ...insert];
|
|
4043
|
+
applied++;
|
|
4044
|
+
continue;
|
|
3652
4045
|
}
|
|
3653
|
-
|
|
4046
|
+
const idx = findLine(op.after);
|
|
4047
|
+
if (idx === -1) {
|
|
4048
|
+
rejected.push({ op, reason: `add anchor not found: ${truncate2(op.after)}` });
|
|
4049
|
+
continue;
|
|
4050
|
+
}
|
|
4051
|
+
lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
|
|
4052
|
+
applied++;
|
|
4053
|
+
} else if (op.op === "delete") {
|
|
4054
|
+
const idx = findLine(op.anchor);
|
|
4055
|
+
if (idx === -1) {
|
|
4056
|
+
rejected.push({ op, reason: `delete anchor not found: ${truncate2(op.anchor)}` });
|
|
4057
|
+
continue;
|
|
4058
|
+
}
|
|
4059
|
+
lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
|
|
4060
|
+
applied++;
|
|
4061
|
+
} else {
|
|
4062
|
+
const idx = findLine(op.anchor);
|
|
4063
|
+
if (idx === -1) {
|
|
4064
|
+
rejected.push({ op, reason: `replace anchor not found: ${truncate2(op.anchor)}` });
|
|
4065
|
+
continue;
|
|
4066
|
+
}
|
|
4067
|
+
if (typeof op.text !== "string") {
|
|
4068
|
+
rejected.push({ op, reason: "replace text missing" });
|
|
4069
|
+
continue;
|
|
4070
|
+
}
|
|
4071
|
+
lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
|
|
4072
|
+
applied++;
|
|
3654
4073
|
}
|
|
3655
4074
|
}
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
4075
|
+
return { surface: lines.join("\n"), applied, rejected };
|
|
4076
|
+
}
|
|
4077
|
+
function patchEditCount(patch) {
|
|
4078
|
+
return patch.ops.length;
|
|
4079
|
+
}
|
|
4080
|
+
function truncate2(s, max = 48) {
|
|
4081
|
+
return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
|
|
4082
|
+
}
|
|
4083
|
+
|
|
4084
|
+
// src/campaign/proposers/skill-opt.ts
|
|
4085
|
+
var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
|
|
4086
|
+
function skillOptProposer(opts) {
|
|
4087
|
+
const evidenceK = opts.evidenceK ?? 3;
|
|
4088
|
+
const defaultBudget = opts.editBudget ?? 3;
|
|
4089
|
+
const directCostLedger = opts.costLedger ?? new CostLedger();
|
|
4090
|
+
async function proposePatches(args) {
|
|
4091
|
+
const userPrompt = buildPatchPrompt({
|
|
4092
|
+
target: opts.target,
|
|
4093
|
+
surface: args.surface,
|
|
4094
|
+
evidence: args.evidence,
|
|
4095
|
+
editBudget: args.editBudget,
|
|
4096
|
+
rejectedBuffer: args.rejectedBuffer,
|
|
4097
|
+
metaNote: args.metaNote,
|
|
4098
|
+
findingsNote: args.findingsNote,
|
|
4099
|
+
count: args.count
|
|
4100
|
+
});
|
|
4101
|
+
const request = {
|
|
4102
|
+
model: opts.model,
|
|
4103
|
+
messages: [
|
|
4104
|
+
{ role: "system", content: SKILLOPT_SYSTEM },
|
|
4105
|
+
{ role: "user", content: userPrompt }
|
|
4106
|
+
],
|
|
4107
|
+
jsonMode: true,
|
|
4108
|
+
temperature: opts.temperature ?? 0.6,
|
|
4109
|
+
maxTokens: opts.maxTokens ?? 4e3
|
|
4110
|
+
};
|
|
4111
|
+
const paid = await (args.costLedger ?? directCostLedger).runPaidCall({
|
|
4112
|
+
channel: "driver",
|
|
4113
|
+
phase: args.costPhase ?? "search.proposal",
|
|
4114
|
+
actor: "skill-opt.propose",
|
|
4115
|
+
model: opts.model,
|
|
4116
|
+
maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
|
|
4117
|
+
signal: args.signal,
|
|
4118
|
+
execute: (signal, callId) => callLlm(request, { ...opts.llm, signal, idempotencyKey: callId }),
|
|
4119
|
+
receipt: costReceiptFromLlm,
|
|
4120
|
+
receiptFromError: costReceiptFromLlmError
|
|
4121
|
+
});
|
|
4122
|
+
if (!paid.succeeded) throw paid.error;
|
|
4123
|
+
const result = paid.value;
|
|
4124
|
+
return parseSkillPatchResponse(result.content, args.count, args.editBudget);
|
|
3661
4125
|
}
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
4126
|
+
return {
|
|
4127
|
+
kind: "skill-opt",
|
|
4128
|
+
proposePatches,
|
|
4129
|
+
async propose(ctx) {
|
|
4130
|
+
if (typeof ctx.currentSurface !== "string") {
|
|
4131
|
+
throw new Error(
|
|
4132
|
+
"skillOptProposer: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
|
|
4133
|
+
);
|
|
4134
|
+
}
|
|
4135
|
+
const surface = ctx.currentSurface;
|
|
4136
|
+
const patches = await proposePatches({
|
|
4137
|
+
surface,
|
|
4138
|
+
evidence: evidenceFromHistory(ctx, evidenceK),
|
|
4139
|
+
editBudget: defaultBudget,
|
|
4140
|
+
rejectedBuffer: [],
|
|
4141
|
+
findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
|
|
4142
|
+
count: ctx.populationSize,
|
|
4143
|
+
signal: ctx.signal,
|
|
4144
|
+
costLedger: ctx.costLedger ?? directCostLedger,
|
|
4145
|
+
costPhase: ctx.costPhase ?? "search.proposal"
|
|
4146
|
+
});
|
|
4147
|
+
const out = [];
|
|
4148
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4149
|
+
for (const patch of patches) {
|
|
4150
|
+
const { surface: candidate, applied } = applySkillPatch(surface, patch);
|
|
4151
|
+
if (applied === 0 || candidate === surface || seen.has(candidate)) continue;
|
|
4152
|
+
seen.add(candidate);
|
|
4153
|
+
out.push({ surface: candidate, label: patch.label, rationale: patch.rationale });
|
|
4154
|
+
if (out.length >= ctx.populationSize) break;
|
|
4155
|
+
}
|
|
4156
|
+
return out;
|
|
4157
|
+
}
|
|
4158
|
+
};
|
|
4159
|
+
}
|
|
4160
|
+
function evidenceFromHistory(ctx, k) {
|
|
4161
|
+
const last = ctx.history.at(-1);
|
|
4162
|
+
if (!last || last.candidates.length === 0) return { weakScenarios: [], weakDimensions: [] };
|
|
4163
|
+
const best = [...last.candidates].sort((a, b) => b.composite - a.composite)[0];
|
|
4164
|
+
if (!best) return { weakScenarios: [], weakDimensions: [] };
|
|
4165
|
+
const weakScenarios = [...best.scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
|
|
4166
|
+
const weakDimensions = Object.entries(best.dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
|
|
4167
|
+
return { weakScenarios, weakDimensions };
|
|
4168
|
+
}
|
|
4169
|
+
function buildPatchPrompt(args) {
|
|
4170
|
+
const lines = [
|
|
4171
|
+
`Skill document governs: ${args.target}.`,
|
|
4172
|
+
"",
|
|
4173
|
+
"Current skill document:",
|
|
4174
|
+
"```",
|
|
4175
|
+
args.surface,
|
|
4176
|
+
"```",
|
|
4177
|
+
"",
|
|
4178
|
+
`Propose ${args.count} candidate patch(es). Each patch is a SMALL bundle of`,
|
|
4179
|
+
`at most ${args.editBudget} op(s). Anchors must be verbatim substrings of`,
|
|
4180
|
+
"existing lines. Prefer adding a specific missing rule or sharpening a vague",
|
|
4181
|
+
"one over deleting; never rewrite the whole document."
|
|
4182
|
+
];
|
|
4183
|
+
if (args.evidence.weakScenarios.length > 0) {
|
|
4184
|
+
lines.push(
|
|
4185
|
+
"",
|
|
4186
|
+
"Weakest scenarios (patch to fix these):",
|
|
4187
|
+
...args.evidence.weakScenarios.map((s) => `- ${s.scenarioId} (${s.composite.toFixed(2)})`)
|
|
3665
4188
|
);
|
|
3666
4189
|
}
|
|
3667
|
-
if (
|
|
3668
|
-
|
|
3669
|
-
|
|
4190
|
+
if (args.evidence.weakDimensions.length > 0) {
|
|
4191
|
+
lines.push(
|
|
4192
|
+
"",
|
|
4193
|
+
"Weakest dimensions (what to improve):",
|
|
4194
|
+
...args.evidence.weakDimensions.map((d) => `- ${d.dimension} (${d.score.toFixed(2)})`)
|
|
3670
4195
|
);
|
|
3671
4196
|
}
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
throw new Error(
|
|
3678
|
-
`compareOptimizationMethods: resamples must be at least ${minimumResamples} for simultaneous confidence ${confidence} across ${opts.methods.length} methods, got ${resamples}`
|
|
4197
|
+
if (args.rejectedBuffer.length > 0) {
|
|
4198
|
+
lines.push(
|
|
4199
|
+
"",
|
|
4200
|
+
"Already tried and REJECTED (do not repeat or restate these edits):",
|
|
4201
|
+
...args.rejectedBuffer.map((e) => `- ${e.label}: ${e.rationale} \u2014 ${e.reason}`)
|
|
3679
4202
|
);
|
|
3680
4203
|
}
|
|
3681
|
-
if (
|
|
3682
|
-
|
|
3683
|
-
"compareOptimizationMethods: optimizationConcurrency must be a positive safe integer"
|
|
3684
|
-
);
|
|
4204
|
+
if (args.findingsNote) {
|
|
4205
|
+
lines.push("", args.findingsNote);
|
|
3685
4206
|
}
|
|
3686
|
-
if (
|
|
3687
|
-
|
|
4207
|
+
if (args.metaNote) {
|
|
4208
|
+
lines.push("", `Strategy note from prior epochs: ${args.metaNote}`);
|
|
3688
4209
|
}
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
4210
|
+
return lines.join("\n");
|
|
4211
|
+
}
|
|
4212
|
+
var SkillPatchParseError = class extends Error {
|
|
4213
|
+
constructor(message) {
|
|
4214
|
+
super(message);
|
|
4215
|
+
this.name = "SkillPatchParseError";
|
|
3693
4216
|
}
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
4217
|
+
};
|
|
4218
|
+
function parseSkillPatchResponse(raw, maxPatches, editBudget) {
|
|
4219
|
+
let text = raw.trim();
|
|
4220
|
+
if (text.startsWith("```")) text = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
|
|
4221
|
+
const start = text.indexOf("{");
|
|
4222
|
+
const end = text.lastIndexOf("}");
|
|
4223
|
+
if (start < 0 || end <= start) {
|
|
4224
|
+
throw new SkillPatchParseError(
|
|
4225
|
+
`parseSkillPatchResponse: response was not valid JSON (no object found): ${snippet(raw)}`
|
|
3697
4226
|
);
|
|
3698
4227
|
}
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
4228
|
+
let parsed;
|
|
4229
|
+
try {
|
|
4230
|
+
parsed = JSON.parse(text.slice(start, end + 1));
|
|
4231
|
+
} catch (err) {
|
|
4232
|
+
throw new SkillPatchParseError(
|
|
4233
|
+
`parseSkillPatchResponse: response was not valid JSON (${err instanceof Error ? err.message : String(err)}): ${snippet(raw)}`
|
|
3702
4234
|
);
|
|
3703
4235
|
}
|
|
4236
|
+
const rawPatches = Array.isArray(parsed.patches) ? parsed.patches : [];
|
|
4237
|
+
const out = [];
|
|
4238
|
+
for (const rp of rawPatches) {
|
|
4239
|
+
if (typeof rp !== "object" || rp === null) continue;
|
|
4240
|
+
const obj = rp;
|
|
4241
|
+
const ops = Array.isArray(obj.ops) ? obj.ops.map(normalizeOp).filter(isOp) : [];
|
|
4242
|
+
if (ops.length === 0) continue;
|
|
4243
|
+
out.push({
|
|
4244
|
+
label: typeof obj.label === "string" ? obj.label : "patch",
|
|
4245
|
+
rationale: typeof obj.rationale === "string" ? obj.rationale : "",
|
|
4246
|
+
ops: ops.slice(0, editBudget)
|
|
4247
|
+
});
|
|
4248
|
+
if (out.length >= maxPatches) break;
|
|
4249
|
+
}
|
|
4250
|
+
return out;
|
|
3704
4251
|
}
|
|
3705
|
-
function
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
4252
|
+
function normalizeOp(raw) {
|
|
4253
|
+
if (typeof raw !== "object" || raw === null) return null;
|
|
4254
|
+
const o = raw;
|
|
4255
|
+
if (o.op === "add") {
|
|
4256
|
+
if (typeof o.text !== "string") return null;
|
|
4257
|
+
const op = { op: "add", text: o.text };
|
|
4258
|
+
if (typeof o.after === "string") op.after = o.after;
|
|
4259
|
+
return op;
|
|
3711
4260
|
}
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
{
|
|
3715
|
-
{ name: "testScenarios", scenarios: opts.testScenarios }
|
|
3716
|
-
];
|
|
3717
|
-
const owner = /* @__PURE__ */ new Map();
|
|
3718
|
-
for (const partition of partitions) {
|
|
3719
|
-
if (!Array.isArray(partition.scenarios) || partition.scenarios.length === 0) {
|
|
3720
|
-
throw new Error(`compareOptimizationMethods: ${partition.name} is empty`);
|
|
3721
|
-
}
|
|
3722
|
-
if (partition.name === "testScenarios" && partition.scenarios.length < 2) {
|
|
3723
|
-
throw new Error(
|
|
3724
|
-
"compareOptimizationMethods: testScenarios requires at least 2 scenarios to estimate uncertainty"
|
|
3725
|
-
);
|
|
3726
|
-
}
|
|
3727
|
-
const local = /* @__PURE__ */ new Set();
|
|
3728
|
-
const duplicates = /* @__PURE__ */ new Set();
|
|
3729
|
-
const overlaps = /* @__PURE__ */ new Map();
|
|
3730
|
-
for (const scenario of partition.scenarios) {
|
|
3731
|
-
if (local.has(scenario.id)) duplicates.add(scenario.id);
|
|
3732
|
-
local.add(scenario.id);
|
|
3733
|
-
const prior = owner.get(scenario.id);
|
|
3734
|
-
if (prior !== void 0 && prior !== partition.name) overlaps.set(scenario.id, prior);
|
|
3735
|
-
}
|
|
3736
|
-
if (duplicates.size > 0) {
|
|
3737
|
-
throw new Error(
|
|
3738
|
-
`compareOptimizationMethods: ${partition.name} contains duplicate scenario id(s) [${[
|
|
3739
|
-
...duplicates
|
|
3740
|
-
].join(", ")}]`
|
|
3741
|
-
);
|
|
3742
|
-
}
|
|
3743
|
-
if (overlaps.size > 0) {
|
|
3744
|
-
const detail = [...overlaps].map(([id, prior]) => `${id} (${prior} \u2229 ${partition.name})`).join(", ");
|
|
3745
|
-
throw new Error(
|
|
3746
|
-
`compareOptimizationMethods: trainScenarios, selectionScenarios, and testScenarios must be pairwise disjoint; overlap: [${detail}]`
|
|
3747
|
-
);
|
|
3748
|
-
}
|
|
3749
|
-
assertCampaignDesign(partition.scenarios, opts.reps ?? 1);
|
|
3750
|
-
for (const id of local) owner.set(id, partition.name);
|
|
4261
|
+
if (o.op === "delete") {
|
|
4262
|
+
if (typeof o.anchor !== "string") return null;
|
|
4263
|
+
return { op: "delete", anchor: o.anchor };
|
|
3751
4264
|
}
|
|
3752
|
-
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
}
|
|
3756
|
-
function slug(name) {
|
|
3757
|
-
return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase() || "method";
|
|
3758
|
-
}
|
|
3759
|
-
function assertConfidence(confidence) {
|
|
3760
|
-
if (!Number.isFinite(confidence) || confidence <= 0 || confidence >= 1) {
|
|
3761
|
-
throw new Error(
|
|
3762
|
-
`compareOptimizationMethods: confidence must be a finite number in (0,1), got ${String(confidence)}`
|
|
3763
|
-
);
|
|
4265
|
+
if (o.op === "replace") {
|
|
4266
|
+
if (typeof o.anchor !== "string" || typeof o.text !== "string") return null;
|
|
4267
|
+
return { op: "replace", anchor: o.anchor, text: o.text };
|
|
3764
4268
|
}
|
|
4269
|
+
return null;
|
|
3765
4270
|
}
|
|
3766
|
-
function
|
|
3767
|
-
|
|
3768
|
-
return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
|
|
3769
|
-
}
|
|
3770
|
-
function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed) {
|
|
3771
|
-
const cloneScenarios = (scenarios) => Object.freeze(scenarios.map((scenario) => structuredClone(scenario)));
|
|
3772
|
-
const judges = opts.judges.map(
|
|
3773
|
-
(judge) => Object.freeze({
|
|
3774
|
-
...judge,
|
|
3775
|
-
dimensions: Object.freeze(
|
|
3776
|
-
judge.dimensions.map((dimension) => Object.freeze({ ...dimension }))
|
|
3777
|
-
)
|
|
3778
|
-
})
|
|
3779
|
-
);
|
|
3780
|
-
return Object.freeze({
|
|
3781
|
-
baselineSurface: structuredClone(opts.baselineSurface),
|
|
3782
|
-
trainScenarios: cloneScenarios(opts.trainScenarios),
|
|
3783
|
-
selectionScenarios: cloneScenarios(opts.selectionScenarios),
|
|
3784
|
-
dispatchWithSurface: opts.dispatchWithSurface,
|
|
3785
|
-
judges: Object.freeze(judges),
|
|
3786
|
-
runDir: `${resolvedRunDir}/optimization/${slug(methodName)}`,
|
|
3787
|
-
seed,
|
|
3788
|
-
runOptions: Object.freeze({ ...opts.optimizationRunOptions ?? {} })
|
|
3789
|
-
});
|
|
3790
|
-
}
|
|
3791
|
-
function costFromLedgerSummary(summary) {
|
|
3792
|
-
const cost = {
|
|
3793
|
-
totalCostUsd: summary.totalCostUsd,
|
|
3794
|
-
accountingComplete: summary.accountingComplete,
|
|
3795
|
-
incompleteReasons: [...summary.incompleteReasons]
|
|
3796
|
-
};
|
|
3797
|
-
assertComparisonCost(cost, "cost ledger");
|
|
3798
|
-
return cost;
|
|
3799
|
-
}
|
|
3800
|
-
function combineCosts(entries) {
|
|
3801
|
-
return {
|
|
3802
|
-
totalCostUsd: entries.reduce((total, entry) => total + entry.cost.totalCostUsd, 0),
|
|
3803
|
-
accountingComplete: entries.every((entry) => entry.cost.accountingComplete),
|
|
3804
|
-
incompleteReasons: entries.flatMap(
|
|
3805
|
-
(entry) => entry.cost.incompleteReasons.map((reason) => `${entry.label}: ${reason}`)
|
|
3806
|
-
)
|
|
3807
|
-
};
|
|
4271
|
+
function isOp(op) {
|
|
4272
|
+
return op !== null;
|
|
3808
4273
|
}
|
|
3809
|
-
function
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
}
|
|
3813
|
-
if (!Number.isFinite(cost.totalCostUsd) || cost.totalCostUsd < 0) {
|
|
3814
|
-
throw new Error(`compareOptimizationMethods: ${label} returned an invalid totalCostUsd`);
|
|
3815
|
-
}
|
|
3816
|
-
if (typeof cost.accountingComplete !== "boolean") {
|
|
3817
|
-
throw new Error(`compareOptimizationMethods: ${label} returned invalid accountingComplete`);
|
|
3818
|
-
}
|
|
3819
|
-
if (!Array.isArray(cost.incompleteReasons) || cost.incompleteReasons.some(
|
|
3820
|
-
(reason) => typeof reason !== "string" || reason.trim().length === 0
|
|
3821
|
-
)) {
|
|
3822
|
-
throw new Error(`compareOptimizationMethods: ${label} returned invalid incompleteReasons`);
|
|
3823
|
-
}
|
|
3824
|
-
if (cost.accountingComplete !== (cost.incompleteReasons.length === 0)) {
|
|
3825
|
-
throw new Error(
|
|
3826
|
-
`compareOptimizationMethods: ${label} returned inconsistent cost completeness and reasons`
|
|
3827
|
-
);
|
|
3828
|
-
}
|
|
4274
|
+
function snippet(s, max = 120) {
|
|
4275
|
+
const t = s.trim().replace(/\s+/g, " ");
|
|
4276
|
+
return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
|
|
3829
4277
|
}
|
|
3830
4278
|
|
|
3831
4279
|
// src/campaign/presets/run-skill-opt.ts
|
|
@@ -4627,14 +5075,14 @@ function scoreboardSummary(rows) {
|
|
|
4627
5075
|
function escapeCell(s) {
|
|
4628
5076
|
return s.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
|
|
4629
5077
|
}
|
|
4630
|
-
function
|
|
5078
|
+
function truncate3(s, max) {
|
|
4631
5079
|
return s.length <= max ? s : `${s.slice(0, Math.max(0, max - 1))}\u2026`;
|
|
4632
5080
|
}
|
|
4633
5081
|
function renderScoreboardMarkdown(rows, opts = {}) {
|
|
4634
5082
|
const maxEv = opts.maxEvidenceChars ?? 160;
|
|
4635
5083
|
const sum = scoreboardSummary(rows);
|
|
4636
5084
|
const pct = (n) => `${Math.round(n * 100)}%`;
|
|
4637
|
-
const ev = (e) => escapeCell(
|
|
5085
|
+
const ev = (e) => escapeCell(truncate3(e.join("; "), maxEv)) || "\u2014";
|
|
4638
5086
|
const out = [`# ${opts.title ?? "Product-flow playback scoreboard"}`, ""];
|
|
4639
5087
|
if (opts.meta) {
|
|
4640
5088
|
for (const [k, v] of Object.entries(opts.meta)) out.push(`- **${k}:** ${v}`);
|
|
@@ -4872,7 +5320,7 @@ async function runLineageLoop(opts) {
|
|
|
4872
5320
|
|
|
4873
5321
|
// src/campaign/presets/run-profile-matrix.ts
|
|
4874
5322
|
import { createHash as createHash6 } from "crypto";
|
|
4875
|
-
import { join as
|
|
5323
|
+
import { join as join4 } from "path";
|
|
4876
5324
|
|
|
4877
5325
|
// src/agent-profile.ts
|
|
4878
5326
|
import { createHash as createHash5 } from "crypto";
|
|
@@ -5137,7 +5585,7 @@ async function runProfileMatrix(opts) {
|
|
|
5137
5585
|
captureSource: opts.captureSource,
|
|
5138
5586
|
storage: opts.storage,
|
|
5139
5587
|
now: opts.now,
|
|
5140
|
-
runDir:
|
|
5588
|
+
runDir: join4(opts.runDir, sanitize(profileId))
|
|
5141
5589
|
});
|
|
5142
5590
|
const axis = harnessAxisOf(profile);
|
|
5143
5591
|
const buildCellIdentity = (cellModel) => buildAgentProfileCell({
|
|
@@ -5420,8 +5868,8 @@ import { promisify } from "util";
|
|
|
5420
5868
|
|
|
5421
5869
|
// src/campaign/proposers/analysis-edit.ts
|
|
5422
5870
|
import { mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
|
|
5423
|
-
import { tmpdir } from "os";
|
|
5424
|
-
import { join as
|
|
5871
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
5872
|
+
import { join as join5 } from "path";
|
|
5425
5873
|
var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
|
|
5426
5874
|
function surfaceToPromptText(surface) {
|
|
5427
5875
|
return typeof surface === "string" ? surface : JSON.stringify(surface);
|
|
@@ -5436,8 +5884,8 @@ function analysisEditProposer(opts) {
|
|
|
5436
5884
|
const phase = ctx.costPhase ?? "search.proposal";
|
|
5437
5885
|
const traces = await opts.resolveTraces(ctx) ?? "";
|
|
5438
5886
|
if (!traces.trim()) throw new Error(opts.noTracesError);
|
|
5439
|
-
const dir = mkdtempSync(
|
|
5440
|
-
const tracePath =
|
|
5887
|
+
const dir = mkdtempSync(join5(tmpdir2(), `${opts.kind}-proposer-`));
|
|
5888
|
+
const tracePath = join5(dir, "traces.jsonl");
|
|
5441
5889
|
writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
|
|
5442
5890
|
`);
|
|
5443
5891
|
let report;
|
|
@@ -8064,8 +8512,8 @@ import {
|
|
|
8064
8512
|
realpathSync,
|
|
8065
8513
|
rmSync
|
|
8066
8514
|
} from "fs";
|
|
8067
|
-
import { devNull, tmpdir as
|
|
8068
|
-
import { basename, dirname as dirname2, isAbsolute as isAbsolute2, join as
|
|
8515
|
+
import { devNull, tmpdir as tmpdir3 } from "os";
|
|
8516
|
+
import { basename, dirname as dirname2, isAbsolute as isAbsolute2, join as join6, relative as relative2, resolve as resolve3, sep } from "path";
|
|
8069
8517
|
var MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024;
|
|
8070
8518
|
var FILE_HASH_CHUNK_BYTES = 1024 * 1024;
|
|
8071
8519
|
var GIT_REPOSITORY_ENV = /* @__PURE__ */ new Set([
|
|
@@ -8194,9 +8642,9 @@ var CANONICAL_GIT_ENV = {
|
|
|
8194
8642
|
GIT_CONFIG_SYSTEM: devNull
|
|
8195
8643
|
};
|
|
8196
8644
|
function patchBytes(git, cwd, baseCommit, candidateCommit) {
|
|
8197
|
-
const scratch = mkdtempSync2(
|
|
8198
|
-
const bareRepo =
|
|
8199
|
-
const emptyTemplate =
|
|
8645
|
+
const scratch = mkdtempSync2(join6(tmpdir3(), "agent-eval-patch-"));
|
|
8646
|
+
const bareRepo = join6(scratch, "repo.git");
|
|
8647
|
+
const emptyTemplate = join6(scratch, "empty-template");
|
|
8200
8648
|
mkdirSync2(emptyTemplate);
|
|
8201
8649
|
try {
|
|
8202
8650
|
const objectFormat = gitObjectHashAlgorithm(candidateCommit);
|
|
@@ -8232,7 +8680,7 @@ function resolveCommit(git, cwd, ref) {
|
|
|
8232
8680
|
}
|
|
8233
8681
|
function unresolvedWorktreePath(surface, worktreeDir) {
|
|
8234
8682
|
if (isAbsolute2(surface.worktreeRef)) return surface.worktreeRef;
|
|
8235
|
-
if (worktreeDir) return
|
|
8683
|
+
if (worktreeDir) return join6(worktreeDir, basename(surface.worktreeRef));
|
|
8236
8684
|
return surface.worktreeRef;
|
|
8237
8685
|
}
|
|
8238
8686
|
function displayGitPath(path) {
|
|
@@ -8308,7 +8756,7 @@ function assertSafeRelativePath(root, path) {
|
|
|
8308
8756
|
if (segment.length === 0 || segment === "." || segment === "..") {
|
|
8309
8757
|
throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`);
|
|
8310
8758
|
}
|
|
8311
|
-
parent =
|
|
8759
|
+
parent = join6(parent, segment);
|
|
8312
8760
|
const stat = lstatSync(parent);
|
|
8313
8761
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
8314
8762
|
throw new WorktreeAdapterError(
|
|
@@ -8568,13 +9016,13 @@ function slug2(label) {
|
|
|
8568
9016
|
}
|
|
8569
9017
|
function gitWorktreeAdapter(opts) {
|
|
8570
9018
|
const git = opts.git ?? defaultGit;
|
|
8571
|
-
const worktreeDir = opts.worktreeDir ??
|
|
9019
|
+
const worktreeDir = opts.worktreeDir ?? join6(opts.repoRoot, ".worktrees");
|
|
8572
9020
|
const branchPrefix = opts.branchPrefix ?? "improve";
|
|
8573
9021
|
return {
|
|
8574
9022
|
async create({ baseRef, label }) {
|
|
8575
9023
|
const id = `${slug2(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
|
|
8576
9024
|
const branch = `${branchPrefix}/${id}`;
|
|
8577
|
-
const path =
|
|
9025
|
+
const path = join6(worktreeDir, id);
|
|
8578
9026
|
const baseCommit = resolveCommit(git, opts.repoRoot, baseRef);
|
|
8579
9027
|
const baseTree = gitText(
|
|
8580
9028
|
git,
|
|
@@ -8683,6 +9131,9 @@ export {
|
|
|
8683
9131
|
neutralizationGate,
|
|
8684
9132
|
sequentialPairedGate,
|
|
8685
9133
|
sequentialDecide,
|
|
9134
|
+
compareOptimizationMethods,
|
|
9135
|
+
costFromLedgerSummary,
|
|
9136
|
+
gepaOptimizationMethod,
|
|
8686
9137
|
rolloutArgumentDiff,
|
|
8687
9138
|
classifyUngroundedLiterals,
|
|
8688
9139
|
LabeledScenarioStoreError,
|
|
@@ -8696,8 +9147,6 @@ export {
|
|
|
8696
9147
|
skillOptProposer,
|
|
8697
9148
|
SkillPatchParseError,
|
|
8698
9149
|
parseSkillPatchResponse,
|
|
8699
|
-
compareOptimizationMethods,
|
|
8700
|
-
costFromLedgerSummary,
|
|
8701
9150
|
runSkillOpt,
|
|
8702
9151
|
gepaReflectionMethod,
|
|
8703
9152
|
gepaParetoMethod,
|
|
@@ -8736,4 +9185,4 @@ export {
|
|
|
8736
9185
|
verifyCodeSurface,
|
|
8737
9186
|
resolveWorktreePath
|
|
8738
9187
|
};
|
|
8739
|
-
//# sourceMappingURL=chunk-
|
|
9188
|
+
//# sourceMappingURL=chunk-VPDOSN3L.js.map
|