@tangle-network/agent-eval 0.119.1 → 0.120.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -3
- package/README.md +0 -26
- package/dist/benchmarks/index.d.ts +11 -5
- package/dist/benchmarks/index.js +4 -4
- package/dist/campaign/index.d.ts +93 -37
- package/dist/campaign/index.js +23 -7
- package/dist/{chunk-PSXJ32SR.js → chunk-32BZXMSO.js} +229 -118
- package/dist/chunk-32BZXMSO.js.map +1 -0
- package/dist/{chunk-XJ7JVCHB.js → chunk-3A246TSA.js} +2 -2
- package/dist/{chunk-XMBOU5W7.js → chunk-JN2FCO5W.js} +4 -5
- package/dist/chunk-JN2FCO5W.js.map +1 -0
- package/dist/{chunk-D7AEXSM5.js → chunk-PICTDURQ.js} +2 -2
- package/dist/{chunk-PAHNGS65.js → chunk-XDIRG3TO.js} +175 -11
- package/dist/chunk-XDIRG3TO.js.map +1 -0
- package/dist/contract/index.d.ts +55 -28
- package/dist/contract/index.js +351 -62
- package/dist/contract/index.js.map +1 -1
- package/dist/index.d.ts +22 -8
- package/dist/index.js +9 -5
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +11 -5
- package/dist/{run-campaign-OBXSN5WK.js → run-campaign-HNFPJET4.js} +2 -2
- package/package.json +1 -6
- package/dist/chunk-PAHNGS65.js.map +0 -1
- package/dist/chunk-PSXJ32SR.js.map +0 -1
- package/dist/chunk-XMBOU5W7.js.map +0 -1
- package/dist/primeintellect/index.d.ts +0 -311
- package/dist/primeintellect/index.js +0 -348
- package/dist/primeintellect/index.js.map +0 -1
- /package/dist/{chunk-XJ7JVCHB.js.map → chunk-3A246TSA.js.map} +0 -0
- /package/dist/{chunk-D7AEXSM5.js.map → chunk-PICTDURQ.js.map} +0 -0
- /package/dist/{run-campaign-OBXSN5WK.js.map → run-campaign-HNFPJET4.js.map} +0 -0
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
|
+
assertCampaignSplitIdentity,
|
|
3
|
+
campaignCoverage,
|
|
4
|
+
canonicalJson,
|
|
2
5
|
contentHash,
|
|
3
6
|
createRunCostLedger,
|
|
7
|
+
formatCoverageFailures,
|
|
4
8
|
fsCampaignStorage,
|
|
5
9
|
resolveRunDir,
|
|
6
10
|
runCampaign,
|
|
7
|
-
|
|
8
|
-
} from "./chunk-
|
|
11
|
+
summarizeAgentReceiptIntegrity
|
|
12
|
+
} from "./chunk-XDIRG3TO.js";
|
|
9
13
|
import {
|
|
10
14
|
clamp01,
|
|
11
15
|
validatePolicyEditCandidateRecord
|
|
@@ -1831,7 +1835,7 @@ function defaultProductionGate(options) {
|
|
|
1831
1835
|
detail: {
|
|
1832
1836
|
candidateUsd: ctx.cost.candidate,
|
|
1833
1837
|
baselineUsd: ctx.cost.baseline,
|
|
1834
|
-
budgetUsd: options.budgetUsd
|
|
1838
|
+
...options.budgetUsd === void 0 ? {} : { budgetUsd: options.budgetUsd }
|
|
1835
1839
|
}
|
|
1836
1840
|
});
|
|
1837
1841
|
if (!budgetPass) {
|
|
@@ -2478,7 +2482,7 @@ function assertCodeSurfaceIdentity(surface) {
|
|
|
2478
2482
|
function codeSurfaceIdentityMaterial(surface) {
|
|
2479
2483
|
assertCodeSurfaceIdentity(surface);
|
|
2480
2484
|
return JSON.stringify({
|
|
2481
|
-
schema: "tangle.code-surface
|
|
2485
|
+
schema: "tangle.code-surface",
|
|
2482
2486
|
baseCommit: surface.baseCommit,
|
|
2483
2487
|
baseTree: surface.baseTree,
|
|
2484
2488
|
candidateTree: surface.candidateTree,
|
|
@@ -2496,6 +2500,33 @@ function surfaceContentHash(surface) {
|
|
|
2496
2500
|
function surfaceHash(surface) {
|
|
2497
2501
|
return surfaceContentHash(surface).slice("sha256:".length, "sha256:".length + 16);
|
|
2498
2502
|
}
|
|
2503
|
+
function renderSurfaceDiff(winnerSurface, baselineSurface) {
|
|
2504
|
+
if (typeof winnerSurface === "string" && typeof baselineSurface === "string") {
|
|
2505
|
+
return [
|
|
2506
|
+
"--- baseline",
|
|
2507
|
+
"+++ winner",
|
|
2508
|
+
...baselineSurface.split("\n").map((line) => `- ${line}`),
|
|
2509
|
+
...winnerSurface.split("\n").map((line) => `+ ${line}`)
|
|
2510
|
+
].join("\n");
|
|
2511
|
+
}
|
|
2512
|
+
const describe = (surface) => {
|
|
2513
|
+
if (typeof surface === "string") return "(prompt surface)";
|
|
2514
|
+
assertCodeSurfaceIdentity(surface);
|
|
2515
|
+
return [
|
|
2516
|
+
`baseCommit=${surface.baseCommit}`,
|
|
2517
|
+
`baseTree=${surface.baseTree}`,
|
|
2518
|
+
`candidateCommit=${surface.candidateCommit}`,
|
|
2519
|
+
`candidateTree=${surface.candidateTree}`,
|
|
2520
|
+
`patch=${surface.patch.sha256}`,
|
|
2521
|
+
`patchBytes=${surface.patch.byteLength}`,
|
|
2522
|
+
...surface.summary ? [surface.summary] : []
|
|
2523
|
+
].join("\n");
|
|
2524
|
+
};
|
|
2525
|
+
return `--- baseline
|
|
2526
|
+
${describe(baselineSurface)}
|
|
2527
|
+
+++ winner
|
|
2528
|
+
${describe(winnerSurface)}`;
|
|
2529
|
+
}
|
|
2499
2530
|
|
|
2500
2531
|
// src/campaign/proposers/gepa.ts
|
|
2501
2532
|
var REFLECTION_SYSTEM = 'You are an expert prompt engineer performing GEPA-style reflective mutation. You are given a prompt surface, its top trials (preserve what works) and its bottom trials (the evidence to fix). For each proposal, reason in this order before writing the payload: (1) LOCALIZE \u2014 point to the exact span of the current surface responsible for a bottom-trial failure; (2) DIAGNOSE the root cause (a missing rule, an ambiguous instruction, an over-broad directive), not just the symptom; (3) propose the MINIMAL, GENERALIZABLE edit that fixes the whole failure class \u2014 state it as a rule the agent should follow, never a patch memorized to the shown trials (that is overfitting and will not transfer to the held-out set); (4) PRESERVE every instruction the top trials depend on \u2014 do not delete or weaken working guidance. Put this localize\u2192diagnose\u2192fix reasoning in each proposal\'s `rationale`. Output ONLY a JSON object of shape {"proposals":[{"label":string,"rationale":string,"payload":string}]} where each `payload` is the FULL improved surface text. No prose outside the JSON.';
|
|
@@ -2747,79 +2778,6 @@ function campaignBreakdown(campaign) {
|
|
|
2747
2778
|
return { dimensions, scenarios };
|
|
2748
2779
|
}
|
|
2749
2780
|
|
|
2750
|
-
// src/campaign/coverage.ts
|
|
2751
|
-
function campaignCoverage(cells, scenarios, reps, requireJudgeScore) {
|
|
2752
|
-
const expectedCellIds = designedCellIds(scenarios, reps);
|
|
2753
|
-
const cellsById = /* @__PURE__ */ new Map();
|
|
2754
|
-
for (const cell of cells) {
|
|
2755
|
-
const matches = cellsById.get(cell.cellId) ?? [];
|
|
2756
|
-
matches.push(cell);
|
|
2757
|
-
cellsById.set(cell.cellId, matches);
|
|
2758
|
-
}
|
|
2759
|
-
const scorableCellIds = [];
|
|
2760
|
-
const unscorableCells = [];
|
|
2761
|
-
for (const cellId of expectedCellIds) {
|
|
2762
|
-
const matches = cellsById.get(cellId) ?? [];
|
|
2763
|
-
if (matches.length === 0) {
|
|
2764
|
-
unscorableCells.push({ cellId, reason: "missing campaign cell" });
|
|
2765
|
-
continue;
|
|
2766
|
-
}
|
|
2767
|
-
if (matches.length > 1) {
|
|
2768
|
-
unscorableCells.push({ cellId, reason: `duplicate campaign cell (${matches.length})` });
|
|
2769
|
-
continue;
|
|
2770
|
-
}
|
|
2771
|
-
const cell = matches[0];
|
|
2772
|
-
const scoreEntries = Object.entries(cell.judgeScores);
|
|
2773
|
-
const successfulScores = scoreEntries.map(([, score]) => score).filter((score) => score.failed !== true && Number.isFinite(score.composite));
|
|
2774
|
-
const nonFiniteScores = scoreEntries.filter(
|
|
2775
|
-
([, score]) => score.failed !== true && (!Number.isFinite(score.composite) || Object.values(score.dimensions).some((value) => !Number.isFinite(value)))
|
|
2776
|
-
);
|
|
2777
|
-
const reasons = [];
|
|
2778
|
-
if (cell.error) reasons.push(cell.error);
|
|
2779
|
-
if (cell.artifact === null || cell.artifact === void 0) reasons.push("missing artifact");
|
|
2780
|
-
if (!cell.error && requireJudgeScore && successfulScores.length === 0) {
|
|
2781
|
-
reasons.push("no successful finite judge score");
|
|
2782
|
-
}
|
|
2783
|
-
if (scoreEntries.some(([, score]) => score.failed === true)) {
|
|
2784
|
-
reasons.push("judge score marked failed");
|
|
2785
|
-
}
|
|
2786
|
-
if (nonFiniteScores.length > 0) {
|
|
2787
|
-
reasons.push(
|
|
2788
|
-
`non-finite judge score: ${nonFiniteScores.map(([name]) => name).sort().join(", ")}`
|
|
2789
|
-
);
|
|
2790
|
-
}
|
|
2791
|
-
if (reasons.length > 0) {
|
|
2792
|
-
unscorableCells.push({ cellId, reason: reasons.join("; ") });
|
|
2793
|
-
} else {
|
|
2794
|
-
scorableCellIds.push(cellId);
|
|
2795
|
-
}
|
|
2796
|
-
}
|
|
2797
|
-
const expected = new Set(expectedCellIds);
|
|
2798
|
-
for (const cell of cells) {
|
|
2799
|
-
if (!expected.has(cell.cellId)) {
|
|
2800
|
-
unscorableCells.push({ cellId: cell.cellId, reason: "unexpected campaign cell" });
|
|
2801
|
-
}
|
|
2802
|
-
}
|
|
2803
|
-
return {
|
|
2804
|
-
complete: unscorableCells.length === 0 && scorableCellIds.length === expectedCellIds.length,
|
|
2805
|
-
expectedCellIds,
|
|
2806
|
-
scorableCellIds,
|
|
2807
|
-
unscorableCells
|
|
2808
|
-
};
|
|
2809
|
-
}
|
|
2810
|
-
function formatCoverageFailures(coverage) {
|
|
2811
|
-
const shown = coverage.unscorableCells.slice(0, 3).map((cell) => `${cell.cellId}: ${cell.reason}`).join("; ");
|
|
2812
|
-
const remainder = coverage.unscorableCells.length - Math.min(3, coverage.unscorableCells.length);
|
|
2813
|
-
return remainder > 0 ? `${shown}; +${remainder} more` : shown || "unknown coverage failure";
|
|
2814
|
-
}
|
|
2815
|
-
function designedCellIds(scenarios, reps) {
|
|
2816
|
-
const ids = [];
|
|
2817
|
-
for (const scenario of scenarios) {
|
|
2818
|
-
for (let rep = 0; rep < reps; rep++) ids.push(`${scenario.id}:${rep}`);
|
|
2819
|
-
}
|
|
2820
|
-
return ids;
|
|
2821
|
-
}
|
|
2822
|
-
|
|
2823
2781
|
// src/campaign/presets/run-optimization.ts
|
|
2824
2782
|
async function runOptimization(opts) {
|
|
2825
2783
|
const { proposer } = opts;
|
|
@@ -2912,6 +2870,7 @@ async function runOptimization(opts) {
|
|
|
2912
2870
|
costLedger,
|
|
2913
2871
|
costPhase: "search.proposal"
|
|
2914
2872
|
});
|
|
2873
|
+
if (proposed.length === 0) break;
|
|
2915
2874
|
const candidates = proposed.map(
|
|
2916
2875
|
(p) => isProposedCandidate(p) ? p : { surface: p, label: "", rationale: "" }
|
|
2917
2876
|
);
|
|
@@ -3089,7 +3048,7 @@ var DEFAULT_DISPATCH_TIMEOUT_MS = 6e5;
|
|
|
3089
3048
|
async function runImprovementLoop(opts) {
|
|
3090
3049
|
if (opts.autoOnPromote === "config") {
|
|
3091
3050
|
throw new Error(
|
|
3092
|
-
"runImprovementLoop: autoOnPromote='config'
|
|
3051
|
+
"runImprovementLoop: autoOnPromote='config' requires isolated deployment, rollback, and independent validation. Use 'pr' or 'none'."
|
|
3093
3052
|
);
|
|
3094
3053
|
}
|
|
3095
3054
|
if (opts.tracing === "off" && opts.proposer) {
|
|
@@ -3122,7 +3081,7 @@ async function runImprovementLoop(opts) {
|
|
|
3122
3081
|
const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS;
|
|
3123
3082
|
const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger });
|
|
3124
3083
|
const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
|
|
3125
|
-
const { runCampaign: runCampaign2 } = await import("./run-campaign-
|
|
3084
|
+
const { runCampaign: runCampaign2 } = await import("./run-campaign-HNFPJET4.js");
|
|
3126
3085
|
const baselineOnHoldout = await runCampaign2({
|
|
3127
3086
|
...opts,
|
|
3128
3087
|
costLedger,
|
|
@@ -3172,15 +3131,18 @@ async function runImprovementLoop(opts) {
|
|
|
3172
3131
|
}
|
|
3173
3132
|
let neutralizedArtifacts;
|
|
3174
3133
|
let neutralizedJudgeScores;
|
|
3134
|
+
let neutralizedOnHoldout;
|
|
3135
|
+
let neutralizedSurface;
|
|
3175
3136
|
if (opts.neutralize && !winnerIsBaseline) {
|
|
3176
|
-
const
|
|
3177
|
-
|
|
3137
|
+
const surface = opts.neutralize(optimization.winnerSurface, opts.baselineSurface);
|
|
3138
|
+
neutralizedSurface = surface;
|
|
3139
|
+
neutralizedOnHoldout = await runCampaign2({
|
|
3178
3140
|
...opts,
|
|
3179
3141
|
costLedger,
|
|
3180
3142
|
costPhase: "holdout.neutralized",
|
|
3181
3143
|
dispatchTimeoutMs,
|
|
3182
3144
|
scenarios: opts.holdoutScenarios,
|
|
3183
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(
|
|
3145
|
+
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
3184
3146
|
runDir: `${opts.runDir}/holdout-neutralized`
|
|
3185
3147
|
});
|
|
3186
3148
|
assertCompleteHoldout("neutralized", neutralizedOnHoldout);
|
|
@@ -3216,8 +3178,7 @@ async function runImprovementLoop(opts) {
|
|
|
3216
3178
|
costPhase: "promotion.gate",
|
|
3217
3179
|
signal: new AbortController().signal
|
|
3218
3180
|
});
|
|
3219
|
-
const
|
|
3220
|
-
const promotedDiff = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface) ? "" : render(optimization.winnerSurface, opts.baselineSurface);
|
|
3181
|
+
const promotedDiff = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface) ? "" : renderSurfaceDiff(optimization.winnerSurface, opts.baselineSurface);
|
|
3221
3182
|
let prResult;
|
|
3222
3183
|
if (opts.autoOnPromote === "pr" && gateResult.decision === "ship") {
|
|
3223
3184
|
prResult = openAutoPr({
|
|
@@ -3232,28 +3193,13 @@ async function runImprovementLoop(opts) {
|
|
|
3232
3193
|
...optimization,
|
|
3233
3194
|
baselineOnHoldout,
|
|
3234
3195
|
winnerOnHoldout,
|
|
3196
|
+
...neutralizedOnHoldout && neutralizedSurface ? { neutralizedOnHoldout, neutralizedSurface } : {},
|
|
3235
3197
|
gateResult,
|
|
3236
3198
|
promotedDiff,
|
|
3237
3199
|
prResult,
|
|
3238
3200
|
cost: costLedger.summary()
|
|
3239
3201
|
};
|
|
3240
3202
|
}
|
|
3241
|
-
function defaultRenderDiff(winnerSurface, baselineSurface) {
|
|
3242
|
-
if (typeof winnerSurface !== "string" || typeof baselineSurface !== "string") {
|
|
3243
|
-
const fmt = (s) => typeof s === "string" ? "(prompt surface)" : `worktree=${s.worktreeRef}${s.baseRef ? ` base=${s.baseRef}` : ""}${s.summary ? `
|
|
3244
|
-
${s.summary}` : ""}`;
|
|
3245
|
-
return `--- baseline
|
|
3246
|
-
${fmt(baselineSurface)}
|
|
3247
|
-
+++ winner
|
|
3248
|
-
${fmt(winnerSurface)}`;
|
|
3249
|
-
}
|
|
3250
|
-
const lines = [];
|
|
3251
|
-
lines.push("--- baseline");
|
|
3252
|
-
lines.push("+++ winner");
|
|
3253
|
-
for (const l of baselineSurface.split("\n")) lines.push(`- ${l}`);
|
|
3254
|
-
for (const l of winnerSurface.split("\n")) lines.push(`+ ${l}`);
|
|
3255
|
-
return lines.join("\n");
|
|
3256
|
-
}
|
|
3257
3203
|
|
|
3258
3204
|
// src/campaign/presets/run-eval.ts
|
|
3259
3205
|
async function runEval(opts) {
|
|
@@ -3278,6 +3224,39 @@ function evolutionaryProposer(opts) {
|
|
|
3278
3224
|
// src/campaign/provenance.ts
|
|
3279
3225
|
import { createHash as createHash2 } from "crypto";
|
|
3280
3226
|
import { join as join2 } from "path";
|
|
3227
|
+
function loopProvenanceArgsFromResult(input) {
|
|
3228
|
+
const { result } = input;
|
|
3229
|
+
return {
|
|
3230
|
+
runId: input.runId,
|
|
3231
|
+
runDir: input.runDir,
|
|
3232
|
+
timestamp: input.timestamp,
|
|
3233
|
+
baselineSurface: input.baselineSurface,
|
|
3234
|
+
winnerSurface: result.winnerSurface,
|
|
3235
|
+
...result.winnerLabel ? { winnerLabel: result.winnerLabel } : {},
|
|
3236
|
+
...result.winnerRationale ? { winnerRationale: result.winnerRationale } : {},
|
|
3237
|
+
baselineSearchCampaign: result.baselineCampaign,
|
|
3238
|
+
generations: result.generations.map(({ record, surfaces }) => ({
|
|
3239
|
+
generationIndex: record.generationIndex,
|
|
3240
|
+
candidates: record.candidates,
|
|
3241
|
+
promoted: record.promoted,
|
|
3242
|
+
surfaces: surfaces.map(({ surfaceHash: surfaceHash2, surface, campaign }) => ({
|
|
3243
|
+
surfaceHash: surfaceHash2,
|
|
3244
|
+
surface,
|
|
3245
|
+
campaign
|
|
3246
|
+
}))
|
|
3247
|
+
})),
|
|
3248
|
+
gate: result.gateResult,
|
|
3249
|
+
baselineOnHoldout: result.baselineOnHoldout,
|
|
3250
|
+
winnerOnHoldout: result.winnerOnHoldout,
|
|
3251
|
+
...result.neutralizedSurface && result.neutralizedOnHoldout ? {
|
|
3252
|
+
neutralizedSurface: result.neutralizedSurface,
|
|
3253
|
+
neutralizedOnHoldout: result.neutralizedOnHoldout
|
|
3254
|
+
} : {},
|
|
3255
|
+
costReceipts: input.costReceipts,
|
|
3256
|
+
totalCostUsd: input.totalCostUsd,
|
|
3257
|
+
totalDurationMs: input.totalDurationMs
|
|
3258
|
+
};
|
|
3259
|
+
}
|
|
3281
3260
|
function meanHoldoutComposite(campaign) {
|
|
3282
3261
|
const xs = [];
|
|
3283
3262
|
for (const cell of campaign.cells) {
|
|
@@ -3288,29 +3267,41 @@ function meanHoldoutComposite(campaign) {
|
|
|
3288
3267
|
return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0;
|
|
3289
3268
|
}
|
|
3290
3269
|
function buildLoopProvenanceRecord(args) {
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3270
|
+
if (!args.runId.trim() || !args.runDir.trim()) {
|
|
3271
|
+
throw new Error("buildLoopProvenanceRecord: runId and runDir must be non-empty");
|
|
3272
|
+
}
|
|
3273
|
+
const timestampMs = Date.parse(args.timestamp);
|
|
3274
|
+
if (!Number.isFinite(timestampMs) || new Date(timestampMs).toISOString() !== args.timestamp) {
|
|
3275
|
+
throw new Error("buildLoopProvenanceRecord: timestamp must be a canonical ISO instant");
|
|
3276
|
+
}
|
|
3277
|
+
const agentReceipts = args.costReceipts.filter((receipt) => receipt.channel === "agent");
|
|
3278
|
+
const integrity = summarizeAgentReceiptIntegrity(agentReceipts);
|
|
3279
|
+
const models = [...new Set(agentReceipts.map((receipt) => receipt.model))].sort();
|
|
3280
|
+
const baselineSearchComposite = campaignMeanComposite(args.baselineSearchCampaign);
|
|
3281
|
+
if (!Number.isFinite(baselineSearchComposite)) {
|
|
3294
3282
|
throw new Error("buildLoopProvenanceRecord: baselineSearchComposite must be finite");
|
|
3295
3283
|
}
|
|
3296
3284
|
const candidates = [];
|
|
3297
3285
|
let incumbentSurfaceHash = surfaceHash(args.baselineSurface);
|
|
3298
|
-
let incumbentComposite =
|
|
3286
|
+
let incumbentComposite = baselineSearchComposite;
|
|
3299
3287
|
let previousGeneration = -1;
|
|
3300
3288
|
for (const gen of args.generations) {
|
|
3301
|
-
if (!Number.isSafeInteger(gen.generationIndex) || gen.generationIndex
|
|
3289
|
+
if (!Number.isSafeInteger(gen.generationIndex) || gen.generationIndex !== previousGeneration + 1) {
|
|
3302
3290
|
throw new Error(
|
|
3303
|
-
"buildLoopProvenanceRecord: generation indices must be
|
|
3291
|
+
"buildLoopProvenanceRecord: generation indices must be contiguous integers starting at zero"
|
|
3304
3292
|
);
|
|
3305
3293
|
}
|
|
3306
3294
|
previousGeneration = gen.generationIndex;
|
|
3295
|
+
if (gen.candidates.length === 0) {
|
|
3296
|
+
throw new Error("buildLoopProvenanceRecord: a recorded generation must contain a candidate");
|
|
3297
|
+
}
|
|
3307
3298
|
if (new Set(gen.promoted).size !== gen.promoted.length || gen.promoted.length > 1) {
|
|
3308
3299
|
throw new Error(
|
|
3309
3300
|
"buildLoopProvenanceRecord: each generation may promote at most one candidate"
|
|
3310
3301
|
);
|
|
3311
3302
|
}
|
|
3312
3303
|
const promotedSet = new Set(gen.promoted);
|
|
3313
|
-
const surfaceByHash = new Map(gen.surfaces.map((
|
|
3304
|
+
const surfaceByHash = new Map(gen.surfaces.map((measured) => [measured.surfaceHash, measured]));
|
|
3314
3305
|
const candidateByHash = new Map(
|
|
3315
3306
|
gen.candidates.map((candidate) => [candidate.surfaceHash, candidate])
|
|
3316
3307
|
);
|
|
@@ -3337,19 +3328,26 @@ function buildLoopProvenanceRecord(args) {
|
|
|
3337
3328
|
incumbentComposite,
|
|
3338
3329
|
promotedSet.has(c.surfaceHash)
|
|
3339
3330
|
);
|
|
3340
|
-
const
|
|
3341
|
-
if (
|
|
3331
|
+
const measured = surfaceByHash.get(c.surfaceHash);
|
|
3332
|
+
if (measured === void 0) {
|
|
3342
3333
|
throw new Error("buildLoopProvenanceRecord: measured candidate is missing its surface");
|
|
3343
3334
|
}
|
|
3335
|
+
const { surface, campaign } = measured;
|
|
3344
3336
|
if (surfaceHash(surface) !== c.surfaceHash) {
|
|
3345
3337
|
throw new Error(
|
|
3346
3338
|
"buildLoopProvenanceRecord: candidate surface hash does not match its surface bytes"
|
|
3347
3339
|
);
|
|
3348
3340
|
}
|
|
3341
|
+
if (campaign.splitDigest !== args.baselineSearchCampaign.splitDigest) {
|
|
3342
|
+
throw new Error(
|
|
3343
|
+
"buildLoopProvenanceRecord: candidate campaign does not match the search split"
|
|
3344
|
+
);
|
|
3345
|
+
}
|
|
3349
3346
|
const entry = {
|
|
3350
3347
|
generation: gen.generationIndex,
|
|
3351
3348
|
surfaceHash: c.surfaceHash,
|
|
3352
3349
|
contentHash: surfaceContentHash(surface),
|
|
3350
|
+
campaignDigest: campaignMeasurementDigest(campaign),
|
|
3353
3351
|
parentSurfaceHash: c.parentSurfaceHash,
|
|
3354
3352
|
parentComposite: c.parentComposite,
|
|
3355
3353
|
eligibleForPromotion: c.eligibleForPromotion,
|
|
@@ -3385,23 +3383,61 @@ function buildLoopProvenanceRecord(args) {
|
|
|
3385
3383
|
}
|
|
3386
3384
|
const baselineHoldoutComposite = meanHoldoutComposite(args.baselineOnHoldout);
|
|
3387
3385
|
const winnerHoldoutComposite = meanHoldoutComposite(args.winnerOnHoldout);
|
|
3388
|
-
|
|
3389
|
-
|
|
3386
|
+
if (args.baselineOnHoldout.splitDigest !== args.winnerOnHoldout.splitDigest) {
|
|
3387
|
+
throw new Error("buildLoopProvenanceRecord: baseline and winner use different holdout splits");
|
|
3388
|
+
}
|
|
3389
|
+
if (args.neutralizedSurface === void 0 !== (args.neutralizedOnHoldout === void 0)) {
|
|
3390
|
+
throw new Error(
|
|
3391
|
+
"buildLoopProvenanceRecord: neutralized surface and campaign must be supplied together"
|
|
3392
|
+
);
|
|
3393
|
+
}
|
|
3394
|
+
if (args.neutralizedOnHoldout && args.neutralizedOnHoldout.splitDigest !== args.baselineOnHoldout.splitDigest) {
|
|
3395
|
+
throw new Error(
|
|
3396
|
+
"buildLoopProvenanceRecord: neutralized campaign uses a different holdout split"
|
|
3397
|
+
);
|
|
3398
|
+
}
|
|
3399
|
+
const neutralizedComposite = args.neutralizedOnHoldout ? meanHoldoutComposite(args.neutralizedOnHoldout) : void 0;
|
|
3400
|
+
const diff = surfaceContentHash(args.baselineSurface) === surfaceContentHash(args.winnerSurface) ? "" : renderSurfaceDiff(args.winnerSurface, args.baselineSurface);
|
|
3401
|
+
const recordWithoutDigest = {
|
|
3402
|
+
schema: "tangle.loop-provenance",
|
|
3390
3403
|
runId: args.runId,
|
|
3391
3404
|
runDir: args.runDir,
|
|
3392
3405
|
timestamp: args.timestamp,
|
|
3393
3406
|
baselineContentHash: surfaceContentHash(args.baselineSurface),
|
|
3394
3407
|
winnerContentHash: surfaceContentHash(args.winnerSurface),
|
|
3395
|
-
diff
|
|
3408
|
+
diff,
|
|
3396
3409
|
candidates,
|
|
3397
|
-
|
|
3410
|
+
evidence: {
|
|
3411
|
+
search: {
|
|
3412
|
+
splitDigest: args.baselineSearchCampaign.splitDigest,
|
|
3413
|
+
baselineCampaignDigest: campaignMeasurementDigest(args.baselineSearchCampaign)
|
|
3414
|
+
},
|
|
3415
|
+
holdout: {
|
|
3416
|
+
splitDigest: args.baselineOnHoldout.splitDigest,
|
|
3417
|
+
baselineCampaignDigest: campaignMeasurementDigest(args.baselineOnHoldout),
|
|
3418
|
+
winnerCampaignDigest: campaignMeasurementDigest(args.winnerOnHoldout),
|
|
3419
|
+
...args.neutralizedSurface && args.neutralizedOnHoldout && neutralizedComposite !== void 0 ? {
|
|
3420
|
+
neutralized: {
|
|
3421
|
+
contentHash: surfaceContentHash(args.neutralizedSurface),
|
|
3422
|
+
campaignDigest: campaignMeasurementDigest(args.neutralizedOnHoldout),
|
|
3423
|
+
composite: neutralizedComposite,
|
|
3424
|
+
lift: neutralizedComposite - baselineHoldoutComposite
|
|
3425
|
+
}
|
|
3426
|
+
} : {}
|
|
3427
|
+
},
|
|
3428
|
+
costReceiptsDigest: canonicalDigest(
|
|
3429
|
+
[...args.costReceipts].sort((left, right) => left.callId.localeCompare(right.callId))
|
|
3430
|
+
)
|
|
3431
|
+
},
|
|
3432
|
+
baselineSearchComposite,
|
|
3398
3433
|
gate: {
|
|
3399
3434
|
decision: args.gate.decision,
|
|
3400
3435
|
reasons: args.gate.reasons,
|
|
3401
3436
|
delta: args.gate.delta,
|
|
3402
3437
|
contributingGates: args.gate.contributingGates.map((g) => ({
|
|
3403
3438
|
name: g.name,
|
|
3404
|
-
passed: g.passed
|
|
3439
|
+
passed: g.passed,
|
|
3440
|
+
detail: durableGateDetail(g.detail)
|
|
3405
3441
|
}))
|
|
3406
3442
|
},
|
|
3407
3443
|
baselineHoldoutComposite,
|
|
@@ -3418,10 +3454,81 @@ function buildLoopProvenanceRecord(args) {
|
|
|
3418
3454
|
totalCostUsd: args.totalCostUsd,
|
|
3419
3455
|
totalDurationMs: args.totalDurationMs
|
|
3420
3456
|
};
|
|
3421
|
-
if (args.winnerLabel)
|
|
3422
|
-
if (args.winnerRationale)
|
|
3457
|
+
if (args.winnerLabel) recordWithoutDigest.winnerLabel = args.winnerLabel;
|
|
3458
|
+
if (args.winnerRationale) recordWithoutDigest.winnerRationale = args.winnerRationale;
|
|
3459
|
+
return {
|
|
3460
|
+
...recordWithoutDigest,
|
|
3461
|
+
recordDigest: canonicalDigest(recordWithoutDigest)
|
|
3462
|
+
};
|
|
3463
|
+
}
|
|
3464
|
+
function campaignMeasurementDigest(campaign) {
|
|
3465
|
+
assertCampaignSplitIdentity(campaign.scenarios, campaign.reps, campaign.splitDigest);
|
|
3466
|
+
return canonicalDigest({
|
|
3467
|
+
schema: "tangle.campaign-measurement",
|
|
3468
|
+
manifestHash: campaign.manifestHash,
|
|
3469
|
+
splitDigest: campaign.splitDigest,
|
|
3470
|
+
seed: campaign.seed,
|
|
3471
|
+
reps: campaign.reps,
|
|
3472
|
+
runDir: campaign.runDir,
|
|
3473
|
+
scenarios: campaign.scenarios,
|
|
3474
|
+
cells: [...campaign.cells].sort((left, right) => left.cellId.localeCompare(right.cellId)).map((cell) => ({
|
|
3475
|
+
manifestHash: cell.manifestHash ?? null,
|
|
3476
|
+
cellId: cell.cellId,
|
|
3477
|
+
scenarioId: cell.scenarioId,
|
|
3478
|
+
rep: cell.rep,
|
|
3479
|
+
generation: cell.generation ?? null,
|
|
3480
|
+
judgeScores: cell.judgeScores,
|
|
3481
|
+
costUsd: cell.costUsd,
|
|
3482
|
+
costEstimated: cell.costEstimated ?? null,
|
|
3483
|
+
costCallIds: [...cell.costCallIds ?? []].sort(),
|
|
3484
|
+
tokenUsage: cell.tokenUsage,
|
|
3485
|
+
resolvedModel: cell.resolvedModel ?? null,
|
|
3486
|
+
durationMs: cell.durationMs,
|
|
3487
|
+
seed: cell.seed,
|
|
3488
|
+
cached: cell.cached,
|
|
3489
|
+
error: cell.error ?? null
|
|
3490
|
+
}))
|
|
3491
|
+
});
|
|
3492
|
+
}
|
|
3493
|
+
function verifyLoopProvenanceRecord(record) {
|
|
3494
|
+
if (record.schema !== "tangle.loop-provenance") {
|
|
3495
|
+
throw new Error("loop provenance has an unsupported schema");
|
|
3496
|
+
}
|
|
3497
|
+
const { recordDigest, ...recordWithoutDigest } = record;
|
|
3498
|
+
if (recordDigest !== canonicalDigest(recordWithoutDigest)) {
|
|
3499
|
+
throw new Error("loop provenance record digest does not match its contents");
|
|
3500
|
+
}
|
|
3423
3501
|
return record;
|
|
3424
3502
|
}
|
|
3503
|
+
function canonicalDigest(value) {
|
|
3504
|
+
const json = JSON.stringify(value, function strictJson(_key, item) {
|
|
3505
|
+
if (typeof item === "number" && !Number.isFinite(item)) {
|
|
3506
|
+
throw new Error("canonical digest cannot encode a non-finite number");
|
|
3507
|
+
}
|
|
3508
|
+
if (typeof item === "bigint" || typeof item === "function" || typeof item === "symbol") {
|
|
3509
|
+
throw new Error(`canonical digest cannot encode ${typeof item}`);
|
|
3510
|
+
}
|
|
3511
|
+
if (item === void 0 && Array.isArray(this)) {
|
|
3512
|
+
throw new Error("canonical digest cannot encode an undefined array item");
|
|
3513
|
+
}
|
|
3514
|
+
if (item instanceof Map || item instanceof Set) {
|
|
3515
|
+
throw new Error(`canonical digest cannot encode ${item instanceof Map ? "Map" : "Set"}`);
|
|
3516
|
+
}
|
|
3517
|
+
return item;
|
|
3518
|
+
});
|
|
3519
|
+
if (json === void 0) throw new Error("canonical digest value is not serializable");
|
|
3520
|
+
return `sha256:${contentHash(JSON.parse(json))}`;
|
|
3521
|
+
}
|
|
3522
|
+
function durableGateDetail(detail) {
|
|
3523
|
+
if (detail === void 0) return null;
|
|
3524
|
+
try {
|
|
3525
|
+
return JSON.parse(canonicalJson(detail));
|
|
3526
|
+
} catch (cause) {
|
|
3527
|
+
throw new Error("buildLoopProvenanceRecord: gate detail must be canonical JSON", {
|
|
3528
|
+
cause
|
|
3529
|
+
});
|
|
3530
|
+
}
|
|
3531
|
+
}
|
|
3425
3532
|
function validateCandidateMeasurement(candidate, expectedParentHash, expectedParentComposite, promoted) {
|
|
3426
3533
|
if (!Number.isFinite(candidate.composite)) {
|
|
3427
3534
|
throw new Error("buildLoopProvenanceRecord: candidate composite must be finite");
|
|
@@ -3748,6 +3855,7 @@ export {
|
|
|
3748
3855
|
codeSurfaceIdentityMaterial,
|
|
3749
3856
|
surfaceContentHash,
|
|
3750
3857
|
surfaceHash,
|
|
3858
|
+
renderSurfaceDiff,
|
|
3751
3859
|
gepaProposer,
|
|
3752
3860
|
extractH2Sections,
|
|
3753
3861
|
countSentenceEdits,
|
|
@@ -3755,13 +3863,16 @@ export {
|
|
|
3755
3863
|
campaignBreakdown,
|
|
3756
3864
|
runOptimization,
|
|
3757
3865
|
runImprovementLoop,
|
|
3758
|
-
defaultRenderDiff,
|
|
3759
3866
|
runEval,
|
|
3760
3867
|
evolutionaryProposer,
|
|
3868
|
+
loopProvenanceArgsFromResult,
|
|
3761
3869
|
buildLoopProvenanceRecord,
|
|
3870
|
+
campaignMeasurementDigest,
|
|
3871
|
+
verifyLoopProvenanceRecord,
|
|
3872
|
+
canonicalDigest,
|
|
3762
3873
|
loopProvenanceSpans,
|
|
3763
3874
|
provenanceRecordPath,
|
|
3764
3875
|
provenanceSpansPath,
|
|
3765
3876
|
emitLoopProvenance
|
|
3766
3877
|
};
|
|
3767
|
-
//# sourceMappingURL=chunk-
|
|
3878
|
+
//# sourceMappingURL=chunk-32BZXMSO.js.map
|