@tangle-network/agent-eval 0.126.7 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +2 -2
- package/dist/analyst/index.d.ts +51 -39
- package/dist/analyst/index.js +4 -4
- package/dist/belief-state/index.d.ts +34 -36
- package/dist/belief-state/index.js +2 -2
- package/dist/benchmarks/index.d.ts +30 -12
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +432 -146
- package/dist/campaign/index.js +11 -9
- package/dist/{chunk-ARU2PZFM.js → chunk-424XHVKV.js} +43 -23
- package/dist/chunk-424XHVKV.js.map +1 -0
- package/dist/{chunk-NUKSVU3W.js → chunk-5DTSBUL2.js} +2 -2
- package/dist/{chunk-BTACVO2W.js → chunk-65QETEKG.js} +542 -258
- package/dist/chunk-65QETEKG.js.map +1 -0
- package/dist/{chunk-CGG5SLH3.js → chunk-ARCFIY7U.js} +2 -2
- package/dist/{chunk-P22LJ3Y2.js → chunk-CLELFRQG.js} +6 -4
- package/dist/chunk-CLELFRQG.js.map +1 -0
- package/dist/{chunk-LUNF2SEL.js → chunk-DJKY2TSY.js} +3 -3
- package/dist/chunk-DPUHNQLN.js +232 -0
- package/dist/chunk-DPUHNQLN.js.map +1 -0
- package/dist/{chunk-IPYXE555.js → chunk-EJGRPCO3.js} +36 -13
- package/dist/chunk-EJGRPCO3.js.map +1 -0
- package/dist/{chunk-K6IAZZ6L.js → chunk-FA3MFYNA.js} +132 -110
- package/dist/chunk-FA3MFYNA.js.map +1 -0
- package/dist/chunk-IG35YDN3.js +614 -0
- package/dist/chunk-IG35YDN3.js.map +1 -0
- package/dist/{chunk-DPZAEKA6.js → chunk-IMGJZBFG.js} +84 -56
- package/dist/chunk-IMGJZBFG.js.map +1 -0
- package/dist/{chunk-H5UD2323.js → chunk-IN455XKS.js} +53 -1
- package/dist/chunk-IN455XKS.js.map +1 -0
- package/dist/{chunk-PJQFMIOX.js → chunk-MHELPNRP.js} +31 -1
- package/dist/chunk-MHELPNRP.js.map +1 -0
- package/dist/{chunk-IILEIWGW.js → chunk-NL33OXEP.js} +41 -36
- package/dist/chunk-NL33OXEP.js.map +1 -0
- package/dist/{chunk-474LBSOX.js → chunk-OD5NQ2CS.js} +41 -16
- package/dist/chunk-OD5NQ2CS.js.map +1 -0
- package/dist/{chunk-QOTFXW5L.js → chunk-P5W7RQKK.js} +2 -2
- package/dist/{chunk-IR3KBHOY.js → chunk-P6FYH6K4.js} +39 -3
- package/dist/chunk-P6FYH6K4.js.map +1 -0
- package/dist/{chunk-J5SQWP6Y.js → chunk-PBE2LOSS.js} +18 -2
- package/dist/chunk-PBE2LOSS.js.map +1 -0
- package/dist/{chunk-E3IADPLG.js → chunk-PRB53E3J.js} +50 -224
- package/dist/chunk-PRB53E3J.js.map +1 -0
- package/dist/{chunk-X4UCIOTZ.js → chunk-S5YLIBFX.js} +2 -2
- package/dist/{chunk-7VYEGHWF.js → chunk-SZLVEKMJ.js} +41 -20
- package/dist/chunk-SZLVEKMJ.js.map +1 -0
- package/dist/{chunk-LKKT3IVV.js → chunk-TSN7JT6D.js} +6 -4
- package/dist/{chunk-LKKT3IVV.js.map → chunk-TSN7JT6D.js.map} +1 -1
- package/dist/{chunk-M7AH34KV.js → chunk-UTA2TGC4.js} +32 -16
- package/dist/chunk-UTA2TGC4.js.map +1 -0
- package/dist/{chunk-MAX3TN3C.js → chunk-UWZZKKU7.js} +7 -19
- package/dist/chunk-UWZZKKU7.js.map +1 -0
- package/dist/{chunk-ZVCHKKOP.js → chunk-WED5KLZQ.js} +39 -15
- package/dist/chunk-WED5KLZQ.js.map +1 -0
- package/dist/{chunk-OCFJACJU.js → chunk-X6VOPMLD.js} +74 -14
- package/dist/chunk-X6VOPMLD.js.map +1 -0
- package/dist/{chunk-MOXWMGPC.js → chunk-Y5CYOL42.js} +159 -70
- package/dist/chunk-Y5CYOL42.js.map +1 -0
- package/dist/{chunk-FO7HEH76.js → chunk-YJBNWCAA.js} +2 -2
- package/dist/chunk-ZPUEEM5U.js +179 -0
- package/dist/chunk-ZPUEEM5U.js.map +1 -0
- package/dist/{chunk-KE2VWPZX.js → chunk-ZUUWPZCV.js} +3 -3
- package/dist/cli.js +4 -4
- package/dist/contract/index.d.ts +421 -124
- package/dist/contract/index.js +242 -115
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +34 -32
- package/dist/control.js +2 -2
- package/dist/hosted/index.d.ts +142 -41
- package/dist/hosted/index.js +23 -1
- package/dist/index.d.ts +364 -255
- package/dist/index.js +315 -226
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.d.ts +32 -32
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +1 -0
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/reporting.d.ts +97 -74
- package/dist/reporting.js +5 -4
- package/dist/rl.d.ts +177 -131
- package/dist/rl.js +232 -126
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +53 -63
- package/dist/rollout/index.js +7 -5
- package/dist/{run-campaign-FFRM3RH5.js → run-campaign-3A2EBYBE.js} +4 -3
- package/dist/supervisor-run/index.d.ts +7 -15
- package/dist/supervisor-run/index.js +2 -2
- package/dist/traces.d.ts +66 -41
- package/dist/traces.js +9 -5
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +2 -2
- package/docs/concepts.md +18 -1
- package/docs/hosted-ingest-spec.md +37 -40
- package/docs/insight-report.md +57 -7
- package/docs/research-report-methodology.md +20 -21
- package/docs/rollout.md +13 -8
- package/package.json +4 -3
- package/dist/chunk-474LBSOX.js.map +0 -1
- package/dist/chunk-7VYEGHWF.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BTACVO2W.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-E3IADPLG.js.map +0 -1
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-IILEIWGW.js.map +0 -1
- package/dist/chunk-IPYXE555.js.map +0 -1
- package/dist/chunk-IR3KBHOY.js.map +0 -1
- package/dist/chunk-J5SQWP6Y.js.map +0 -1
- package/dist/chunk-K6IAZZ6L.js.map +0 -1
- package/dist/chunk-M7AH34KV.js.map +0 -1
- package/dist/chunk-MAX3TN3C.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-OCFJACJU.js.map +0 -1
- package/dist/chunk-P22LJ3Y2.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-ZVCHKKOP.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- /package/dist/{chunk-NUKSVU3W.js.map → chunk-5DTSBUL2.js.map} +0 -0
- /package/dist/{chunk-CGG5SLH3.js.map → chunk-ARCFIY7U.js.map} +0 -0
- /package/dist/{chunk-LUNF2SEL.js.map → chunk-DJKY2TSY.js.map} +0 -0
- /package/dist/{chunk-QOTFXW5L.js.map → chunk-P5W7RQKK.js.map} +0 -0
- /package/dist/{chunk-X4UCIOTZ.js.map → chunk-S5YLIBFX.js.map} +0 -0
- /package/dist/{chunk-FO7HEH76.js.map → chunk-YJBNWCAA.js.map} +0 -0
- /package/dist/{chunk-KE2VWPZX.js.map → chunk-ZUUWPZCV.js.map} +0 -0
- /package/dist/{run-campaign-FFRM3RH5.js.map → run-campaign-3A2EBYBE.js.map} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
summaryTable
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-IMGJZBFG.js";
|
|
4
4
|
import {
|
|
5
5
|
VerificationError
|
|
6
6
|
} from "./chunk-ONWEPEDO.js";
|
|
@@ -30,34 +30,46 @@ function evaluateReleaseConfidence(input) {
|
|
|
30
30
|
const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios);
|
|
31
31
|
const searchScores = scoresFor(runs, "search");
|
|
32
32
|
const holdoutScores = scoresFor(runs, "holdout");
|
|
33
|
-
const
|
|
33
|
+
const runScores = runs.map(runSplitScore).filter(isFiniteNumber);
|
|
34
34
|
const traceScores = traces.map((t) => t.score).filter(isFiniteNumber);
|
|
35
|
-
const scoreUniverse =
|
|
36
|
-
const
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
35
|
+
const scoreUniverse = runs.length > 0 ? runScores : traceScores;
|
|
36
|
+
const qualityRuns = runs.filter((run) => runSplitScore(run) !== void 0);
|
|
37
|
+
const unscoredRuns = runs.filter(
|
|
38
|
+
(run) => runSplitScore(run) === void 0 && !hasExplicitTaskFailure(run) && !isFailedTerminalOutcome(run.terminalOutcome)
|
|
39
|
+
).length;
|
|
40
|
+
const passOutcomes = runs.length > 0 ? runs.map((run) => runPassOutcome(run, thresholds.failureScoreThreshold)) : traces.map((trace) => tracePassOutcome(trace, thresholds.failureScoreThreshold));
|
|
41
|
+
const reliabilityRows = runs.length > 0 ? runs.map((run) => terminalSuccess(run.terminalOutcome)) : traces.map((trace) => trace.ok);
|
|
42
|
+
const unclassifiedTerminalRuns = reliabilityRows.filter((outcome) => outcome === void 0).length;
|
|
43
|
+
const terminalFailureRuns = reliabilityRows.filter((outcome) => outcome === false).length;
|
|
44
|
+
const reliabilityRate = reliabilityRows.length === 0 || unclassifiedTerminalRuns > 0 ? null : (reliabilityRows.length - terminalFailureRuns) / reliabilityRows.length;
|
|
45
|
+
const searchRuns = qualityRuns.filter((r) => r.splitTag === "search").length;
|
|
46
|
+
const holdoutRuns = qualityRuns.filter((r) => r.splitTag === "holdout").length;
|
|
47
|
+
const failed = failedRows(runs, traces, thresholds.failureScoreThreshold);
|
|
48
|
+
const searchMeanScore = meanOrNull(searchScores);
|
|
49
|
+
const holdoutMeanScore = meanOrNull(holdoutScores);
|
|
50
|
+
const runCosts = runs.flatMap(
|
|
51
|
+
(run) => run.costProvenance.kind === "uncaptured" ? [] : [run.costProvenance.usd]
|
|
52
|
+
);
|
|
53
|
+
const traceCosts = traces.map((trace) => trace.costUsd).filter(isFiniteNumber);
|
|
54
|
+
const meanCostUsd = runs.length > 0 ? runCosts.length === runs.length ? meanOrNull(runCosts) : null : meanOrNull(traceCosts);
|
|
55
|
+
const wallTimes = runs.length > 0 ? runs.map((run) => run.wallMs) : traces.map((trace) => trace.durationMs).filter(isFiniteNumber);
|
|
40
56
|
const metrics = {
|
|
41
57
|
scenarioCount,
|
|
42
58
|
searchRuns,
|
|
43
59
|
holdoutRuns,
|
|
44
|
-
|
|
45
|
-
|
|
60
|
+
unscoredRuns,
|
|
61
|
+
unclassifiedTerminalRuns,
|
|
62
|
+
terminalFailureRuns,
|
|
63
|
+
reliabilityRate,
|
|
64
|
+
passRate: passOutcomeRate(passOutcomes),
|
|
65
|
+
meanScore: meanOrNull(scoreUniverse),
|
|
46
66
|
searchMeanScore,
|
|
47
67
|
holdoutMeanScore,
|
|
48
|
-
overfitGap:
|
|
49
|
-
meanCostUsd
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
p95WallMs: percentile(
|
|
54
|
-
[...runs.map((r) => r.wallMs), ...traces.map((t) => t.durationMs).filter(isFiniteNumber)],
|
|
55
|
-
0.95
|
|
56
|
-
),
|
|
57
|
-
failedRows: failedRows(runs, traces, thresholds.failureScoreThreshold).length,
|
|
58
|
-
failuresWithAsi: failedRows(runs, traces, thresholds.failureScoreThreshold).filter(
|
|
59
|
-
(row) => row.hasAsi
|
|
60
|
-
).length,
|
|
68
|
+
overfitGap: diffOrNull(searchMeanScore, holdoutMeanScore),
|
|
69
|
+
meanCostUsd,
|
|
70
|
+
p95WallMs: percentileOrNull(wallTimes, 0.95),
|
|
71
|
+
failedRows: failed.length,
|
|
72
|
+
failuresWithAsi: failed.filter((row) => row.hasAsi).length,
|
|
61
73
|
singleShotTraces: traces.filter((t) => t.turnCount === 1).length,
|
|
62
74
|
multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,
|
|
63
75
|
splitCounts,
|
|
@@ -68,10 +80,11 @@ function evaluateReleaseConfidence(input) {
|
|
|
68
80
|
const issues = [];
|
|
69
81
|
checkCorpus(input, thresholds, metrics, issues);
|
|
70
82
|
checkQuality(thresholds, metrics, issues);
|
|
83
|
+
checkReliability(metrics, issues);
|
|
71
84
|
checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues);
|
|
72
85
|
checkDiagnostics(thresholds, metrics, issues);
|
|
73
86
|
checkEfficiency(thresholds, metrics, issues);
|
|
74
|
-
const axes = buildAxes(metrics, thresholds,
|
|
87
|
+
const axes = buildAxes(metrics, thresholds, issues);
|
|
75
88
|
const status = issues.some((i) => i.severity === "critical") ? "fail" : issues.length > 0 ? "warn" : "pass";
|
|
76
89
|
return {
|
|
77
90
|
target: input.target,
|
|
@@ -141,7 +154,23 @@ function checkQuality(thresholds, metrics, issues) {
|
|
|
141
154
|
detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.`
|
|
142
155
|
});
|
|
143
156
|
}
|
|
144
|
-
if (metrics.
|
|
157
|
+
if (metrics.unscoredRuns > 0) {
|
|
158
|
+
issues.push({
|
|
159
|
+
axis: "quality",
|
|
160
|
+
severity: "critical",
|
|
161
|
+
code: "unscored_runs",
|
|
162
|
+
detail: `${metrics.unscoredRuns} supplied run(s) have no task result.`
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
if (metrics.passRate === null || metrics.meanScore === null) {
|
|
166
|
+
issues.push({
|
|
167
|
+
axis: "quality",
|
|
168
|
+
severity: "critical",
|
|
169
|
+
code: "missing_quality_scores",
|
|
170
|
+
detail: "No task-quality scores are available for pass-rate and mean-score checks."
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
if (metrics.passRate !== null && metrics.passRate < thresholds.minPassRate) {
|
|
145
174
|
issues.push({
|
|
146
175
|
axis: "quality",
|
|
147
176
|
severity: "critical",
|
|
@@ -149,7 +178,7 @@ function checkQuality(thresholds, metrics, issues) {
|
|
|
149
178
|
detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.`
|
|
150
179
|
});
|
|
151
180
|
}
|
|
152
|
-
if (metrics.meanScore < thresholds.minMeanScore) {
|
|
181
|
+
if (metrics.meanScore !== null && metrics.meanScore < thresholds.minMeanScore) {
|
|
153
182
|
issues.push({
|
|
154
183
|
axis: "quality",
|
|
155
184
|
severity: "critical",
|
|
@@ -158,6 +187,24 @@ function checkQuality(thresholds, metrics, issues) {
|
|
|
158
187
|
});
|
|
159
188
|
}
|
|
160
189
|
}
|
|
190
|
+
function checkReliability(metrics, issues) {
|
|
191
|
+
if (metrics.reliabilityRate === null) {
|
|
192
|
+
issues.push({
|
|
193
|
+
axis: "reliability",
|
|
194
|
+
severity: "critical",
|
|
195
|
+
code: "missing_reliability_evidence",
|
|
196
|
+
detail: metrics.unclassifiedTerminalRuns > 0 ? `${metrics.unclassifiedTerminalRuns} supplied run(s) have no classified terminal result.` : "No classified terminal results are available."
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
if (metrics.terminalFailureRuns > 0) {
|
|
200
|
+
issues.push({
|
|
201
|
+
axis: "reliability",
|
|
202
|
+
severity: "critical",
|
|
203
|
+
code: "terminal_run_failures",
|
|
204
|
+
detail: `${metrics.terminalFailureRuns} run(s) ended failed, cancelled, or incomplete.`
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
}
|
|
161
208
|
function checkGeneralization(gateDecision, thresholds, metrics, issues) {
|
|
162
209
|
if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {
|
|
163
210
|
issues.push({
|
|
@@ -167,7 +214,7 @@ function checkGeneralization(gateDecision, thresholds, metrics, issues) {
|
|
|
167
214
|
detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.`
|
|
168
215
|
});
|
|
169
216
|
}
|
|
170
|
-
if (
|
|
217
|
+
if (metrics.overfitGap !== null && metrics.overfitGap > thresholds.maxOverfitGap) {
|
|
171
218
|
issues.push({
|
|
172
219
|
axis: "generalization",
|
|
173
220
|
severity: "critical",
|
|
@@ -196,7 +243,14 @@ function checkDiagnostics(thresholds, metrics, issues) {
|
|
|
196
243
|
}
|
|
197
244
|
}
|
|
198
245
|
function checkEfficiency(thresholds, metrics, issues) {
|
|
199
|
-
if (metrics.meanCostUsd
|
|
246
|
+
if (Number.isFinite(thresholds.maxMeanCostUsd) && metrics.meanCostUsd === null) {
|
|
247
|
+
issues.push({
|
|
248
|
+
axis: "efficiency",
|
|
249
|
+
severity: "critical",
|
|
250
|
+
code: "missing_cost",
|
|
251
|
+
detail: "A finite cost limit was configured but no cost evidence is available."
|
|
252
|
+
});
|
|
253
|
+
} else if (metrics.meanCostUsd !== null && metrics.meanCostUsd > thresholds.maxMeanCostUsd) {
|
|
200
254
|
issues.push({
|
|
201
255
|
axis: "efficiency",
|
|
202
256
|
severity: "critical",
|
|
@@ -204,7 +258,14 @@ function checkEfficiency(thresholds, metrics, issues) {
|
|
|
204
258
|
detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.`
|
|
205
259
|
});
|
|
206
260
|
}
|
|
207
|
-
if (metrics.p95WallMs
|
|
261
|
+
if (Number.isFinite(thresholds.maxP95WallMs) && metrics.p95WallMs === null) {
|
|
262
|
+
issues.push({
|
|
263
|
+
axis: "efficiency",
|
|
264
|
+
severity: "critical",
|
|
265
|
+
code: "missing_latency",
|
|
266
|
+
detail: "A finite latency limit was configured but no latency evidence is available."
|
|
267
|
+
});
|
|
268
|
+
} else if (metrics.p95WallMs !== null && metrics.p95WallMs > thresholds.maxP95WallMs) {
|
|
208
269
|
issues.push({
|
|
209
270
|
axis: "efficiency",
|
|
210
271
|
severity: "critical",
|
|
@@ -213,7 +274,7 @@ function checkEfficiency(thresholds, metrics, issues) {
|
|
|
213
274
|
});
|
|
214
275
|
}
|
|
215
276
|
}
|
|
216
|
-
function buildAxes(metrics, thresholds,
|
|
277
|
+
function buildAxes(metrics, thresholds, issues) {
|
|
217
278
|
return [
|
|
218
279
|
axis(
|
|
219
280
|
"corpus",
|
|
@@ -224,13 +285,19 @@ function buildAxes(metrics, thresholds, gateDecision, issues) {
|
|
|
224
285
|
axis(
|
|
225
286
|
"quality",
|
|
226
287
|
issues,
|
|
227
|
-
Math.min(metrics.passRate, metrics.meanScore),
|
|
288
|
+
metrics.passRate === null || metrics.meanScore === null ? null : Math.min(metrics.passRate, metrics.meanScore),
|
|
228
289
|
`passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`
|
|
229
290
|
),
|
|
291
|
+
axis(
|
|
292
|
+
"reliability",
|
|
293
|
+
issues,
|
|
294
|
+
metrics.reliabilityRate,
|
|
295
|
+
`successRate=${fmt(metrics.reliabilityRate)} terminalFailures=${metrics.terminalFailureRuns} unclassified=${metrics.unclassifiedTerminalRuns}`
|
|
296
|
+
),
|
|
230
297
|
axis(
|
|
231
298
|
"generalization",
|
|
232
299
|
issues,
|
|
233
|
-
|
|
300
|
+
gapScore(metrics.overfitGap, thresholds.maxOverfitGap),
|
|
234
301
|
`holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`
|
|
235
302
|
),
|
|
236
303
|
axis(
|
|
@@ -250,7 +317,7 @@ function buildAxes(metrics, thresholds, gateDecision, issues) {
|
|
|
250
317
|
function axis(name, issues, score, detail) {
|
|
251
318
|
const own = issues.filter((i) => i.axis === name);
|
|
252
319
|
const status = own.some((i) => i.severity === "critical") ? "fail" : own.length > 0 ? "warn" : "pass";
|
|
253
|
-
return { name, status, score: bounded(score), detail };
|
|
320
|
+
return { name, status, score: score === null ? null : bounded(score), detail };
|
|
254
321
|
}
|
|
255
322
|
function countScenarioSplits(scenarios) {
|
|
256
323
|
const counts = { train: 0, dev: 0, test: 0, holdout: 0 };
|
|
@@ -268,14 +335,13 @@ function countDomains(scenarios) {
|
|
|
268
335
|
function countFailureModes(runs, traces, threshold) {
|
|
269
336
|
const out = {};
|
|
270
337
|
for (const run of runs) {
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
const mode = run.failureMode ?? "low_score";
|
|
338
|
+
if (runPassOutcome(run, threshold) === false) {
|
|
339
|
+
const mode = run.failureMode ?? run.failureClass ?? "low_score";
|
|
274
340
|
out[mode] = (out[mode] ?? 0) + 1;
|
|
275
341
|
}
|
|
276
342
|
}
|
|
277
343
|
for (const trace of traces) {
|
|
278
|
-
if (trace
|
|
344
|
+
if (tracePassOutcome(trace, threshold) === false) {
|
|
279
345
|
const mode = trace.failureMode ?? (trace.ok === false ? "not_ok" : "low_score");
|
|
280
346
|
out[mode] = (out[mode] ?? 0) + 1;
|
|
281
347
|
}
|
|
@@ -295,59 +361,77 @@ function countResponsibleSurfaces(traces) {
|
|
|
295
361
|
function failedRows(runs, traces, threshold) {
|
|
296
362
|
const out = [];
|
|
297
363
|
for (const run of runs) {
|
|
298
|
-
|
|
299
|
-
if (run.failureMode || score !== void 0 && score < threshold) {
|
|
364
|
+
if (runPassOutcome(run, threshold) === false) {
|
|
300
365
|
const asiMetric = run.outcome.raw.asi;
|
|
301
366
|
out.push({ hasAsi: typeof asiMetric === "number" && asiMetric > 0 });
|
|
302
367
|
}
|
|
303
368
|
}
|
|
304
369
|
for (const trace of traces) {
|
|
305
|
-
if (trace
|
|
370
|
+
if (tracePassOutcome(trace, threshold) === false) {
|
|
306
371
|
out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 });
|
|
307
372
|
}
|
|
308
373
|
}
|
|
309
374
|
return out;
|
|
310
375
|
}
|
|
311
|
-
function
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
return
|
|
376
|
+
function passOutcomeRate(outcomes) {
|
|
377
|
+
const classified = outcomes.filter((outcome) => outcome !== null);
|
|
378
|
+
if (classified.length === 0) return null;
|
|
379
|
+
return classified.filter(Boolean).length / classified.length;
|
|
380
|
+
}
|
|
381
|
+
function runPassOutcome(run, threshold) {
|
|
382
|
+
if (hasExplicitTaskFailure(run)) return false;
|
|
383
|
+
const score = runSplitScore(run);
|
|
384
|
+
return score === void 0 ? null : score >= threshold;
|
|
385
|
+
}
|
|
386
|
+
function hasExplicitTaskFailure(run) {
|
|
387
|
+
return run.failureClass !== void 0 && run.failureClass !== "success" || run.failureMode !== void 0;
|
|
388
|
+
}
|
|
389
|
+
function tracePassOutcome(trace, threshold) {
|
|
390
|
+
if (trace.failureMode !== void 0) return false;
|
|
391
|
+
if (trace.ok === false) return false;
|
|
392
|
+
if (isFiniteNumber(trace.score)) return trace.score >= threshold;
|
|
393
|
+
return trace.ok === true ? true : null;
|
|
394
|
+
}
|
|
395
|
+
function isFailedTerminalOutcome(outcome) {
|
|
396
|
+
return outcome === "failed" || outcome === "cancelled" || outcome === "incomplete";
|
|
397
|
+
}
|
|
398
|
+
function terminalSuccess(outcome) {
|
|
399
|
+
if (outcome === "succeeded") return true;
|
|
400
|
+
if (isFailedTerminalOutcome(outcome)) return false;
|
|
401
|
+
return void 0;
|
|
323
402
|
}
|
|
324
403
|
function scoresFor(runs, split) {
|
|
325
|
-
return runs.filter((run) => run.splitTag === split).map(
|
|
404
|
+
return runs.filter((run) => run.splitTag === split).map(runSplitScore).filter(isFiniteNumber);
|
|
326
405
|
}
|
|
327
|
-
function
|
|
328
|
-
|
|
406
|
+
function runSplitScore(run) {
|
|
407
|
+
const score = run.splitTag === "holdout" ? run.outcome.holdoutScore : run.outcome.searchScore;
|
|
408
|
+
return isFiniteNumber(score) ? score : void 0;
|
|
409
|
+
}
|
|
410
|
+
function meanOrNull(xs) {
|
|
411
|
+
if (xs.length === 0) return null;
|
|
329
412
|
return xs.reduce((sum, x) => sum + x, 0) / xs.length;
|
|
330
413
|
}
|
|
331
|
-
function
|
|
332
|
-
if (xs.length === 0) return
|
|
414
|
+
function percentileOrNull(xs, p) {
|
|
415
|
+
if (xs.length === 0) return null;
|
|
333
416
|
const sorted = [...xs].sort((a, b) => a - b);
|
|
334
417
|
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
|
|
335
418
|
}
|
|
336
419
|
function isFiniteNumber(value) {
|
|
337
420
|
return typeof value === "number" && Number.isFinite(value);
|
|
338
421
|
}
|
|
339
|
-
function
|
|
340
|
-
if (
|
|
422
|
+
function diffOrNull(a, b) {
|
|
423
|
+
if (a === null || b === null) return null;
|
|
341
424
|
return a - b;
|
|
342
425
|
}
|
|
343
426
|
function gapScore(gap, maxGap) {
|
|
344
|
-
if (
|
|
427
|
+
if (gap === null) return null;
|
|
345
428
|
if (maxGap <= 0) return gap <= 0 ? 1 : 0;
|
|
346
429
|
return bounded(1 - Math.max(0, gap) / maxGap);
|
|
347
430
|
}
|
|
348
431
|
function efficiencyScore(metrics, thresholds) {
|
|
349
|
-
const cost = Number.isFinite(thresholds.maxMeanCostUsd)
|
|
350
|
-
const latency = Number.isFinite(thresholds.maxP95WallMs)
|
|
432
|
+
const cost = Number.isFinite(thresholds.maxMeanCostUsd) ? metrics.meanCostUsd === null ? null : bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12)) : 1;
|
|
433
|
+
const latency = Number.isFinite(thresholds.maxP95WallMs) ? metrics.p95WallMs === null ? null : bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12)) : 1;
|
|
434
|
+
if (cost === null || latency === null) return null;
|
|
351
435
|
return Math.min(cost, latency);
|
|
352
436
|
}
|
|
353
437
|
function bounded(x) {
|
|
@@ -361,7 +445,7 @@ function renderSummary(target, status, metrics, issues) {
|
|
|
361
445
|
return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(",")}`;
|
|
362
446
|
}
|
|
363
447
|
function fmt(x) {
|
|
364
|
-
if (
|
|
448
|
+
if (x === null) return "n/a";
|
|
365
449
|
return x.toFixed(4);
|
|
366
450
|
}
|
|
367
451
|
|
|
@@ -371,8 +455,8 @@ function bootstrapCi(baseline, candidate, options = {}) {
|
|
|
371
455
|
const iterations = options.iterations ?? 1e3;
|
|
372
456
|
const minTotal = options.minTotalSamples ?? 6;
|
|
373
457
|
const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate));
|
|
374
|
-
const baselineMean =
|
|
375
|
-
const candidateMean =
|
|
458
|
+
const baselineMean = mean(baseline);
|
|
459
|
+
const candidateMean = mean(candidate);
|
|
376
460
|
const delta = candidateMean - baselineMean;
|
|
377
461
|
if (baseline.length + candidate.length < minTotal || baseline.length === 0 || candidate.length === 0) {
|
|
378
462
|
return {
|
|
@@ -390,7 +474,7 @@ function bootstrapCi(baseline, candidate, options = {}) {
|
|
|
390
474
|
for (let i = 0; i < iterations; i++) {
|
|
391
475
|
const bResample = resample(baseline, rng);
|
|
392
476
|
const cResample = resample(candidate, rng);
|
|
393
|
-
deltas[i] =
|
|
477
|
+
deltas[i] = mean(cResample) - mean(bResample);
|
|
394
478
|
}
|
|
395
479
|
deltas.sort((a, b) => a - b);
|
|
396
480
|
const lowerIdx = Math.floor(alpha / 2 * iterations);
|
|
@@ -413,7 +497,7 @@ function bootstrapCi(baseline, candidate, options = {}) {
|
|
|
413
497
|
verdict
|
|
414
498
|
};
|
|
415
499
|
}
|
|
416
|
-
function
|
|
500
|
+
function mean(xs) {
|
|
417
501
|
if (xs.length === 0) return 0;
|
|
418
502
|
let s = 0;
|
|
419
503
|
for (const x of xs) s += x;
|
|
@@ -496,13 +580,15 @@ function renderReleaseReport(scorecard, options = {}) {
|
|
|
496
580
|
lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`);
|
|
497
581
|
lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`);
|
|
498
582
|
lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`);
|
|
583
|
+
lines.push(`| Unscored runs | ${scorecard.metrics.unscoredRuns} |`);
|
|
584
|
+
lines.push(`| Terminal failures | ${scorecard.metrics.terminalFailureRuns} |`);
|
|
499
585
|
lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`);
|
|
500
586
|
lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`);
|
|
501
587
|
lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`);
|
|
502
588
|
lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`);
|
|
503
589
|
lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`);
|
|
504
590
|
lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`);
|
|
505
|
-
lines.push(`| p95 wall time | ${
|
|
591
|
+
lines.push(`| p95 wall time | ${duration(scorecard.metrics.p95WallMs)} |`);
|
|
506
592
|
lines.push("");
|
|
507
593
|
if (scorecard.issues.length > 0) {
|
|
508
594
|
lines.push("## Issues");
|
|
@@ -561,10 +647,13 @@ function entries(values) {
|
|
|
561
647
|
return Object.entries(values).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
562
648
|
}
|
|
563
649
|
function pct(value) {
|
|
564
|
-
return
|
|
650
|
+
return value === null ? "n/a" : `${(value * 100).toFixed(1)}%`;
|
|
565
651
|
}
|
|
566
652
|
function num(value) {
|
|
567
|
-
return
|
|
653
|
+
return value === null ? "n/a" : value.toFixed(3);
|
|
654
|
+
}
|
|
655
|
+
function duration(value) {
|
|
656
|
+
return value === null ? "n/a" : `${Math.round(value)} ms`;
|
|
568
657
|
}
|
|
569
658
|
|
|
570
659
|
export {
|
|
@@ -574,4 +663,4 @@ export {
|
|
|
574
663
|
judgeReplayGate,
|
|
575
664
|
renderReleaseReport
|
|
576
665
|
};
|
|
577
|
-
//# sourceMappingURL=chunk-
|
|
666
|
+
//# sourceMappingURL=chunk-Y5CYOL42.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/release-confidence.ts","../src/promotion-gate.ts","../src/release-report.ts"],"sourcesContent":["/**\n * Release confidence gate.\n *\n * This is the production-facing composition layer over the lower-level\n * primitives:\n * - Dataset manifests prove corpus/version coverage.\n * - RunRecord rows prove reproducible search/holdout outcomes.\n * - Multi-shot trace evidence carries turn counts and ASI diagnostics.\n * - HeldOutGate decisions remain the paired promotion authority.\n *\n * The gate is intentionally pure and conservative. Missing declared evidence\n * fails closed instead of being treated as a neutral zero.\n */\n\nimport type { DatasetManifest, DatasetScenario, DatasetSplit } from './dataset'\nimport { VerificationError } from './errors'\nimport type { GateDecision } from './held-out-gate'\nimport type { RunRecord, RunSplitTag } from './run-record'\n\n/** Severity of an actionable finding attached to a run/trace. */\nexport type AsiSeverity = 'info' | 'warning' | 'error' | 'critical'\n\n/** Actionable side-info — a diagnosed finding the loop can act on. */\nexport interface ActionableSideInfo {\n /** Stable expectation/check id when available. */\n expectationId?: string\n /** Human-readable diagnosis of what happened. */\n message: string\n severity?: AsiSeverity\n /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */\n evidence?: string\n /** Prompt/tool/context surface likely responsible. */\n responsibleSurface?: string\n /** Suggested fix in natural language. */\n suggestion?: string\n /** Whether this expectation was satisfied. Defaults to false for ASI rows. */\n matched?: boolean\n metadata?: Record<string, unknown>\n}\n\nexport type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail'\nexport type ReleaseConfidenceAxisName =\n | 'corpus'\n | 'quality'\n | 'reliability'\n | 'generalization'\n | 'diagnostics'\n | 'efficiency'\n\nexport interface ReleaseTraceEvidence {\n scenarioId: string\n candidateId?: string\n split?: RunSplitTag\n score?: number\n ok?: boolean\n turnCount?: number\n costUsd?: number\n durationMs?: number\n failureMode?: string\n asi?: ActionableSideInfo[]\n metadata?: Record<string, unknown>\n}\n\nexport interface ReleaseConfidenceThresholds {\n /** Require a Dataset manifest or explicit scenarios. Default true. */\n requireCorpus?: boolean\n minScenarioCount?: number\n minSearchRuns?: number\n minHoldoutRuns?: number\n /** Require at least one holdout scenario/run. Default true. */\n requireHoldout?: boolean\n minPassRate?: number\n minMeanScore?: number\n /** Search mean may exceed holdout mean by at most this much. */\n maxOverfitGap?: number\n maxMeanCostUsd?: number\n maxP95WallMs?: number\n /** Low-score/failed rows must carry ASI. Default true. */\n requireAsiForFailures?: boolean\n /** Score below this is considered a failure for ASI coverage. Default 0.5. */\n failureScoreThreshold?: number\n}\n\nexport interface ReleaseConfidenceInput {\n target: string\n candidateId?: string\n baselineId?: string\n dataset?: DatasetManifest\n scenarios?: readonly DatasetScenario[]\n runs?: readonly RunRecord[]\n traces?: readonly ReleaseTraceEvidence[]\n gateDecision?: GateDecision | null\n thresholds?: ReleaseConfidenceThresholds\n}\n\nexport interface ReleaseConfidenceAxis {\n name: ReleaseConfidenceAxisName\n status: ReleaseConfidenceStatus\n score: number | null\n detail: string\n}\n\nexport interface ReleaseConfidenceIssue {\n axis: ReleaseConfidenceAxisName\n severity: 'critical' | 'warning'\n code: string\n detail: string\n}\n\nexport interface ReleaseConfidenceMetrics {\n scenarioCount: number\n /** Search rows with a finite search score. */\n searchRuns: number\n /** Holdout rows with a finite holdout score. */\n holdoutRuns: number\n /** Runs with neither a split-matched score nor an explicit task failure. */\n unscoredRuns: number\n /** Run rows, or trace rows when no runs exist, with no classified terminal result. */\n unclassifiedTerminalRuns: number\n /** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */\n terminalFailureRuns: number\n /** Success fraction when every run or fallback trace row has a classified result. */\n reliabilityRate: number | null\n passRate: number | null\n meanScore: number | null\n searchMeanScore: number | null\n holdoutMeanScore: number | null\n overfitGap: number | null\n meanCostUsd: number | null\n p95WallMs: number | null\n failedRows: number\n failuresWithAsi: number\n singleShotTraces: number\n multiShotTraces: number\n splitCounts: Record<DatasetSplit, number>\n domainCounts: Record<string, number>\n failureModeCounts: Record<string, number>\n responsibleSurfaceCounts: Record<string, number>\n}\n\nexport interface ReleaseConfidenceScorecard {\n target: string\n candidateId: string | null\n baselineId: string | null\n status: ReleaseConfidenceStatus\n promote: boolean\n axes: ReleaseConfidenceAxis[]\n issues: ReleaseConfidenceIssue[]\n metrics: ReleaseConfidenceMetrics\n dataset: DatasetManifest | null\n gateDecision: GateDecision | null\n summary: string\n}\n\nconst DEFAULT_THRESHOLDS: Required<ReleaseConfidenceThresholds> = {\n requireCorpus: true,\n minScenarioCount: 1,\n minSearchRuns: 1,\n minHoldoutRuns: 1,\n requireHoldout: true,\n minPassRate: 0.8,\n minMeanScore: 0.7,\n maxOverfitGap: 0.15,\n maxMeanCostUsd: Number.POSITIVE_INFINITY,\n maxP95WallMs: Number.POSITIVE_INFINITY,\n requireAsiForFailures: true,\n failureScoreThreshold: 0.5,\n}\n\nexport function evaluateReleaseConfidence(\n input: ReleaseConfidenceInput,\n): ReleaseConfidenceScorecard {\n const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds }\n const candidateId = input.candidateId ?? null\n const runs = filterCandidate(input.runs ?? [], candidateId, input.baselineId)\n const traces = filterTraceCandidate(input.traces ?? [], candidateId, input.baselineId)\n const scenarios = input.scenarios ?? []\n const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length\n const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios)\n const searchScores = scoresFor(runs, 'search')\n const holdoutScores = scoresFor(runs, 'holdout')\n const runScores = runs.map(runSplitScore).filter(isFiniteNumber)\n const traceScores = traces.map((t) => t.score).filter(isFiniteNumber)\n const scoreUniverse = runs.length > 0 ? runScores : traceScores\n const qualityRuns = runs.filter((run) => runSplitScore(run) !== undefined)\n const unscoredRuns = runs.filter(\n (run) =>\n runSplitScore(run) === undefined &&\n !hasExplicitTaskFailure(run) &&\n !isFailedTerminalOutcome(run.terminalOutcome),\n ).length\n const passOutcomes =\n runs.length > 0\n ? runs.map((run) => runPassOutcome(run, thresholds.failureScoreThreshold))\n : traces.map((trace) => tracePassOutcome(trace, thresholds.failureScoreThreshold))\n const reliabilityRows =\n runs.length > 0\n ? runs.map((run) => terminalSuccess(run.terminalOutcome))\n : traces.map((trace) => trace.ok)\n const unclassifiedTerminalRuns = reliabilityRows.filter((outcome) => outcome === undefined).length\n const terminalFailureRuns = reliabilityRows.filter((outcome) => outcome === false).length\n const reliabilityRate =\n reliabilityRows.length === 0 || unclassifiedTerminalRuns > 0\n ? null\n : (reliabilityRows.length - terminalFailureRuns) / reliabilityRows.length\n const searchRuns = qualityRuns.filter((r) => r.splitTag === 'search').length\n const holdoutRuns = qualityRuns.filter((r) => r.splitTag === 'holdout').length\n const failed = failedRows(runs, traces, thresholds.failureScoreThreshold)\n const searchMeanScore = meanOrNull(searchScores)\n const holdoutMeanScore = meanOrNull(holdoutScores)\n const runCosts = runs.flatMap((run) =>\n run.costProvenance.kind === 'uncaptured' ? [] : [run.costProvenance.usd],\n )\n const traceCosts = traces.map((trace) => trace.costUsd).filter(isFiniteNumber)\n const meanCostUsd =\n runs.length > 0\n ? runCosts.length === runs.length\n ? meanOrNull(runCosts)\n : null\n : meanOrNull(traceCosts)\n const wallTimes =\n runs.length > 0\n ? runs.map((run) => run.wallMs)\n : traces.map((trace) => trace.durationMs).filter(isFiniteNumber)\n const metrics: ReleaseConfidenceMetrics = {\n scenarioCount,\n searchRuns,\n holdoutRuns,\n unscoredRuns,\n unclassifiedTerminalRuns,\n terminalFailureRuns,\n reliabilityRate,\n passRate: passOutcomeRate(passOutcomes),\n meanScore: meanOrNull(scoreUniverse),\n searchMeanScore,\n holdoutMeanScore,\n overfitGap: diffOrNull(searchMeanScore, holdoutMeanScore),\n meanCostUsd,\n p95WallMs: percentileOrNull(wallTimes, 0.95),\n failedRows: failed.length,\n failuresWithAsi: failed.filter((row) => row.hasAsi).length,\n singleShotTraces: traces.filter((t) => t.turnCount === 1).length,\n multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,\n splitCounts,\n domainCounts: countDomains(scenarios),\n failureModeCounts: countFailureModes(runs, traces, thresholds.failureScoreThreshold),\n responsibleSurfaceCounts: countResponsibleSurfaces(traces),\n }\n\n const issues: ReleaseConfidenceIssue[] = []\n checkCorpus(input, thresholds, metrics, issues)\n checkQuality(thresholds, metrics, issues)\n checkReliability(metrics, issues)\n checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues)\n checkDiagnostics(thresholds, metrics, issues)\n checkEfficiency(thresholds, metrics, issues)\n\n const axes = buildAxes(metrics, thresholds, issues)\n const status = issues.some((i) => i.severity === 'critical')\n ? 'fail'\n : issues.length > 0\n ? 'warn'\n : 'pass'\n\n return {\n target: input.target,\n candidateId,\n baselineId: input.baselineId ?? null,\n status,\n promote: status === 'pass' && (input.gateDecision ? input.gateDecision.promote : true),\n axes,\n issues,\n metrics,\n dataset: input.dataset ?? null,\n gateDecision: input.gateDecision ?? null,\n summary: renderSummary(input.target, status, metrics, issues),\n }\n}\n\nexport function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard {\n const scorecard = evaluateReleaseConfidence(input)\n if (scorecard.status === 'fail') {\n throw new VerificationError(scorecard.summary)\n }\n return scorecard\n}\n\nfunction filterCandidate(\n runs: readonly RunRecord[],\n candidateId: string | null,\n baselineId?: string,\n): RunRecord[] {\n if (candidateId) return runs.filter((r) => r.candidateId === candidateId)\n if (baselineId) return runs.filter((r) => r.candidateId !== baselineId)\n return [...runs]\n}\n\nfunction filterTraceCandidate(\n traces: readonly ReleaseTraceEvidence[],\n candidateId: string | null,\n baselineId?: string,\n): ReleaseTraceEvidence[] {\n if (candidateId)\n return traces.filter((t) => t.candidateId === undefined || t.candidateId === candidateId)\n if (baselineId)\n return traces.filter((t) => t.candidateId === undefined || t.candidateId !== baselineId)\n return [...traces]\n}\n\nfunction checkCorpus(\n input: ReleaseConfidenceInput,\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) {\n issues.push({\n axis: 'corpus',\n severity: 'critical',\n code: 'missing_corpus',\n detail: 'No Dataset manifest or scenarios supplied.',\n })\n }\n if (metrics.scenarioCount < thresholds.minScenarioCount) {\n issues.push({\n axis: 'corpus',\n severity: 'critical',\n code: 'few_scenarios',\n detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.`,\n })\n }\n if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) {\n issues.push({\n axis: 'corpus',\n severity: 'critical',\n code: 'missing_holdout_split',\n detail: 'Corpus has no holdout scenarios.',\n })\n }\n}\n\nfunction checkQuality(\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (metrics.searchRuns < thresholds.minSearchRuns) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'few_search_runs',\n detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.`,\n })\n }\n if (metrics.unscoredRuns > 0) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'unscored_runs',\n detail: `${metrics.unscoredRuns} supplied run(s) have no task result.`,\n })\n }\n if (metrics.passRate === null || metrics.meanScore === null) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'missing_quality_scores',\n detail: 'No task-quality scores are available for pass-rate and mean-score checks.',\n })\n }\n if (metrics.passRate !== null && metrics.passRate < thresholds.minPassRate) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'low_pass_rate',\n detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.`,\n })\n }\n if (metrics.meanScore !== null && metrics.meanScore < thresholds.minMeanScore) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'low_mean_score',\n detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.`,\n })\n }\n}\n\nfunction checkReliability(\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (metrics.reliabilityRate === null) {\n issues.push({\n axis: 'reliability',\n severity: 'critical',\n code: 'missing_reliability_evidence',\n detail:\n metrics.unclassifiedTerminalRuns > 0\n ? `${metrics.unclassifiedTerminalRuns} supplied run(s) have no classified terminal result.`\n : 'No classified terminal results are available.',\n })\n }\n if (metrics.terminalFailureRuns > 0) {\n issues.push({\n axis: 'reliability',\n severity: 'critical',\n code: 'terminal_run_failures',\n detail: `${metrics.terminalFailureRuns} run(s) ended failed, cancelled, or incomplete.`,\n })\n }\n}\n\nfunction checkGeneralization(\n gateDecision: GateDecision | null,\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {\n issues.push({\n axis: 'generalization',\n severity: 'critical',\n code: 'few_holdout_runs',\n detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.`,\n })\n }\n if (metrics.overfitGap !== null && metrics.overfitGap > thresholds.maxOverfitGap) {\n issues.push({\n axis: 'generalization',\n severity: 'critical',\n code: 'overfit_gap',\n detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.`,\n })\n }\n if (gateDecision && !gateDecision.promote) {\n issues.push({\n axis: 'generalization',\n severity: 'critical',\n code: `gate_${gateDecision.rejectionCode ?? 'reject'}`,\n detail: gateDecision.reason,\n })\n }\n}\n\nfunction checkDiagnostics(\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (!thresholds.requireAsiForFailures) return\n if (metrics.failedRows > metrics.failuresWithAsi) {\n issues.push({\n axis: 'diagnostics',\n severity: 'critical',\n code: 'missing_failure_asi',\n detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`,\n })\n }\n}\n\nfunction checkEfficiency(\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (Number.isFinite(thresholds.maxMeanCostUsd) && metrics.meanCostUsd === null) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'missing_cost',\n detail: 'A finite cost limit was configured but no cost evidence is available.',\n })\n } else if (metrics.meanCostUsd !== null && metrics.meanCostUsd > thresholds.maxMeanCostUsd) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'cost_budget',\n detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.`,\n })\n }\n if (Number.isFinite(thresholds.maxP95WallMs) && metrics.p95WallMs === null) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'missing_latency',\n detail: 'A finite latency limit was configured but no latency evidence is available.',\n })\n } else if (metrics.p95WallMs !== null && metrics.p95WallMs > thresholds.maxP95WallMs) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'latency_budget',\n detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.`,\n })\n }\n}\n\nfunction buildAxes(\n metrics: ReleaseConfidenceMetrics,\n thresholds: Required<ReleaseConfidenceThresholds>,\n issues: ReleaseConfidenceIssue[],\n): ReleaseConfidenceAxis[] {\n return [\n axis(\n 'corpus',\n issues,\n bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)),\n `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`,\n ),\n axis(\n 'quality',\n issues,\n metrics.passRate === null || metrics.meanScore === null\n ? null\n : Math.min(metrics.passRate, metrics.meanScore),\n `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`,\n ),\n axis(\n 'reliability',\n issues,\n metrics.reliabilityRate,\n `successRate=${fmt(metrics.reliabilityRate)} terminalFailures=${metrics.terminalFailureRuns} unclassified=${metrics.unclassifiedTerminalRuns}`,\n ),\n axis(\n 'generalization',\n issues,\n gapScore(metrics.overfitGap, thresholds.maxOverfitGap),\n `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`,\n ),\n axis(\n 'diagnostics',\n issues,\n metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows,\n `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`,\n ),\n axis(\n 'efficiency',\n issues,\n efficiencyScore(metrics, thresholds),\n `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`,\n ),\n ]\n}\n\nfunction axis(\n name: ReleaseConfidenceAxisName,\n issues: ReleaseConfidenceIssue[],\n score: number | null,\n detail: string,\n): ReleaseConfidenceAxis {\n const own = issues.filter((i) => i.axis === name)\n const status = own.some((i) => i.severity === 'critical')\n ? 'fail'\n : own.length > 0\n ? 'warn'\n : 'pass'\n return { name, status, score: score === null ? null : bounded(score), detail }\n}\n\nfunction countScenarioSplits(scenarios: readonly DatasetScenario[]): Record<DatasetSplit, number> {\n const counts: Record<DatasetSplit, number> = { train: 0, dev: 0, test: 0, holdout: 0 }\n for (const scenario of scenarios) counts[scenario.split ?? 'train']++\n return counts\n}\n\nfunction countDomains(scenarios: readonly DatasetScenario[]): Record<string, number> {\n const out: Record<string, number> = {}\n for (const scenario of scenarios) {\n const domain = scenario.tags?.domain ?? scenario.tags?.category ?? 'uncategorized'\n out[domain] = (out[domain] ?? 0) + 1\n }\n return out\n}\n\nfunction countFailureModes(\n runs: readonly RunRecord[],\n traces: readonly ReleaseTraceEvidence[],\n threshold: number,\n): Record<string, number> {\n const out: Record<string, number> = {}\n for (const run of runs) {\n if (runPassOutcome(run, threshold) === false) {\n const mode = run.failureMode ?? run.failureClass ?? 'low_score'\n out[mode] = (out[mode] ?? 0) + 1\n }\n }\n for (const trace of traces) {\n if (tracePassOutcome(trace, threshold) === false) {\n const mode = trace.failureMode ?? (trace.ok === false ? 'not_ok' : 'low_score')\n out[mode] = (out[mode] ?? 0) + 1\n }\n }\n return out\n}\n\nfunction countResponsibleSurfaces(traces: readonly ReleaseTraceEvidence[]): Record<string, number> {\n const out: Record<string, number> = {}\n for (const trace of traces) {\n for (const asi of trace.asi ?? []) {\n const surface = asi.responsibleSurface ?? 'unknown'\n out[surface] = (out[surface] ?? 0) + 1\n }\n }\n return out\n}\n\nfunction failedRows(\n runs: readonly RunRecord[],\n traces: readonly ReleaseTraceEvidence[],\n threshold: number,\n): Array<{ hasAsi: boolean }> {\n const out: Array<{ hasAsi: boolean }> = []\n for (const run of runs) {\n if (runPassOutcome(run, threshold) === false) {\n const asiMetric = run.outcome.raw.asi\n out.push({ hasAsi: typeof asiMetric === 'number' && asiMetric > 0 })\n }\n }\n for (const trace of traces) {\n if (tracePassOutcome(trace, threshold) === false) {\n out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 })\n }\n }\n return out\n}\n\nfunction passOutcomeRate(outcomes: readonly (boolean | null)[]): number | null {\n const classified = outcomes.filter((outcome): outcome is boolean => outcome !== null)\n if (classified.length === 0) return null\n return classified.filter(Boolean).length / classified.length\n}\n\nfunction runPassOutcome(run: RunRecord, threshold: number): boolean | null {\n if (hasExplicitTaskFailure(run)) return false\n const score = runSplitScore(run)\n return score === undefined ? null : score >= threshold\n}\n\nfunction hasExplicitTaskFailure(run: RunRecord): boolean {\n return (\n (run.failureClass !== undefined && run.failureClass !== 'success') ||\n run.failureMode !== undefined\n )\n}\n\nfunction tracePassOutcome(trace: ReleaseTraceEvidence, threshold: number): boolean | null {\n if (trace.failureMode !== undefined) return false\n if (trace.ok === false) return false\n if (isFiniteNumber(trace.score)) return trace.score >= threshold\n return trace.ok === true ? true : null\n}\n\nfunction isFailedTerminalOutcome(\n outcome: RunRecord['terminalOutcome'],\n): outcome is 'failed' | 'cancelled' | 'incomplete' {\n return outcome === 'failed' || outcome === 'cancelled' || outcome === 'incomplete'\n}\n\nfunction terminalSuccess(outcome: RunRecord['terminalOutcome']): boolean | undefined {\n if (outcome === 'succeeded') return true\n if (isFailedTerminalOutcome(outcome)) return false\n return undefined\n}\n\nfunction scoresFor(runs: readonly RunRecord[], split: RunSplitTag): number[] {\n return runs\n .filter((run) => run.splitTag === split)\n .map(runSplitScore)\n .filter(isFiniteNumber)\n}\n\nfunction runSplitScore(run: RunRecord): number | undefined {\n const score = run.splitTag === 'holdout' ? run.outcome.holdoutScore : run.outcome.searchScore\n return isFiniteNumber(score) ? score : undefined\n}\n\nfunction meanOrNull(xs: readonly number[]): number | null {\n if (xs.length === 0) return null\n return xs.reduce((sum, x) => sum + x, 0) / xs.length\n}\n\nfunction percentileOrNull(xs: readonly number[], p: number): number | null {\n if (xs.length === 0) return null\n const sorted = [...xs].sort((a, b) => a - b)\n return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))]!\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === 'number' && Number.isFinite(value)\n}\n\nfunction diffOrNull(a: number | null, b: number | null): number | null {\n if (a === null || b === null) return null\n return a - b\n}\n\nfunction gapScore(gap: number | null, maxGap: number): number | null {\n if (gap === null) return null\n if (maxGap <= 0) return gap <= 0 ? 1 : 0\n return bounded(1 - Math.max(0, gap) / maxGap)\n}\n\nfunction efficiencyScore(\n metrics: ReleaseConfidenceMetrics,\n thresholds: Required<ReleaseConfidenceThresholds>,\n): number | null {\n const cost = Number.isFinite(thresholds.maxMeanCostUsd)\n ? metrics.meanCostUsd === null\n ? null\n : bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12))\n : 1\n const latency = Number.isFinite(thresholds.maxP95WallMs)\n ? metrics.p95WallMs === null\n ? null\n : bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12))\n : 1\n if (cost === null || latency === null) return null\n return Math.min(cost, latency)\n}\n\nfunction bounded(x: number): number {\n if (!Number.isFinite(x)) return 0\n return Math.max(0, Math.min(1, x))\n}\n\nfunction renderSummary(\n target: string,\n status: ReleaseConfidenceStatus,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): string {\n const prefix = `release confidence ${status}: ${target}`\n const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`\n if (issues.length === 0) return `${prefix}; ${metricText}`\n return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(',')}`\n}\n\nfunction fmt(x: number | null): string {\n if (x === null) return 'n/a'\n return x.toFixed(4)\n}\n","/**\n * Bootstrap-CI promotion gate.\n *\n * In any iterative-improvement loop (GEPA, prompt evolution, dataset\n * curation), the question is \"did this generation actually improve, or are\n * we celebrating noise?\". With small N and noisy outcomes, point-estimate\n * deltas lie. Bootstrap confidence intervals tell the operator whether the\n * delta is real before code or prompts get promoted.\n *\n * This module is pure functions — no I/O, no model calls. Easy to unit-test\n * and to compose into any verdict gate.\n *\n * Default gate:\n * - Bootstrap mean baseline vs candidate (1k resamples).\n * - Compute the delta distribution; pass if the lower CI bound > 0.\n * - Tunable confidence (default 95%) and resample count.\n *\n * Verdict semantics intentionally match the existing `experiments.jsonl`\n * vocabulary:\n * - ADVANCE: candidate's CI lower bound > baseline mean (real win)\n * - KEEP: overlap, but candidate point estimate >= baseline (neutral)\n * - REVERT: candidate's CI upper bound < baseline mean (real regression)\n * - INCONCLUSIVE: not enough samples or CI straddles zero with no signal\n */\n\nexport type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE'\n\nexport interface BootstrapResult {\n baselineMean: number\n candidateMean: number\n /** candidateMean - baselineMean, point estimate. */\n delta: number\n /** Lower bound of the (1 - alpha) CI on the delta. */\n ciLower: number\n /** Upper bound of the (1 - alpha) CI on the delta. */\n ciUpper: number\n /** Number of bootstrap resamples used. */\n iterations: number\n alpha: number\n verdict: Verdict\n}\n\nexport interface BootstrapOptions {\n /** Confidence level alpha (default 0.05 → 95% CI). */\n alpha?: number\n /** Number of resamples (default 1000). */\n iterations?: number\n /**\n * Minimum total samples (baseline + candidate) below which we always\n * return INCONCLUSIVE — bootstrap with too few samples is meaningless.\n * Default 6 (combined).\n */\n minTotalSamples?: number\n /** RNG seed for reproducibility. Default: Math.random. */\n seed?: number\n}\n\n/**\n * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.\n *\n * Uses simple percentile bootstrap on the difference of resampled means.\n * That's the standard non-parametric primitive — no distributional\n * assumptions, robust to skew, easy to reason about.\n */\nexport function bootstrapCi(\n baseline: number[],\n candidate: number[],\n options: BootstrapOptions = {},\n): BootstrapResult {\n const alpha = options.alpha ?? 0.05\n const iterations = options.iterations ?? 1000\n const minTotal = options.minTotalSamples ?? 6\n const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate))\n\n const baselineMean = mean(baseline)\n const candidateMean = mean(candidate)\n const delta = candidateMean - baselineMean\n\n if (\n baseline.length + candidate.length < minTotal ||\n baseline.length === 0 ||\n candidate.length === 0\n ) {\n return {\n baselineMean,\n candidateMean,\n delta,\n ciLower: -Infinity,\n ciUpper: Infinity,\n iterations: 0,\n alpha,\n verdict: 'INCONCLUSIVE',\n }\n }\n\n const deltas: number[] = new Array(iterations)\n for (let i = 0; i < iterations; i++) {\n const bResample = resample(baseline, rng)\n const cResample = resample(candidate, rng)\n deltas[i] = mean(cResample) - mean(bResample)\n }\n deltas.sort((a, b) => a - b)\n const lowerIdx = Math.floor((alpha / 2) * iterations)\n const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1\n const ciLower = deltas[Math.max(0, lowerIdx)]!\n const ciUpper = deltas[Math.min(iterations - 1, upperIdx)]!\n\n let verdict: Verdict\n if (ciLower > 0) verdict = 'ADVANCE'\n else if (ciUpper < 0) verdict = 'REVERT'\n else if (delta >= 0) verdict = 'KEEP'\n else verdict = 'INCONCLUSIVE'\n\n return {\n baselineMean,\n candidateMean,\n delta,\n ciLower,\n ciUpper,\n iterations,\n alpha,\n verdict,\n }\n}\n\nfunction mean(xs: number[]): number {\n if (xs.length === 0) return 0\n let s = 0\n for (const x of xs) s += x\n return s / xs.length\n}\n\nfunction resample(xs: number[], rng: () => number): number[] {\n const out = new Array(xs.length)\n for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)]\n return out\n}\n\n/** Mulberry32 — fast deterministic PRNG. Stable across runs given the same seed. */\nfunction mulberry32(seed: number): () => number {\n let t = seed >>> 0\n return () => {\n t += 0x6d2b79f5\n let r = t\n r = Math.imul(r ^ (r >>> 15), r | 1)\n r ^= r + Math.imul(r ^ (r >>> 7), r | 61)\n return ((r ^ (r >>> 14)) >>> 0) / 4294967296\n }\n}\n\n/** Stable seed derived from the inputs — same data → same CI bounds. */\nfunction hashSeed(a: number[], b: number[]): number {\n let h = 2166136261\n for (const x of [...a, ...b]) {\n const view = new Float64Array([x])\n const bytes = new Uint8Array(view.buffer)\n for (const byte of bytes) {\n h ^= byte\n h = Math.imul(h, 16777619)\n }\n }\n return h >>> 0\n}\n\n/**\n * Judge-replay promotion gate.\n *\n * The cheap inner-loop judge that drives an evolution run is by definition\n * fast and noisy. When you're about to promote a winning variant to the\n * canonical default, you want a STRONGER judge (a more expensive model, a\n * human grader, a separately-trained reward model) to confirm the win\n * generalises beyond the inner loop.\n *\n * This helper takes raw winner + baseline outputs, scores both through the\n * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger\n * judge agrees the winner is real with the configured confidence. Doesn't\n * matter what shape your \"output\" is — pass a string, an object, anything\n * the judge can read.\n */\nexport interface JudgeReplayGateArgs<TOutput> {\n baselineOutputs: TOutput[]\n candidateOutputs: TOutput[]\n /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */\n judge: (output: TOutput) => Promise<number> | number\n alpha?: number\n iterations?: number\n /** RNG seed for reproducibility. */\n seed?: number\n /** Maximum concurrent judge calls. Default 4. */\n judgeConcurrency?: number\n}\n\n/**\n * Confirm a candidate's win with a stronger judge: score baseline and candidate outputs independently, then bootstrap a CI to verify the lift generalises beyond the inner loop.\n */\nexport async function judgeReplayGate<TOutput>(\n args: JudgeReplayGateArgs<TOutput>,\n): Promise<BootstrapResult & { baselineSamples: number; candidateSamples: number }> {\n const concurrency = args.judgeConcurrency ?? 4\n const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency)\n const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency)\n const ci = bootstrapCi(baselineScores, candidateScores, {\n ...(args.alpha !== undefined ? { alpha: args.alpha } : {}),\n ...(args.iterations !== undefined ? { iterations: args.iterations } : {}),\n ...(args.seed !== undefined ? { seed: args.seed } : {}),\n })\n return {\n ...ci,\n baselineSamples: baselineScores.length,\n candidateSamples: candidateScores.length,\n }\n}\n\nasync function scoreAll<TOutput>(\n outputs: TOutput[],\n judge: (output: TOutput) => Promise<number> | number,\n concurrency: number,\n): Promise<number[]> {\n const results: number[] = new Array(outputs.length)\n let next = 0\n async function worker(): Promise<void> {\n while (true) {\n const i = next++\n if (i >= outputs.length) return\n const v = await judge(outputs[i]!)\n results[i] = Number.isFinite(v) ? v : 0\n }\n }\n await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()))\n return results\n}\n","import type { ReleaseConfidenceScorecard } from './release-confidence'\nimport type { RunRecord } from './run-record'\nimport { summaryTable } from './summary-report'\n\nexport interface RenderReleaseReportOptions {\n title?: string\n runs?: readonly RunRecord[]\n comparator?: string\n traceAnalystFindings?: readonly string[]\n nextActions?: readonly string[]\n}\n\nexport function renderReleaseReport(\n scorecard: ReleaseConfidenceScorecard,\n options: RenderReleaseReportOptions = {},\n): string {\n const title = options.title ?? `Release Report: ${scorecard.target}`\n const lines: string[] = []\n lines.push(`# ${title}`)\n lines.push('')\n lines.push(`Status: **${scorecard.status.toUpperCase()}**`)\n lines.push(`Promote: **${scorecard.promote ? 'yes' : 'no'}**`)\n if (scorecard.candidateId) lines.push(`Candidate: \\`${scorecard.candidateId}\\``)\n if (scorecard.baselineId) lines.push(`Baseline: \\`${scorecard.baselineId}\\``)\n lines.push('')\n lines.push(scorecard.summary)\n lines.push('')\n\n lines.push('## Metrics')\n lines.push('')\n lines.push('| Metric | Value |')\n lines.push('|---|---:|')\n lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`)\n lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`)\n lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`)\n lines.push(`| Unscored runs | ${scorecard.metrics.unscoredRuns} |`)\n lines.push(`| Terminal failures | ${scorecard.metrics.terminalFailureRuns} |`)\n lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`)\n lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`)\n lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`)\n lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`)\n lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`)\n lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`)\n lines.push(`| p95 wall time | ${duration(scorecard.metrics.p95WallMs)} |`)\n lines.push('')\n\n if (scorecard.issues.length > 0) {\n lines.push('## Issues')\n lines.push('')\n for (const issue of scorecard.issues) {\n lines.push(`- **${issue.severity}** \\`${issue.code}\\` (${issue.axis}): ${issue.detail}`)\n }\n lines.push('')\n }\n\n const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts)\n if (surfaces.length > 0) {\n lines.push('## Responsible Surfaces')\n lines.push('')\n for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`)\n lines.push('')\n }\n\n const failures = entries(scorecard.metrics.failureModeCounts)\n if (failures.length > 0) {\n lines.push('## Failure Modes')\n lines.push('')\n for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`)\n lines.push('')\n }\n\n if (options.runs && options.runs.length > 0) {\n lines.push('## Run Summary')\n lines.push('')\n lines.push(\n summaryTable([...options.runs], {\n comparator: options.comparator ?? scorecard.baselineId ?? undefined,\n split: 'holdout',\n }).markdown,\n )\n lines.push('')\n }\n\n if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {\n lines.push('## TraceAnalyst Findings')\n lines.push('')\n for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`)\n lines.push('')\n }\n\n const nextActions = options.nextActions ?? defaultNextActions(scorecard)\n if (nextActions.length > 0) {\n lines.push('## Next Actions')\n lines.push('')\n for (const action of nextActions) lines.push(`- ${action}`)\n lines.push('')\n }\n\n return `${lines.join('\\n').trimEnd()}\\n`\n}\n\nfunction defaultNextActions(scorecard: ReleaseConfidenceScorecard): string[] {\n if (scorecard.promote) return ['Promote the candidate and keep canaries enabled.']\n return scorecard.issues\n .filter((issue) => issue.severity === 'critical')\n .map((issue) => `Resolve ${issue.code}: ${issue.detail}`)\n}\n\nfunction entries(values: Record<string, number>): Array<[string, number]> {\n return Object.entries(values)\n .filter(([, count]) => count > 0)\n .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))\n}\n\nfunction pct(value: number | null): string {\n return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`\n}\n\nfunction num(value: number | null): string {\n return value === null ? 'n/a' : value.toFixed(3)\n}\n\nfunction duration(value: number | null): string {\n return value === null ? 'n/a' : `${Math.round(value)} ms`\n}\n"],"mappings":";;;;;;;;AA0JA,IAAM,qBAA4D;AAAA,EAChE,eAAe;AAAA,EACf,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,gBAAgB;AAAA,EAChB,gBAAgB;AAAA,EAChB,aAAa;AAAA,EACb,cAAc;AAAA,EACd,eAAe;AAAA,EACf,gBAAgB,OAAO;AAAA,EACvB,cAAc,OAAO;AAAA,EACrB,uBAAuB;AAAA,EACvB,uBAAuB;AACzB;AAEO,SAAS,0BACd,OAC4B;AAC5B,QAAM,aAAa,EAAE,GAAG,oBAAoB,GAAG,MAAM,WAAW;AAChE,QAAM,cAAc,MAAM,eAAe;AACzC,QAAM,OAAO,gBAAgB,MAAM,QAAQ,CAAC,GAAG,aAAa,MAAM,UAAU;AAC5E,QAAM,SAAS,qBAAqB,MAAM,UAAU,CAAC,GAAG,aAAa,MAAM,UAAU;AACrF,QAAM,YAAY,MAAM,aAAa,CAAC;AACtC,QAAM,gBAAgB,MAAM,SAAS,iBAAiB,UAAU;AAChE,QAAM,cAAc,MAAM,SAAS,eAAe,oBAAoB,SAAS;AAC/E,QAAM,eAAe,UAAU,MAAM,QAAQ;AAC7C,QAAM,gBAAgB,UAAU,MAAM,SAAS;AAC/C,QAAM,YAAY,KAAK,IAAI,aAAa,EAAE,OAAO,cAAc;AAC/D,QAAM,cAAc,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,cAAc;AACpE,QAAM,gBAAgB,KAAK,SAAS,IAAI,YAAY;AACpD,QAAM,cAAc,KAAK,OAAO,CAAC,QAAQ,cAAc,GAAG,MAAM,MAAS;AACzE,QAAM,eAAe,KAAK;AAAA,IACxB,CAAC,QACC,cAAc,GAAG,MAAM,UACvB,CAAC,uBAAuB,GAAG,KAC3B,CAAC,wBAAwB,IAAI,eAAe;AAAA,EAChD,EAAE;AACF,QAAM,eACJ,KAAK,SAAS,IACV,KAAK,IAAI,CAAC,QAAQ,eAAe,KAAK,WAAW,qBAAqB,CAAC,IACvE,OAAO,IAAI,CAAC,UAAU,iBAAiB,OAAO,WAAW,qBAAqB,CAAC;AACrF,QAAM,kBACJ,KAAK,SAAS,IACV,KAAK,IAAI,CAAC,QAAQ,gBAAgB,IAAI,eAAe,CAAC,IACtD,OAAO,IAAI,CAAC,UAAU,MAAM,EAAE;AACpC,QAAM,2BAA2B,gBAAgB,OAAO,CAAC,YAAY,YAAY,MAAS,EAAE;AAC5F,QAAM,sBAAsB,gBAAgB,OAAO,CAAC,YAAY,YAAY,KAAK,EAAE;AACnF,QAAM,kBACJ,gBAAgB,WAAW,KAAK,2BAA2B,IACvD,QACC,gBAAgB,SAAS,uBAAuB,gBAAgB;AACvE,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,EAAE,aAAa,QAAQ,EAAE;AACtE,QAAM,cAAc,YAAY,OAAO,CAAC,MAAM,EAAE,aAAa,SAAS,EAAE;AACxE,QAAM,SAAS,WAAW,MAAM,QAAQ,WAAW,qBAAqB;AACxE,QAAM,kBAAkB,WAAW,YAAY;AAC/C,QAAM,mBAAmB,WAAW,aAAa;AACjD,QAAM,WAAW,KAAK;AAAA,IAAQ,CAAC,QAC7B,IAAI,eAAe,SAAS,eAAe,CAAC,IAAI,CAAC,IAAI,eAAe,GAAG;AAAA,EACzE;AACA,QAAM,aAAa,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO,EAAE,OAAO,cAAc;AAC7E,QAAM,cACJ,KAAK,SAAS,IACV,SAAS,WAAW,KAAK,SACvB,WAAW,QAAQ,IACnB,OACF,WAAW,UAAU;AAC3B,QAAM,YACJ,KAAK,SAAS,IACV,KAAK,IAAI,CAAC,QAAQ,IAAI,MAAM,IAC5B,OAAO,IAAI,CAAC,UAAU,MAAM,UAAU,EAAE,OAAO,cAAc;AACnE,QAAM,UAAoC;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,gBAAgB,YAAY;AAAA,IACtC,WAAW,WAAW,aAAa;AAAA,IACnC;AAAA,IACA;AAAA,IACA,YAAY,WAAW,iBAAiB,gBAAgB;AAAA,IACxD;AAAA,IACA,WAAW,iBAAiB,WAAW,IAAI;AAAA,IAC3C,YAAY,OAAO;AAAA,IACnB,iBAAiB,OAAO,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE;AAAA,IACpD,kBAAkB,OAAO,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE;AAAA,IAC1D,iBAAiB,OAAO,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,CAAC,EAAE;AAAA,IAC9D;AAAA,IACA,cAAc,aAAa,SAAS;AAAA,IACpC,mBAAmB,kBAAkB,MAAM,QAAQ,WAAW,qBAAqB;AAAA,IACnF,0BAA0B,yBAAyB,MAAM;AAAA,EAC3D;AAEA,QAAM,SAAmC,CAAC;AAC1C,cAAY,OAAO,YAAY,SAAS,MAAM;AAC9C,eAAa,YAAY,SAAS,MAAM;AACxC,mBAAiB,SAAS,MAAM;AAChC,sBAAoB,MAAM,gBAAgB,MAAM,YAAY,SAAS,MAAM;AAC3E,mBAAiB,YAAY,SAAS,MAAM;AAC5C,kBAAgB,YAAY,SAAS,MAAM;AAE3C,QAAM,OAAO,UAAU,SAAS,YAAY,MAAM;AAClD,QAAM,SAAS,OAAO,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU,IACvD,SACA,OAAO,SAAS,IACd,SACA;AAEN,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd;AAAA,IACA,YAAY,MAAM,cAAc;AAAA,IAChC;AAAA,IACA,SAAS,WAAW,WAAW,MAAM,eAAe,MAAM,aAAa,UAAU;AAAA,IACjF;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,MAAM,WAAW;AAAA,IAC1B,cAAc,MAAM,gBAAgB;AAAA,IACpC,SAAS,cAAc,MAAM,QAAQ,QAAQ,SAAS,MAAM;AAAA,EAC9D;AACF;AAEO,SAAS,wBAAwB,OAA2D;AACjG,QAAM,YAAY,0BAA0B,KAAK;AACjD,MAAI,UAAU,WAAW,QAAQ;AAC/B,UAAM,IAAI,kBAAkB,UAAU,OAAO;AAAA,EAC/C;AACA,SAAO;AACT;AAEA,SAAS,gBACP,MACA,aACA,YACa;AACb,MAAI,YAAa,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,WAAW;AACxE,MAAI,WAAY,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAU;AACtE,SAAO,CAAC,GAAG,IAAI;AACjB;AAEA,SAAS,qBACP,QACA,aACA,YACwB;AACxB,MAAI;AACF,WAAO,OAAO,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAa,EAAE,gBAAgB,WAAW;AAC1F,MAAI;AACF,WAAO,OAAO,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAa,EAAE,gBAAgB,UAAU;AACzF,SAAO,CAAC,GAAG,MAAM;AACnB;AAEA,SAAS,YACP,OACA,YACA,SACA,QACM;AACN,MAAI,WAAW,iBAAiB,CAAC,MAAM,YAAY,MAAM,WAAW,UAAU,OAAO,GAAG;AACtF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,gBAAgB,WAAW,kBAAkB;AACvD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,aAAa,sBAAsB,WAAW,gBAAgB;AAAA,IACnF,CAAC;AAAA,EACH;AACA,MAAI,WAAW,kBAAkB,QAAQ,YAAY,YAAY,GAAG;AAClE,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACF;AAEA,SAAS,aACP,YACA,SACA,QACM;AACN,MAAI,QAAQ,aAAa,WAAW,eAAe;AACjD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,UAAU,wBAAwB,WAAW,aAAa;AAAA,IAC/E,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,eAAe,GAAG;AAC5B,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,YAAY;AAAA,IACjC,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,aAAa,QAAQ,QAAQ,cAAc,MAAM;AAC3D,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,aAAa,QAAQ,QAAQ,WAAW,WAAW,aAAa;AAC1E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,YAAY,IAAI,QAAQ,QAAQ,CAAC,MAAM,IAAI,WAAW,WAAW,CAAC;AAAA,IAC5E,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,cAAc,QAAQ,QAAQ,YAAY,WAAW,cAAc;AAC7E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,aAAa,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,WAAW,YAAY,CAAC;AAAA,IAC/E,CAAC;AAAA,EACH;AACF;AAEA,SAAS,iBACP,SACA,QACM;AACN,MAAI,QAAQ,oBAAoB,MAAM;AACpC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QACE,QAAQ,2BAA2B,IAC/B,GAAG,QAAQ,wBAAwB,yDACnC;AAAA,IACR,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,sBAAsB,GAAG;AACnC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,mBAAmB;AAAA,IACxC,CAAC;AAAA,EACH;AACF;AAEA,SAAS,oBACP,cACA,YACA,SACA,QACM;AACN,MAAI,WAAW,kBAAkB,QAAQ,cAAc,WAAW,gBAAgB;AAChF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,WAAW,yBAAyB,WAAW,cAAc;AAAA,IAClF,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,eAAe,QAAQ,QAAQ,aAAa,WAAW,eAAe;AAChF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,sBAAsB,IAAI,QAAQ,UAAU,CAAC,MAAM,IAAI,WAAW,aAAa,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH;AACA,MAAI,gBAAgB,CAAC,aAAa,SAAS;AACzC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM,QAAQ,aAAa,iBAAiB,QAAQ;AAAA,MACpD,QAAQ,aAAa;AAAA,IACvB,CAAC;AAAA,EACH;AACF;AAEA,SAAS,iBACP,YACA,SACA,QACM;AACN,MAAI,CAAC,WAAW,sBAAuB;AACvC,MAAI,QAAQ,aAAa,QAAQ,iBAAiB;AAChD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,aAAa,QAAQ,eAAe;AAAA,IACzD,CAAC;AAAA,EACH;AACF;AAEA,SAAS,gBACP,YACA,SACA,QACM;AACN,MAAI,OAAO,SAAS,WAAW,cAAc,KAAK,QAAQ,gBAAgB,MAAM;AAC9E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH,WAAW,QAAQ,gBAAgB,QAAQ,QAAQ,cAAc,WAAW,gBAAgB;AAC1F,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,eAAe,IAAI,QAAQ,WAAW,CAAC,MAAM,IAAI,WAAW,cAAc,CAAC;AAAA,IACrF,CAAC;AAAA,EACH;AACA,MAAI,OAAO,SAAS,WAAW,YAAY,KAAK,QAAQ,cAAc,MAAM;AAC1E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH,WAAW,QAAQ,cAAc,QAAQ,QAAQ,YAAY,WAAW,cAAc;AACpF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,aAAa,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,WAAW,YAAY,CAAC;AAAA,IAC/E,CAAC;AAAA,EACH;AACF;AAEA,SAAS,UACP,SACA,YACA,QACyB;AACzB,SAAO;AAAA,IACL;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ,QAAQ,gBAAgB,KAAK,IAAI,GAAG,WAAW,gBAAgB,CAAC;AAAA,MACxE,GAAG,QAAQ,aAAa,uBAAuB,QAAQ,YAAY,OAAO;AAAA,IAC5E;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ,aAAa,QAAQ,QAAQ,cAAc,OAC/C,OACA,KAAK,IAAI,QAAQ,UAAU,QAAQ,SAAS;AAAA,MAChD,YAAY,IAAI,QAAQ,QAAQ,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAAA,IACvE;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,eAAe,IAAI,QAAQ,eAAe,CAAC,qBAAqB,QAAQ,mBAAmB,iBAAiB,QAAQ,wBAAwB;AAAA,IAC9I;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,YAAY,WAAW,aAAa;AAAA,MACrD,eAAe,QAAQ,WAAW,eAAe,IAAI,QAAQ,UAAU,CAAC;AAAA,IAC1E;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ,eAAe,IAAI,IAAI,QAAQ,kBAAkB,QAAQ;AAAA,MACjE,mBAAmB,QAAQ,eAAe,IAAI,QAAQ,UAAU;AAAA,IAClE;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,gBAAgB,SAAS,UAAU;AAAA,MACnC,eAAe,IAAI,QAAQ,WAAW,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAAA,IAC7E;AAAA,EACF;AACF;AAEA,SAAS,KACP,MACA,QACA,OACA,QACuB;AACvB,QAAM,MAAM,OAAO,OAAO,CAAC,MAAM,EAAE,SAAS,IAAI;AAChD,QAAM,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU,IACpD,SACA,IAAI,SAAS,IACX,SACA;AACN,SAAO,EAAE,MAAM,QAAQ,OAAO,UAAU,OAAO,OAAO,QAAQ,KAAK,GAAG,OAAO;AAC/E;AAEA,SAAS,oBAAoB,WAAqE;AAChG,QAAM,SAAuC,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,EAAE;AACrF,aAAW,YAAY,UAAW,QAAO,SAAS,SAAS,OAAO;AAClE,SAAO;AACT;AAEA,SAAS,aAAa,WAA+D;AACnF,QAAM,MAA8B,CAAC;AACrC,aAAW,YAAY,WAAW;AAChC,UAAM,SAAS,SAAS,MAAM,UAAU,SAAS,MAAM,YAAY;AACnE,QAAI,MAAM,KAAK,IAAI,MAAM,KAAK,KAAK;AAAA,EACrC;AACA,SAAO;AACT;AAEA,SAAS,kBACP,MACA,QACA,WACwB;AACxB,QAAM,MAA8B,CAAC;AACrC,aAAW,OAAO,MAAM;AACtB,QAAI,eAAe,KAAK,SAAS,MAAM,OAAO;AAC5C,YAAM,OAAO,IAAI,eAAe,IAAI,gBAAgB;AACpD,UAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK;AAAA,IACjC;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,QAAI,iBAAiB,OAAO,SAAS,MAAM,OAAO;AAChD,YAAM,OAAO,MAAM,gBAAgB,MAAM,OAAO,QAAQ,WAAW;AACnE,UAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK;AAAA,IACjC;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,yBAAyB,QAAiE;AACjG,QAAM,MAA8B,CAAC;AACrC,aAAW,SAAS,QAAQ;AAC1B,eAAW,OAAO,MAAM,OAAO,CAAC,GAAG;AACjC,YAAM,UAAU,IAAI,sBAAsB;AAC1C,UAAI,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK;AAAA,IACvC;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,WACP,MACA,QACA,WAC4B;AAC5B,QAAM,MAAkC,CAAC;AACzC,aAAW,OAAO,MAAM;AACtB,QAAI,eAAe,KAAK,SAAS,MAAM,OAAO;AAC5C,YAAM,YAAY,IAAI,QAAQ,IAAI;AAClC,UAAI,KAAK,EAAE,QAAQ,OAAO,cAAc,YAAY,YAAY,EAAE,CAAC;AAAA,IACrE;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,QAAI,iBAAiB,OAAO,SAAS,MAAM,OAAO;AAChD,UAAI,KAAK,EAAE,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE,CAAC;AAAA,IACnD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,UAAsD;AAC7E,QAAM,aAAa,SAAS,OAAO,CAAC,YAAgC,YAAY,IAAI;AACpF,MAAI,WAAW,WAAW,EAAG,QAAO;AACpC,SAAO,WAAW,OAAO,OAAO,EAAE,SAAS,WAAW;AACxD;AAEA,SAAS,eAAe,KAAgB,WAAmC;AACzE,MAAI,uBAAuB,GAAG,EAAG,QAAO;AACxC,QAAM,QAAQ,cAAc,GAAG;AAC/B,SAAO,UAAU,SAAY,OAAO,SAAS;AAC/C;AAEA,SAAS,uBAAuB,KAAyB;AACvD,SACG,IAAI,iBAAiB,UAAa,IAAI,iBAAiB,aACxD,IAAI,gBAAgB;AAExB;AAEA,SAAS,iBAAiB,OAA6B,WAAmC;AACxF,MAAI,MAAM,gBAAgB,OAAW,QAAO;AAC5C,MAAI,MAAM,OAAO,MAAO,QAAO;AAC/B,MAAI,eAAe,MAAM,KAAK,EAAG,QAAO,MAAM,SAAS;AACvD,SAAO,MAAM,OAAO,OAAO,OAAO;AACpC;AAEA,SAAS,wBACP,SACkD;AAClD,SAAO,YAAY,YAAY,YAAY,eAAe,YAAY;AACxE;AAEA,SAAS,gBAAgB,SAA4D;AACnF,MAAI,YAAY,YAAa,QAAO;AACpC,MAAI,wBAAwB,OAAO,EAAG,QAAO;AAC7C,SAAO;AACT;AAEA,SAAS,UAAU,MAA4B,OAA8B;AAC3E,SAAO,KACJ,OAAO,CAAC,QAAQ,IAAI,aAAa,KAAK,EACtC,IAAI,aAAa,EACjB,OAAO,cAAc;AAC1B;AAEA,SAAS,cAAc,KAAoC;AACzD,QAAM,QAAQ,IAAI,aAAa,YAAY,IAAI,QAAQ,eAAe,IAAI,QAAQ;AAClF,SAAO,eAAe,KAAK,IAAI,QAAQ;AACzC;AAEA,SAAS,WAAW,IAAsC;AACxD,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,SAAO,GAAG,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG;AAChD;AAEA,SAAS,iBAAiB,IAAuB,GAA0B;AACzE,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,QAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,SAAO,OAAO,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,OAAO,MAAM,IAAI,CAAC,CAAC,CAAC;AAC1F;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,WAAW,GAAkB,GAAiC;AACrE,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,SAAO,IAAI;AACb;AAEA,SAAS,SAAS,KAAoB,QAA+B;AACnE,MAAI,QAAQ,KAAM,QAAO;AACzB,MAAI,UAAU,EAAG,QAAO,OAAO,IAAI,IAAI;AACvC,SAAO,QAAQ,IAAI,KAAK,IAAI,GAAG,GAAG,IAAI,MAAM;AAC9C;AAEA,SAAS,gBACP,SACA,YACe;AACf,QAAM,OAAO,OAAO,SAAS,WAAW,cAAc,IAClD,QAAQ,gBAAgB,OACtB,OACA,QAAQ,WAAW,iBAAiB,KAAK,IAAI,QAAQ,aAAa,KAAK,CAAC,IAC1E;AACJ,QAAM,UAAU,OAAO,SAAS,WAAW,YAAY,IACnD,QAAQ,cAAc,OACpB,OACA,QAAQ,WAAW,eAAe,KAAK,IAAI,QAAQ,WAAW,KAAK,CAAC,IACtE;AACJ,MAAI,SAAS,QAAQ,YAAY,KAAM,QAAO;AAC9C,SAAO,KAAK,IAAI,MAAM,OAAO;AAC/B;AAEA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,cACP,QACA,QACA,SACA,QACQ;AACR,QAAM,SAAS,sBAAsB,MAAM,KAAK,MAAM;AACtD,QAAM,aAAa,aAAa,QAAQ,aAAa,eAAe,QAAQ,UAAU,gBAAgB,QAAQ,WAAW,aAAa,IAAI,QAAQ,QAAQ,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAC/L,MAAI,OAAO,WAAW,EAAG,QAAO,GAAG,MAAM,KAAK,UAAU;AACxD,SAAO,GAAG,MAAM,KAAK,UAAU,YAAY,OAAO,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC;AAChF;AAEA,SAAS,IAAI,GAA0B;AACrC,MAAI,MAAM,KAAM,QAAO;AACvB,SAAO,EAAE,QAAQ,CAAC;AACpB;;;ACrqBO,SAAS,YACd,UACA,WACA,UAA4B,CAAC,GACZ;AACjB,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,WAAW,QAAQ,mBAAmB;AAC5C,QAAM,MAAM,WAAW,QAAQ,QAAQ,SAAS,UAAU,SAAS,CAAC;AAEpE,QAAM,eAAe,KAAK,QAAQ;AAClC,QAAM,gBAAgB,KAAK,SAAS;AACpC,QAAM,QAAQ,gBAAgB;AAE9B,MACE,SAAS,SAAS,UAAU,SAAS,YACrC,SAAS,WAAW,KACpB,UAAU,WAAW,GACrB;AACA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,SAAS;AAAA,MACT,YAAY;AAAA,MACZ;AAAA,MACA,SAAS;AAAA,IACX;AAAA,EACF;AAEA,QAAM,SAAmB,IAAI,MAAM,UAAU;AAC7C,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,UAAM,YAAY,SAAS,UAAU,GAAG;AACxC,UAAM,YAAY,SAAS,WAAW,GAAG;AACzC,WAAO,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,SAAS;AAAA,EAC9C;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAC5D,QAAM,UAAU,OAAO,KAAK,IAAI,GAAG,QAAQ,CAAC;AAC5C,QAAM,UAAU,OAAO,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC;AAEzD,MAAI;AACJ,MAAI,UAAU,EAAG,WAAU;AAAA,WAClB,UAAU,EAAG,WAAU;AAAA,WACvB,SAAS,EAAG,WAAU;AAAA,MAC1B,WAAU;AAEf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,KAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,MAAI,IAAI;AACR,aAAW,KAAK,GAAI,MAAK;AACzB,SAAO,IAAI,GAAG;AAChB;AAEA,SAAS,SAAS,IAAc,KAA6B;AAC3D,QAAM,MAAM,IAAI,MAAM,GAAG,MAAM;AAC/B,WAAS,IAAI,GAAG,IAAI,GAAG,QAAQ,IAAK,KAAI,CAAC,IAAI,GAAG,KAAK,MAAM,IAAI,IAAI,GAAG,MAAM,CAAC;AAC7E,SAAO;AACT;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,IAAI,SAAS;AACjB,SAAO,MAAM;AACX,SAAK;AACL,QAAI,IAAI;AACR,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AACF;AAGA,SAAS,SAAS,GAAa,GAAqB;AAClD,MAAI,IAAI;AACR,aAAW,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG;AAC5B,UAAM,OAAO,IAAI,aAAa,CAAC,CAAC,CAAC;AACjC,UAAM,QAAQ,IAAI,WAAW,KAAK,MAAM;AACxC,eAAW,QAAQ,OAAO;AACxB,WAAK;AACL,UAAI,KAAK,KAAK,GAAG,QAAQ;AAAA,IAC3B;AAAA,EACF;AACA,SAAO,MAAM;AACf;AAiCA,eAAsB,gBACpB,MACkF;AAClF,QAAM,cAAc,KAAK,oBAAoB;AAC7C,QAAM,iBAAiB,MAAM,SAAS,KAAK,iBAAiB,KAAK,OAAO,WAAW;AACnF,QAAM,kBAAkB,MAAM,SAAS,KAAK,kBAAkB,KAAK,OAAO,WAAW;AACrF,QAAM,KAAK,YAAY,gBAAgB,iBAAiB;AAAA,IACtD,GAAI,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,IACxD,GAAI,KAAK,eAAe,SAAY,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;AAAA,IACvE,GAAI,KAAK,SAAS,SAAY,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC;AAAA,EACvD,CAAC;AACD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,iBAAiB,eAAe;AAAA,IAChC,kBAAkB,gBAAgB;AAAA,EACpC;AACF;AAEA,eAAe,SACb,SACA,OACA,aACmB;AACnB,QAAM,UAAoB,IAAI,MAAM,QAAQ,MAAM;AAClD,MAAI,OAAO;AACX,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,QAAQ,OAAQ;AACzB,YAAM,IAAI,MAAM,MAAM,QAAQ,CAAC,CAAE;AACjC,cAAQ,CAAC,IAAI,OAAO,SAAS,CAAC,IAAI,IAAI;AAAA,IACxC;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAClF,SAAO;AACT;;;AC1NO,SAAS,oBACd,WACA,UAAsC,CAAC,GAC/B;AACR,QAAM,QAAQ,QAAQ,SAAS,mBAAmB,UAAU,MAAM;AAClE,QAAM,QAAkB,CAAC;AACzB,QAAM,KAAK,KAAK,KAAK,EAAE;AACvB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,aAAa,UAAU,OAAO,YAAY,CAAC,IAAI;AAC1D,QAAM,KAAK,cAAc,UAAU,UAAU,QAAQ,IAAI,IAAI;AAC7D,MAAI,UAAU,YAAa,OAAM,KAAK,gBAAgB,UAAU,WAAW,IAAI;AAC/E,MAAI,UAAU,WAAY,OAAM,KAAK,eAAe,UAAU,UAAU,IAAI;AAC5E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,UAAU,OAAO;AAC5B,QAAM,KAAK,EAAE;AAEb,QAAM,KAAK,YAAY;AACvB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,oBAAoB;AAC/B,QAAM,KAAK,YAAY;AACvB,QAAM,KAAK,iBAAiB,UAAU,QAAQ,aAAa,IAAI;AAC/D,QAAM,KAAK,mBAAmB,UAAU,QAAQ,UAAU,IAAI;AAC9D,QAAM,KAAK,oBAAoB,UAAU,QAAQ,WAAW,IAAI;AAChE,QAAM,KAAK,qBAAqB,UAAU,QAAQ,YAAY,IAAI;AAClE,QAAM,KAAK,yBAAyB,UAAU,QAAQ,mBAAmB,IAAI;AAC7E,QAAM,KAAK,iBAAiB,IAAI,UAAU,QAAQ,QAAQ,CAAC,IAAI;AAC/D,QAAM,KAAK,kBAAkB,IAAI,UAAU,QAAQ,SAAS,CAAC,IAAI;AACjE,QAAM,KAAK,mBAAmB,IAAI,UAAU,QAAQ,eAAe,CAAC,IAAI;AACxE,QAAM,KAAK,oBAAoB,IAAI,UAAU,QAAQ,gBAAgB,CAAC,IAAI;AAC1E,QAAM,KAAK,mBAAmB,IAAI,UAAU,QAAQ,UAAU,CAAC,IAAI;AACnE,QAAM,KAAK,kBAAkB,IAAI,UAAU,QAAQ,WAAW,CAAC,IAAI;AACnE,QAAM,KAAK,qBAAqB,SAAS,UAAU,QAAQ,SAAS,CAAC,IAAI;AACzE,QAAM,KAAK,EAAE;AAEb,MAAI,UAAU,OAAO,SAAS,GAAG;AAC/B,UAAM,KAAK,WAAW;AACtB,UAAM,KAAK,EAAE;AACb,eAAW,SAAS,UAAU,QAAQ;AACpC,YAAM,KAAK,OAAO,MAAM,QAAQ,QAAQ,MAAM,IAAI,OAAO,MAAM,IAAI,MAAM,MAAM,MAAM,EAAE;AAAA,IACzF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,WAAW,QAAQ,UAAU,QAAQ,wBAAwB;AACnE,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,KAAK,yBAAyB;AACpC,UAAM,KAAK,EAAE;AACb,eAAW,CAAC,SAAS,KAAK,KAAK,SAAU,OAAM,KAAK,KAAK,OAAO,KAAK,KAAK,EAAE;AAC5E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,WAAW,QAAQ,UAAU,QAAQ,iBAAiB;AAC5D,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,KAAK,kBAAkB;AAC7B,UAAM,KAAK,EAAE;AACb,eAAW,CAAC,MAAM,KAAK,KAAK,SAAU,OAAM,KAAK,KAAK,IAAI,KAAK,KAAK,EAAE;AACtE,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,QAAQ,QAAQ,QAAQ,KAAK,SAAS,GAAG;AAC3C,UAAM,KAAK,gBAAgB;AAC3B,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,aAAa,CAAC,GAAG,QAAQ,IAAI,GAAG;AAAA,QAC9B,YAAY,QAAQ,cAAc,UAAU,cAAc;AAAA,QAC1D,OAAO;AAAA,MACT,CAAC,EAAE;AAAA,IACL;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,QAAQ,wBAAwB,QAAQ,qBAAqB,SAAS,GAAG;AAC3E,UAAM,KAAK,0BAA0B;AACrC,UAAM,KAAK,EAAE;AACb,eAAW,WAAW,QAAQ,qBAAsB,OAAM,KAAK,KAAK,OAAO,EAAE;AAC7E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,cAAc,QAAQ,eAAe,mBAAmB,SAAS;AACvE,MAAI,YAAY,SAAS,GAAG;AAC1B,UAAM,KAAK,iBAAiB;AAC5B,UAAM,KAAK,EAAE;AACb,eAAW,UAAU,YAAa,OAAM,KAAK,KAAK,MAAM,EAAE;AAC1D,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,SAAO,GAAG,MAAM,KAAK,IAAI,EAAE,QAAQ,CAAC;AAAA;AACtC;AAEA,SAAS,mBAAmB,WAAiD;AAC3E,MAAI,UAAU,QAAS,QAAO,CAAC,kDAAkD;AACjF,SAAO,UAAU,OACd,OAAO,CAAC,UAAU,MAAM,aAAa,UAAU,EAC/C,IAAI,CAAC,UAAU,WAAW,MAAM,IAAI,KAAK,MAAM,MAAM,EAAE;AAC5D;AAEA,SAAS,QAAQ,QAAyD;AACxE,SAAO,OAAO,QAAQ,MAAM,EACzB,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,QAAQ,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;AAC3D;AAEA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAC7D;AAEA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,QAAQ,MAAM,QAAQ,CAAC;AACjD;AAEA,SAAS,SAAS,OAA8B;AAC9C,SAAO,UAAU,OAAO,QAAQ,GAAG,KAAK,MAAM,KAAK,CAAC;AACtD;","names":[]}
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
costReceiptFromLlm,
|
|
6
6
|
costReceiptFromLlmError,
|
|
7
7
|
maximumChargeForLlmRequest
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-PBE2LOSS.js";
|
|
9
9
|
import {
|
|
10
10
|
CostLedger
|
|
11
11
|
} from "./chunk-WS3NZZQQ.js";
|
|
@@ -1053,4 +1053,4 @@ export {
|
|
|
1053
1053
|
startServer,
|
|
1054
1054
|
startServerAsync
|
|
1055
1055
|
};
|
|
1056
|
-
//# sourceMappingURL=chunk-
|
|
1056
|
+
//# sourceMappingURL=chunk-YJBNWCAA.js.map
|