@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/release-report-Cz9NKH39.js

@@ -0,0 +1,603 @@
+import { c as VerificationError, s as ValidationError } from "./errors-8YnH8WlF.js";
+import { s as validateRunRecord } from "./run-record-BuoE80Dq.js";
+import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
+import { t as FAILURE_CLASSES } from "./schema-CRhEY1SO.js";
+import { a as summaryTable } from "./summary-report-BNs5nmXI.js";
+//#region src/release-confidence.ts
+const DEFAULT_THRESHOLDS = {
+	requireCorpus: true,
+	minScenarioCount: 1,
+	minSearchRuns: 1,
+	minHoldoutRuns: 1,
+	requireHoldout: true,
+	minPassRate: .8,
+	minMeanScore: .7,
+	maxOverfitGap: .15,
+	maxMeanCostUsd: Number.POSITIVE_INFINITY,
+	maxP95WallMs: Number.POSITIVE_INFINITY,
+	requireAsiForFailures: true,
+	failureScoreThreshold: .5
+};
+function evaluateReleaseConfidence(input) {
+	const thresholds = {
+		...DEFAULT_THRESHOLDS,
+		...input.thresholds
+	};
+	const candidateId = input.candidateId ?? null;
+	const runs = filterCandidate((input.runs ?? []).map(validateRunRecord), candidateId, input.baselineId);
+	const traces = filterTraceCandidate((input.traces ?? []).map(validateReleaseTraceEvidence), candidateId, input.baselineId);
+	const scenarios = input.scenarios ?? [];
+	const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length;
+	const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios);
+	const searchScores = scoresFor(runs, "search");
+	const holdoutScores = scoresFor(runs, "holdout");
+	const runScores = runs.map(runSplitScore).filter(isFiniteNumber);
+	const traceScores = traces.map((t) => t.score).filter(isFiniteNumber);
+	const scoreUniverse = runs.length > 0 ? runScores : traceScores;
+	const qualityRuns = runs.filter((run) => runSplitScore(run) !== void 0);
+	const unscoredRuns = runs.filter((run) => runSplitScore(run) === void 0 && !hasExplicitTaskFailure(run) && !isFailedTerminalOutcome(run.terminalOutcome)).length;
+	const honestRuns = runs.filter((run) => !isRealnessGated(run));
+	const passOutcomes = runs.length > 0 ? honestRuns.map((run) => runPassOutcome(run, thresholds.failureScoreThreshold)) : traces.map((trace) => tracePassOutcome(trace, thresholds.failureScoreThreshold));
+	const reliabilityRows = runs.length > 0 ? runs.map((run) => terminalSuccess(run.terminalOutcome)) : traces.map((trace) => trace.ok);
+	const unclassifiedTerminalRuns = reliabilityRows.filter((outcome) => outcome === void 0).length;
+	const terminalFailureRuns = reliabilityRows.filter((outcome) => outcome === false).length;
+	const reliabilityRate = reliabilityRows.length === 0 || unclassifiedTerminalRuns > 0 ? null : (reliabilityRows.length - terminalFailureRuns) / reliabilityRows.length;
+	const searchRuns = qualityRuns.filter((r) => r.splitTag === "search").length;
+	const holdoutRuns = qualityRuns.filter((r) => r.splitTag === "holdout").length;
+	const failed = failedRows(runs, traces, thresholds.failureScoreThreshold);
+	const searchMeanScore = meanOrNull(searchScores);
+	const holdoutMeanScore = meanOrNull(holdoutScores);
+	const runCosts = runs.flatMap((run) => run.costProvenance.kind === "uncaptured" ? [] : [run.costProvenance.usd]);
+	const traceCosts = traces.map((trace) => trace.costUsd).filter(isFiniteNumber);
+	const meanCostUsd = runs.length > 0 ? runCosts.length === runs.length ? meanOrNull(runCosts) : null : meanOrNull(traceCosts);
+	const wallTimes = runs.length > 0 ? runs.map((run) => run.wallMs) : traces.map((trace) => trace.durationMs).filter(isFiniteNumber);
+	const metrics = {
+		scenarioCount,
+		searchRuns,
+		holdoutRuns,
+		unscoredRuns,
+		unclassifiedTerminalRuns,
+		terminalFailureRuns,
+		reliabilityRate,
+		passRate: passOutcomeRate(passOutcomes),
+		realnessGatedRuns: runs.length - honestRuns.length,
+		meanScore: meanOrNull(scoreUniverse),
+		searchMeanScore,
+		holdoutMeanScore,
+		overfitGap: diffOrNull(searchMeanScore, holdoutMeanScore),
+		meanCostUsd,
+		p95WallMs: percentileOrNull(wallTimes, .95),
+		failedRows: failed.length,
+		failuresWithAsi: failed.filter((row) => row.hasAsi).length,
+		singleShotTraces: traces.filter((t) => t.turnCount === 1).length,
+		multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,
+		splitCounts,
+		domainCounts: countDomains(scenarios),
+		failureClassCounts: countFailureClasses(runs, traces, thresholds.failureScoreThreshold),
+		responsibleSurfaceCounts: countResponsibleSurfaces(traces)
+	};
+	const issues = [];
+	checkCorpus(input, thresholds, metrics, issues);
+	checkQuality(thresholds, metrics, issues);
+	checkReliability(metrics, issues);
+	checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues);
+	checkDiagnostics(thresholds, metrics, issues);
+	checkEfficiency(thresholds, metrics, issues);
+	const axes = buildAxes(metrics, thresholds, issues);
+	const status = issues.some((i) => i.severity === "critical") ? "fail" : issues.length > 0 ? "warn" : "pass";
+	return {
+		target: input.target,
+		candidateId,
+		baselineId: input.baselineId ?? null,
+		status,
+		promote: status === "pass" && (input.gateDecision ? input.gateDecision.promote : true),
+		axes,
+		issues,
+		metrics,
+		dataset: input.dataset ?? null,
+		gateDecision: input.gateDecision ?? null,
+		summary: renderSummary(input.target, status, metrics, issues)
+	};
+}
+function assertReleaseConfidence(input) {
+	const scorecard = evaluateReleaseConfidence(input);
+	if (scorecard.status === "fail") throw new VerificationError(scorecard.summary);
+	return scorecard;
+}
+function filterCandidate(runs, candidateId, baselineId) {
+	if (candidateId) return runs.filter((r) => r.candidateId === candidateId);
+	if (baselineId) return runs.filter((r) => r.candidateId !== baselineId);
+	return [...runs];
+}
+function filterTraceCandidate(traces, candidateId, baselineId) {
+	if (candidateId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId === candidateId);
+	if (baselineId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId !== baselineId);
+	return [...traces];
+}
+function validateReleaseTraceEvidence(trace, index) {
+	const value = trace;
+	if (Object.hasOwn(value, "failureMode")) throw new ValidationError(`traces[${index}].failureMode is not supported; use canonical failureClass`);
+	if (trace.failureClass !== void 0 && !FAILURE_CLASSES.includes(trace.failureClass)) throw new ValidationError(`traces[${index}].failureClass must be one of ${FAILURE_CLASSES.join(", ")}`);
+	return trace;
+}
+function checkCorpus(input, thresholds, metrics, issues) {
+	if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) issues.push({
+		axis: "corpus",
+		severity: "critical",
+		code: "missing_corpus",
+		detail: "No Dataset manifest or scenarios supplied."
+	});
+	if (metrics.scenarioCount < thresholds.minScenarioCount) issues.push({
+		axis: "corpus",
+		severity: "critical",
+		code: "few_scenarios",
+		detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.`
+	});
+	if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) issues.push({
+		axis: "corpus",
+		severity: "critical",
+		code: "missing_holdout_split",
+		detail: "Corpus has no holdout scenarios."
+	});
+}
+function checkQuality(thresholds, metrics, issues) {
+	if (metrics.searchRuns < thresholds.minSearchRuns) issues.push({
+		axis: "quality",
+		severity: "critical",
+		code: "few_search_runs",
+		detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.`
+	});
+	if (metrics.unscoredRuns > 0) issues.push({
+		axis: "quality",
+		severity: "critical",
+		code: "unscored_runs",
+		detail: `${metrics.unscoredRuns} supplied run(s) have no task result.`
+	});
+	if (metrics.passRate === null || metrics.meanScore === null) issues.push({
+		axis: "quality",
+		severity: "critical",
+		code: "missing_quality_scores",
+		detail: "No task-quality scores are available for pass-rate and mean-score checks."
+	});
+	if (metrics.passRate !== null && metrics.passRate < thresholds.minPassRate) issues.push({
+		axis: "quality",
+		severity: "critical",
+		code: "low_pass_rate",
+		detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.`
+	});
+	if (metrics.meanScore !== null && metrics.meanScore < thresholds.minMeanScore) issues.push({
+		axis: "quality",
+		severity: "critical",
+		code: "low_mean_score",
+		detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.`
+	});
+}
+function checkReliability(metrics, issues) {
+	if (metrics.reliabilityRate === null) issues.push({
+		axis: "reliability",
+		severity: "critical",
+		code: "missing_reliability_evidence",
+		detail: metrics.unclassifiedTerminalRuns > 0 ? `${metrics.unclassifiedTerminalRuns} supplied run(s) have no classified terminal result.` : "No classified terminal results are available."
+	});
+	if (metrics.terminalFailureRuns > 0) issues.push({
+		axis: "reliability",
+		severity: "critical",
+		code: "terminal_run_failures",
+		detail: `${metrics.terminalFailureRuns} run(s) ended failed, cancelled, or incomplete.`
+	});
+}
+function checkGeneralization(gateDecision, thresholds, metrics, issues) {
+	if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) issues.push({
+		axis: "generalization",
+		severity: "critical",
+		code: "few_holdout_runs",
+		detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.`
+	});
+	if (metrics.overfitGap !== null && metrics.overfitGap > thresholds.maxOverfitGap) issues.push({
+		axis: "generalization",
+		severity: "critical",
+		code: "overfit_gap",
+		detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.`
+	});
+	if (gateDecision && !gateDecision.promote) issues.push({
+		axis: "generalization",
+		severity: "critical",
+		code: `gate_${gateDecision.rejectionCode ?? "reject"}`,
+		detail: gateDecision.reason
+	});
+}
+function checkDiagnostics(thresholds, metrics, issues) {
+	if (!thresholds.requireAsiForFailures) return;
+	if (metrics.failedRows > metrics.failuresWithAsi) issues.push({
+		axis: "diagnostics",
+		severity: "critical",
+		code: "missing_failure_asi",
+		detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`
+	});
+}
+function checkEfficiency(thresholds, metrics, issues) {
+	if (Number.isFinite(thresholds.maxMeanCostUsd) && metrics.meanCostUsd === null) issues.push({
+		axis: "efficiency",
+		severity: "critical",
+		code: "missing_cost",
+		detail: "A finite cost limit was configured but no cost evidence is available."
+	});
+	else if (metrics.meanCostUsd !== null && metrics.meanCostUsd > thresholds.maxMeanCostUsd) issues.push({
+		axis: "efficiency",
+		severity: "critical",
+		code: "cost_budget",
+		detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.`
+	});
+	if (Number.isFinite(thresholds.maxP95WallMs) && metrics.p95WallMs === null) issues.push({
+		axis: "efficiency",
+		severity: "critical",
+		code: "missing_latency",
+		detail: "A finite latency limit was configured but no latency evidence is available."
+	});
+	else if (metrics.p95WallMs !== null && metrics.p95WallMs > thresholds.maxP95WallMs) issues.push({
+		axis: "efficiency",
+		severity: "critical",
+		code: "latency_budget",
+		detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.`
+	});
+}
+function buildAxes(metrics, thresholds, issues) {
+	return [
+		axis("corpus", issues, bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)), `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`),
+		axis("quality", issues, metrics.passRate === null || metrics.meanScore === null ? null : Math.min(metrics.passRate, metrics.meanScore), `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`),
+		axis("reliability", issues, metrics.reliabilityRate, `successRate=${fmt(metrics.reliabilityRate)} terminalFailures=${metrics.terminalFailureRuns} unclassified=${metrics.unclassifiedTerminalRuns}`),
+		axis("generalization", issues, gapScore(metrics.overfitGap, thresholds.maxOverfitGap), `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`),
+		axis("diagnostics", issues, metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows, `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`),
+		axis("efficiency", issues, efficiencyScore(metrics, thresholds), `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`)
+	];
+}
+function axis(name, issues, score, detail) {
+	const own = issues.filter((i) => i.axis === name);
+	return {
+		name,
+		status: own.some((i) => i.severity === "critical") ? "fail" : own.length > 0 ? "warn" : "pass",
+		score: score === null ? null : bounded(score),
+		detail
+	};
+}
+function countScenarioSplits(scenarios) {
+	const counts = {
+		train: 0,
+		dev: 0,
+		test: 0,
+		holdout: 0
+	};
+	for (const scenario of scenarios) counts[scenario.split ?? "train"]++;
+	return counts;
+}
+function countDomains(scenarios) {
+	const out = {};
+	for (const scenario of scenarios) {
+		const domain = scenario.tags?.domain ?? scenario.tags?.category ?? "uncategorized";
+		out[domain] = (out[domain] ?? 0) + 1;
+	}
+	return out;
+}
+function countFailureClasses(runs, traces, threshold) {
+	const out = {};
+	for (const run of runs) if (runPassOutcome(run, threshold) === false) {
+		const failureClass = run.failureClass !== void 0 && run.failureClass !== "success" ? run.failureClass : "unknown";
+		out[failureClass] = (out[failureClass] ?? 0) + 1;
+	}
+	for (const trace of traces) if (tracePassOutcome(trace, threshold) === false) {
+		const failureClass = trace.failureClass !== void 0 && trace.failureClass !== "success" ? trace.failureClass : "unknown";
+		out[failureClass] = (out[failureClass] ?? 0) + 1;
+	}
+	return out;
+}
+function countResponsibleSurfaces(traces) {
+	const out = {};
+	for (const trace of traces) for (const asi of trace.asi ?? []) {
+		const surface = asi.responsibleSurface ?? "unknown";
+		out[surface] = (out[surface] ?? 0) + 1;
+	}
+	return out;
+}
+function failedRows(runs, traces, threshold) {
+	const out = [];
+	for (const run of runs) if (runPassOutcome(run, threshold) === false) {
+		const asiMetric = run.outcome.raw.asi;
+		out.push({ hasAsi: typeof asiMetric === "number" && asiMetric > 0 });
+	}
+	for (const trace of traces) if (tracePassOutcome(trace, threshold) === false) out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 });
+	return out;
+}
+function passOutcomeRate(outcomes) {
+	const classified = outcomes.filter((outcome) => outcome !== null);
+	if (classified.length === 0) return null;
+	return classified.filter(Boolean).length / classified.length;
+}
+function runPassOutcome(run, threshold) {
+	if (hasExplicitTaskFailure(run)) return false;
+	const score = runSplitScore(run);
+	return score === void 0 ? null : score >= threshold;
+}
+function hasExplicitTaskFailure(run) {
+	return run.failureClass !== void 0 && run.failureClass !== "success";
+}
+function tracePassOutcome(trace, threshold) {
+	if (trace.failureClass !== void 0 && trace.failureClass !== "success") return false;
+	if (trace.ok === false) return false;
+	if (isFiniteNumber(trace.score)) return trace.score >= threshold;
+	return trace.ok === true ? true : null;
+}
+function isFailedTerminalOutcome(outcome) {
+	return outcome === "failed" || outcome === "cancelled" || outcome === "incomplete";
+}
+function terminalSuccess(outcome) {
+	if (outcome === "succeeded") return true;
+	if (isFailedTerminalOutcome(outcome)) return false;
+}
+function scoresFor(runs, split) {
+	return runs.filter((run) => run.splitTag === split).map(runSplitScore).filter(isFiniteNumber);
+}
+/**
+* RAW and split-exact (`observedSplitScore`): this feeds the per-split means,
+* the overfit gap, and the pass threshold — descriptions of what the runs
+* reported. A gamed run inflating them is visible next to `realnessGatedRuns`,
+* and the promotion number (`passRate`) excludes gated runs entirely.
+*/
+function runSplitScore(run) {
+	const score = observedSplitScore(run, run.splitTag === "holdout" ? "holdout" : "search");
+	return isFiniteNumber(score) ? score : void 0;
+}
+function meanOrNull(xs) {
+	if (xs.length === 0) return null;
+	return xs.reduce((sum, x) => sum + x, 0) / xs.length;
+}
+function percentileOrNull(xs, p) {
+	if (xs.length === 0) return null;
+	const sorted = [...xs].sort((a, b) => a - b);
+	return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
+}
+function isFiniteNumber(value) {
+	return typeof value === "number" && Number.isFinite(value);
+}
+function diffOrNull(a, b) {
+	if (a === null || b === null) return null;
+	return a - b;
+}
+function gapScore(gap, maxGap) {
+	if (gap === null) return null;
+	if (maxGap <= 0) return gap <= 0 ? 1 : 0;
+	return bounded(1 - Math.max(0, gap) / maxGap);
+}
+function efficiencyScore(metrics, thresholds) {
+	const cost = Number.isFinite(thresholds.maxMeanCostUsd) ? metrics.meanCostUsd === null ? null : bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12)) : 1;
+	const latency = Number.isFinite(thresholds.maxP95WallMs) ? metrics.p95WallMs === null ? null : bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12)) : 1;
+	if (cost === null || latency === null) return null;
+	return Math.min(cost, latency);
+}
+function bounded(x) {
+	if (!Number.isFinite(x)) return 0;
+	return Math.max(0, Math.min(1, x));
+}
+function renderSummary(target, status, metrics, issues) {
+	const prefix = `release confidence ${status}: ${target}`;
+	const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`;
+	if (issues.length === 0) return `${prefix}; ${metricText}`;
+	return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(",")}`;
+}
+function fmt(x) {
+	if (x === null) return "n/a";
+	return x.toFixed(4);
+}
+//#endregion
+//#region src/promotion-gate.ts
+/**
+* Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+*
+* Uses simple percentile bootstrap on the difference of resampled means.
+* That's the standard non-parametric primitive — no distributional
+* assumptions, robust to skew, easy to reason about.
+*/
+function bootstrapCi(baseline, candidate, options = {}) {
+	const alpha = options.alpha ?? .05;
+	const iterations = options.iterations ?? 1e3;
+	const minTotal = options.minTotalSamples ?? 6;
+	const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate));
+	const baselineMean = mean(baseline);
+	const candidateMean = mean(candidate);
+	const delta = candidateMean - baselineMean;
+	if (baseline.length + candidate.length < minTotal || baseline.length === 0 || candidate.length === 0) return {
+		baselineMean,
+		candidateMean,
+		delta,
+		ciLower: -Infinity,
+		ciUpper: Infinity,
+		iterations: 0,
+		alpha,
+		verdict: "INCONCLUSIVE"
+	};
+	const deltas = new Array(iterations);
+	for (let i = 0; i < iterations; i++) {
+		const bResample = resample(baseline, rng);
+		deltas[i] = mean(resample(candidate, rng)) - mean(bResample);
+	}
+	deltas.sort((a, b) => a - b);
+	const lowerIdx = Math.floor(alpha / 2 * iterations);
+	const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1;
+	const ciLower = deltas[Math.max(0, lowerIdx)];
+	const ciUpper = deltas[Math.min(iterations - 1, upperIdx)];
+	let verdict;
+	if (ciLower > 0) verdict = "ADVANCE";
+	else if (ciUpper < 0) verdict = "REVERT";
+	else if (delta >= 0) verdict = "KEEP";
+	else verdict = "INCONCLUSIVE";
+	return {
+		baselineMean,
+		candidateMean,
+		delta,
+		ciLower,
+		ciUpper,
+		iterations,
+		alpha,
+		verdict
+	};
+}
+function mean(xs) {
+	if (xs.length === 0) return 0;
+	let s = 0;
+	for (const x of xs) s += x;
+	return s / xs.length;
+}
+function resample(xs, rng) {
+	const out = new Array(xs.length);
+	for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)];
+	return out;
+}
+/** Mulberry32 — fast deterministic PRNG. Stable across runs given the same seed. */
+function mulberry32(seed) {
+	let t = seed >>> 0;
+	return () => {
+		t += 1831565813;
+		let r = t;
+		r = Math.imul(r ^ r >>> 15, r | 1);
+		r ^= r + Math.imul(r ^ r >>> 7, r | 61);
+		return ((r ^ r >>> 14) >>> 0) / 4294967296;
+	};
+}
+/** Stable seed derived from the inputs — same data → same CI bounds. */
+function hashSeed(a, b) {
+	let h = 2166136261;
+	for (const x of [...a, ...b]) {
+		const view = new Float64Array([x]);
+		const bytes = new Uint8Array(view.buffer);
+		for (const byte of bytes) {
+			h ^= byte;
+			h = Math.imul(h, 16777619);
+		}
+	}
+	return h >>> 0;
+}
+/**
+* Confirm a candidate's win with a stronger judge: score baseline and candidate outputs independently, then bootstrap a CI to verify the lift generalises beyond the inner loop.
+*/
+async function judgeReplayGate(args) {
+	const concurrency = args.judgeConcurrency ?? 4;
+	const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency);
+	const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency);
+	return {
+		...bootstrapCi(baselineScores, candidateScores, {
+			...args.alpha !== void 0 ? { alpha: args.alpha } : {},
+			...args.iterations !== void 0 ? { iterations: args.iterations } : {},
+			...args.seed !== void 0 ? { seed: args.seed } : {}
+		}),
+		baselineSamples: baselineScores.length,
+		candidateSamples: candidateScores.length
+	};
+}
+async function scoreAll(outputs, judge, concurrency) {
+	const results = new Array(outputs.length);
+	let next = 0;
+	async function worker() {
+		while (true) {
+			const i = next++;
+			if (i >= outputs.length) return;
+			const v = await judge(outputs[i]);
+			results[i] = Number.isFinite(v) ? v : 0;
+		}
+	}
+	await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+	return results;
+}
+//#endregion
+//#region src/release-report.ts
+function renderReleaseReport(scorecard, options = {}) {
+	const title = options.title ?? `Release Report: ${scorecard.target}`;
+	const lines = [];
+	lines.push(`# ${title}`);
+	lines.push("");
+	lines.push(`Status: **${scorecard.status.toUpperCase()}**`);
+	lines.push(`Promote: **${scorecard.promote ? "yes" : "no"}**`);
+	if (scorecard.candidateId) lines.push(`Candidate: \`${scorecard.candidateId}\``);
+	if (scorecard.baselineId) lines.push(`Baseline: \`${scorecard.baselineId}\``);
+	lines.push("");
+	lines.push(scorecard.summary);
+	lines.push("");
+	lines.push("## Metrics");
+	lines.push("");
+	lines.push("| Metric | Value |");
+	lines.push("|---|---:|");
+	lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`);
+	lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`);
+	lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`);
+	lines.push(`| Unscored runs | ${scorecard.metrics.unscoredRuns} |`);
+	lines.push(`| Terminal failures | ${scorecard.metrics.terminalFailureRuns} |`);
+	lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`);
+	lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`);
+	lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`);
+	lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`);
+	lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`);
+	lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`);
+	lines.push(`| p95 wall time | ${duration(scorecard.metrics.p95WallMs)} |`);
+	lines.push("");
+	if (scorecard.issues.length > 0) {
+		lines.push("## Issues");
+		lines.push("");
+		for (const issue of scorecard.issues) lines.push(`- **${issue.severity}** \`${issue.code}\` (${issue.axis}): ${issue.detail}`);
+		lines.push("");
+	}
+	const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts);
+	if (surfaces.length > 0) {
+		lines.push("## Responsible Surfaces");
+		lines.push("");
+		for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`);
+		lines.push("");
+	}
+	const failures = entries(scorecard.metrics.failureClassCounts);
+	if (failures.length > 0) {
+		lines.push("## Failure Classes");
+		lines.push("");
+		for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`);
+		lines.push("");
+	}
+	if (options.runs && options.runs.length > 0) {
+		lines.push("## Run Summary");
+		lines.push("");
+		lines.push(summaryTable([...options.runs], {
+			comparator: options.comparator ?? scorecard.baselineId ?? void 0,
+			split: "holdout"
+		}).markdown);
+		lines.push("");
+	}
+	if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {
+		lines.push("## TraceAnalyst Findings");
+		lines.push("");
+		for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`);
+		lines.push("");
+	}
+	const nextActions = options.nextActions ?? defaultNextActions(scorecard);
+	if (nextActions.length > 0) {
+		lines.push("## Next Actions");
+		lines.push("");
+		for (const action of nextActions) lines.push(`- ${action}`);
+		lines.push("");
+	}
+	return `${lines.join("\n").trimEnd()}\n`;
+}
+function defaultNextActions(scorecard) {
+	if (scorecard.promote) return ["Promote the candidate and keep canaries enabled."];
+	return scorecard.issues.filter((issue) => issue.severity === "critical").map((issue) => `Resolve ${issue.code}: ${issue.detail}`);
+}
+function entries(values) {
+	return Object.entries(values).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+}
+function pct(value) {
+	return value === null ? "n/a" : `${(value * 100).toFixed(1)}%`;
+}
+function num(value) {
+	return value === null ? "n/a" : value.toFixed(3);
+}
+function duration(value) {
+	return value === null ? "n/a" : `${Math.round(value)} ms`;
+}
+//#endregion
+export { evaluateReleaseConfidence as a, assertReleaseConfidence as i, bootstrapCi as n, judgeReplayGate as r, renderReleaseReport as t };
+
+//# sourceMappingURL=release-report-Cz9NKH39.js.map