@tangle-network/agent-eval 0.128.2 → 0.130.0

package/dist/analyze-runs-C1CavBMk.js

@@ -0,0 +1,1067 @@
+import { s as validateRunRecord } from "./run-record-BuoE80Dq.js";
+import { A as spearmanR, C as pairedTTest, E as pearsonR, L as continuousAgreement, O as requiredPairedSampleSize, b as pairedMde, v as pairedBootstrap, y as pairedCohensDz } from "./statistics-CnnxdpOg.js";
+import { r as pairRunRecords } from "./paired-arms-D9D0wXj2.js";
+import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
+import { o as llmSpans } from "./query-Di7eEQ79.js";
+import { r as paretoChart } from "./summary-report-BNs5nmXI.js";
+//#region src/contamination-guard.ts
+function checkCanaries(output, scenarios) {
+	const leaks = [];
+	for (const s of scenarios) {
+		if (!s.canary) continue;
+		if (output.includes(s.canary)) leaks.push({
+			scenarioId: s.id,
+			canary: s.canary,
+			evidence: excerpt(output, s.canary)
+		});
+	}
+	return leaks;
+}
+/**
+* Behavioral canary: tests the AGENT, not the eval grep.
+*
+*   - `forbiddenPattern` PRESENT in output ⇒ AGENT EMITTED BAD CONTENT ⇒ LEAK.
+*   - `forbiddenPattern` ABSENT from output ⇒ AGENT HELD THE LINE       ⇒ PASS.
+*
+* Use when running known-bad-prompt scenarios against the agent under
+* test and you want to know if the agent misbehaved. The classical
+* {@link checkCanaries} / {@link import('./canary').runCanaries | runCanaries}
+* test whether the eval check fires when the bad output is forced
+* into the eval flow — different question, different answer.
+*
+* Pattern resolution order (first match wins):
+*   1. `scenario.forbiddenPattern` — if it parses as `/body/flags`,
+*      treated as a regex; otherwise a literal substring.
+*   2. `scenario.canary` — literal substring fallback so the helper
+*      works on existing scenario fixtures.
+*
+* Returns `null` when nothing forbidden was found OR the scenario
+* declared no pattern.
+*/
+function checkBehavioralCanary(output, scenario) {
+	const pattern = scenario.forbiddenPattern ?? scenario.canary;
+	if (!pattern) return null;
+	const hit = matchForbidden(output, pattern);
+	if (!hit) return null;
+	return {
+		scenarioId: scenario.id,
+		canary: pattern,
+		evidence: excerpt(output, hit)
+	};
+}
+/**
+* Behavioral canary over many (scenario, output) pairs. Sibling to
+* {@link import('./canary').runCanaries | runCanaries} — same idea
+* (run-many → report) but the question being answered is "did the
+* AGENT misbehave?" rather than "did the EVAL grep fire?".
+*
+* Returns one `CanaryLeak` per pair where the agent's output
+* contained its scenario's `forbiddenPattern` (or `canary` fallback).
+*/
+function runBehavioralCanaries(cases) {
+	const leaks = [];
+	for (const c of cases) {
+		const leak = checkBehavioralCanary(c.output, c.scenario);
+		if (leak) leaks.push({
+			...leak,
+			runId: c.runId ?? leak.runId
+		});
+	}
+	return leaks;
+}
+/**
+* Resolve a forbidden-pattern string to the matched substring inside
+* `output`. `/body/flags` notation is interpreted as a regex; anything
+* else is a literal substring.
+*/
+function matchForbidden(output, pattern) {
+	const re = tryParseRegex(pattern);
+	if (re) {
+		const m = output.match(re);
+		return m && m[0].length > 0 ? m[0] : null;
+	}
+	return output.includes(pattern) ? pattern : null;
+}
+function tryParseRegex(pattern) {
+	if (pattern.length < 2 || pattern[0] !== "/") return null;
+	const last = pattern.lastIndexOf("/");
+	if (last <= 0) return null;
+	const body = pattern.slice(1, last);
+	const flags = pattern.slice(last + 1);
+	if (!/^[gimsuy]*$/.test(flags)) return null;
+	try {
+		return new RegExp(body, flags);
+	} catch {
+		return null;
+	}
+}
+/**
+* Scan the LLM-output history in a corpus; returns every case where a
+* canary from a known scenario appeared in agent output. Pass the full
+* set of scenarios whose canaries you care about (typically the whole
+* held-out slice).
+*/
+async function canaryLeakView(store, scenarios) {
+	const targets = scenarios.filter((s) => !!s.canary);
+	if (targets.length === 0) return [];
+	const spans = await llmSpans(store);
+	const leaks = [];
+	for (const span of spans) {
+		const output = span.output ?? "";
+		for (const s of targets) if (s.canary && output.includes(s.canary)) leaks.push({
+			scenarioId: s.id,
+			canary: s.canary,
+			runId: span.runId,
+			evidence: excerpt(output, s.canary)
+		});
+	}
+	return leaks;
+}
+var HoldoutAuditor = class {
+	scenarios;
+	accessLog = [];
+	constructor(scenarios) {
+		this.scenarios = scenarios;
+	}
+	/** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */
+	get(scenarioId, purpose) {
+		if (purpose !== "evaluation" && purpose !== "debugging") throw new Error(`HoldoutAuditor.get: purpose must be 'evaluation' or 'debugging', got ${purpose}`);
+		const s = this.scenarios.find((x) => x.id === scenarioId);
+		if (!s) throw new Error(`holdout scenario "${scenarioId}" not found`);
+		this.accessLog.push({
+			scenarioId,
+			purpose,
+			at: Date.now()
+		});
+		return s;
+	}
+	getAccessLog() {
+		return this.accessLog;
+	}
+};
+function excerpt(source, needle) {
+	const at = source.indexOf(needle);
+	if (at < 0) return "";
+	const start = Math.max(0, at - 30);
+	const end = Math.min(source.length, at + needle.length + 30);
+	return (start > 0 ? "…" : "") + source.slice(start, end) + (end < source.length ? "…" : "");
+}
+//#endregion
+//#region src/contract/analyze-runs.ts
+/** Summarize runtime facts without interpreting task quality or promotion readiness. */
+function summarizeExecution(opts) {
+	const runs = opts.runs.map(validateRunRecord);
+	return {
+		execution: computeExecutionInsight(runs, opts.histogramBins ?? 12),
+		costProvenance: summarizeCostProvenance(runs)
+	};
+}
+async function analyzeRuns(opts) {
+	const runs = opts.runs.map(validateRunRecord);
+	const bins = opts.histogramBins ?? 12;
+	const threshold = opts.decisionThreshold ?? .02;
+	const split = resolveSplit(runs, opts.split ?? "auto");
+	const compositeWithIds = runs.map((r) => ({
+		runId: r.runId,
+		score: compositeOf(r, split)
+	})).filter((p) => Number.isFinite(p.score));
+	const composite = distributionOf(compositeWithIds.map((p) => p.score), bins, compositeWithIds);
+	const perDimension = computePerDimension(runs, bins);
+	const { execution, costProvenance: provenance } = summarizeExecution({
+		runs,
+		histogramBins: bins
+	});
+	const knownCostRuns = runs.filter((run) => run.costProvenance.kind !== "uncaptured");
+	const costs = knownCostRuns.map((r) => r.costUsd).filter(isFiniteNumber);
+	const costDist = distributionOf(costs, bins);
+	const pareto = paretoChart(knownCostRuns, { split });
+	const degraded = {};
+	if (provenance.uncaptured.n > 0) degraded.cost = diagnoseCostCoverage(runs, provenance);
+	else if (costs.length === 0 || costs.every((c) => c === 0)) degraded.cost = `all ${runs.length} explicitly observed or estimated USD values are $0`;
+	if (pareto.points.length < 2) degraded.pareto = pareto.points.length === 0 ? "no candidates — Pareto unavailable" : "single candidate — Pareto is a single point, not a frontier";
+	const costQuality = {
+		cost: costDist,
+		pareto,
+		provenance,
+		...degraded.cost || degraded.pareto ? { degraded } : {}
+	};
+	const judges = computeJudgeInsights(runs);
+	const interRater = opts.raterScores ? computeInterRater(opts.raterScores) : void 0;
+	const lift = computeLift(runs, opts.baselineCandidateId, opts.candidateCandidateId, split);
+	const failureClusters = opts.analyst ? await computeFailureClusters(runs, opts.analyst, split) : void 0;
+	const failureClasses = computeFailureClasses(runs, split);
+	const contamination = opts.canaryScenarios ? computeContamination(runs, opts.canaryScenarios) : void 0;
+	const outcomeCorrelation = opts.outcomeSignal ? computeOutcomeCorrelation(runs, opts.outcomeSignal, split) : void 0;
+	const release = buildReleaseScorecard(composite, lift, contamination);
+	const priorPeriodComparison = opts.baselineRuns ? computePriorPeriodComparison(runs, opts.baselineRuns, split, opts.baselineLabel) : void 0;
+	const recommendations = buildRecommendations({
+		composite,
+		judges,
+		interRater,
+		lift,
+		failureClusters,
+		failureClasses,
+		contamination,
+		outcomeCorrelation,
+		priorPeriodComparison,
+		threshold
+	});
+	return {
+		n: runs.length,
+		execution,
+		composite,
+		perDimension,
+		costQuality,
+		judges,
+		interRater,
+		lift,
+		failureClusters,
+		contamination,
+		outcomeCorrelation,
+		release,
+		...failureClasses ? { failureClasses } : {},
+		...priorPeriodComparison ? { priorPeriodComparison } : {},
+		recommendations
+	};
+}
+function computeExecutionInsight(runs, bins) {
+	const aggregateRows = runs.flatMap((run) => {
+		const usage = aggregateTokenUsage(run);
+		return usage ? [{
+			usage,
+			costUsd: finiteRaw(run, "aggregate_cost_usd")
+		}] : [];
+	});
+	const aggregateCosts = aggregateRows.flatMap((row) => row.costUsd !== void 0 ? [row.costUsd] : []);
+	const modelCounts = /* @__PURE__ */ new Map();
+	let executionErrorRuns = 0;
+	let executionErrorEvents = 0;
+	let errorReportingRuns = 0;
+	let errorSpanEvents = 0;
+	let errorSpanReportingRuns = 0;
+	const terminalOutcomes = {
+		succeeded: 0,
+		failed: 0,
+		cancelled: 0,
+		incomplete: 0,
+		unknown: 0
+	};
+	const errorsByTerminalOutcome = {
+		succeeded: {
+			withErrors: 0,
+			withoutErrors: 0,
+			unreported: 0
+		},
+		failed: {
+			withErrors: 0,
+			withoutErrors: 0,
+			unreported: 0
+		},
+		cancelled: {
+			withErrors: 0,
+			withoutErrors: 0,
+			unreported: 0
+		},
+		incomplete: {
+			withErrors: 0,
+			withoutErrors: 0,
+			unreported: 0
+		},
+		unknown: {
+			withErrors: 0,
+			withoutErrors: 0,
+			unreported: 0
+		}
+	};
+	let modelCallRuns = 0;
+	let modelCallEvents = 0;
+	let modelCallReportingRuns = 0;
+	for (const run of runs) {
+		modelCounts.set(run.model, (modelCounts.get(run.model) ?? 0) + 1);
+		const terminalOutcome = run.terminalOutcome;
+		terminalOutcomes[terminalOutcome] += 1;
+		const modelCalls = nonNegativeCountRaw(run, "llm_span_count");
+		if (modelCalls !== void 0) {
+			modelCallEvents += modelCalls;
+			modelCallReportingRuns += 1;
+		}
+		const usage = run.tokenUsage;
+		if ((modelCalls ?? 0) > 0 || usage.input > 0 || usage.output > 0 || (usage.cached ?? 0) > 0 || (usage.cacheWrite ?? 0) > 0) modelCallRuns += 1;
+		const errorEvents = reportedExecutionErrorEvents(run);
+		if (errorEvents !== void 0) {
+			executionErrorEvents += errorEvents;
+			errorReportingRuns += 1;
+			if (errorEvents > 0) {
+				executionErrorRuns += 1;
+				errorsByTerminalOutcome[terminalOutcome].withErrors += 1;
+			} else errorsByTerminalOutcome[terminalOutcome].withoutErrors += 1;
+		} else errorsByTerminalOutcome[terminalOutcome].unreported += 1;
+		const reportedErrorSpans = nonNegativeCountRaw(run, "error_span_count");
+		if (reportedErrorSpans !== void 0) {
+			errorSpanEvents += reportedErrorSpans;
+			errorSpanReportingRuns += 1;
+		}
+	}
+	return {
+		durationMs: distributionOf(runs.map((run) => run.wallMs), bins),
+		queueMs: distributionOf(runs.filter((run) => run.queueMs !== void 0).map((run) => run.queueMs), bins),
+		tokenUsage: summarizeTokenUsage(runs.map((run) => run.tokenUsage), bins),
+		aggregateUsage: {
+			runs: aggregateRows.length,
+			tokenUsage: summarizeTokenUsage(aggregateRows.map((row) => row.usage), bins),
+			costUsd: distributionOf(aggregateCosts, bins),
+			totalCostUsd: aggregateCosts.reduce((total, value) => total + value, 0)
+		},
+		models: [...modelCounts.entries()].map(([model, count]) => ({
+			model,
+			runs: count
+		})).sort((left, right) => right.runs - left.runs || left.model.localeCompare(right.model)),
+		modelCalls: {
+			runs: modelCallRuns,
+			events: modelCallEvents,
+			reportingRuns: modelCallReportingRuns
+		},
+		executionErrors: {
+			runs: executionErrorRuns,
+			fraction: errorReportingRuns > 0 ? executionErrorRuns / errorReportingRuns : null,
+			events: executionErrorEvents,
+			reportingRuns: errorReportingRuns,
+			errorSpanEvents,
+			errorSpanReportingRuns,
+			byTerminalOutcome: errorsByTerminalOutcome
+		},
+		terminalOutcomes
+	};
+}
+function reportedExecutionErrorEvents(run) {
+	return nonNegativeCountRaw(run, "execution_error_count");
+}
+function nonNegativeCountRaw(run, key) {
+	const value = finiteRaw(run, key);
+	return value !== void 0 && Number.isInteger(value) && value >= 0 ? value : void 0;
+}
+function summarizeTokenUsage(usages, bins) {
+	const reasoning = usages.flatMap((usage) => usage.reasoning !== void 0 ? [usage.reasoning] : []);
+	const cached = usages.flatMap((usage) => usage.cached !== void 0 ? [usage.cached] : []);
+	const cacheWrite = usages.flatMap((usage) => usage.cacheWrite !== void 0 ? [usage.cacheWrite] : []);
+	return {
+		input: distributionOf(usages.map((usage) => usage.input), bins),
+		output: distributionOf(usages.map((usage) => usage.output), bins),
+		reasoning: distributionOf(reasoning, bins),
+		cached: distributionOf(cached, bins),
+		cacheWrite: distributionOf(cacheWrite, bins),
+		totals: {
+			input: usages.reduce((total, usage) => total + usage.input, 0),
+			output: usages.reduce((total, usage) => total + usage.output, 0),
+			reasoning: reasoning.reduce((total, value) => total + value, 0),
+			cached: cached.reduce((total, value) => total + value, 0),
+			cacheWrite: cacheWrite.reduce((total, value) => total + value, 0)
+		}
+	};
+}
+function aggregateTokenUsage(run) {
+	const input = finiteRaw(run, "aggregate_prompt_tokens");
+	const output = finiteRaw(run, "aggregate_completion_tokens");
+	const reasoning = finiteRaw(run, "aggregate_reasoning_tokens");
+	const cached = finiteRaw(run, "aggregate_cached_tokens");
+	const cacheWrite = finiteRaw(run, "aggregate_cache_write_tokens");
+	if (input === void 0 && output === void 0 && reasoning === void 0 && cached === void 0 && cacheWrite === void 0) return void 0;
+	return {
+		input: input ?? 0,
+		output: output ?? 0,
+		...reasoning !== void 0 ? { reasoning } : {},
+		...cached !== void 0 ? { cached } : {},
+		...cacheWrite !== void 0 ? { cacheWrite } : {}
+	};
+}
+function finiteRaw(run, key) {
+	const value = run.outcome.raw[key];
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : void 0;
+}
+function summarizeCostProvenance(runs) {
+	const summary = {
+		observed: {
+			n: 0,
+			totalUsd: 0
+		},
+		estimated: {
+			n: 0,
+			totalUsd: 0
+		},
+		uncaptured: { n: 0 },
+		knownFraction: 0
+	};
+	for (const run of runs) {
+		const cost = run.costProvenance;
+		if (cost.kind === "uncaptured") summary.uncaptured.n += 1;
+		else {
+			summary[cost.kind].n += 1;
+			summary[cost.kind].totalUsd += cost.usd;
+		}
+	}
+	const known = summary.observed.n + summary.estimated.n;
+	summary.knownFraction = runs.length > 0 ? known / runs.length : 0;
+	return summary;
+}
+function diagnoseCostCoverage(runs, provenance) {
+	const uncaptured = provenance.uncaptured.n;
+	const known = provenance.observed.n + provenance.estimated.n;
+	if (uncaptured === runs.length) return `USD cost uncaptured for all ${runs.length} runs — no observed or estimated USD values; token and wall-time metrics remain available.`;
+	return `USD cost uncaptured for ${uncaptured}/${runs.length} runs; excluded those rows from cost statistics (${known}/${runs.length} retained: ${provenance.observed.n} observed, ${provenance.estimated.n} estimated).`;
+}
+/**
+* Model-free task-failure tally.
+*
+* Explicit non-success classes are task-failure evidence.
+* A low task score without a class is counted as `unknown`.
+*/
+function computeFailureClasses(runs, split) {
+	const counts = /* @__PURE__ */ new Map();
+	for (const r of runs) {
+		if (!isTaskFailure(r, split)) continue;
+		const key = r.failureClass !== void 0 && r.failureClass !== "success" ? r.failureClass : "unknown";
+		counts.set(key, (counts.get(key) ?? 0) + 1);
+	}
+	if (counts.size === 0) return void 0;
+	const n = runs.length;
+	return [...counts.entries()].map(([failureClass, count]) => ({
+		failureClass,
+		count,
+		share: n > 0 ? count / n : 0
+	})).sort((a, b) => b.count - a.count || a.failureClass.localeCompare(b.failureClass));
+}
+function computePriorPeriodComparison(current, baseline, split, windowLabel) {
+	if (current.length === 0 || baseline.length === 0) return void 0;
+	const metrics = {};
+	const directions = {};
+	const compositeCurrent = current.map((r) => compositeOf(r, split)).filter(Number.isFinite);
+	const compositeBaseline = baseline.map((r) => compositeOf(r, split)).filter(Number.isFinite);
+	if (compositeCurrent.length > 0 && compositeBaseline.length > 0) {
+		metrics.composite = welchCompare(compositeBaseline, compositeCurrent);
+		directions.composite = "higher-is-better";
+	}
+	const costCurrent = knownCostValues(current);
+	const costBaseline = knownCostValues(baseline);
+	if (costCurrent.length > 0 && costBaseline.length > 0) {
+		metrics.cost = welchCompare(costBaseline, costCurrent);
+		directions.cost = "lower-is-better";
+	}
+	const durCurrent = current.map((r) => r.wallMs).filter(Number.isFinite);
+	const durBaseline = baseline.map((r) => r.wallMs).filter(Number.isFinite);
+	if (durCurrent.length > 0 && durBaseline.length > 0) {
+		metrics.duration = welchCompare(durBaseline, durCurrent);
+		directions.duration = "lower-is-better";
+	}
+	const tokCurrent = current.map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0)).filter(Number.isFinite);
+	const tokBaseline = baseline.map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0)).filter(Number.isFinite);
+	if (tokCurrent.length > 0 && tokBaseline.length > 0) {
+		metrics.tokenUsage = welchCompare(tokBaseline, tokCurrent);
+		directions.tokenUsage = "lower-is-better";
+	}
+	const dimsCurrent = collectPerDimension(current);
+	const dimsBaseline = collectPerDimension(baseline);
+	for (const dim of Object.keys(dimsCurrent)) {
+		const b = dimsBaseline[dim];
+		const c = dimsCurrent[dim];
+		if (!b || b.length === 0 || !c || c.length === 0) continue;
+		metrics[`dim.${dim}`] = welchCompare(b, c);
+		directions[`dim.${dim}`] = "higher-is-better";
+	}
+	const regressedMetrics = [];
+	const improvedMetrics = [];
+	for (const [name, delta] of Object.entries(metrics)) {
+		if (!delta.significant) continue;
+		if ((directions[name] ?? "higher-is-better") === "higher-is-better" ? delta.delta > 0 : delta.delta < 0) improvedMetrics.push(name);
+		else regressedMetrics.push(name);
+	}
+	return {
+		baselineN: baseline.length,
+		currentN: current.length,
+		...windowLabel ? { windowLabel } : {},
+		metrics,
+		regressedMetrics,
+		improvedMetrics
+	};
+}
+function knownCostValues(runs) {
+	return runs.filter((run) => run.costProvenance.kind !== "uncaptured").map((run) => run.costUsd).filter(isFiniteNumber);
+}
+function isFiniteNumber(value) {
+	return typeof value === "number" && Number.isFinite(value);
+}
+/** Collect per-dimension values across runs (from outcome.judgeScores.perDimMean). */
+function collectPerDimension(runs) {
+	const out = {};
+	for (const r of runs) {
+		const perDim = r.outcome.judgeScores?.perDimMean;
+		if (!perDim) continue;
+		for (const [dim, value] of Object.entries(perDim)) {
+			if (!Number.isFinite(value)) continue;
+			if (!out[dim]) out[dim] = [];
+			out[dim].push(value);
+		}
+	}
+	return out;
+}
+/** Two-sample Welch comparison: unequal-variance t-test + CI on the delta
+*  + Cohen's d (pooled stddev). Significance = p < 0.05 AND |d| >= 0.2. */
+function welchCompare(baseline, current) {
+	const baselineMean = mean(baseline);
+	const currentMean = mean(current);
+	const baselineVar = sampleVariance(baseline, baselineMean);
+	const currentVar = sampleVariance(current, currentMean);
+	const baselineN = baseline.length;
+	const currentN = current.length;
+	const delta = currentMean - baselineMean;
+	const se = Math.sqrt(baselineVar / baselineN + currentVar / currentN);
+	const halfWidth = 1.96 * (se > 0 ? se : 0);
+	const ci95 = [delta - halfWidth, delta + halfWidth];
+	const t = se > 0 ? delta / se : 0;
+	const pValue = se > 0 ? 2 * (1 - standardNormalCdf(Math.abs(t))) : 1;
+	const pooledStddev = Math.sqrt(((baselineN - 1) * baselineVar + (currentN - 1) * currentVar) / Math.max(1, baselineN + currentN - 2));
+	const cohensD = pooledStddev > 0 ? delta / pooledStddev : 0;
+	return {
+		current: currentMean,
+		baseline: baselineMean,
+		delta,
+		ci95,
+		pValue,
+		cohensD,
+		baselineN,
+		currentN,
+		significant: pValue < .05 && Math.abs(cohensD) >= .2
+	};
+}
+function sampleVariance(xs, xsMean) {
+	if (xs.length < 2) return 0;
+	let s = 0;
+	for (const x of xs) s += (x - xsMean) ** 2;
+	return s / (xs.length - 1);
+}
+/** Abramowitz & Stegun approximation to Φ(z). Maximum error ~7.5e-8. */
+function standardNormalCdf(z) {
+	const a1 = .254829592;
+	const a2 = -.284496736;
+	const a3 = 1.421413741;
+	const a4 = -1.453152027;
+	const a5 = 1.061405429;
+	const p = .3275911;
+	const sign = z < 0 ? -1 : 1;
+	const x = Math.abs(z) / Math.SQRT2;
+	const t = 1 / (1 + p * x);
+	return .5 * (1 + sign * (1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x)));
+}
+function resolveSplit(runs, pref) {
+	if (pref !== "auto") return pref;
+	return runs.some((r) => Number.isFinite(observedSplitScore(r, "holdout"))) ? "holdout" : "search";
+}
+/**
+* RAW (`observedSplitScore`): `analyzeRuns` describes what a set of runs
+* reported, and every downstream reader of this composite — distributions,
+* per-candidate summaries, the reward-hacking correlation — needs the ungated
+* number to see an inflated run at all.
+*/
+function compositeOf(run, split) {
+	const score = observedSplitScore(run, split);
+	return Number.isFinite(score) ? score : NaN;
+}
+function distributionOf(values, bins, withIds) {
+	if (values.length === 0) return {
+		n: 0,
+		mean: null,
+		p50: null,
+		p95: null,
+		stddev: null,
+		min: null,
+		max: null,
+		histogram: []
+	};
+	const sorted = [...values].sort((a, b) => a - b);
+	const n = sorted.length;
+	const mean = sorted.reduce((s, v) => s + v, 0) / n;
+	const variance = sorted.reduce((s, v) => s + (v - mean) ** 2, 0) / n;
+	const stddev = Math.sqrt(variance);
+	const tailRuns = withIds ? [...withIds].sort((a, b) => a.score - b.score).slice(0, Math.min(5, withIds.length)) : void 0;
+	return {
+		n,
+		mean,
+		p50: percentile(sorted, .5),
+		p95: percentile(sorted, .95),
+		stddev,
+		min: sorted[0],
+		max: sorted[n - 1],
+		histogram: histogram(sorted, bins),
+		...tailRuns ? { tailRuns } : {}
+	};
+}
+function percentile(sorted, q) {
+	if (sorted.length === 0) return 0;
+	if (sorted.length === 1) return sorted[0];
+	const idx = (sorted.length - 1) * q;
+	const lo = Math.floor(idx);
+	const hi = Math.ceil(idx);
+	if (lo === hi) return sorted[lo];
+	const w = idx - lo;
+	return sorted[lo] * (1 - w) + sorted[hi] * w;
+}
+/** Even-width histogram over the value range. Returns inclusive-lo /
+*  exclusive-hi bins (closed on right for the last bin) compatible with
+*  the substrate's `GainDistributionBin` shape. */
+function histogram(sorted, bins) {
+	if (sorted.length === 0 || bins < 1) return [];
+	const min = sorted[0];
+	const max = sorted[sorted.length - 1];
+	if (min === max) return [{
+		lo: min,
+		hi: max,
+		count: sorted.length
+	}];
+	const width = (max - min) / bins;
+	const out = [];
+	for (let i = 0; i < bins; i++) {
+		const lo = min + i * width;
+		const hi = i === bins - 1 ? max : lo + width;
+		out.push({
+			lo,
+			hi,
+			count: 0
+		});
+	}
+	for (const v of sorted) {
+		const idx = Math.min(bins - 1, Math.floor((v - min) / width));
+		out[idx].count++;
+	}
+	return out;
+}
+function computePerDimension(runs, bins) {
+	const byDim = /* @__PURE__ */ new Map();
+	for (const run of runs) {
+		const scores = run.outcome.judgeScores;
+		if (!scores) continue;
+		for (const [dim, value] of Object.entries(scores.perDimMean ?? {})) {
+			if (!Number.isFinite(value)) continue;
+			const arr = byDim.get(dim) ?? [];
+			arr.push(value);
+			byDim.set(dim, arr);
+		}
+	}
+	const out = {};
+	for (const [dim, values] of byDim) out[dim] = distributionOf(values, bins);
+	return out;
+}
+function computeJudgeInsights(runs) {
+	const out = {};
+	const byJudge = /* @__PURE__ */ new Map();
+	for (const run of runs) {
+		const scores = run.outcome.judgeScores;
+		if (!scores?.perJudge) continue;
+		for (const [judgeId, dims] of Object.entries(scores.perJudge)) {
+			const dimValues = Object.values(dims).filter(Number.isFinite);
+			if (dimValues.length === 0) continue;
+			const judgeMean = dimValues.reduce((s, v) => s + v, 0) / dimValues.length;
+			const arr = byJudge.get(judgeId) ?? [];
+			arr.push(judgeMean);
+			byJudge.set(judgeId, arr);
+		}
+	}
+	for (const [judgeId, values] of byJudge) out[judgeId] = {
+		n: values.length,
+		meanScore: values.reduce((s, v) => s + v, 0) / values.length
+	};
+	return out;
+}
+function computeInterRater(ratings) {
+	const byRun = /* @__PURE__ */ new Map();
+	for (const r of ratings) {
+		if (!Number.isFinite(r.score)) continue;
+		const list = byRun.get(r.runId) ?? [];
+		list.push({
+			rater: r.rater,
+			score: r.score
+		});
+		byRun.set(r.runId, list);
+	}
+	const raters = new Set(ratings.map((r) => r.rater));
+	const jointlyRated = [];
+	for (const [runId, ratersForRun] of byRun) {
+		const seen = new Set(ratersForRun.map((r) => r.rater));
+		let all = true;
+		for (const r of raters) if (!seen.has(r)) all = false;
+		if (all) jointlyRated.push(runId);
+	}
+	if (raters.size < 2 || jointlyRated.length === 0) return void 0;
+	const raterList = [...raters].sort();
+	const perPair = {};
+	for (let i = 0; i < raterList.length; i++) for (let j = i + 1; j < raterList.length; j++) {
+		const a = raterList[i];
+		const b = raterList[j];
+		const aScores = [];
+		const bScores = [];
+		for (const runId of jointlyRated) {
+			const ratersForRun = byRun.get(runId);
+			const sa = ratersForRun.find((r) => r.rater === a)?.score;
+			const sb = ratersForRun.find((r) => r.rater === b)?.score;
+			if (sa !== void 0 && sb !== void 0) {
+				aScores.push(sa);
+				bScores.push(sb);
+			}
+		}
+		const agreement = continuousAgreement(aScores.map((score, index) => [score, bScores[index]]), { bootstrap: 0 });
+		perPair[`${a}::${b}`] = agreement.weightedKappa;
+	}
+	const agreement = continuousAgreement(jointlyRated.map((runId) => {
+		const ratingsByRater = new Map(byRun.get(runId).map((rating) => [rating.rater, rating.score]));
+		return raterList.map((rater) => ratingsByRater.get(rater));
+	}), { bootstrap: 0 });
+	const disagreementCases = jointlyRated.map((runId) => {
+		const ratersForRun = byRun.get(runId);
+		const scores = ratersForRun.map((r) => r.score);
+		return {
+			runId,
+			ratings: ratersForRun,
+			range: Math.max(...scores) - Math.min(...scores)
+		};
+	}).sort((a, b) => b.range - a.range).slice(0, 20);
+	return {
+		raters: raters.size,
+		jointlyRated: jointlyRated.length,
+		kappa: Number.isFinite(agreement.weightedKappa) ? agreement.weightedKappa : 0,
+		icc: agreement.icc,
+		pearson: agreement.pearson,
+		spearman: agreement.spearman,
+		perPair,
+		disagreementCases
+	};
+}
+function computeLift(runs, baselineId, candidateId, split) {
+	let bId = baselineId;
+	let cId = candidateId;
+	if (!bId || !cId) {
+		const ids = [...new Set(runs.map((r) => r.candidateId))];
+		if (ids.length !== 2) return void 0;
+		const [idA, idB] = ids;
+		const scoresA = finiteCompositeScores(runs.filter((run) => run.candidateId === idA), split);
+		const scoresB = finiteCompositeScores(runs.filter((run) => run.candidateId === idB), split);
+		if (scoresA.length === 0 || scoresB.length === 0) return void 0;
+		const meanA = mean(scoresA);
+		const meanB = mean(scoresB);
+		bId = meanA <= meanB ? idA : idB;
+		cId = meanA <= meanB ? idB : idA;
+	}
+	const baseline = runs.filter((r) => r.candidateId === bId);
+	const candidate = runs.filter((r) => r.candidateId === cId);
+	if (baseline.length === 0 || candidate.length === 0) return void 0;
+	const pairing = pairRunRecords(baseline.filter((run) => Number.isFinite(compositeOf(run, split))), candidate.filter((run) => Number.isFinite(compositeOf(run, split))));
+	const pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split));
+	const pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split));
+	if (pairedBaseline.length === 0) return void 0;
+	const baselineMean = mean(pairedBaseline);
+	const candidateMean = mean(pairedCandidate);
+	const delta = candidateMean - baselineMean;
+	const bootstrap = pairedBootstrap(pairedBaseline, pairedCandidate, {
+		confidence: .95,
+		resamples: 2e3,
+		statistic: "mean"
+	});
+	const tTest = pairedTTest(pairedBaseline, pairedCandidate);
+	const d = pairedCohensDz(pairedBaseline, pairedCandidate);
+	const mde = pairedMde({
+		nPaired: pairedBaseline.length,
+		power: .8,
+		alpha: .05
+	});
+	const requiredN = d === null || d === 0 ? null : requiredPairedSampleSize({
+		effect: Math.abs(d),
+		power: .8,
+		alpha: .05
+	});
+	return {
+		baselineMean,
+		candidateMean,
+		delta,
+		ci95: [bootstrap.low, bootstrap.high],
+		pValue: tTest.p,
+		n: pairedBaseline.length,
+		unpairedBaseline: pairing.unpairedBaseline.length,
+		unpairedCandidate: pairing.unpairedTreatment.length,
+		cohensD: d,
+		mde,
+		requiredN
+	};
+}
+function mean(arr) {
+	return arr.length === 0 ? 0 : arr.reduce((s, v) => s + v, 0) / arr.length;
+}
+async function computeFailureClusters(runs, analyst, split) {
+	const failed = runs.filter((run) => isTaskFailure(run, split));
+	if (failed.length === 0) return {
+		clusters: [],
+		totalFailures: 0
+	};
+	const clusters = /* @__PURE__ */ new Map();
+	for (const run of failed) try {
+		const result = await analyst.run(run.runId, { runRecord: run });
+		for (const finding of result.findings) {
+			const key = finding.area || finding.analyst_id || "unclassified";
+			const c = clusters.get(key) ?? {
+				exemplars: [],
+				share: 0
+			};
+			if (c.exemplars.length < 5) c.exemplars.push(run.runId);
+			clusters.set(key, c);
+		}
+	} catch {
+		const c = clusters.get("analyst-error") ?? {
+			exemplars: [],
+			share: 0
+		};
+		if (c.exemplars.length < 5) c.exemplars.push(run.runId);
+		clusters.set("analyst-error", c);
+	}
+	const clusterList = [...clusters.entries()].map(([id, c]) => ({
+		id,
+		name: id,
+		share: c.exemplars.length / failed.length,
+		exemplars: c.exemplars
+	}));
+	clusterList.sort((a, b) => b.share - a.share);
+	return {
+		clusters: clusterList,
+		totalFailures: failed.length
+	};
+}
+function finiteCompositeScores(runs, split) {
+	return runs.map((run) => compositeOf(run, split)).filter(Number.isFinite);
+}
+function isTaskFailure(run, split) {
+	if (run.failureClass !== void 0 && run.failureClass !== "success") return true;
+	const score = compositeOf(run, split);
+	return Number.isFinite(score) && score < .5;
+}
+function computeContamination(runs, canaries) {
+	let leaks = 0;
+	const details = [];
+	for (const run of runs) {
+		const output = stringifyOutput(run);
+		if (!output) continue;
+		const leaksHere = checkCanaries(output, canaries);
+		for (const leak of leaksHere) {
+			leaks++;
+			details.push({
+				runId: run.runId,
+				canary: leak.canary,
+				matched: leak.evidence
+			});
+		}
+	}
+	return {
+		leaks,
+		holdoutAuditPassed: leaks === 0,
+		details
+	};
+}
+function stringifyOutput(run) {
+	const metadata = run.metadata;
+	if (typeof metadata?.output === "string") return metadata.output;
+	if (typeof metadata?.text === "string") return metadata.text;
+}
+function computeOutcomeCorrelation(runs, outcome, split) {
+	const xs = [];
+	const ys = [];
+	for (const run of runs) {
+		const y = outcome.valueByRunId[run.runId];
+		if (y === void 0 || !Number.isFinite(y)) continue;
+		const x = compositeOf(run, split);
+		if (!Number.isFinite(x)) continue;
+		xs.push(x);
+		ys.push(y);
+	}
+	if (xs.length < 3) return void 0;
+	const p = pearsonR(xs, ys);
+	const s = spearmanR(xs, ys);
+	const meanX = mean(xs);
+	const meanY = mean(ys);
+	let num = 0;
+	let denom = 0;
+	for (let i = 0; i < xs.length; i++) {
+		num += (xs[i] - meanX) * (ys[i] - meanY);
+		denom += (xs[i] - meanX) ** 2;
+	}
+	const slope = denom === 0 ? 0 : num / denom;
+	const intercept = meanY - slope * meanX;
+	const ssTot = ys.reduce((a, y) => a + (y - meanY) ** 2, 0);
+	const ssRes = ys.reduce((a, y, i) => a + (y - (intercept + slope * xs[i])) ** 2, 0);
+	const r2 = ssTot === 0 ? 0 : 1 - ssRes / ssTot;
+	return {
+		metric: outcome.metric,
+		n: xs.length,
+		pearson: p,
+		spearman: s,
+		rewardModel: {
+			intercept,
+			slope,
+			r2
+		}
+	};
+}
+function buildReleaseScorecard(composite, lift, contamination) {
+	const axes = [];
+	const liftPass = lift === void 0 ? "not_evaluated" : lift.ci95[0] > 0 ? "pass" : lift.delta > 0 ? "warn" : "fail";
+	axes.push({
+		name: "quality-lift",
+		status: liftPass,
+		detail: lift ? `delta=${lift.delta.toFixed(3)}, CI95=[${lift.ci95[0].toFixed(3)}, ${lift.ci95[1].toFixed(3)}], n=${lift.n}` : "no baseline/candidate pair available"
+	});
+	const contamPass = contamination === void 0 ? "not_evaluated" : contamination.leaks === 0 ? "pass" : "fail";
+	axes.push({
+		name: "contamination",
+		status: contamPass,
+		detail: contamination ? `${contamination.leaks} canary leak(s)` : "no canaries supplied"
+	});
+	axes.push(composite.n === 0 ? {
+		name: "composite-distribution",
+		status: "not_evaluated",
+		detail: "no task-quality scores available"
+	} : {
+		name: "composite-distribution",
+		status: composite.mean !== null && composite.mean >= .5 ? "pass" : composite.mean !== null && composite.mean >= .3 ? "warn" : "fail",
+		detail: composite.mean === null || composite.p50 === null || composite.p95 === null ? "task-quality distribution is internally incomplete" : `mean=${composite.mean.toFixed(3)}, p50=${composite.p50.toFixed(3)}, p95=${composite.p95.toFixed(3)} over n=${composite.n}`
+	});
+	return {
+		status: axes.some((a) => a.status === "fail") ? "fail" : axes.some((a) => a.status === "warn" || a.status === "not_evaluated") ? "warn" : "pass",
+		axes,
+		issues: []
+	};
+}
+function buildRecommendations(ctx) {
+	const out = [];
+	if (ctx.priorPeriodComparison) {
+		const ppc = ctx.priorPeriodComparison;
+		const label = ppc.windowLabel ?? "baseline period";
+		for (const name of ppc.regressedMetrics) {
+			const d = ppc.metrics[name];
+			if (!d) continue;
+			out.push({
+				priority: "critical",
+				kind: "investigate",
+				title: `${name} regressed from ${d.baseline.toFixed(3)} → ${d.current.toFixed(3)} vs ${label}`,
+				detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). The regression is statistically significant at p<0.05 with at-least-small effect size.`,
+				evidencePath: `priorPeriodComparison.metrics.${name}`
+			});
+		}
+		for (const name of ppc.improvedMetrics) {
+			const d = ppc.metrics[name];
+			if (!d) continue;
+			out.push({
+				priority: "low",
+				kind: "ship",
+				title: `${name} improved from ${d.baseline.toFixed(3)} → ${d.current.toFixed(3)} vs ${label}`,
+				detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). Statistically significant improvement worth flagging.`,
+				evidencePath: `priorPeriodComparison.metrics.${name}`
+			});
+		}
+	}
+	if (ctx.composite.n > 0 && ctx.composite.mean !== null && ctx.composite.p50 !== null && ctx.composite.p95 !== null) {
+		if (ctx.composite.mean < .3) {
+			const tail = ctx.composite.tailRuns ?? [];
+			const names = tail.slice(0, 5).map((t) => `${t.runId}=${t.score.toFixed(3)}`).join(", ");
+			out.push({
+				priority: "critical",
+				kind: "investigate",
+				title: `Composite mean ${ctx.composite.mean.toFixed(3)} is below the 0.3 floor — the agent is broken on this corpus`,
+				detail: tail.length > 0 ? `Worst ${tail.length} run${tail.length === 1 ? "" : "s"} to inspect first: ${names}. Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.` : `Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`,
+				evidencePath: "composite.tailRuns"
+			});
+		} else if (ctx.composite.mean < .5) {
+			const tail = ctx.composite.tailRuns ?? [];
+			const names = tail.slice(0, 3).map((t) => `${t.runId}=${t.score.toFixed(3)}`).join(", ");
+			out.push({
+				priority: "high",
+				kind: "investigate",
+				title: `Composite mean ${ctx.composite.mean.toFixed(3)} is below 0.5 — investigate the lower tail before claiming the agent is healthy`,
+				detail: tail.length > 0 ? `Worst ${tail.length} run${tail.length === 1 ? "" : "s"}: ${names}. Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.` : `Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`,
+				evidencePath: "composite.tailRuns"
+			});
+		}
+	}
+	if (ctx.failureClasses && ctx.failureClasses.length > 0) {
+		const top = ctx.failureClasses[0];
+		if (top.count >= 3 && top.share >= .15) out.push({
+			priority: top.share >= .25 ? "high" : "medium",
+			kind: "investigate",
+			title: `'${top.failureClass}' is the dominant failure class — ${top.count} runs (${(top.share * 100).toFixed(0)}% of the corpus)`,
+			detail: `The mean composite can look acceptable while one failure class dominates the lower tail. ${top.count} of ${ctx.composite.n} runs failed with '${top.failureClass}'${ctx.failureClasses.length > 1 ? ` (next: '${ctx.failureClasses[1].failureClass}' ×${ctx.failureClasses[1].count})` : ""}. Fix this cause first.`,
+			evidencePath: "failureClasses"
+		});
+	}
+	if (Object.keys(ctx.judges).length === 0 && ctx.composite.n > 0) out.push({
+		priority: "medium",
+		kind: "expand-corpus",
+		title: "No judge scores recorded — per-dimension + calibration insights unavailable",
+		detail: "Records have no `outcome.judgeScores`. To unlock perDimension, judges, and calibration, attach a Judge run during your eval pass and populate `outcome.judgeScores.perJudge[judgeName][dimension] = score`. See `docs/insight-report.md` for the expected shape.",
+		evidencePath: "judges"
+	});
+	if (ctx.lift) {
+		const pairedEffect = ctx.lift.cohensD === null ? "undefined (zero delta variance)" : ctx.lift.cohensD.toFixed(2);
+		const requiredRuns = ctx.lift.requiredN === null ? "not estimable" : `~${ctx.lift.requiredN} paired runs`;
+		const decisive = ctx.lift.ci95[0] > ctx.threshold;
+		const inconclusive = ctx.lift.ci95[0] <= ctx.threshold && ctx.lift.ci95[1] > ctx.threshold;
+		if (decisive) out.push({
+			priority: "critical",
+			kind: "ship",
+			title: `Ship — lift ${ctx.lift.delta.toFixed(3)} (95% CI ${ctx.lift.ci95[0].toFixed(3)}..${ctx.lift.ci95[1].toFixed(3)})`,
+			detail: `Holdout lift exceeds threshold ${ctx.threshold} with 95% bootstrap confidence (n=${ctx.lift.n}, p=${ctx.lift.pValue.toFixed(4)}, paired d=${pairedEffect}).`,
+			evidencePath: "lift"
+		});
+		else if (inconclusive) out.push({
+			priority: "high",
+			kind: "expand-corpus",
+			title: `Inconclusive — required sample is ${requiredRuns} (have ${ctx.lift.n}) at current effect size`,
+			detail: `CI straddles threshold. Current MDE at 80% power is ${ctx.lift.mde.toFixed(3)}; observed delta is ${ctx.lift.delta.toFixed(3)}.`,
+			evidencePath: "lift"
+		});
+		else out.push({
+			priority: "critical",
+			kind: "hold",
+			title: `Hold — lift CI lower bound ${ctx.lift.ci95[0].toFixed(3)} is at or below threshold ${ctx.threshold}`,
+			detail: `Bootstrap CI provides no statistical evidence the candidate is better. Consider tightening the mutation or expanding the holdout.`,
+			evidencePath: "lift"
+		});
+	}
+	if (ctx.contamination && ctx.contamination.leaks > 0) out.push({
+		priority: "critical",
+		kind: "fix",
+		title: `${ctx.contamination.leaks} canary leak${ctx.contamination.leaks === 1 ? "" : "s"} detected`,
+		detail: `Holdout integrity is compromised. The lift number is unreliable until you investigate.`,
+		evidencePath: "contamination"
+	});
+	if (ctx.interRater && ctx.interRater.kappa < .5) out.push({
+		priority: "high",
+		kind: "recalibrate",
+		title: `Inter-rater weighted kappa ${ctx.interRater.kappa.toFixed(2)} is below 0.5`,
+		detail: "Raters disagree on what good looks like. Review the largest disagreement cases and refine the rubric before automating these decisions.",
+		evidencePath: "interRater"
+	});
+	if (ctx.failureClusters && ctx.failureClusters.clusters.length > 0) {
+		const top = ctx.failureClusters.clusters[0];
+		out.push({
+			priority: "high",
+			kind: "investigate",
+			title: `Top failure cluster: ${top.name} (${(top.share * 100).toFixed(0)}% of failures)`,
+			detail: `${ctx.failureClusters.totalFailures} runs failed. The largest cluster groups ${top.exemplars.length} exemplars under '${top.name}'.`,
+			evidencePath: "failureClusters.clusters[0]"
+		});
+	}
+	if (ctx.outcomeCorrelation && Math.abs(ctx.outcomeCorrelation.spearman) < .3) out.push({
+		priority: "medium",
+		kind: "recalibrate",
+		title: `Judge scores decoupled from ${ctx.outcomeCorrelation.metric} (Spearman ρ=${ctx.outcomeCorrelation.spearman.toFixed(2)})`,
+		detail: `Your judges score what they were trained to score, but it isn't predicting downstream ${ctx.outcomeCorrelation.metric}. Consider retraining the judge against ${ctx.outcomeCorrelation.metric} as the gold signal.`,
+		evidencePath: "outcomeCorrelation"
+	});
+	return out;
+}
+//#endregion
+export { checkBehavioralCanary as a, canaryLeakView as i, summarizeExecution as n, checkCanaries as o, HoldoutAuditor as r, runBehavioralCanaries as s, analyzeRuns as t };
+
+//# sourceMappingURL=analyze-runs-C1CavBMk.js.map