@tangle-network/agent-eval 0.129.0 → 0.130.1

package/dist/summary-report-BNs5nmXI.js

@@ -0,0 +1,862 @@
+import { r as hashJson, t as canonicalize } from "./pre-registration-DakwTRXk.js";
+import { N as wilcoxonSignedRank, a as confidenceInterval, b as pairedMde, t as benjaminiHochberg, v as pairedBootstrap, y as pairedCohensDz } from "./statistics-CnnxdpOg.js";
+import { r as pairRunRecords } from "./paired-arms-D9D0wXj2.js";
+import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
+//#region src/summary-report.ts
+/**
+* Table 1 helper. Buckets runs by `candidateId`, computes mean +
+* bootstrap CI on the chosen split, and (when a comparator is given)
+* BH-adjusted Wilcoxon p + paired Cohen's dz versus that comparator.
+*/
+function summaryTable(runs, opts = {}) {
+	const split = opts.split ?? "holdout";
+	const confidence = opts.confidence ?? .95;
+	const fdr = opts.fdr ?? .05;
+	const comparator = opts.comparator ?? null;
+	const byCandidate = /* @__PURE__ */ new Map();
+	for (const r of runs) {
+		if (r.splitTag !== split) continue;
+		const v = observedSplitScore(r, split);
+		if (typeof v !== "number" || !Number.isFinite(v)) continue;
+		const bucket = byCandidate.get(r.candidateId) ?? {
+			runs: [],
+			scores: []
+		};
+		bucket.runs.push(r);
+		bucket.scores.push(v);
+		byCandidate.set(r.candidateId, bucket);
+	}
+	const candidateIds = [...byCandidate.keys()].sort();
+	const compRuns = comparator ? byCandidate.get(comparator) : void 0;
+	const tentative = [];
+	for (const id of candidateIds) {
+		const bucket = byCandidate.get(id);
+		const ci = confidenceInterval(bucket.scores, confidence);
+		let rawP = null;
+		let d = null;
+		let pairedN = null;
+		let unpairedCandidateN = null;
+		let unpairedComparatorN = null;
+		if (comparator && compRuns && id !== comparator) {
+			const paired = pairScoresByKey(bucket.runs, compRuns.runs, split);
+			pairedN = paired.before.length;
+			unpairedCandidateN = paired.unpairedCandidateN;
+			unpairedComparatorN = paired.unpairedComparatorN;
+			if (paired.before.length >= 6) rawP = wilcoxonSignedRank(paired.before, paired.after).p;
+			d = pairedCohensDz(paired.before, paired.after);
+		}
+		tentative.push({
+			candidateId: id,
+			n: bucket.scores.length,
+			mean: ci.mean,
+			ciLow: ci.lower,
+			ciHigh: ci.upper,
+			qValue: rawP,
+			cohensD: d,
+			pairedN,
+			unpairedCandidateN,
+			unpairedComparatorN,
+			rawP
+		});
+	}
+	if (comparator) {
+		const idxs = [];
+		const ps = [];
+		for (let i = 0; i < tentative.length; i++) {
+			const r = tentative[i];
+			if (r.candidateId === comparator) continue;
+			if (r.rawP === null || !Number.isFinite(r.rawP)) continue;
+			idxs.push(i);
+			ps.push(r.rawP);
+		}
+		if (ps.length > 0) {
+			const { qValues } = benjaminiHochberg(ps, fdr);
+			for (let k = 0; k < idxs.length; k++) tentative[idxs[k]].qValue = qValues[k];
+		}
+	}
+	const rows = tentative.map(({ rawP: _rawP, ...rest }) => rest);
+	return {
+		rows,
+		comparator,
+		split,
+		markdown: renderSummaryTableMarkdown(rows, comparator, split)
+	};
+}
+function pairScoresByKey(candidate, baseline, split) {
+	const scoreOf = (run) => {
+		const v = observedSplitScore(run, split);
+		return typeof v === "number" && Number.isFinite(v) ? v : void 0;
+	};
+	const scoredCandidate = candidate.filter((run) => scoreOf(run) !== void 0);
+	const pairing = pairRunRecords(baseline.filter((run) => scoreOf(run) !== void 0), scoredCandidate);
+	return {
+		before: pairing.pairs.map((pair) => scoreOf(pair.baseline)),
+		after: pairing.pairs.map((pair) => scoreOf(pair.treatment)),
+		unpairedCandidateN: pairing.unpairedTreatment.length,
+		unpairedComparatorN: pairing.unpairedBaseline.length
+	};
+}
+function renderSummaryTableMarkdown(rows, comparator, split) {
+	const lines = [];
+	const cmpLabel = comparator ? ` (vs ${comparator})` : "";
+	lines.push(`Summary Table — ${split} split${cmpLabel}`);
+	lines.push("");
+	lines.push("| Candidate | N | Pairs | Unpaired | Mean | 95% CI | q (BH) | Cohen's dz |");
+	lines.push("|---|---:|---:|---:|---:|---|---:|---:|");
+	for (const r of rows) {
+		const ci = `[${fmt(r.ciLow)}, ${fmt(r.ciHigh)}]`;
+		const q = r.qValue !== null ? r.qValue.toFixed(4) : "—";
+		const d = r.cohensD !== null ? r.cohensD.toFixed(3) : "—";
+		const paired = r.pairedN ?? "—";
+		const unpaired = r.unpairedCandidateN === null || r.unpairedComparatorN === null ? "—" : `${r.unpairedCandidateN}/${r.unpairedComparatorN}`;
+		lines.push(`| ${r.candidateId} | ${r.n} | ${paired} | ${unpaired} | ${fmt(r.mean)} | ${ci} | ${q} | ${d} |`);
+	}
+	return lines.join("\n");
+}
+/**
+* Cost vs quality scatter spec. `gateDecisions` is keyed by
+* candidate id; if present, every point picks up the gate verdict
+* for overlay.
+*/
+function paretoChart(runs, opts = {}) {
+	const split = opts.split ?? "holdout";
+	const buckets = /* @__PURE__ */ new Map();
+	for (const r of runs) {
+		if (r.splitTag !== split) continue;
+		const v = observedSplitScore(r, split);
+		if (typeof v !== "number" || !Number.isFinite(v)) continue;
+		if (r.costUsd === null) continue;
+		const bucket = buckets.get(r.candidateId) ?? {
+			cost: [],
+			quality: []
+		};
+		bucket.cost.push(r.costUsd);
+		bucket.quality.push(v);
+		buckets.set(r.candidateId, bucket);
+	}
+	const points = [];
+	for (const [candidateId, bucket] of buckets.entries()) points.push({
+		candidateId,
+		cost: avg(bucket.cost),
+		quality: avg(bucket.quality),
+		n: bucket.cost.length,
+		onFrontier: false,
+		gate: opts.gateDecisions?.[candidateId] ? gateLabel(opts.gateDecisions[candidateId]) : void 0
+	});
+	for (const p of points) p.onFrontier = !points.some((q) => q !== p && dominates(q, p));
+	return {
+		kind: "pareto-cost-quality",
+		split,
+		axes: {
+			x: "costUsd",
+			y: "score"
+		},
+		points
+	};
+}
+function dominates(a, b) {
+	return a.cost <= b.cost && a.quality >= b.quality && (a.cost < b.cost || a.quality > b.quality);
+}
+function gateLabel(d) {
+	return d.promote ? "promote" : "reject";
+}
+/**
+* Held-out improvement distribution: per-pair delta (candidate −
+* comparator), histogrammed. Includes the bootstrap CI on the median
+* delta — same primitive the promotion gate uses.
+*/
+function gainHistogram(runs, candidateId, comparator, opts = {}) {
+	const split = opts.split ?? "holdout";
+	const binCount = opts.bins ?? 11;
+	if (binCount < 1) throw new Error("gainHistogram: bins must be ≥ 1");
+	const { before, after, unpairedCandidateN, unpairedComparatorN } = pairScoresByKey(runs.filter((r) => r.candidateId === candidateId && r.splitTag === split), runs.filter((r) => r.candidateId === comparator && r.splitTag === split), split);
+	const n = before.length;
+	if (n === 0) return {
+		kind: "gain-distribution",
+		candidateId,
+		comparator,
+		split,
+		n: 0,
+		unpairedCandidateN,
+		unpairedComparatorN,
+		bins: [],
+		median: null,
+		ci: null
+	};
+	const deltas = before.map((b, i) => after[i] - b);
+	const sortedDeltas = [...deltas].sort((a, b) => a - b);
+	const median = medianOfSorted(sortedDeltas);
+	const min = sortedDeltas[0];
+	const max = sortedDeltas[sortedDeltas.length - 1];
+	const bound = Math.max(Math.abs(min), Math.abs(max), 1e-6);
+	const lo = -bound;
+	const width = (bound - lo) / binCount;
+	const bins = [];
+	for (let i = 0; i < binCount; i++) bins.push({
+		lo: lo + i * width,
+		hi: lo + (i + 1) * width,
+		count: 0
+	});
+	for (const d of deltas) {
+		let idx = Math.floor((d - lo) / width);
+		if (idx < 0) idx = 0;
+		if (idx >= binCount) idx = binCount - 1;
+		bins[idx].count += 1;
+	}
+	const ci = pairedBootstrap(before, after, {
+		confidence: opts.confidence ?? .95,
+		resamples: opts.resamples ?? 2e3,
+		statistic: "median",
+		seed: opts.seed
+	});
+	return {
+		kind: "gain-distribution",
+		candidateId,
+		comparator,
+		split,
+		n,
+		unpairedCandidateN,
+		unpairedComparatorN,
+		bins,
+		median,
+		ci: {
+			low: ci.low,
+			high: ci.high
+		}
+	};
+}
+/**
+* Hard floor below which a paired comparison is treated as uninformative
+* regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank
+* exact tables; below this the test has no power to separate effect sizes.
+*/
+const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
+/**
+* Internal: paired posterior summary on (candidate − comparator) deltas.
+*
+* Returns the bootstrap CI on the median (matching `gainHistogram`) plus
+* Bayesian-bootstrap posterior summaries Pr(Δ>0) and Pr(Δ∈ROPE) computed
+* from Dirichlet(1, ..., 1) weights on the observed paired deltas
+* (Rubin 1981), and the
+* minimum detectable paired effect at the configured power and α.
+*
+* `null` is returned when no paired observations exist; callers must
+* gate on `n` before consuming the bootstrap statistics.
+*/
+function pairedPosterior(runs, candidateId, comparator, opts) {
+	const { before, after } = pairScoresByKey(runs.filter((r) => r.candidateId === candidateId && r.splitTag === opts.split), runs.filter((r) => r.candidateId === comparator && r.splitTag === opts.split), opts.split);
+	const n = before.length;
+	if (n === 0) return null;
+	const deltas = before.map((b, i) => after[i] - b);
+	const meanDelta = deltas.reduce((s, x) => s + x, 0) / n;
+	const medianDelta = medianOfSorted([...deltas].sort((a, b) => a - b));
+	const sdDelta = stdev(deltas, meanDelta);
+	const ci = pairedBootstrap(before, after, {
+		confidence: opts.confidence,
+		resamples: 2e3,
+		statistic: "median",
+		seed: opts.seed
+	});
+	const meanSamples = bayesianBootstrapMeanSamples(deltas, 2e3, opts.seed);
+	const prGreaterThanZero = meanSamples.length === 0 ? 0 : meanSamples.filter((s) => s > 0).length / meanSamples.length;
+	const prInRope = opts.rope === null || meanSamples.length === 0 ? null : meanSamples.filter((s) => s >= opts.rope.low && s <= opts.rope.high).length / meanSamples.length;
+	const dStandardised = pairedMde({
+		nPaired: n,
+		alpha: opts.mdeAlpha,
+		power: opts.mdePower
+	});
+	const mde = sdDelta === 0 ? 0 : dStandardised * sdDelta;
+	return {
+		n,
+		meanDelta,
+		medianDelta,
+		sdDelta,
+		ci: {
+			low: ci.low,
+			high: ci.high
+		},
+		prGreaterThanZero,
+		prInRope,
+		mde
+	};
+}
+function bayesianBootstrapMeanSamples(deltas, resamples, seed) {
+	const n = deltas.length;
+	if (n === 0) return [];
+	if (n === 1) return new Array(resamples).fill(deltas[0]);
+	const rng = seedRng(seed);
+	const samples = new Array(resamples);
+	for (let b = 0; b < resamples; b++) {
+		let weightedSum = 0;
+		let weightSum = 0;
+		for (let k = 0; k < n; k++) {
+			const weight = -Math.log(Math.max(Number.MIN_VALUE, rng()));
+			weightedSum += deltas[k] * weight;
+			weightSum += weight;
+		}
+		samples[b] = weightedSum / weightSum;
+	}
+	return samples;
+}
+function seedRng(seed) {
+	if (seed === void 0) return Math.random;
+	let s = seed >>> 0;
+	return () => {
+		s = s + 1831565813 >>> 0;
+		let t = s;
+		t = Math.imul(t ^ t >>> 15, t | 1);
+		t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+		return ((t ^ t >>> 14) >>> 0) / 4294967296;
+	};
+}
+function stdev(xs, mean) {
+	if (xs.length < 2) return 0;
+	let sse = 0;
+	for (const x of xs) sse += (x - mean) ** 2;
+	return Math.sqrt(sse / (xs.length - 1));
+}
+/**
+* Executive research report for CPO / AI-lead / launch-review consumption.
+*
+* Composes:
+*   - `summaryTable`         marginal stats with BH-FDR-adjusted q-values
+*   - `paretoChart`           cost-vs-quality frontier with gate overlay
+*   - `gainHistogram`         per-candidate paired-delta distribution
+*   - paired posterior (this file): bootstrap CI on median, Bayesian-bootstrap Pr(Δ>0),
+*                              Pr(Δ∈ROPE), MDE at the configured power
+*
+* Decisions are made on paired evidence — never on marginal means alone —
+* and respect any held-out gate decision the caller passes through. The
+* report embeds a SHA-256 fingerprint of the input run set and, optionally,
+* the hash of a preregistered protocol so a downstream reader can verify
+* provenance and that the analysis was the preregistered one.
+*
+* Async because the fingerprint uses Web Crypto via `hashJson`; deterministic
+* for any fixed `runs`, `seed`, and ROPE.
+*/
+async function researchReport(runs, opts = {}) {
+	const split = opts.split ?? "holdout";
+	const comparator = opts.comparator ?? null;
+	const confidence = opts.confidence ?? .95;
+	const fdr = opts.fdr ?? .05;
+	const minPairs = Math.max(opts.minPairs ?? 20, 6);
+	const rope = opts.rope ?? null;
+	const mdePower = opts.mdePower ?? .8;
+	const mdeAlpha = opts.mdeAlpha ?? fdr;
+	const title = opts.title ?? "Agent Evaluation Research Report";
+	const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+	const preregistrationHash = opts.preregistrationHash ?? null;
+	if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) throw new Error(`researchReport: rope must satisfy low ≤ high with finite bounds, got ${JSON.stringify(rope)}`);
+	const summary = summaryTable(runs, {
+		comparator: comparator ?? void 0,
+		split,
+		confidence,
+		fdr
+	});
+	const pareto = paretoChart(runs, {
+		split,
+		gateDecisions: opts.gateDecisions
+	});
+	const candidateIds = opts.candidateIds ?? summary.rows.map((r) => r.candidateId).filter((id) => id !== comparator);
+	const gains = comparator ? candidateIds.map((id) => gainHistogram(runs, id, comparator, {
+		split,
+		confidence,
+		seed: opts.seed
+	})) : [];
+	const gainByCandidate = new Map(gains.map((g) => [g.candidateId, g]));
+	const paretoByCandidate = new Map(pareto.points.map((p) => [p.candidateId, p]));
+	const posteriorByCandidate = /* @__PURE__ */ new Map();
+	if (comparator) for (const id of candidateIds) posteriorByCandidate.set(id, pairedPosterior(runs, id, comparator, {
+		split,
+		confidence,
+		seed: opts.seed,
+		rope,
+		mdePower,
+		mdeAlpha
+	}));
+	const candidates = summary.rows.map((row) => {
+		const gain = gainByCandidate.get(row.candidateId);
+		const point = paretoByCandidate.get(row.candidateId);
+		const posterior = posteriorByCandidate.get(row.candidateId) ?? null;
+		const gateDecision = opts.gateDecisions?.[row.candidateId];
+		const classified = classifyCandidate(row, {
+			comparator,
+			posterior,
+			point,
+			gateDecision,
+			split,
+			fdr,
+			minPairs,
+			rope
+		});
+		return {
+			candidateId: row.candidateId,
+			n: row.n,
+			mean: row.mean,
+			ciLow: row.ciLow,
+			ciHigh: row.ciHigh,
+			qValue: row.qValue,
+			cohensD: row.cohensD,
+			meanDeltaVsComparator: posterior ? posterior.meanDelta : null,
+			pairedN: posterior?.n ?? gain?.n ?? 0,
+			medianGain: posterior ? posterior.medianDelta : gain ? gain.median : null,
+			meanGain: posterior ? posterior.meanDelta : null,
+			gainCi: posterior ? posterior.ci : gain ? gain.ci : null,
+			prGreaterThanZero: posterior ? posterior.prGreaterThanZero : null,
+			prInRope: posterior ? posterior.prInRope : null,
+			mde: posterior ? posterior.mde : null,
+			onParetoFrontier: point?.onFrontier ?? false,
+			gate: gateDecision ? gateLabel(gateDecision) : void 0,
+			decision: classified.decision,
+			decisionReason: classified.reason
+		};
+	}).sort((a, b) => {
+		const decisionRank = decisionWeight(b.decision) - decisionWeight(a.decision);
+		if (decisionRank !== 0) return decisionRank;
+		return b.mean - a.mean;
+	});
+	const recommendation = buildRecommendation(candidates, {
+		comparator,
+		failureClusters: opts.failureClusters,
+		rope,
+		minPairs,
+		preregistrationHash
+	});
+	const executiveSummary = buildExecutiveSummary(candidates, recommendation, {
+		comparator,
+		split,
+		failureClusters: opts.failureClusters,
+		preregistrationHash
+	});
+	const methodology = buildMethodology({
+		split,
+		comparator,
+		fdr,
+		minPairs,
+		rope,
+		confidence,
+		mdePower,
+		mdeAlpha
+	});
+	const runFingerprint = await hashJson(canonicalize({
+		triples: runs.filter((r) => r.splitTag === split).map((r) => ({
+			runId: r.runId,
+			candidateId: r.candidateId,
+			splitTag: r.splitTag
+		})).sort((a, b) => a.runId.localeCompare(b.runId)),
+		comparator,
+		split
+	}));
+	const markdown = renderResearchMarkdown({
+		title,
+		generatedAt,
+		split,
+		comparator,
+		rope,
+		runFingerprint,
+		preregistrationHash,
+		executiveSummary,
+		recommendation,
+		candidates,
+		summary,
+		pareto,
+		gains,
+		methodology,
+		failureClusters: opts.failureClusters
+	});
+	const html = renderResearchHtml(markdown, title);
+	return {
+		kind: "agent-eval-research-report",
+		title,
+		generatedAt,
+		split,
+		comparator,
+		runFingerprint,
+		preregistrationHash,
+		rope,
+		executiveSummary,
+		recommendation,
+		candidates,
+		summary,
+		charts: {
+			pareto,
+			gains
+		},
+		methodology,
+		failureClusters: opts.failureClusters,
+		markdown,
+		html
+	};
+}
+function buildMethodology(ctx) {
+	const assumptions = [
+		"Pairs are matched by (experimentId, scenarioId, seed); input order does not affect the comparison.",
+		"Paired deltas are exchangeable conditional on the matched scenario — no mid-run distribution shift.",
+		`Decisions are pre-specified at fdr=${ctx.fdr}, minPairs=${ctx.minPairs}, confidence=${ctx.confidence}; deviating from these post-hoc invalidates the false-discovery control.`
+	];
+	if (ctx.rope) assumptions.push(`The Region of Practical Equivalence ${formatRope(ctx.rope)} is supplied by the domain owner; equivalent verdicts are only meaningful if that range is treated as the standing definition of "no material difference."`);
+	if (ctx.comparator === null) assumptions.push("No comparator was configured; this run is descriptive, not causal.");
+	return {
+		assumptions,
+		methods: [
+			"Marginal scores summarised with BH-FDR-adjusted Wilcoxon signed-rank q-values and paired Cohen's dz via summaryTable.",
+			"Paired evidence summarised with a frequentist bootstrap CI on the median delta and a Dirichlet-weight Bayesian bootstrap for Pr(Δ>0) and Pr(Δ∈ROPE) on the mean delta.",
+			`Minimum detectable effect reported per candidate at α=${ctx.mdeAlpha} (two-sided), power=${ctx.mdePower}, standardised by the observed paired-delta SD.`,
+			"Pareto frontier flagged as a separate axis (cost vs quality); a candidate can be on-frontier without winning the paired test.",
+			"Held-out gate decisions, when supplied, override the statistical verdict in the reject direction."
+		],
+		alternatives: [
+			"Paired t-test rejected: not robust to the heavy-tailed score distributions common in agent benchmarks.",
+			"Unpaired Mann–Whitney rejected: matched scenarios make pairing free; unpaired throws away that variance reduction.",
+			"Sequential / always-valid inference (e-values, mSPRT) is the right tool for iterative sweeps and is out of scope for this single-look report — preregister and run once, or wrap this report in an alpha-spending schedule.",
+			"Hierarchical Bayesian shrinkage across many candidates is future work; the current ranking uses raw paired statistics."
+		],
+		whenNotToApply: [
+			`Paired N below 6 on any candidate — the bootstrap CI is degenerate.`,
+			"Comparator chosen post-hoc by inspecting the same data; q-values are no longer false-discovery-controlled.",
+			"Scenarios not drawn under a stable preregistered protocol; the report can describe the data but cannot anchor a launch decision.",
+			"Score distributions with mid-run shift (judge model swap, rubric change, infra outage) — pair exchangeability is violated."
+		],
+		citations: [
+			"Benjamini, Y. & Hochberg, Y. (1995). Controlling the false discovery rate: a practical and powerful approach to multiple testing. JRSS B, 57(1), 289–300.",
+			"Wilcoxon, F. (1945). Individual comparisons by ranking methods. Biometrics Bulletin, 1(6), 80–83.",
+			"Efron, B. (1979). Bootstrap methods: another look at the jackknife. Annals of Statistics, 7(1), 1–26.",
+			"Rubin, D. B. (1981). The Bayesian bootstrap. Annals of Statistics, 9(1), 130–134.",
+			"Kruschke, J. K. (2018). Rejecting or accepting parameter values in Bayesian estimation. Advances in Methods and Practices in Psychological Science, 1(2), 270–280. (ROPE.)"
+		]
+	};
+}
+function formatRope(rope) {
+	return `[${fmt(rope.low)}, ${fmt(rope.high)}]`;
+}
+function classifyCandidate(row, ctx) {
+	if (ctx.comparator && row.candidateId === ctx.comparator) return {
+		decision: "hold",
+		reason: "Comparator baseline."
+	};
+	if (ctx.gateDecision && !ctx.gateDecision.promote) return {
+		decision: "reject",
+		reason: `Held-out gate returned ${ctx.gateDecision.rejectionCode ?? "rejected"}: ${ctx.gateDecision.reason}`
+	};
+	if (!ctx.comparator) return {
+		decision: ctx.point?.onFrontier ? "hold" : "needs_more_data",
+		reason: "No comparator configured; report ranks candidates but cannot anchor a promotion call."
+	};
+	if (!ctx.posterior || ctx.posterior.n < 6) return {
+		decision: "needs_more_data",
+		reason: `Only ${ctx.posterior?.n ?? 0} paired observations; below hard floor of 6 for any paired inference.`
+	};
+	const ci = ctx.posterior.ci;
+	if (ctx.split === "search") return {
+		decision: "hold",
+		reason: "Search-split evidence is descriptive; release decisions require matched holdout runs."
+	};
+	if (ctx.rope && ci.low >= ctx.rope.low && ci.high <= ctx.rope.high) return {
+		decision: "equivalent",
+		reason: `Paired-delta CI [${fmt(ci.low)}, ${fmt(ci.high)}] is fully inside ROPE ${formatRope(ctx.rope)}; candidate is practically equivalent to comparator.`
+	};
+	const gainPositive = ci.low > 0;
+	if (ci.high < 0) return {
+		decision: "reject",
+		reason: `Paired-delta CI [${fmt(ci.low)}, ${fmt(ci.high)}] lies entirely below zero.`
+	};
+	if (ctx.posterior.n < ctx.minPairs) return {
+		decision: "needs_more_data",
+		reason: `Only ${ctx.posterior.n} paired observations; minimum detectable effect at this N is ${fmt(ctx.posterior.mde)} score units (need ≥ ${ctx.minPairs} pairs to issue a directional verdict).`
+	};
+	if (row.qValue !== null && row.qValue <= ctx.fdr && gainPositive) return {
+		decision: "promote",
+		reason: `BH-adjusted q=${fmt(row.qValue)} ≤ ${ctx.fdr} and paired-delta CI [${fmt(ci.low)}, ${fmt(ci.high)}] excludes zero; Pr(Δ>0)=${fmt(ctx.posterior.prGreaterThanZero)}.`
+	};
+	return {
+		decision: "hold",
+		reason: `Pr(Δ>0)=${fmt(ctx.posterior.prGreaterThanZero)} but CI [${fmt(ci.low)}, ${fmt(ci.high)}] crosses zero; effect not decisive at fdr=${ctx.fdr}.`
+	};
+}
+function buildRecommendation(candidates, ctx) {
+	const nonComparator = candidates.filter((c) => c.candidateId !== ctx.comparator);
+	const bestPromote = nonComparator.find((c) => c.decision === "promote");
+	const bestEquivalent = nonComparator.find((c) => c.decision === "equivalent");
+	const chosen = bestPromote ?? bestEquivalent ?? nonComparator[0] ?? null;
+	const decision = bestPromote ? "promote" : nonComparator.some((c) => c.decision === "needs_more_data") ? "needs_more_data" : bestEquivalent ? "equivalent" : nonComparator.some((c) => c.decision === "hold") ? "hold" : "reject";
+	const rationale = [];
+	const risks = [];
+	const nextActions = [];
+	if (chosen) {
+		rationale.push(`${chosen.candidateId}: ${chosen.decisionReason}`);
+		if (chosen.gainCi) {
+			const probSummary = chosen.prGreaterThanZero !== null ? `, Pr(Δ>0)=${fmt(chosen.prGreaterThanZero)}` : "";
+			rationale.push(`Median paired gain CI: [${fmt(chosen.gainCi.low)}, ${fmt(chosen.gainCi.high)}]${probSummary}.`);
+		}
+		if (chosen.mde !== null && Number.isFinite(chosen.mde)) rationale.push(`MDE at current paired N=${chosen.pairedN}: ${fmt(chosen.mde)} score units.`);
+	}
+	if (!ctx.comparator) {
+		risks.push("No comparator was configured; verdict is descriptive, not causal.");
+		nextActions.push("Re-run with a stable comparator candidate for paired inference.");
+	}
+	if (!ctx.preregistrationHash) {
+		risks.push("No preregistration hash supplied; readers cannot verify the analysis was specified before data inspection.");
+		nextActions.push("Sign a HypothesisManifest before the next sweep and pass `preregistrationHash` so the report cites it.");
+	}
+	if (ctx.rope === null && nonComparator.length > 0) {
+		risks.push("No ROPE configured; the report cannot distinguish \"equivalent\" from \"inconclusive\".");
+		nextActions.push("Define a domain-specific Region of Practical Equivalence and pass it to lock in the equivalence threshold.");
+	}
+	const inconclusive = nonComparator.filter((c) => c.decision === "needs_more_data");
+	if (inconclusive.length > 0) {
+		const worst = inconclusive.reduce((a, b) => b.pairedN < a.pairedN ? b : a);
+		risks.push(`${inconclusive.length} candidate(s) below soft floor (${ctx.minPairs} pairs); thinnest is ${worst.candidateId} with ${worst.pairedN}.`);
+		nextActions.push(`Collect at least ${ctx.minPairs - worst.pairedN} more matched holdout runs for ${worst.candidateId}.`);
+	}
+	const rejected = nonComparator.filter((c) => c.decision === "reject");
+	if (rejected.length > 0) risks.push(`${rejected.length} candidate(s) failed the paired test or held-out gate; do not ship those variants.`);
+	if (ctx.failureClusters && ctx.failureClusters.clusters.length > 0) {
+		const top = ctx.failureClusters.clusters[0];
+		risks.push(`Top failure cluster: ${top.failureClass} across ${top.runCount} run(s).`);
+		nextActions.push("Prioritize the largest failure cluster before broad rollout.");
+	}
+	if (decision === "promote") nextActions.push("Ship behind the existing promotion gate and monitor canaries.");
+	else if (decision === "hold") nextActions.push("Keep current production candidate while expanding holdout evidence.");
+	else if (decision === "equivalent") nextActions.push("Either keep the comparator (no quality regression) or promote on cost/latency grounds — equivalence does not justify either; the choice is a product decision, not a stats one.");
+	else if (decision === "reject") nextActions.push("Do not promote this sweep; inspect failures and generate a revised candidate.");
+	return {
+		decision,
+		candidateId: chosen?.candidateId ?? null,
+		rationale,
+		risks,
+		nextActions
+	};
+}
+function buildExecutiveSummary(candidates, recommendation, ctx) {
+	const lines = [];
+	const nonComparator = candidates.filter((c) => c.candidateId !== ctx.comparator);
+	lines.push(`Evaluated ${nonComparator.length} candidate(s) on the ${ctx.split} split${ctx.comparator ? ` against ${ctx.comparator}` : ""}.`);
+	lines.push(`Recommendation: ${recommendation.decision}${recommendation.candidateId ? ` ${recommendation.candidateId}` : ""}.`);
+	const promoted = nonComparator.filter((c) => c.decision === "promote").length;
+	const held = nonComparator.filter((c) => c.decision === "hold").length;
+	const equivalent = nonComparator.filter((c) => c.decision === "equivalent").length;
+	const rejected = nonComparator.filter((c) => c.decision === "reject").length;
+	const more = nonComparator.filter((c) => c.decision === "needs_more_data").length;
+	lines.push(`Decision mix: ${promoted} promote, ${equivalent} equivalent, ${held} hold, ${rejected} reject, ${more} need more data.`);
+	const frontier = nonComparator.filter((c) => c.onParetoFrontier).map((c) => c.candidateId);
+	if (frontier.length > 0) lines.push(`Pareto-frontier candidates: ${frontier.join(", ")}.`);
+	if (ctx.failureClusters) lines.push(`Failure clustering found ${ctx.failureClusters.totalFailures}/${ctx.failureClusters.totalRuns} failed runs across ${ctx.failureClusters.clusters.length} reportable cluster(s).`);
+	lines.push(ctx.preregistrationHash ? `Preregistered analysis: ${ctx.preregistrationHash.slice(0, 12)}…` : "Analysis is post-hoc — no preregistration hash supplied.");
+	return lines;
+}
+function renderResearchMarkdown(report) {
+	const lines = [];
+	lines.push(`# ${report.title}`);
+	lines.push("");
+	lines.push(`**Generated:** ${report.generatedAt}`);
+	lines.push(`**Primary split:** ${report.split}`);
+	lines.push(`**Comparator:** ${report.comparator ?? "not configured"}`);
+	lines.push(`**ROPE:** ${report.rope ? formatRope(report.rope) : "not configured"}`);
+	lines.push(`**Run fingerprint:** \`${report.runFingerprint}\``);
+	lines.push(`**Preregistration:** ${report.preregistrationHash ? `\`${report.preregistrationHash}\`` : "none"}`);
+	lines.push("");
+	lines.push("## Executive Summary");
+	lines.push("");
+	for (const item of report.executiveSummary) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("## Recommendation");
+	lines.push("");
+	lines.push(`**Decision:** ${report.recommendation.decision}`);
+	lines.push(`**Candidate:** ${report.recommendation.candidateId ?? "N/A"}`);
+	lines.push("");
+	lines.push("### Rationale");
+	lines.push("");
+	for (const item of report.recommendation.rationale) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("### Risks");
+	lines.push("");
+	for (const item of report.recommendation.risks.length ? report.recommendation.risks : ["No material report-level risks detected."]) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("### Next Actions");
+	lines.push("");
+	for (const item of report.recommendation.nextActions) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("## Candidate Decision Table");
+	lines.push("");
+	lines.push("| Candidate | Decision | Mean | Δ̄ | Pr(Δ>0) | q | dz | Paired N | Median Gain CI | MDE | Pareto | Gate |");
+	lines.push("|---|---|---:|---:|---:|---:|---:|---:|---|---:|---|---|");
+	for (const c of report.candidates) {
+		const delta = c.meanDeltaVsComparator === null ? "-" : signed(c.meanDeltaVsComparator);
+		const prGt = c.prGreaterThanZero === null ? "-" : c.prGreaterThanZero.toFixed(3);
+		const q = c.qValue !== null ? c.qValue.toFixed(4) : "-";
+		const d = c.cohensD !== null ? c.cohensD.toFixed(3) : "-";
+		const gain = c.gainCi ? `[${fmt(c.gainCi.low)}, ${fmt(c.gainCi.high)}]` : "-";
+		const mde = c.mde === null || !Number.isFinite(c.mde) ? "-" : fmt(c.mde);
+		lines.push(`| ${c.candidateId} | ${c.decision} | ${fmt(c.mean)} | ${delta} | ${prGt} | ${q} | ${d} | ${c.pairedN} | ${gain} | ${mde} | ${c.onParetoFrontier ? "yes" : "no"} | ${c.gate ?? "-"} |`);
+	}
+	lines.push("");
+	lines.push("## Statistical Summary");
+	lines.push("");
+	lines.push(report.summary.markdown);
+	lines.push("");
+	lines.push("## Methodology");
+	lines.push("");
+	lines.push("### Assumptions");
+	lines.push("");
+	for (const item of report.methodology.assumptions) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("### Methods");
+	lines.push("");
+	for (const item of report.methodology.methods) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("### Alternatives Considered");
+	lines.push("");
+	for (const item of report.methodology.alternatives) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("### When NOT To Apply");
+	lines.push("");
+	for (const item of report.methodology.whenNotToApply) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("### Citations");
+	lines.push("");
+	for (const item of report.methodology.citations) lines.push(`- ${item}`);
+	lines.push("");
+	lines.push("## Chart Specs");
+	lines.push("");
+	lines.push("The report carries JSON chart specs for Pareto cost/quality and paired gain histograms.");
+	lines.push("");
+	lines.push("```json");
+	lines.push(JSON.stringify({
+		pareto: report.pareto,
+		gains: report.gains
+	}, null, 2));
+	lines.push("```");
+	if (report.failureClusters) {
+		lines.push("");
+		lines.push("## Failure Clusters");
+		lines.push("");
+		lines.push("| Failure Class | Runs | Scenarios | Tool | Example |");
+		lines.push("|---|---:|---:|---|---|");
+		for (const c of report.failureClusters.clusters.slice(0, 10)) lines.push(`| ${c.failureClass} | ${c.runCount} | ${c.scenarioIds.length} | ${c.toolName ?? "-"} | ${escapePipes(c.exampleError ?? c.exampleRunId)} |`);
+	}
+	return lines.join("\n");
+}
+function renderResearchHtml(markdown, title) {
+	const body = markdownToHtml(markdown);
+	return [
+		"<!doctype html>",
+		"<html lang=\"en\">",
+		"<head>",
+		"<meta charset=\"utf-8\">",
+		"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">",
+		`<title>${escapeHtml(title)}</title>`,
+		"<style>",
+		"body{font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,\"Segoe UI\",sans-serif;margin:0;color:#172026;background:#f7f8f8;}",
+		"main{max-width:1080px;margin:0 auto;padding:40px 24px 64px;background:#fff;min-height:100vh;}",
+		"h1{font-size:34px;line-height:1.15;margin:0 0 20px;}h2{margin-top:34px;border-top:1px solid #d9dfdf;padding-top:22px;}h3{margin-top:22px;}",
+		"p,li{line-height:1.55;}table{border-collapse:collapse;width:100%;margin:16px 0;font-size:14px;}th,td{border:1px solid #d9dfdf;padding:8px;text-align:left;}th{background:#eef2f2;}",
+		"code,pre{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;}pre{overflow:auto;background:#111827;color:#f9fafb;padding:16px;border-radius:6px;}",
+		"</style>",
+		"</head>",
+		"<body><main>",
+		body,
+		"</main></body></html>"
+	].join("\n");
+}
+function markdownToHtml(markdown) {
+	const lines = markdown.split("\n");
+	const html = [];
+	let inList = false;
+	let inCode = false;
+	let code = [];
+	let table = [];
+	const flushList = () => {
+		if (inList) {
+			html.push("</ul>");
+			inList = false;
+		}
+	};
+	const flushTable = () => {
+		if (table.length === 0) return;
+		html.push(renderMarkdownTable(table));
+		table = [];
+	};
+	for (const line of lines) {
+		if (line.startsWith("```")) {
+			if (inCode) {
+				html.push(`<pre><code>${escapeHtml(code.join("\n"))}</code></pre>`);
+				code = [];
+				inCode = false;
+			} else {
+				flushList();
+				flushTable();
+				inCode = true;
+			}
+			continue;
+		}
+		if (inCode) {
+			code.push(line);
+			continue;
+		}
+		if (line.startsWith("|")) {
+			flushList();
+			table.push(line);
+			continue;
+		}
+		flushTable();
+		if (line.startsWith("- ")) {
+			if (!inList) {
+				html.push("<ul>");
+				inList = true;
+			}
+			html.push(`<li>${inlineMarkdown(line.slice(2))}</li>`);
+			continue;
+		}
+		flushList();
+		if (line.startsWith("# ")) html.push(`<h1>${inlineMarkdown(line.slice(2))}</h1>`);
+		else if (line.startsWith("## ")) html.push(`<h2>${inlineMarkdown(line.slice(3))}</h2>`);
+		else if (line.startsWith("### ")) html.push(`<h3>${inlineMarkdown(line.slice(4))}</h3>`);
+		else if (line.trim() === "") html.push("");
+		else html.push(`<p>${inlineMarkdown(line)}</p>`);
+	}
+	flushList();
+	flushTable();
+	return html.join("\n");
+}
+function renderMarkdownTable(lines) {
+	const rows = lines.filter((line) => !/^\|[-:\s|]+\|$/.test(line)).map((line) => line.slice(1, -1).split("|").map((cell) => inlineMarkdown(cell.trim())));
+	if (rows.length === 0) return "";
+	const [head, ...body] = rows;
+	return `<table><thead><tr>${head.map((cell) => `<th>${cell}</th>`).join("")}</tr></thead><tbody>${body.map((row) => `<tr>${row.map((cell) => `<td>${cell}</td>`).join("")}</tr>`).join("\n")}</tbody></table>`;
+}
+function inlineMarkdown(s) {
+	return escapeHtml(s).replace(/\*\*([^*]+)\*\*/g, "<strong>$1</strong>");
+}
+function escapeHtml(s) {
+	return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function escapePipes(s) {
+	return s.replace(/\|/g, "\\|");
+}
+function decisionWeight(decision) {
+	if (decision === "promote") return 5;
+	if (decision === "equivalent") return 4;
+	if (decision === "hold") return 3;
+	if (decision === "needs_more_data") return 2;
+	return 1;
+}
+function signed(x) {
+	return `${x >= 0 ? "+" : ""}${fmt(x)}`;
+}
+function avg(xs) {
+	if (xs.length === 0) return NaN;
+	return xs.reduce((s, x) => s + x, 0) / xs.length;
+}
+function medianOfSorted(sorted) {
+	if (sorted.length === 0) return 0;
+	const mid = Math.floor(sorted.length / 2);
+	return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function fmt(x) {
+	if (!Number.isFinite(x)) return String(x);
+	return x.toFixed(4);
+}
+//#endregion
+export { summaryTable as a, researchReport as i, gainHistogram as n, paretoChart as r, RESEARCH_REPORT_HARD_PAIR_FLOOR as t };
+
+//# sourceMappingURL=summary-report-BNs5nmXI.js.map