@tangle-network/agent-eval 0.20.11 → 0.21.0

package/dist/chunk-3IX6QTB7.js

@@ -0,0 +1,1349 @@
+import {
+  benjaminiHochberg,
+  cohensD,
+  confidenceInterval,
+  pairedBootstrap,
+  pairedMde,
+  wilcoxonSignedRank
+} from "./chunk-KRR4VMH7.js";
+
+// src/release-confidence.ts
+var DEFAULT_THRESHOLDS = {
+  requireCorpus: true,
+  minScenarioCount: 1,
+  minSearchRuns: 1,
+  minHoldoutRuns: 1,
+  requireHoldout: true,
+  minPassRate: 0.8,
+  minMeanScore: 0.7,
+  maxOverfitGap: 0.15,
+  maxMeanCostUsd: Number.POSITIVE_INFINITY,
+  maxP95WallMs: Number.POSITIVE_INFINITY,
+  requireAsiForFailures: true,
+  failureScoreThreshold: 0.5
+};
+function releaseTraceEvidenceFromMultiShotTrials(trials) {
+  return trials.map((trial) => ({
+    scenarioId: trial.scenarioId,
+    candidateId: trial.variantId,
+    split: trial.split === "holdout" ? "holdout" : trial.split === "dev" ? "dev" : "search",
+    score: trial.score,
+    ok: trial.ok,
+    turnCount: Array.isArray(trial.trace?.turns) ? trial.trace.turns.length : void 0,
+    costUsd: trial.cost,
+    durationMs: trial.durationMs,
+    failureMode: trial.error ? "runtime_error" : void 0,
+    asi: trial.asi,
+    metadata: trial.metadata
+  }));
+}
+function evaluateReleaseConfidence(input) {
+  const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds };
+  const candidateId = input.candidateId ?? null;
+  const runs = filterCandidate(input.runs ?? [], candidateId, input.baselineId);
+  const traces = filterTraceCandidate(input.traces ?? [], candidateId, input.baselineId);
+  const scenarios = input.scenarios ?? [];
+  const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length;
+  const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios);
+  const searchScores = scoresFor(runs, "search");
+  const holdoutScores = scoresFor(runs, "holdout");
+  const allScores = [...searchScores, ...holdoutScores];
+  const traceScores = traces.map((t) => t.score).filter(isFiniteNumber);
+  const scoreUniverse = allScores.length > 0 ? allScores : traceScores;
+  const searchRuns = runs.filter((r) => r.splitTag === "search").length;
+  const holdoutRuns = runs.filter((r) => r.splitTag === "holdout").length;
+  const searchMeanScore = mean(searchScores);
+  const holdoutMeanScore = mean(holdoutScores);
+  const metrics = {
+    scenarioCount,
+    searchRuns,
+    holdoutRuns,
+    passRate: passRate(runs, traces, thresholds.failureScoreThreshold),
+    meanScore: mean(scoreUniverse),
+    searchMeanScore,
+    holdoutMeanScore,
+    overfitGap: safeDiff(searchMeanScore, holdoutMeanScore),
+    meanCostUsd: mean([...runs.map((r) => r.costUsd), ...traces.map((t) => t.costUsd).filter(isFiniteNumber)]),
+    p95WallMs: percentile([...runs.map((r) => r.wallMs), ...traces.map((t) => t.durationMs).filter(isFiniteNumber)], 0.95),
+    failedRows: failedRows(runs, traces, thresholds.failureScoreThreshold).length,
+    failuresWithAsi: failedRows(runs, traces, thresholds.failureScoreThreshold).filter((row) => row.hasAsi).length,
+    singleShotTraces: traces.filter((t) => t.turnCount === 1).length,
+    multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,
+    splitCounts,
+    domainCounts: countDomains(scenarios),
+    failureModeCounts: countFailureModes(runs, traces, thresholds.failureScoreThreshold),
+    responsibleSurfaceCounts: countResponsibleSurfaces(traces)
+  };
+  const issues = [];
+  checkCorpus(input, thresholds, metrics, issues);
+  checkQuality(thresholds, metrics, issues);
+  checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues);
+  checkDiagnostics(thresholds, metrics, issues);
+  checkEfficiency(thresholds, metrics, issues);
+  const axes = buildAxes(metrics, thresholds, input.gateDecision ?? null, issues);
+  const status = issues.some((i) => i.severity === "critical") ? "fail" : issues.length > 0 ? "warn" : "pass";
+  return {
+    target: input.target,
+    candidateId,
+    baselineId: input.baselineId ?? null,
+    status,
+    promote: status === "pass" && (input.gateDecision ? input.gateDecision.promote : true),
+    axes,
+    issues,
+    metrics,
+    dataset: input.dataset ?? null,
+    gateDecision: input.gateDecision ?? null,
+    summary: renderSummary(input.target, status, metrics, issues)
+  };
+}
+function assertReleaseConfidence(input) {
+  const scorecard = evaluateReleaseConfidence(input);
+  if (scorecard.status === "fail") {
+    throw new Error(scorecard.summary);
+  }
+  return scorecard;
+}
+function filterCandidate(runs, candidateId, baselineId) {
+  if (candidateId) return runs.filter((r) => r.candidateId === candidateId);
+  if (baselineId) return runs.filter((r) => r.candidateId !== baselineId);
+  return [...runs];
+}
+function filterTraceCandidate(traces, candidateId, baselineId) {
+  if (candidateId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId === candidateId);
+  if (baselineId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId !== baselineId);
+  return [...traces];
+}
+function checkCorpus(input, thresholds, metrics, issues) {
+  if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) {
+    issues.push({ axis: "corpus", severity: "critical", code: "missing_corpus", detail: "No Dataset manifest or scenarios supplied." });
+  }
+  if (metrics.scenarioCount < thresholds.minScenarioCount) {
+    issues.push({ axis: "corpus", severity: "critical", code: "few_scenarios", detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.` });
+  }
+  if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) {
+    issues.push({ axis: "corpus", severity: "critical", code: "missing_holdout_split", detail: "Corpus has no holdout scenarios." });
+  }
+}
+function checkQuality(thresholds, metrics, issues) {
+  if (metrics.searchRuns < thresholds.minSearchRuns) {
+    issues.push({ axis: "quality", severity: "critical", code: "few_search_runs", detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.` });
+  }
+  if (metrics.passRate < thresholds.minPassRate) {
+    issues.push({ axis: "quality", severity: "critical", code: "low_pass_rate", detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.` });
+  }
+  if (metrics.meanScore < thresholds.minMeanScore) {
+    issues.push({ axis: "quality", severity: "critical", code: "low_mean_score", detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.` });
+  }
+}
+function checkGeneralization(gateDecision, thresholds, metrics, issues) {
+  if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {
+    issues.push({ axis: "generalization", severity: "critical", code: "few_holdout_runs", detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.` });
+  }
+  if (Number.isFinite(metrics.overfitGap) && metrics.overfitGap > thresholds.maxOverfitGap) {
+    issues.push({ axis: "generalization", severity: "critical", code: "overfit_gap", detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.` });
+  }
+  if (gateDecision && !gateDecision.promote) {
+    issues.push({ axis: "generalization", severity: "critical", code: `gate_${gateDecision.rejectionCode ?? "reject"}`, detail: gateDecision.reason });
+  }
+}
+function checkDiagnostics(thresholds, metrics, issues) {
+  if (!thresholds.requireAsiForFailures) return;
+  if (metrics.failedRows > metrics.failuresWithAsi) {
+    issues.push({
+      axis: "diagnostics",
+      severity: "critical",
+      code: "missing_failure_asi",
+      detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`
+    });
+  }
+}
+function checkEfficiency(thresholds, metrics, issues) {
+  if (metrics.meanCostUsd > thresholds.maxMeanCostUsd) {
+    issues.push({ axis: "efficiency", severity: "critical", code: "cost_budget", detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.` });
+  }
+  if (metrics.p95WallMs > thresholds.maxP95WallMs) {
+    issues.push({ axis: "efficiency", severity: "critical", code: "latency_budget", detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.` });
+  }
+}
+function buildAxes(metrics, thresholds, gateDecision, issues) {
+  return [
+    axis("corpus", issues, bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)), `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`),
+    axis("quality", issues, Math.min(metrics.passRate, metrics.meanScore), `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`),
+    axis("generalization", issues, gateDecision && !gateDecision.promote ? 0 : gapScore(metrics.overfitGap, thresholds.maxOverfitGap), `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`),
+    axis("diagnostics", issues, metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows, `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`),
+    axis("efficiency", issues, efficiencyScore(metrics, thresholds), `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`)
+  ];
+}
+function axis(name, issues, score, detail) {
+  const own = issues.filter((i) => i.axis === name);
+  const status = own.some((i) => i.severity === "critical") ? "fail" : own.length > 0 ? "warn" : "pass";
+  return { name, status, score: bounded(score), detail };
+}
+function countScenarioSplits(scenarios) {
+  const counts = { train: 0, dev: 0, test: 0, holdout: 0 };
+  for (const scenario of scenarios) counts[scenario.split ?? "train"]++;
+  return counts;
+}
+function countDomains(scenarios) {
+  const out = {};
+  for (const scenario of scenarios) {
+    const domain = scenario.tags?.domain ?? scenario.tags?.category ?? "uncategorized";
+    out[domain] = (out[domain] ?? 0) + 1;
+  }
+  return out;
+}
+function countFailureModes(runs, traces, threshold) {
+  const out = {};
+  for (const run of runs) {
+    const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+    if (run.failureMode || score !== void 0 && score < threshold) {
+      const mode = run.failureMode ?? "low_score";
+      out[mode] = (out[mode] ?? 0) + 1;
+    }
+  }
+  for (const trace of traces) {
+    if (trace.failureMode || trace.ok === false || trace.score !== void 0 && trace.score < threshold) {
+      const mode = trace.failureMode ?? (trace.ok === false ? "not_ok" : "low_score");
+      out[mode] = (out[mode] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function countResponsibleSurfaces(traces) {
+  const out = {};
+  for (const trace of traces) {
+    for (const asi of trace.asi ?? []) {
+      const surface = asi.responsibleSurface ?? "unknown";
+      out[surface] = (out[surface] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function failedRows(runs, traces, threshold) {
+  const out = [];
+  for (const run of runs) {
+    const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+    if (run.failureMode || score !== void 0 && score < threshold) {
+      const asiMetric = run.outcome.raw.asi;
+      out.push({ hasAsi: typeof asiMetric === "number" && asiMetric > 0 });
+    }
+  }
+  for (const trace of traces) {
+    if (trace.failureMode || trace.ok === false || trace.score !== void 0 && trace.score < threshold) {
+      out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 });
+    }
+  }
+  return out;
+}
+function passRate(runs, traces, threshold) {
+  const outcomes = [
+    ...runs.map((run) => {
+      const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+      return !run.failureMode && score !== void 0 && score >= threshold;
+    }),
+    ...traces.map((trace) => trace.ok !== false && (trace.score === void 0 || trace.score >= threshold))
+  ];
+  if (outcomes.length === 0) return 0;
+  return outcomes.filter(Boolean).length / outcomes.length;
+}
+function scoresFor(runs, split) {
+  return runs.filter((run) => run.splitTag === split).map((run) => split === "holdout" ? run.outcome.holdoutScore : run.outcome.searchScore).filter(isFiniteNumber);
+}
+function mean(xs) {
+  if (xs.length === 0) return Number.NaN;
+  return xs.reduce((sum, x) => sum + x, 0) / xs.length;
+}
+function percentile(xs, p) {
+  if (xs.length === 0) return Number.NaN;
+  const sorted = [...xs].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
+}
+function isFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+function safeDiff(a, b) {
+  if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN;
+  return a - b;
+}
+function gapScore(gap, maxGap) {
+  if (!Number.isFinite(gap)) return 0;
+  if (maxGap <= 0) return gap <= 0 ? 1 : 0;
+  return bounded(1 - Math.max(0, gap) / maxGap);
+}
+function efficiencyScore(metrics, thresholds) {
+  const cost = Number.isFinite(thresholds.maxMeanCostUsd) && Number.isFinite(metrics.meanCostUsd) ? bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12)) : 1;
+  const latency = Number.isFinite(thresholds.maxP95WallMs) && Number.isFinite(metrics.p95WallMs) ? bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12)) : 1;
+  return Math.min(cost, latency);
+}
+function bounded(x) {
+  if (!Number.isFinite(x)) return 0;
+  return Math.max(0, Math.min(1, x));
+}
+function renderSummary(target, status, metrics, issues) {
+  const prefix = `release confidence ${status}: ${target}`;
+  const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`;
+  if (issues.length === 0) return `${prefix}; ${metricText}`;
+  return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(",")}`;
+}
+function fmt(x) {
+  if (!Number.isFinite(x)) return String(x);
+  return x.toFixed(4);
+}
+
+// src/pre-registration.ts
+function canonicalize(v) {
+  if (v === null || typeof v !== "object") return v;
+  if (Array.isArray(v)) return v.map(canonicalize);
+  const keys = Object.keys(v).sort();
+  const out = {};
+  for (const k of keys) out[k] = canonicalize(v[k]);
+  return out;
+}
+async function hashJson(obj) {
+  const canonical = canonicalize(obj);
+  const bytes = new TextEncoder().encode(JSON.stringify(canonical));
+  const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
+  return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+async function signManifest(m) {
+  const hash = await hashJson(m);
+  return { ...m, contentHash: hash, algo: "sha256-content" };
+}
+async function verifyManifest(m) {
+  const { contentHash, algo: _algo, ...rest } = m;
+  void _algo;
+  const resigned = await signManifest(rest);
+  return resigned.contentHash === contentHash;
+}
+async function evaluateHypothesis(manifest, observed) {
+  if (!await verifyManifest(manifest)) {
+    throw new Error("evaluateHypothesis: manifest content hash mismatch (tampered)");
+  }
+  const reasons = [];
+  const directionOk = manifest.direction === "increase" ? observed.effect > 0 : observed.effect < 0;
+  if (!directionOk) reasons.push("wrong_direction");
+  if (Math.abs(observed.effect) < manifest.minEffect) reasons.push("effect_too_small");
+  if (observed.pValue >= manifest.alpha) reasons.push("not_significant");
+  if (observed.n < manifest.preRegisteredN) reasons.push("undersampled");
+  return {
+    manifest,
+    observedN: observed.n,
+    observedEffect: observed.effect,
+    observedPValue: observed.pValue,
+    confirmed: reasons.length === 0,
+    rejectionReasons: reasons
+  };
+}
+
+// src/summary-report.ts
+function summaryTable(runs, opts = {}) {
+  const split = opts.split ?? "holdout";
+  const confidence = opts.confidence ?? 0.95;
+  const fdr = opts.fdr ?? 0.05;
+  const comparator = opts.comparator ?? null;
+  const scoreField = split === "holdout" ? "holdoutScore" : "searchScore";
+  const byCandidate = /* @__PURE__ */ new Map();
+  for (const r of runs) {
+    if (r.splitTag !== split) continue;
+    const v = r.outcome[scoreField];
+    if (typeof v !== "number" || !Number.isFinite(v)) continue;
+    const bucket = byCandidate.get(r.candidateId) ?? { runs: [], scores: [] };
+    bucket.runs.push(r);
+    bucket.scores.push(v);
+    byCandidate.set(r.candidateId, bucket);
+  }
+  const candidateIds = [...byCandidate.keys()].sort();
+  const compRuns = comparator ? byCandidate.get(comparator) : void 0;
+  const tentative = [];
+  for (const id of candidateIds) {
+    const bucket = byCandidate.get(id);
+    const ci = confidenceInterval(bucket.scores, confidence);
+    let rawP = Number.NaN;
+    let d = Number.NaN;
+    if (comparator && compRuns && id !== comparator) {
+      const paired = pairScoresByKey(bucket.runs, compRuns.runs, scoreField);
+      if (paired.before.length >= 6) {
+        rawP = wilcoxonSignedRank(paired.before, paired.after).p;
+      }
+      d = cohensD(compRuns.scores, bucket.scores);
+    }
+    tentative.push({
+      candidateId: id,
+      n: bucket.scores.length,
+      mean: ci.mean,
+      ciLow: ci.lower,
+      ciHigh: ci.upper,
+      qValue: rawP,
+      cohensD: d,
+      rawP
+    });
+  }
+  if (comparator) {
+    const idxs = [];
+    const ps = [];
+    for (let i = 0; i < tentative.length; i++) {
+      const r = tentative[i];
+      if (r.candidateId === comparator) continue;
+      if (!Number.isFinite(r.rawP)) continue;
+      idxs.push(i);
+      ps.push(r.rawP);
+    }
+    if (ps.length > 0) {
+      const { qValues } = benjaminiHochberg(ps, fdr);
+      for (let k = 0; k < idxs.length; k++) {
+        tentative[idxs[k]].qValue = qValues[k];
+      }
+    }
+  }
+  const rows = tentative.map(({ rawP: _rawP, ...rest }) => rest);
+  const markdown = renderSummaryTableMarkdown(rows, comparator, split);
+  return { rows, comparator, split, markdown };
+}
+function pairScoresByKey(candidate, baseline, scoreField) {
+  const baseIdx = /* @__PURE__ */ new Map();
+  for (const r of baseline) {
+    const v = r.outcome[scoreField];
+    if (typeof v === "number" && Number.isFinite(v)) {
+      baseIdx.set(`${r.experimentId}::${r.seed}`, v);
+    }
+  }
+  const before = [];
+  const after = [];
+  for (const r of candidate) {
+    const v = r.outcome[scoreField];
+    if (typeof v !== "number" || !Number.isFinite(v)) continue;
+    const key = `${r.experimentId}::${r.seed}`;
+    const b = baseIdx.get(key);
+    if (b === void 0) continue;
+    before.push(b);
+    after.push(v);
+  }
+  return { before, after };
+}
+function renderSummaryTableMarkdown(rows, comparator, split) {
+  const lines = [];
+  const cmpLabel = comparator ? ` (vs ${comparator})` : "";
+  lines.push(`Summary Table \u2014 ${split} split${cmpLabel}`);
+  lines.push("");
+  lines.push("| Candidate | N | Mean | 95% CI | q (BH) | Cohen's d |");
+  lines.push("|---|---:|---:|---|---:|---:|");
+  for (const r of rows) {
+    const ci = `[${fmt2(r.ciLow)}, ${fmt2(r.ciHigh)}]`;
+    const q = Number.isFinite(r.qValue) ? r.qValue.toFixed(4) : "\u2014";
+    const d = Number.isFinite(r.cohensD) ? r.cohensD.toFixed(3) : "\u2014";
+    lines.push(`| ${r.candidateId} | ${r.n} | ${fmt2(r.mean)} | ${ci} | ${q} | ${d} |`);
+  }
+  return lines.join("\n");
+}
+function paretoChart(runs, opts = {}) {
+  const split = opts.split ?? "holdout";
+  const scoreField = split === "holdout" ? "holdoutScore" : "searchScore";
+  const buckets = /* @__PURE__ */ new Map();
+  for (const r of runs) {
+    if (r.splitTag !== split) continue;
+    const v = r.outcome[scoreField];
+    if (typeof v !== "number" || !Number.isFinite(v)) continue;
+    const bucket = buckets.get(r.candidateId) ?? { cost: [], quality: [] };
+    bucket.cost.push(r.costUsd);
+    bucket.quality.push(v);
+    buckets.set(r.candidateId, bucket);
+  }
+  const points = [];
+  for (const [candidateId, bucket] of buckets.entries()) {
+    points.push({
+      candidateId,
+      cost: avg(bucket.cost),
+      quality: avg(bucket.quality),
+      n: bucket.cost.length,
+      onFrontier: false,
+      gate: opts.gateDecisions?.[candidateId] ? gateLabel(opts.gateDecisions[candidateId]) : void 0
+    });
+  }
+  for (const p of points) {
+    p.onFrontier = !points.some((q) => q !== p && dominates(q, p));
+  }
+  return {
+    kind: "pareto-cost-quality",
+    split,
+    axes: { x: "costUsd", y: "score" },
+    points
+  };
+}
+function dominates(a, b) {
+  return a.cost <= b.cost && a.quality >= b.quality && (a.cost < b.cost || a.quality > b.quality);
+}
+function gateLabel(d) {
+  if (d.promote) return "promote";
+  if (d.rejectionCode === "few_runs") return "reject_few_runs";
+  if (d.rejectionCode === "negative_delta") return "reject_negative_delta";
+  if (d.rejectionCode === "overfit_gap") return "reject_overfit_gap";
+  return null;
+}
+function gainHistogram(runs, candidateId, comparator, opts = {}) {
+  const split = opts.split ?? "holdout";
+  const scoreField = split === "holdout" ? "holdoutScore" : "searchScore";
+  const binCount = opts.bins ?? 11;
+  if (binCount < 1) throw new Error("gainHistogram: bins must be \u2265 1");
+  const candidate = runs.filter((r) => r.candidateId === candidateId && r.splitTag === split);
+  const baseline = runs.filter((r) => r.candidateId === comparator && r.splitTag === split);
+  const { before, after } = pairScoresByKey(candidate, baseline, scoreField);
+  const n = before.length;
+  if (n === 0) {
+    return {
+      kind: "gain-distribution",
+      candidateId,
+      comparator,
+      split,
+      n: 0,
+      bins: [],
+      median: 0,
+      ci: { low: 0, high: 0 }
+    };
+  }
+  const deltas = before.map((b, i) => after[i] - b);
+  const sortedDeltas = [...deltas].sort((a, b) => a - b);
+  const median = medianOfSorted(sortedDeltas);
+  const min = sortedDeltas[0];
+  const max = sortedDeltas[sortedDeltas.length - 1];
+  const bound = Math.max(Math.abs(min), Math.abs(max), 1e-6);
+  const lo = -bound;
+  const hi = bound;
+  const width = (hi - lo) / binCount;
+  const bins = [];
+  for (let i = 0; i < binCount; i++) {
+    bins.push({ lo: lo + i * width, hi: lo + (i + 1) * width, count: 0 });
+  }
+  for (const d of deltas) {
+    let idx = Math.floor((d - lo) / width);
+    if (idx < 0) idx = 0;
+    if (idx >= binCount) idx = binCount - 1;
+    bins[idx].count += 1;
+  }
+  const ci = pairedBootstrap(before, after, {
+    confidence: opts.confidence ?? 0.95,
+    resamples: opts.resamples ?? 2e3,
+    statistic: "median",
+    seed: opts.seed
+  });
+  return {
+    kind: "gain-distribution",
+    candidateId,
+    comparator,
+    split,
+    n,
+    bins,
+    median,
+    ci: { low: ci.low, high: ci.high }
+  };
+}
+var RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
+function pairedPosterior(runs, candidateId, comparator, opts) {
+  const scoreField = opts.split === "holdout" ? "holdoutScore" : "searchScore";
+  const candidate = runs.filter((r) => r.candidateId === candidateId && r.splitTag === opts.split);
+  const baseline = runs.filter((r) => r.candidateId === comparator && r.splitTag === opts.split);
+  const { before, after } = pairScoresByKey(candidate, baseline, scoreField);
+  const n = before.length;
+  if (n === 0) return null;
+  const deltas = before.map((b, i) => after[i] - b);
+  const meanDelta = deltas.reduce((s, x) => s + x, 0) / n;
+  const sortedDeltas = [...deltas].sort((a, b) => a - b);
+  const medianDelta = medianOfSorted(sortedDeltas);
+  const sdDelta = stdev(deltas, meanDelta);
+  const ci = pairedBootstrap(before, after, {
+    confidence: opts.confidence,
+    resamples: 2e3,
+    statistic: "median",
+    seed: opts.seed
+  });
+  const meanSamples = bootstrapMeanSamples(deltas, 2e3, opts.seed);
+  const prGreaterThanZero = meanSamples.length === 0 ? 0 : meanSamples.filter((s) => s > 0).length / meanSamples.length;
+  const prInRope = opts.rope === null || meanSamples.length === 0 ? null : meanSamples.filter((s) => s >= opts.rope.low && s <= opts.rope.high).length / meanSamples.length;
+  const dStandardised = pairedMde({ nPaired: n, alpha: opts.mdeAlpha, power: opts.mdePower });
+  const mde = sdDelta === 0 ? 0 : dStandardised * sdDelta;
+  return {
+    n,
+    meanDelta,
+    medianDelta,
+    sdDelta,
+    ci: { low: ci.low, high: ci.high },
+    prGreaterThanZero,
+    prInRope,
+    mde
+  };
+}
+function bootstrapMeanSamples(deltas, resamples, seed) {
+  const n = deltas.length;
+  if (n === 0) return [];
+  if (n === 1) return new Array(resamples).fill(deltas[0]);
+  const rng = seedRng(seed);
+  const samples = new Array(resamples);
+  for (let b = 0; b < resamples; b++) {
+    let sum = 0;
+    for (let k = 0; k < n; k++) sum += deltas[Math.floor(rng() * n)];
+    samples[b] = sum / n;
+  }
+  return samples;
+}
+function seedRng(seed) {
+  if (seed === void 0) return Math.random;
+  let s = seed >>> 0;
+  return () => {
+    s = s + 1831565813 >>> 0;
+    let t = s;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+function stdev(xs, mean3) {
+  if (xs.length < 2) return 0;
+  let sse = 0;
+  for (const x of xs) sse += (x - mean3) ** 2;
+  return Math.sqrt(sse / (xs.length - 1));
+}
+async function researchReport(runs, opts = {}) {
+  const split = opts.split ?? "holdout";
+  const comparator = opts.comparator ?? null;
+  const confidence = opts.confidence ?? 0.95;
+  const fdr = opts.fdr ?? 0.05;
+  const minPairs = Math.max(opts.minPairs ?? 20, RESEARCH_REPORT_HARD_PAIR_FLOOR);
+  const rope = opts.rope ?? null;
+  const mdePower = opts.mdePower ?? 0.8;
+  const mdeAlpha = opts.mdeAlpha ?? fdr;
+  const title = opts.title ?? "Agent Evaluation Research Report";
+  const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  const preregistrationHash = opts.preregistrationHash ?? null;
+  if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {
+    throw new Error(`researchReport: rope must satisfy low \u2264 high with finite bounds, got ${JSON.stringify(rope)}`);
+  }
+  const summary = summaryTable(runs, {
+    comparator: comparator ?? void 0,
+    split,
+    confidence,
+    fdr
+  });
+  const pareto = paretoChart(runs, { split, gateDecisions: opts.gateDecisions });
+  const candidateIds = opts.candidateIds ?? summary.rows.map((r) => r.candidateId).filter((id) => id !== comparator);
+  const gains = comparator ? candidateIds.map((id) => gainHistogram(runs, id, comparator, {
+    split,
+    confidence,
+    seed: opts.seed
+  })) : [];
+  const gainByCandidate = new Map(gains.map((g) => [g.candidateId, g]));
+  const paretoByCandidate = new Map(pareto.points.map((p) => [p.candidateId, p]));
+  const posteriorByCandidate = /* @__PURE__ */ new Map();
+  if (comparator) {
+    for (const id of candidateIds) {
+      posteriorByCandidate.set(id, pairedPosterior(runs, id, comparator, {
+        split,
+        confidence,
+        seed: opts.seed,
+        rope,
+        mdePower,
+        mdeAlpha
+      }));
+    }
+  }
+  const candidates = summary.rows.map((row) => {
+    const gain = gainByCandidate.get(row.candidateId);
+    const point = paretoByCandidate.get(row.candidateId);
+    const posterior = posteriorByCandidate.get(row.candidateId) ?? null;
+    const classified = classifyCandidate(row, {
+      comparator,
+      posterior,
+      point,
+      fdr,
+      minPairs,
+      rope
+    });
+    return {
+      candidateId: row.candidateId,
+      n: row.n,
+      mean: row.mean,
+      ciLow: row.ciLow,
+      ciHigh: row.ciHigh,
+      qValue: row.qValue,
+      cohensD: row.cohensD,
+      meanDeltaVsComparator: posterior ? posterior.meanDelta : null,
+      pairedN: posterior?.n ?? gain?.n ?? 0,
+      medianGain: posterior ? posterior.medianDelta : gain ? gain.median : null,
+      meanGain: posterior ? posterior.meanDelta : null,
+      gainCi: posterior ? posterior.ci : gain ? gain.ci : null,
+      prGreaterThanZero: posterior ? posterior.prGreaterThanZero : null,
+      prInRope: posterior ? posterior.prInRope : null,
+      mde: posterior ? posterior.mde : null,
+      onParetoFrontier: point?.onFrontier ?? false,
+      gate: point?.gate,
+      decision: classified.decision,
+      decisionReason: classified.reason
+    };
+  }).sort((a, b) => {
+    const decisionRank = decisionWeight(b.decision) - decisionWeight(a.decision);
+    if (decisionRank !== 0) return decisionRank;
+    return b.mean - a.mean;
+  });
+  const recommendation = buildRecommendation(candidates, {
+    comparator,
+    failureClusters: opts.failureClusters,
+    rope,
+    minPairs,
+    preregistrationHash
+  });
+  const executiveSummary = buildExecutiveSummary(candidates, recommendation, {
+    comparator,
+    split,
+    failureClusters: opts.failureClusters,
+    preregistrationHash
+  });
+  const methodology = buildMethodology({ split, comparator, fdr, minPairs, rope, confidence, mdePower, mdeAlpha });
+  const runFingerprint = await hashJson(canonicalize({
+    triples: runs.filter((r) => r.splitTag === split).map((r) => ({ runId: r.runId, candidateId: r.candidateId, splitTag: r.splitTag })).sort((a, b) => a.runId.localeCompare(b.runId)),
+    comparator,
+    split
+  }));
+  const markdown = renderResearchMarkdown({
+    title,
+    generatedAt,
+    split,
+    comparator,
+    rope,
+    runFingerprint,
+    preregistrationHash,
+    executiveSummary,
+    recommendation,
+    candidates,
+    summary,
+    pareto,
+    gains,
+    methodology,
+    failureClusters: opts.failureClusters
+  });
+  const html = renderResearchHtml(markdown, title);
+  return {
+    kind: "agent-eval-research-report",
+    title,
+    generatedAt,
+    split,
+    comparator,
+    runFingerprint,
+    preregistrationHash,
+    rope,
+    executiveSummary,
+    recommendation,
+    candidates,
+    summary,
+    charts: { pareto, gains },
+    methodology,
+    failureClusters: opts.failureClusters,
+    markdown,
+    html
+  };
+}
+function buildMethodology(ctx) {
+  const assumptions = [
+    "Pairs are matched by (experimentId, seed); the candidate and comparator see the same scenarios in the same order.",
+    "Paired deltas are exchangeable conditional on the matched scenario \u2014 no mid-run distribution shift.",
+    `Decisions are pre-specified at fdr=${ctx.fdr}, minPairs=${ctx.minPairs}, confidence=${ctx.confidence}; deviating from these post-hoc invalidates the false-discovery control.`
+  ];
+  if (ctx.rope) {
+    assumptions.push(`The Region of Practical Equivalence ${formatRope(ctx.rope)} is supplied by the domain owner; equivalent verdicts are only meaningful if that range is treated as the standing definition of "no material difference."`);
+  }
+  if (ctx.comparator === null) {
+    assumptions.push("No comparator was configured; this run is descriptive, not causal.");
+  }
+  const methods = [
+    "Marginal scores summarised with BH-FDR-adjusted Wilcoxon signed-rank q-values and Cohen's d via summaryTable.",
+    "Paired evidence summarised with bootstrap CI on the median delta and Bayesian-bootstrap-style Pr(\u0394>0) and Pr(\u0394\u2208ROPE) on the mean delta.",
+    `Minimum detectable effect reported per candidate at \u03B1=${ctx.mdeAlpha} (two-sided), power=${ctx.mdePower}, standardised by the observed paired-delta SD.`,
+    "Pareto frontier flagged as a separate axis (cost vs quality); a candidate can be on-frontier without winning the paired test.",
+    "Held-out gate decisions, when supplied, override the statistical verdict in the reject direction."
+  ];
+  const alternatives = [
+    "Paired t-test rejected: not robust to the heavy-tailed score distributions common in agent benchmarks.",
+    "Unpaired Mann\u2013Whitney rejected: matched scenarios make pairing free; unpaired throws away that variance reduction.",
+    "Sequential / always-valid inference (e-values, mSPRT) is the right tool for iterative sweeps and is out of scope for this single-look report \u2014 preregister and run once, or wrap this report in an alpha-spending schedule.",
+    "Hierarchical Bayesian shrinkage across many candidates is future work; the current ranking uses raw paired statistics."
+  ];
+  const whenNotToApply = [
+    `Paired N below ${RESEARCH_REPORT_HARD_PAIR_FLOOR} on any candidate \u2014 the bootstrap CI is degenerate.`,
+    "Comparator chosen post-hoc by inspecting the same data; q-values are no longer false-discovery-controlled.",
+    "Scenarios not drawn under a stable preregistered protocol; the report can describe the data but cannot anchor a launch decision.",
+    "Score distributions with mid-run shift (judge model swap, rubric change, infra outage) \u2014 pair exchangeability is violated."
+  ];
+  const citations = [
+    "Benjamini, Y. & Hochberg, Y. (1995). Controlling the false discovery rate: a practical and powerful approach to multiple testing. JRSS B, 57(1), 289\u2013300.",
+    "Wilcoxon, F. (1945). Individual comparisons by ranking methods. Biometrics Bulletin, 1(6), 80\u201383.",
+    "Efron, B. (1979). Bootstrap methods: another look at the jackknife. Annals of Statistics, 7(1), 1\u201326.",
+    "Rubin, D. B. (1981). The Bayesian bootstrap. Annals of Statistics, 9(1), 130\u2013134.",
+    "Kruschke, J. K. (2018). Rejecting or accepting parameter values in Bayesian estimation. Advances in Methods and Practices in Psychological Science, 1(2), 270\u2013280. (ROPE.)"
+  ];
+  return { assumptions, methods, alternatives, whenNotToApply, citations };
+}
+function formatRope(rope) {
+  return `[${fmt2(rope.low)}, ${fmt2(rope.high)}]`;
+}
+function classifyCandidate(row, ctx) {
+  if (ctx.comparator && row.candidateId === ctx.comparator) {
+    return { decision: "hold", reason: "Comparator baseline." };
+  }
+  if (!ctx.comparator) {
+    return {
+      decision: ctx.point?.onFrontier ? "hold" : "needs_more_data",
+      reason: "No comparator configured; report ranks candidates but cannot anchor a promotion call."
+    };
+  }
+  if (ctx.point?.gate && ctx.point.gate !== "promote") {
+    return { decision: "reject", reason: `Held-out gate returned ${ctx.point.gate}.` };
+  }
+  if (!ctx.posterior || ctx.posterior.n < RESEARCH_REPORT_HARD_PAIR_FLOOR) {
+    return {
+      decision: "needs_more_data",
+      reason: `Only ${ctx.posterior?.n ?? 0} paired observations; below hard floor of ${RESEARCH_REPORT_HARD_PAIR_FLOOR} for any paired inference.`
+    };
+  }
+  const ci = ctx.posterior.ci;
+  if (ctx.rope && ci.low >= ctx.rope.low && ci.high <= ctx.rope.high) {
+    return {
+      decision: "equivalent",
+      reason: `Paired-delta CI [${fmt2(ci.low)}, ${fmt2(ci.high)}] is fully inside ROPE ${formatRope(ctx.rope)}; candidate is practically equivalent to comparator.`
+    };
+  }
+  const significant = Number.isFinite(row.qValue) && row.qValue <= ctx.fdr;
+  const gainPositive = ci.low > 0;
+  const gainNegative = ci.high < 0;
+  if (gainNegative) {
+    return { decision: "reject", reason: `Paired-delta CI [${fmt2(ci.low)}, ${fmt2(ci.high)}] lies entirely below zero.` };
+  }
+  if (ctx.posterior.n < ctx.minPairs) {
+    return {
+      decision: "needs_more_data",
+      reason: `Only ${ctx.posterior.n} paired observations; minimum detectable effect at this N is ${fmt2(ctx.posterior.mde)} score units (need \u2265 ${ctx.minPairs} pairs to issue a directional verdict).`
+    };
+  }
+  if (significant && gainPositive) {
+    return {
+      decision: "promote",
+      reason: `BH-adjusted q=${fmt2(row.qValue)} \u2264 ${ctx.fdr} and paired-delta CI [${fmt2(ci.low)}, ${fmt2(ci.high)}] excludes zero; Pr(\u0394>0)=${fmt2(ctx.posterior.prGreaterThanZero)}.`
+    };
+  }
+  return {
+    decision: "hold",
+    reason: `Pr(\u0394>0)=${fmt2(ctx.posterior.prGreaterThanZero)} but CI [${fmt2(ci.low)}, ${fmt2(ci.high)}] crosses zero; effect not decisive at fdr=${ctx.fdr}.`
+  };
+}
+function buildRecommendation(candidates, ctx) {
+  const nonComparator = candidates.filter((c) => c.candidateId !== ctx.comparator);
+  const bestPromote = nonComparator.find((c) => c.decision === "promote");
+  const bestEquivalent = nonComparator.find((c) => c.decision === "equivalent");
+  const chosen = bestPromote ?? bestEquivalent ?? nonComparator[0] ?? null;
+  const decision = bestPromote ? "promote" : nonComparator.some((c) => c.decision === "needs_more_data") ? "needs_more_data" : bestEquivalent ? "equivalent" : nonComparator.some((c) => c.decision === "hold") ? "hold" : "reject";
+  const rationale = [];
+  const risks = [];
+  const nextActions = [];
+  if (chosen) {
+    rationale.push(`${chosen.candidateId}: ${chosen.decisionReason}`);
+    if (chosen.gainCi) {
+      const probSummary = chosen.prGreaterThanZero !== null ? `, Pr(\u0394>0)=${fmt2(chosen.prGreaterThanZero)}` : "";
+      rationale.push(`Median paired gain CI: [${fmt2(chosen.gainCi.low)}, ${fmt2(chosen.gainCi.high)}]${probSummary}.`);
+    }
+    if (chosen.mde !== null && Number.isFinite(chosen.mde)) {
+      rationale.push(`MDE at current paired N=${chosen.pairedN}: ${fmt2(chosen.mde)} score units.`);
+    }
+  }
+  if (!ctx.comparator) {
+    risks.push("No comparator was configured; verdict is descriptive, not causal.");
+    nextActions.push("Re-run with a stable comparator candidate for paired inference.");
+  }
+  if (!ctx.preregistrationHash) {
+    risks.push("No preregistration hash supplied; readers cannot verify the analysis was specified before data inspection.");
+    nextActions.push("Sign a HypothesisManifest before the next sweep and pass `preregistrationHash` so the report cites it.");
+  }
+  if (ctx.rope === null && nonComparator.length > 0) {
+    risks.push('No ROPE configured; the report cannot distinguish "equivalent" from "inconclusive".');
+    nextActions.push("Define a domain-specific Region of Practical Equivalence and pass it to lock in the equivalence threshold.");
+  }
+  const inconclusive = nonComparator.filter((c) => c.decision === "needs_more_data");
+  if (inconclusive.length > 0) {
+    const worst = inconclusive.reduce((a, b) => b.pairedN < a.pairedN ? b : a);
+    risks.push(`${inconclusive.length} candidate(s) below soft floor (${ctx.minPairs} pairs); thinnest is ${worst.candidateId} with ${worst.pairedN}.`);
+    nextActions.push(`Collect at least ${ctx.minPairs - worst.pairedN} more matched holdout runs for ${worst.candidateId}.`);
+  }
+  const rejected = nonComparator.filter((c) => c.decision === "reject");
+  if (rejected.length > 0) {
+    risks.push(`${rejected.length} candidate(s) failed the paired test or held-out gate; do not ship those variants.`);
+  }
+  if (ctx.failureClusters && ctx.failureClusters.clusters.length > 0) {
+    const top = ctx.failureClusters.clusters[0];
+    risks.push(`Top failure cluster: ${top.failureClass} across ${top.runCount} run(s).`);
+    nextActions.push("Prioritize the largest failure cluster before broad rollout.");
+  }
+  if (decision === "promote") {
+    nextActions.push("Ship behind the existing promotion gate and monitor canaries.");
+  } else if (decision === "hold") {
+    nextActions.push("Keep current production candidate while expanding holdout evidence.");
+  } else if (decision === "equivalent") {
+    nextActions.push("Either keep the comparator (no quality regression) or promote on cost/latency grounds \u2014 equivalence does not justify either; the choice is a product decision, not a stats one.");
+  } else if (decision === "reject") {
+    nextActions.push("Do not promote this sweep; inspect failures and generate a revised candidate.");
+  }
+  return {
+    decision,
+    candidateId: chosen?.candidateId ?? null,
+    rationale,
+    risks,
+    nextActions
+  };
+}
+function buildExecutiveSummary(candidates, recommendation, ctx) {
+  const lines = [];
+  const nonComparator = candidates.filter((c) => c.candidateId !== ctx.comparator);
+  lines.push(`Evaluated ${nonComparator.length} candidate(s) on the ${ctx.split} split${ctx.comparator ? ` against ${ctx.comparator}` : ""}.`);
+  lines.push(`Recommendation: ${recommendation.decision}${recommendation.candidateId ? ` ${recommendation.candidateId}` : ""}.`);
+  const promoted = nonComparator.filter((c) => c.decision === "promote").length;
+  const held = nonComparator.filter((c) => c.decision === "hold").length;
+  const equivalent = nonComparator.filter((c) => c.decision === "equivalent").length;
+  const rejected = nonComparator.filter((c) => c.decision === "reject").length;
+  const more = nonComparator.filter((c) => c.decision === "needs_more_data").length;
+  lines.push(`Decision mix: ${promoted} promote, ${equivalent} equivalent, ${held} hold, ${rejected} reject, ${more} need more data.`);
+  const frontier = nonComparator.filter((c) => c.onParetoFrontier).map((c) => c.candidateId);
+  if (frontier.length > 0) lines.push(`Pareto-frontier candidates: ${frontier.join(", ")}.`);
+  if (ctx.failureClusters) {
+    lines.push(`Failure clustering found ${ctx.failureClusters.totalFailures}/${ctx.failureClusters.totalRuns} failed runs across ${ctx.failureClusters.clusters.length} reportable cluster(s).`);
+  }
+  lines.push(ctx.preregistrationHash ? `Preregistered analysis: ${ctx.preregistrationHash.slice(0, 12)}\u2026` : "Analysis is post-hoc \u2014 no preregistration hash supplied.");
+  return lines;
+}
+function renderResearchMarkdown(report) {
+  const lines = [];
+  lines.push(`# ${report.title}`);
+  lines.push("");
+  lines.push(`**Generated:** ${report.generatedAt}`);
+  lines.push(`**Primary split:** ${report.split}`);
+  lines.push(`**Comparator:** ${report.comparator ?? "not configured"}`);
+  lines.push(`**ROPE:** ${report.rope ? formatRope(report.rope) : "not configured"}`);
+  lines.push(`**Run fingerprint:** \`${report.runFingerprint}\``);
+  lines.push(`**Preregistration:** ${report.preregistrationHash ? `\`${report.preregistrationHash}\`` : "none"}`);
+  lines.push("");
+  lines.push("## Executive Summary");
+  lines.push("");
+  for (const item of report.executiveSummary) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("## Recommendation");
+  lines.push("");
+  lines.push(`**Decision:** ${report.recommendation.decision}`);
+  lines.push(`**Candidate:** ${report.recommendation.candidateId ?? "N/A"}`);
+  lines.push("");
+  lines.push("### Rationale");
+  lines.push("");
+  for (const item of report.recommendation.rationale) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("### Risks");
+  lines.push("");
+  for (const item of report.recommendation.risks.length ? report.recommendation.risks : ["No material report-level risks detected."]) {
+    lines.push(`- ${item}`);
+  }
+  lines.push("");
+  lines.push("### Next Actions");
+  lines.push("");
+  for (const item of report.recommendation.nextActions) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("## Candidate Decision Table");
+  lines.push("");
+  lines.push("| Candidate | Decision | Mean | \u0394\u0304 | Pr(\u0394>0) | q | d | Paired N | Median Gain CI | MDE | Pareto | Gate |");
+  lines.push("|---|---|---:|---:|---:|---:|---:|---:|---|---:|---|---|");
+  for (const c of report.candidates) {
+    const delta = c.meanDeltaVsComparator === null ? "-" : signed(c.meanDeltaVsComparator);
+    const prGt = c.prGreaterThanZero === null ? "-" : c.prGreaterThanZero.toFixed(3);
+    const q = Number.isFinite(c.qValue) ? c.qValue.toFixed(4) : "-";
+    const d = Number.isFinite(c.cohensD) ? c.cohensD.toFixed(3) : "-";
+    const gain = c.gainCi ? `[${fmt2(c.gainCi.low)}, ${fmt2(c.gainCi.high)}]` : "-";
+    const mde = c.mde === null || !Number.isFinite(c.mde) ? "-" : fmt2(c.mde);
+    lines.push(`| ${c.candidateId} | ${c.decision} | ${fmt2(c.mean)} | ${delta} | ${prGt} | ${q} | ${d} | ${c.pairedN} | ${gain} | ${mde} | ${c.onParetoFrontier ? "yes" : "no"} | ${c.gate ?? "-"} |`);
+  }
+  lines.push("");
+  lines.push("## Statistical Summary");
+  lines.push("");
+  lines.push(report.summary.markdown);
+  lines.push("");
+  lines.push("## Methodology");
+  lines.push("");
+  lines.push("### Assumptions");
+  lines.push("");
+  for (const item of report.methodology.assumptions) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("### Methods");
+  lines.push("");
+  for (const item of report.methodology.methods) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("### Alternatives Considered");
+  lines.push("");
+  for (const item of report.methodology.alternatives) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("### When NOT To Apply");
+  lines.push("");
+  for (const item of report.methodology.whenNotToApply) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("### Citations");
+  lines.push("");
+  for (const item of report.methodology.citations) lines.push(`- ${item}`);
+  lines.push("");
+  lines.push("## Chart Specs");
+  lines.push("");
+  lines.push("The report carries JSON chart specs for Pareto cost/quality and paired gain histograms.");
+  lines.push("");
+  lines.push("```json");
+  lines.push(JSON.stringify({ pareto: report.pareto, gains: report.gains }, null, 2));
+  lines.push("```");
+  if (report.failureClusters) {
+    lines.push("");
+    lines.push("## Failure Clusters");
+    lines.push("");
+    lines.push("| Failure Class | Runs | Scenarios | Tool | Example |");
+    lines.push("|---|---:|---:|---|---|");
+    for (const c of report.failureClusters.clusters.slice(0, 10)) {
+      lines.push(`| ${c.failureClass} | ${c.runCount} | ${c.scenarioIds.length} | ${c.toolName ?? "-"} | ${escapePipes(c.exampleError ?? c.exampleRunId)} |`);
+    }
+  }
+  return lines.join("\n");
+}
+function renderResearchHtml(markdown, title) {
+  const body = markdownToHtml(markdown);
+  return [
+    "<!doctype html>",
+    '<html lang="en">',
+    "<head>",
+    '<meta charset="utf-8">',
+    '<meta name="viewport" content="width=device-width, initial-scale=1">',
+    `<title>${escapeHtml(title)}</title>`,
+    "<style>",
+    'body{font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;margin:0;color:#172026;background:#f7f8f8;}',
+    "main{max-width:1080px;margin:0 auto;padding:40px 24px 64px;background:#fff;min-height:100vh;}",
+    "h1{font-size:34px;line-height:1.15;margin:0 0 20px;}h2{margin-top:34px;border-top:1px solid #d9dfdf;padding-top:22px;}h3{margin-top:22px;}",
+    "p,li{line-height:1.55;}table{border-collapse:collapse;width:100%;margin:16px 0;font-size:14px;}th,td{border:1px solid #d9dfdf;padding:8px;text-align:left;}th{background:#eef2f2;}",
+    "code,pre{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;}pre{overflow:auto;background:#111827;color:#f9fafb;padding:16px;border-radius:6px;}",
+    "</style>",
+    "</head>",
+    "<body><main>",
+    body,
+    "</main></body></html>"
+  ].join("\n");
+}
+function markdownToHtml(markdown) {
+  const lines = markdown.split("\n");
+  const html = [];
+  let inList = false;
+  let inCode = false;
+  let code = [];
+  let table = [];
+  const flushList = () => {
+    if (inList) {
+      html.push("</ul>");
+      inList = false;
+    }
+  };
+  const flushTable = () => {
+    if (table.length === 0) return;
+    html.push(renderMarkdownTable(table));
+    table = [];
+  };
+  for (const line of lines) {
+    if (line.startsWith("```")) {
+      if (inCode) {
+        html.push(`<pre><code>${escapeHtml(code.join("\n"))}</code></pre>`);
+        code = [];
+        inCode = false;
+      } else {
+        flushList();
+        flushTable();
+        inCode = true;
+      }
+      continue;
+    }
+    if (inCode) {
+      code.push(line);
+      continue;
+    }
+    if (line.startsWith("|")) {
+      flushList();
+      table.push(line);
+      continue;
+    }
+    flushTable();
+    if (line.startsWith("- ")) {
+      if (!inList) {
+        html.push("<ul>");
+        inList = true;
+      }
+      html.push(`<li>${inlineMarkdown(line.slice(2))}</li>`);
+      continue;
+    }
+    flushList();
+    if (line.startsWith("# ")) html.push(`<h1>${inlineMarkdown(line.slice(2))}</h1>`);
+    else if (line.startsWith("## ")) html.push(`<h2>${inlineMarkdown(line.slice(3))}</h2>`);
+    else if (line.startsWith("### ")) html.push(`<h3>${inlineMarkdown(line.slice(4))}</h3>`);
+    else if (line.trim() === "") html.push("");
+    else html.push(`<p>${inlineMarkdown(line)}</p>`);
+  }
+  flushList();
+  flushTable();
+  return html.join("\n");
+}
+function renderMarkdownTable(lines) {
+  const rows = lines.filter((line) => !/^\|[-:\s|]+\|$/.test(line)).map((line) => line.slice(1, -1).split("|").map((cell) => inlineMarkdown(cell.trim())));
+  if (rows.length === 0) return "";
+  const [head, ...body] = rows;
+  const th = head.map((cell) => `<th>${cell}</th>`).join("");
+  const trs = body.map((row) => `<tr>${row.map((cell) => `<td>${cell}</td>`).join("")}</tr>`).join("\n");
+  return `<table><thead><tr>${th}</tr></thead><tbody>${trs}</tbody></table>`;
+}
+function inlineMarkdown(s) {
+  return escapeHtml(s).replace(/\*\*([^*]+)\*\*/g, "<strong>$1</strong>");
+}
+function escapeHtml(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+function escapePipes(s) {
+  return s.replace(/\|/g, "\\|");
+}
+function decisionWeight(decision) {
+  if (decision === "promote") return 5;
+  if (decision === "equivalent") return 4;
+  if (decision === "hold") return 3;
+  if (decision === "needs_more_data") return 2;
+  return 1;
+}
+function signed(x) {
+  return `${x >= 0 ? "+" : ""}${fmt2(x)}`;
+}
+function avg(xs) {
+  if (xs.length === 0) return Number.NaN;
+  return xs.reduce((s, x) => s + x, 0) / xs.length;
+}
+function medianOfSorted(sorted) {
+  if (sorted.length === 0) return 0;
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function fmt2(x) {
+  if (!Number.isFinite(x)) return String(x);
+  return x.toFixed(4);
+}
+
+// src/release-report.ts
+function renderReleaseReport(scorecard, options = {}) {
+  const title = options.title ?? `Release Report: ${scorecard.target}`;
+  const lines = [];
+  lines.push(`# ${title}`);
+  lines.push("");
+  lines.push(`Status: **${scorecard.status.toUpperCase()}**`);
+  lines.push(`Promote: **${scorecard.promote ? "yes" : "no"}**`);
+  if (scorecard.candidateId) lines.push(`Candidate: \`${scorecard.candidateId}\``);
+  if (scorecard.baselineId) lines.push(`Baseline: \`${scorecard.baselineId}\``);
+  lines.push("");
+  lines.push(scorecard.summary);
+  lines.push("");
+  lines.push("## Metrics");
+  lines.push("");
+  lines.push("| Metric | Value |");
+  lines.push("|---|---:|");
+  lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`);
+  lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`);
+  lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`);
+  lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`);
+  lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`);
+  lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`);
+  lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`);
+  lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`);
+  lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`);
+  lines.push(`| p95 wall time | ${Math.round(scorecard.metrics.p95WallMs)} ms |`);
+  lines.push("");
+  if (scorecard.issues.length > 0) {
+    lines.push("## Issues");
+    lines.push("");
+    for (const issue of scorecard.issues) {
+      lines.push(`- **${issue.severity}** \`${issue.code}\` (${issue.axis}): ${issue.detail}`);
+    }
+    lines.push("");
+  }
+  const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts);
+  if (surfaces.length > 0) {
+    lines.push("## Responsible Surfaces");
+    lines.push("");
+    for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`);
+    lines.push("");
+  }
+  const failures = entries(scorecard.metrics.failureModeCounts);
+  if (failures.length > 0) {
+    lines.push("## Failure Modes");
+    lines.push("");
+    for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`);
+    lines.push("");
+  }
+  if (options.runs && options.runs.length > 0) {
+    lines.push("## Run Summary");
+    lines.push("");
+    lines.push(summaryTable([...options.runs], {
+      comparator: options.comparator ?? scorecard.baselineId ?? void 0,
+      split: "holdout"
+    }).markdown);
+    lines.push("");
+  }
+  if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {
+    lines.push("## TraceAnalyst Findings");
+    lines.push("");
+    for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`);
+    lines.push("");
+  }
+  const nextActions = options.nextActions ?? defaultNextActions(scorecard);
+  if (nextActions.length > 0) {
+    lines.push("## Next Actions");
+    lines.push("");
+    for (const action of nextActions) lines.push(`- ${action}`);
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd() + "\n";
+}
+function defaultNextActions(scorecard) {
+  if (scorecard.promote) return ["Promote the candidate and keep canaries enabled."];
+  return scorecard.issues.filter((issue) => issue.severity === "critical").map((issue) => `Resolve ${issue.code}: ${issue.detail}`);
+}
+function entries(values) {
+  return Object.entries(values).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+}
+function pct(value) {
+  return Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : "n/a";
+}
+function num(value) {
+  return Number.isFinite(value) ? value.toFixed(3) : "n/a";
+}
+
+// src/promotion-gate.ts
+function bootstrapCi(baseline, candidate, options = {}) {
+  const alpha = options.alpha ?? 0.05;
+  const iterations = options.iterations ?? 1e3;
+  const minTotal = options.minTotalSamples ?? 6;
+  const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate));
+  const baselineMean = mean2(baseline);
+  const candidateMean = mean2(candidate);
+  const delta = candidateMean - baselineMean;
+  if (baseline.length + candidate.length < minTotal || baseline.length === 0 || candidate.length === 0) {
+    return {
+      baselineMean,
+      candidateMean,
+      delta,
+      ciLower: -Infinity,
+      ciUpper: Infinity,
+      iterations: 0,
+      alpha,
+      verdict: "INCONCLUSIVE"
+    };
+  }
+  const deltas = new Array(iterations);
+  for (let i = 0; i < iterations; i++) {
+    const bResample = resample(baseline, rng);
+    const cResample = resample(candidate, rng);
+    deltas[i] = mean2(cResample) - mean2(bResample);
+  }
+  deltas.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * iterations);
+  const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1;
+  const ciLower = deltas[Math.max(0, lowerIdx)];
+  const ciUpper = deltas[Math.min(iterations - 1, upperIdx)];
+  let verdict;
+  if (ciLower > 0) verdict = "ADVANCE";
+  else if (ciUpper < 0) verdict = "REVERT";
+  else if (delta >= 0) verdict = "KEEP";
+  else verdict = "INCONCLUSIVE";
+  return {
+    baselineMean,
+    candidateMean,
+    delta,
+    ciLower,
+    ciUpper,
+    iterations,
+    alpha,
+    verdict
+  };
+}
+function mean2(xs) {
+  if (xs.length === 0) return 0;
+  let s = 0;
+  for (const x of xs) s += x;
+  return s / xs.length;
+}
+function resample(xs, rng) {
+  const out = new Array(xs.length);
+  for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)];
+  return out;
+}
+function mulberry32(seed) {
+  let t = seed >>> 0;
+  return () => {
+    t += 1831565813;
+    let r = t;
+    r = Math.imul(r ^ r >>> 15, r | 1);
+    r ^= r + Math.imul(r ^ r >>> 7, r | 61);
+    return ((r ^ r >>> 14) >>> 0) / 4294967296;
+  };
+}
+function hashSeed(a, b) {
+  let h = 2166136261;
+  for (const x of [...a, ...b]) {
+    const view = new Float64Array([x]);
+    const bytes = new Uint8Array(view.buffer);
+    for (const byte of bytes) {
+      h ^= byte;
+      h = Math.imul(h, 16777619);
+    }
+  }
+  return h >>> 0;
+}
+async function judgeReplayGate(args) {
+  const concurrency = args.judgeConcurrency ?? 4;
+  const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency);
+  const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency);
+  const ci = bootstrapCi(baselineScores, candidateScores, {
+    ...args.alpha !== void 0 ? { alpha: args.alpha } : {},
+    ...args.iterations !== void 0 ? { iterations: args.iterations } : {},
+    ...args.seed !== void 0 ? { seed: args.seed } : {}
+  });
+  return {
+    ...ci,
+    baselineSamples: baselineScores.length,
+    candidateSamples: candidateScores.length
+  };
+}
+async function scoreAll(outputs, judge, concurrency) {
+  const results = new Array(outputs.length);
+  let next = 0;
+  async function worker() {
+    while (true) {
+      const i = next++;
+      if (i >= outputs.length) return;
+      const v = await judge(outputs[i]);
+      results[i] = Number.isFinite(v) ? v : 0;
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+  return results;
+}
+
+export {
+  releaseTraceEvidenceFromMultiShotTrials,
+  evaluateReleaseConfidence,
+  assertReleaseConfidence,
+  canonicalize,
+  hashJson,
+  signManifest,
+  verifyManifest,
+  evaluateHypothesis,
+  summaryTable,
+  paretoChart,
+  gainHistogram,
+  RESEARCH_REPORT_HARD_PAIR_FLOOR,
+  researchReport,
+  renderReleaseReport,
+  bootstrapCi,
+  judgeReplayGate
+};
+//# sourceMappingURL=chunk-3IX6QTB7.js.map