@tangle-network/agent-eval 0.21.0 → 0.22.0

package/dist/chunk-UAND2LOT.js

@@ -0,0 +1,738 @@
+import {
+  summaryTable
+} from "./chunk-IOXMGMHQ.js";
+
+// src/release-confidence.ts
+var DEFAULT_THRESHOLDS = {
+  requireCorpus: true,
+  minScenarioCount: 1,
+  minSearchRuns: 1,
+  minHoldoutRuns: 1,
+  requireHoldout: true,
+  minPassRate: 0.8,
+  minMeanScore: 0.7,
+  maxOverfitGap: 0.15,
+  maxMeanCostUsd: Number.POSITIVE_INFINITY,
+  maxP95WallMs: Number.POSITIVE_INFINITY,
+  requireAsiForFailures: true,
+  failureScoreThreshold: 0.5
+};
+function releaseTraceEvidenceFromMultiShotTrials(trials) {
+  return trials.map((trial) => ({
+    scenarioId: trial.scenarioId,
+    candidateId: trial.variantId,
+    split: trial.split === "holdout" ? "holdout" : trial.split === "dev" ? "dev" : "search",
+    score: trial.score,
+    ok: trial.ok,
+    turnCount: Array.isArray(trial.trace?.turns) ? trial.trace.turns.length : void 0,
+    costUsd: trial.cost,
+    durationMs: trial.durationMs,
+    failureMode: trial.error ? "runtime_error" : void 0,
+    asi: trial.asi,
+    metadata: trial.metadata
+  }));
+}
+function evaluateReleaseConfidence(input) {
+  const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds };
+  const candidateId = input.candidateId ?? null;
+  const runs = filterCandidate(input.runs ?? [], candidateId, input.baselineId);
+  const traces = filterTraceCandidate(input.traces ?? [], candidateId, input.baselineId);
+  const scenarios = input.scenarios ?? [];
+  const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length;
+  const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios);
+  const searchScores = scoresFor(runs, "search");
+  const holdoutScores = scoresFor(runs, "holdout");
+  const allScores = [...searchScores, ...holdoutScores];
+  const traceScores = traces.map((t) => t.score).filter(isFiniteNumber);
+  const scoreUniverse = allScores.length > 0 ? allScores : traceScores;
+  const searchRuns = runs.filter((r) => r.splitTag === "search").length;
+  const holdoutRuns = runs.filter((r) => r.splitTag === "holdout").length;
+  const searchMeanScore = mean(searchScores);
+  const holdoutMeanScore = mean(holdoutScores);
+  const metrics = {
+    scenarioCount,
+    searchRuns,
+    holdoutRuns,
+    passRate: passRate(runs, traces, thresholds.failureScoreThreshold),
+    meanScore: mean(scoreUniverse),
+    searchMeanScore,
+    holdoutMeanScore,
+    overfitGap: safeDiff(searchMeanScore, holdoutMeanScore),
+    meanCostUsd: mean([...runs.map((r) => r.costUsd), ...traces.map((t) => t.costUsd).filter(isFiniteNumber)]),
+    p95WallMs: percentile([...runs.map((r) => r.wallMs), ...traces.map((t) => t.durationMs).filter(isFiniteNumber)], 0.95),
+    failedRows: failedRows(runs, traces, thresholds.failureScoreThreshold).length,
+    failuresWithAsi: failedRows(runs, traces, thresholds.failureScoreThreshold).filter((row) => row.hasAsi).length,
+    singleShotTraces: traces.filter((t) => t.turnCount === 1).length,
+    multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,
+    splitCounts,
+    domainCounts: countDomains(scenarios),
+    failureModeCounts: countFailureModes(runs, traces, thresholds.failureScoreThreshold),
+    responsibleSurfaceCounts: countResponsibleSurfaces(traces)
+  };
+  const issues = [];
+  checkCorpus(input, thresholds, metrics, issues);
+  checkQuality(thresholds, metrics, issues);
+  checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues);
+  checkDiagnostics(thresholds, metrics, issues);
+  checkEfficiency(thresholds, metrics, issues);
+  const axes = buildAxes(metrics, thresholds, input.gateDecision ?? null, issues);
+  const status = issues.some((i) => i.severity === "critical") ? "fail" : issues.length > 0 ? "warn" : "pass";
+  return {
+    target: input.target,
+    candidateId,
+    baselineId: input.baselineId ?? null,
+    status,
+    promote: status === "pass" && (input.gateDecision ? input.gateDecision.promote : true),
+    axes,
+    issues,
+    metrics,
+    dataset: input.dataset ?? null,
+    gateDecision: input.gateDecision ?? null,
+    summary: renderSummary(input.target, status, metrics, issues)
+  };
+}
+function assertReleaseConfidence(input) {
+  const scorecard = evaluateReleaseConfidence(input);
+  if (scorecard.status === "fail") {
+    throw new Error(scorecard.summary);
+  }
+  return scorecard;
+}
+function filterCandidate(runs, candidateId, baselineId) {
+  if (candidateId) return runs.filter((r) => r.candidateId === candidateId);
+  if (baselineId) return runs.filter((r) => r.candidateId !== baselineId);
+  return [...runs];
+}
+function filterTraceCandidate(traces, candidateId, baselineId) {
+  if (candidateId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId === candidateId);
+  if (baselineId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId !== baselineId);
+  return [...traces];
+}
+function checkCorpus(input, thresholds, metrics, issues) {
+  if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) {
+    issues.push({ axis: "corpus", severity: "critical", code: "missing_corpus", detail: "No Dataset manifest or scenarios supplied." });
+  }
+  if (metrics.scenarioCount < thresholds.minScenarioCount) {
+    issues.push({ axis: "corpus", severity: "critical", code: "few_scenarios", detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.` });
+  }
+  if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) {
+    issues.push({ axis: "corpus", severity: "critical", code: "missing_holdout_split", detail: "Corpus has no holdout scenarios." });
+  }
+}
+function checkQuality(thresholds, metrics, issues) {
+  if (metrics.searchRuns < thresholds.minSearchRuns) {
+    issues.push({ axis: "quality", severity: "critical", code: "few_search_runs", detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.` });
+  }
+  if (metrics.passRate < thresholds.minPassRate) {
+    issues.push({ axis: "quality", severity: "critical", code: "low_pass_rate", detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.` });
+  }
+  if (metrics.meanScore < thresholds.minMeanScore) {
+    issues.push({ axis: "quality", severity: "critical", code: "low_mean_score", detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.` });
+  }
+}
+function checkGeneralization(gateDecision, thresholds, metrics, issues) {
+  if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {
+    issues.push({ axis: "generalization", severity: "critical", code: "few_holdout_runs", detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.` });
+  }
+  if (Number.isFinite(metrics.overfitGap) && metrics.overfitGap > thresholds.maxOverfitGap) {
+    issues.push({ axis: "generalization", severity: "critical", code: "overfit_gap", detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.` });
+  }
+  if (gateDecision && !gateDecision.promote) {
+    issues.push({ axis: "generalization", severity: "critical", code: `gate_${gateDecision.rejectionCode ?? "reject"}`, detail: gateDecision.reason });
+  }
+}
+function checkDiagnostics(thresholds, metrics, issues) {
+  if (!thresholds.requireAsiForFailures) return;
+  if (metrics.failedRows > metrics.failuresWithAsi) {
+    issues.push({
+      axis: "diagnostics",
+      severity: "critical",
+      code: "missing_failure_asi",
+      detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`
+    });
+  }
+}
+function checkEfficiency(thresholds, metrics, issues) {
+  if (metrics.meanCostUsd > thresholds.maxMeanCostUsd) {
+    issues.push({ axis: "efficiency", severity: "critical", code: "cost_budget", detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.` });
+  }
+  if (metrics.p95WallMs > thresholds.maxP95WallMs) {
+    issues.push({ axis: "efficiency", severity: "critical", code: "latency_budget", detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.` });
+  }
+}
+function buildAxes(metrics, thresholds, gateDecision, issues) {
+  return [
+    axis("corpus", issues, bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)), `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`),
+    axis("quality", issues, Math.min(metrics.passRate, metrics.meanScore), `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`),
+    axis("generalization", issues, gateDecision && !gateDecision.promote ? 0 : gapScore(metrics.overfitGap, thresholds.maxOverfitGap), `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`),
+    axis("diagnostics", issues, metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows, `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`),
+    axis("efficiency", issues, efficiencyScore(metrics, thresholds), `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`)
+  ];
+}
+function axis(name, issues, score, detail) {
+  const own = issues.filter((i) => i.axis === name);
+  const status = own.some((i) => i.severity === "critical") ? "fail" : own.length > 0 ? "warn" : "pass";
+  return { name, status, score: bounded(score), detail };
+}
+function countScenarioSplits(scenarios) {
+  const counts = { train: 0, dev: 0, test: 0, holdout: 0 };
+  for (const scenario of scenarios) counts[scenario.split ?? "train"]++;
+  return counts;
+}
+function countDomains(scenarios) {
+  const out = {};
+  for (const scenario of scenarios) {
+    const domain = scenario.tags?.domain ?? scenario.tags?.category ?? "uncategorized";
+    out[domain] = (out[domain] ?? 0) + 1;
+  }
+  return out;
+}
+function countFailureModes(runs, traces, threshold) {
+  const out = {};
+  for (const run of runs) {
+    const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+    if (run.failureMode || score !== void 0 && score < threshold) {
+      const mode = run.failureMode ?? "low_score";
+      out[mode] = (out[mode] ?? 0) + 1;
+    }
+  }
+  for (const trace of traces) {
+    if (trace.failureMode || trace.ok === false || trace.score !== void 0 && trace.score < threshold) {
+      const mode = trace.failureMode ?? (trace.ok === false ? "not_ok" : "low_score");
+      out[mode] = (out[mode] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function countResponsibleSurfaces(traces) {
+  const out = {};
+  for (const trace of traces) {
+    for (const asi of trace.asi ?? []) {
+      const surface = asi.responsibleSurface ?? "unknown";
+      out[surface] = (out[surface] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function failedRows(runs, traces, threshold) {
+  const out = [];
+  for (const run of runs) {
+    const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+    if (run.failureMode || score !== void 0 && score < threshold) {
+      const asiMetric = run.outcome.raw.asi;
+      out.push({ hasAsi: typeof asiMetric === "number" && asiMetric > 0 });
+    }
+  }
+  for (const trace of traces) {
+    if (trace.failureMode || trace.ok === false || trace.score !== void 0 && trace.score < threshold) {
+      out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 });
+    }
+  }
+  return out;
+}
+function passRate(runs, traces, threshold) {
+  const outcomes = [
+    ...runs.map((run) => {
+      const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+      return !run.failureMode && score !== void 0 && score >= threshold;
+    }),
+    ...traces.map((trace) => trace.ok !== false && (trace.score === void 0 || trace.score >= threshold))
+  ];
+  if (outcomes.length === 0) return 0;
+  return outcomes.filter(Boolean).length / outcomes.length;
+}
+function scoresFor(runs, split) {
+  return runs.filter((run) => run.splitTag === split).map((run) => split === "holdout" ? run.outcome.holdoutScore : run.outcome.searchScore).filter(isFiniteNumber);
+}
+function mean(xs) {
+  if (xs.length === 0) return Number.NaN;
+  return xs.reduce((sum, x) => sum + x, 0) / xs.length;
+}
+function percentile(xs, p) {
+  if (xs.length === 0) return Number.NaN;
+  const sorted = [...xs].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
+}
+function isFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+function safeDiff(a, b) {
+  if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN;
+  return a - b;
+}
+function gapScore(gap, maxGap) {
+  if (!Number.isFinite(gap)) return 0;
+  if (maxGap <= 0) return gap <= 0 ? 1 : 0;
+  return bounded(1 - Math.max(0, gap) / maxGap);
+}
+function efficiencyScore(metrics, thresholds) {
+  const cost = Number.isFinite(thresholds.maxMeanCostUsd) && Number.isFinite(metrics.meanCostUsd) ? bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12)) : 1;
+  const latency = Number.isFinite(thresholds.maxP95WallMs) && Number.isFinite(metrics.p95WallMs) ? bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12)) : 1;
+  return Math.min(cost, latency);
+}
+function bounded(x) {
+  if (!Number.isFinite(x)) return 0;
+  return Math.max(0, Math.min(1, x));
+}
+function renderSummary(target, status, metrics, issues) {
+  const prefix = `release confidence ${status}: ${target}`;
+  const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`;
+  if (issues.length === 0) return `${prefix}; ${metricText}`;
+  return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(",")}`;
+}
+function fmt(x) {
+  if (!Number.isFinite(x)) return String(x);
+  return x.toFixed(4);
+}
+
+// src/meta-eval/rubric-predictive-validity.ts
+async function rubricPredictiveValidity(input) {
+  const minSamples = input.minSamples ?? 8;
+  const reduction = input.reduction ?? "latest";
+  const resamples = input.bootstrapResamples ?? 500;
+  const rng = makeRng(input.seed);
+  const outcomes = await input.outcomes.list();
+  const outcomesByRun = /* @__PURE__ */ new Map();
+  for (const o of outcomes) {
+    const arr = outcomesByRun.get(o.runId) ?? [];
+    arr.push(o);
+    outcomesByRun.set(o.runId, arr);
+  }
+  const observedRubrics = /* @__PURE__ */ new Set();
+  for (const r of input.runs) {
+    for (const k of Object.keys(r.outcome.raw)) observedRubrics.add(k);
+  }
+  const rubrics = input.rubrics ?? [...observedRubrics];
+  const buckets = [];
+  for (const r of rubrics) {
+    for (const o of input.outcomeMetrics) {
+      buckets.push({ rubric: r, outcome: o, xs: [], ys: [] });
+    }
+  }
+  let joined = 0;
+  let skipped = 0;
+  for (const run of input.runs) {
+    const os = outcomesByRun.get(run.runId);
+    if (!os || os.length === 0) {
+      skipped++;
+      continue;
+    }
+    let joinedThisRun = false;
+    for (const r of rubrics) {
+      const x = run.outcome.raw[r];
+      if (typeof x !== "number" || !Number.isFinite(x)) continue;
+      for (const o of input.outcomeMetrics) {
+        const values = os.map((row) => row.metrics[o]).filter((v) => typeof v === "number" && Number.isFinite(v));
+        if (values.length === 0) continue;
+        const y = reduce(values, os, o, reduction);
+        if (y === null) continue;
+        const bucket = buckets.find((b) => b.rubric === r && b.outcome === o);
+        bucket.xs.push(x);
+        bucket.ys.push(y);
+        joinedThisRun = true;
+      }
+    }
+    if (joinedThisRun) joined++;
+  }
+  const pairs = [];
+  for (const b of buckets) {
+    if (b.xs.length < minSamples) continue;
+    const pearson = pearsonR(b.xs, b.ys);
+    const spearman = pearsonR(rankWithTies(b.xs), rankWithTies(b.ys));
+    const ci = bootstrapCi(b.xs, b.ys, resamples, rng);
+    const verdict = Math.abs(spearman) >= 0.7 ? "load_bearing" : Math.abs(spearman) >= 0.4 ? "informative" : "decorative";
+    pairs.push({
+      rubric: b.rubric,
+      outcome: b.outcome,
+      n: b.xs.length,
+      pearson,
+      spearman,
+      ci95: ci,
+      verdict
+    });
+  }
+  const byRubric = /* @__PURE__ */ new Map();
+  for (const p of pairs) {
+    const arr = byRubric.get(p.rubric) ?? [];
+    arr.push(p);
+    byRubric.set(p.rubric, arr);
+  }
+  const ranked = [...byRubric.entries()].map(([rubric, ps]) => {
+    const best = ps.reduce((a, b) => Math.abs(b.spearman) > Math.abs(a.spearman) ? b : a);
+    return {
+      rubric,
+      bestOutcome: best.outcome,
+      spearman: best.spearman,
+      pearson: best.pearson,
+      n: best.n,
+      verdict: best.verdict
+    };
+  }).sort((a, b) => Math.abs(b.spearman) - Math.abs(a.spearman));
+  const rubricsWithoutData = rubrics.filter((r) => !byRubric.has(r));
+  return { pairs, ranked, joinedSamples: joined, skippedRuns: skipped, rubricsWithoutData };
+}
+function reduce(values, outcomes, metric, kind) {
+  if (values.length === 0) return null;
+  if (kind === "mean") return values.reduce((s, v) => s + v, 0) / values.length;
+  if (kind === "max") return Math.max(...values);
+  const sorted = [...outcomes].filter((o) => typeof o.metrics[metric] === "number").sort((a, b) => b.capturedAt - a.capturedAt);
+  return sorted[0]?.metrics[metric] ?? null;
+}
+function pearsonR(a, b) {
+  if (a.length !== b.length || a.length < 2) return Number.NaN;
+  const ma = a.reduce((s, v) => s + v, 0) / a.length;
+  const mb = b.reduce((s, v) => s + v, 0) / b.length;
+  let num2 = 0, da = 0, db = 0;
+  for (let i = 0; i < a.length; i++) {
+    const xa = a[i] - ma;
+    const xb = b[i] - mb;
+    num2 += xa * xb;
+    da += xa * xa;
+    db += xb * xb;
+  }
+  if (da === 0 || db === 0) return da === 0 && db === 0 ? 1 : 0;
+  return num2 / Math.sqrt(da * db);
+}
+function rankWithTies(xs) {
+  const indexed = xs.map((v, i) => ({ v, i })).sort((a, b) => a.v - b.v);
+  const r = new Array(xs.length);
+  for (let i = 0; i < indexed.length; ) {
+    let j = i;
+    while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
+    const avg = (i + j + 2) / 2;
+    for (let k = i; k <= j; k++) r[indexed[k].i] = avg;
+    i = j + 1;
+  }
+  return r;
+}
+function bootstrapCi(xs, ys, iterations, rng) {
+  const n = xs.length;
+  if (n < 3) return { low: Number.NaN, high: Number.NaN };
+  const samples = [];
+  for (let b = 0; b < iterations; b++) {
+    const rx = new Array(n);
+    const ry = new Array(n);
+    for (let i = 0; i < n; i++) {
+      const idx = Math.floor(rng() * n);
+      rx[i] = xs[idx];
+      ry[i] = ys[idx];
+    }
+    const r = pearsonR(rx, ry);
+    if (Number.isFinite(r)) samples.push(r);
+  }
+  samples.sort((a, b) => a - b);
+  if (samples.length === 0) return { low: Number.NaN, high: Number.NaN };
+  return {
+    low: samples[Math.floor(0.025 * samples.length)],
+    high: samples[Math.min(samples.length - 1, Math.floor(0.975 * samples.length))]
+  };
+}
+function makeRng(seed) {
+  if (seed === void 0) return Math.random;
+  let s = seed >>> 0;
+  return () => {
+    s = s + 1831565813 >>> 0;
+    let t = s;
+    t = Math.imul(t ^ t >>> 15, t | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+
+// src/sequential.ts
+function pairedEvalueSequence(deltas, opts = {}) {
+  const c = opts.bound ?? 1;
+  const alpha = opts.alpha ?? 0.05;
+  const initialShrink = opts.initialBetShrinkage ?? 0.5;
+  const rope = opts.rope ?? null;
+  if (c <= 0) throw new Error("pairedEvalueSequence: bound must be > 0");
+  if (alpha <= 0 || alpha >= 1) throw new Error("pairedEvalueSequence: alpha must be in (0,1)");
+  if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {
+    throw new Error("pairedEvalueSequence: rope must satisfy low \u2264 high");
+  }
+  const steps = [];
+  let clipped = false;
+  let evalue = 1;
+  let decisionFiredAt = null;
+  let sum = 0;
+  let sumSq = 0;
+  let count = 0;
+  for (let i = 0; i < deltas.length; i++) {
+    let d = deltas[i];
+    if (d < -c || d > c) {
+      d = Math.max(-c, Math.min(c, d));
+      clipped = true;
+    }
+    const muHat = count === 0 ? 0 : sum / count;
+    const varHat = count === 0 ? c * c : Math.max(1e-12, sumSq / count - muHat * muHat);
+    const t = i + 1;
+    const shrink = initialShrink * Math.min(1, count / 32);
+    let lambda = muHat / (varHat + c * c) * shrink;
+    const lambdaMax = 0.99 / c;
+    if (lambda > lambdaMax) lambda = lambdaMax;
+    if (lambda < -lambdaMax) lambda = -lambdaMax;
+    evalue = evalue * (1 + lambda * d);
+    if (!Number.isFinite(evalue) || evalue < 0) evalue = 0;
+    sum += d;
+    sumSq += d * d;
+    count += 1;
+    const pValue = Math.min(1, 1 / Math.max(evalue, 1e-300));
+    const cs = empiricalBernsteinCs(sum, sumSq, count, c, alpha);
+    let decision = "continue";
+    if (rope && cs.low >= rope.low && cs.high <= rope.high) decision = "equivalent";
+    else if (evalue >= 2 / alpha && muHat > 0) decision = "promote_now";
+    else if (evalue >= 2 / alpha && muHat < 0) decision = "reject_now";
+    else if (rope && cs.high < rope.low) decision = "reject_now";
+    if (decision !== "continue" && decisionFiredAt === null) decisionFiredAt = t;
+    steps.push({ t, delta: d, evalue, pValue, csLow: cs.low, csHigh: cs.high, decision });
+  }
+  const finalDecision = steps.length === 0 ? "continue" : steps[steps.length - 1].decision;
+  return { steps, finalDecision, decisionFiredAt, clipped };
+}
+function evaluateInterimReleaseConfidence(input) {
+  const candidates = input.deltaSeries.map((s) => {
+    const seq = pairedEvalueSequence(s.deltas, {
+      alpha: input.alpha,
+      bound: input.bound,
+      rope: input.rope
+    });
+    const last = seq.steps[seq.steps.length - 1];
+    return {
+      candidateId: s.candidateId,
+      decision: seq.finalDecision,
+      decisionFiredAt: seq.decisionFiredAt,
+      finalEvalue: last?.evalue ?? 1,
+      finalPValue: last?.pValue ?? 1,
+      pairs: seq.steps.length,
+      csLow: last?.csLow ?? Number.NEGATIVE_INFINITY,
+      csHigh: last?.csHigh ?? Number.POSITIVE_INFINITY
+    };
+  });
+  const promote = candidates.find((c) => c.decision === "promote_now");
+  if (promote) return { candidates, recommendation: { decision: "promote_now", candidateId: promote.candidateId } };
+  const live = candidates.find((c) => c.decision === "continue");
+  if (live) return { candidates, recommendation: { decision: "continue", candidateId: null } };
+  const equiv = candidates.find((c) => c.decision === "equivalent");
+  if (equiv) return { candidates, recommendation: { decision: "equivalent", candidateId: equiv.candidateId } };
+  return { candidates, recommendation: { decision: "reject_now", candidateId: null } };
+}
+function empiricalBernsteinCs(sum, sumSq, n, bound, alpha) {
+  if (n === 0) return { low: -bound, high: bound };
+  const mean3 = sum / n;
+  const variance = Math.max(0, sumSq / n - mean3 * mean3);
+  const psi = Math.log(2 / alpha) + 1.7 * Math.log(Math.log(Math.max(Math.E, n)) + 1);
+  const radius = Math.sqrt(2 * variance * psi / n) + 3 * bound * psi / n;
+  return { low: mean3 - radius, high: mean3 + radius };
+}
+
+// src/release-report.ts
+function renderReleaseReport(scorecard, options = {}) {
+  const title = options.title ?? `Release Report: ${scorecard.target}`;
+  const lines = [];
+  lines.push(`# ${title}`);
+  lines.push("");
+  lines.push(`Status: **${scorecard.status.toUpperCase()}**`);
+  lines.push(`Promote: **${scorecard.promote ? "yes" : "no"}**`);
+  if (scorecard.candidateId) lines.push(`Candidate: \`${scorecard.candidateId}\``);
+  if (scorecard.baselineId) lines.push(`Baseline: \`${scorecard.baselineId}\``);
+  lines.push("");
+  lines.push(scorecard.summary);
+  lines.push("");
+  lines.push("## Metrics");
+  lines.push("");
+  lines.push("| Metric | Value |");
+  lines.push("|---|---:|");
+  lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`);
+  lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`);
+  lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`);
+  lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`);
+  lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`);
+  lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`);
+  lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`);
+  lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`);
+  lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`);
+  lines.push(`| p95 wall time | ${Math.round(scorecard.metrics.p95WallMs)} ms |`);
+  lines.push("");
+  if (scorecard.issues.length > 0) {
+    lines.push("## Issues");
+    lines.push("");
+    for (const issue of scorecard.issues) {
+      lines.push(`- **${issue.severity}** \`${issue.code}\` (${issue.axis}): ${issue.detail}`);
+    }
+    lines.push("");
+  }
+  const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts);
+  if (surfaces.length > 0) {
+    lines.push("## Responsible Surfaces");
+    lines.push("");
+    for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`);
+    lines.push("");
+  }
+  const failures = entries(scorecard.metrics.failureModeCounts);
+  if (failures.length > 0) {
+    lines.push("## Failure Modes");
+    lines.push("");
+    for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`);
+    lines.push("");
+  }
+  if (options.runs && options.runs.length > 0) {
+    lines.push("## Run Summary");
+    lines.push("");
+    lines.push(summaryTable([...options.runs], {
+      comparator: options.comparator ?? scorecard.baselineId ?? void 0,
+      split: "holdout"
+    }).markdown);
+    lines.push("");
+  }
+  if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {
+    lines.push("## TraceAnalyst Findings");
+    lines.push("");
+    for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`);
+    lines.push("");
+  }
+  const nextActions = options.nextActions ?? defaultNextActions(scorecard);
+  if (nextActions.length > 0) {
+    lines.push("## Next Actions");
+    lines.push("");
+    for (const action of nextActions) lines.push(`- ${action}`);
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd() + "\n";
+}
+function defaultNextActions(scorecard) {
+  if (scorecard.promote) return ["Promote the candidate and keep canaries enabled."];
+  return scorecard.issues.filter((issue) => issue.severity === "critical").map((issue) => `Resolve ${issue.code}: ${issue.detail}`);
+}
+function entries(values) {
+  return Object.entries(values).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+}
+function pct(value) {
+  return Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : "n/a";
+}
+function num(value) {
+  return Number.isFinite(value) ? value.toFixed(3) : "n/a";
+}
+
+// src/promotion-gate.ts
+function bootstrapCi2(baseline, candidate, options = {}) {
+  const alpha = options.alpha ?? 0.05;
+  const iterations = options.iterations ?? 1e3;
+  const minTotal = options.minTotalSamples ?? 6;
+  const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate));
+  const baselineMean = mean2(baseline);
+  const candidateMean = mean2(candidate);
+  const delta = candidateMean - baselineMean;
+  if (baseline.length + candidate.length < minTotal || baseline.length === 0 || candidate.length === 0) {
+    return {
+      baselineMean,
+      candidateMean,
+      delta,
+      ciLower: -Infinity,
+      ciUpper: Infinity,
+      iterations: 0,
+      alpha,
+      verdict: "INCONCLUSIVE"
+    };
+  }
+  const deltas = new Array(iterations);
+  for (let i = 0; i < iterations; i++) {
+    const bResample = resample(baseline, rng);
+    const cResample = resample(candidate, rng);
+    deltas[i] = mean2(cResample) - mean2(bResample);
+  }
+  deltas.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * iterations);
+  const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1;
+  const ciLower = deltas[Math.max(0, lowerIdx)];
+  const ciUpper = deltas[Math.min(iterations - 1, upperIdx)];
+  let verdict;
+  if (ciLower > 0) verdict = "ADVANCE";
+  else if (ciUpper < 0) verdict = "REVERT";
+  else if (delta >= 0) verdict = "KEEP";
+  else verdict = "INCONCLUSIVE";
+  return {
+    baselineMean,
+    candidateMean,
+    delta,
+    ciLower,
+    ciUpper,
+    iterations,
+    alpha,
+    verdict
+  };
+}
+function mean2(xs) {
+  if (xs.length === 0) return 0;
+  let s = 0;
+  for (const x of xs) s += x;
+  return s / xs.length;
+}
+function resample(xs, rng) {
+  const out = new Array(xs.length);
+  for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)];
+  return out;
+}
+function mulberry32(seed) {
+  let t = seed >>> 0;
+  return () => {
+    t += 1831565813;
+    let r = t;
+    r = Math.imul(r ^ r >>> 15, r | 1);
+    r ^= r + Math.imul(r ^ r >>> 7, r | 61);
+    return ((r ^ r >>> 14) >>> 0) / 4294967296;
+  };
+}
+function hashSeed(a, b) {
+  let h = 2166136261;
+  for (const x of [...a, ...b]) {
+    const view = new Float64Array([x]);
+    const bytes = new Uint8Array(view.buffer);
+    for (const byte of bytes) {
+      h ^= byte;
+      h = Math.imul(h, 16777619);
+    }
+  }
+  return h >>> 0;
+}
+async function judgeReplayGate(args) {
+  const concurrency = args.judgeConcurrency ?? 4;
+  const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency);
+  const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency);
+  const ci = bootstrapCi2(baselineScores, candidateScores, {
+    ...args.alpha !== void 0 ? { alpha: args.alpha } : {},
+    ...args.iterations !== void 0 ? { iterations: args.iterations } : {},
+    ...args.seed !== void 0 ? { seed: args.seed } : {}
+  });
+  return {
+    ...ci,
+    baselineSamples: baselineScores.length,
+    candidateSamples: candidateScores.length
+  };
+}
+async function scoreAll(outputs, judge, concurrency) {
+  const results = new Array(outputs.length);
+  let next = 0;
+  async function worker() {
+    while (true) {
+      const i = next++;
+      if (i >= outputs.length) return;
+      const v = await judge(outputs[i]);
+      results[i] = Number.isFinite(v) ? v : 0;
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+  return results;
+}
+
+export {
+  releaseTraceEvidenceFromMultiShotTrials,
+  evaluateReleaseConfidence,
+  assertReleaseConfidence,
+  rubricPredictiveValidity,
+  pairedEvalueSequence,
+  evaluateInterimReleaseConfidence,
+  renderReleaseReport,
+  bootstrapCi2 as bootstrapCi,
+  judgeReplayGate
+};
+//# sourceMappingURL=chunk-UAND2LOT.js.map