@tangle-network/agent-eval 0.21.0 → 0.23.0

package/dist/chunk-7EAUOUQS.js

@@ -0,0 +1,495 @@
+import {
+  summaryTable
+} from "./chunk-IOXMGMHQ.js";
+
+// src/release-confidence.ts
+var DEFAULT_THRESHOLDS = {
+  requireCorpus: true,
+  minScenarioCount: 1,
+  minSearchRuns: 1,
+  minHoldoutRuns: 1,
+  requireHoldout: true,
+  minPassRate: 0.8,
+  minMeanScore: 0.7,
+  maxOverfitGap: 0.15,
+  maxMeanCostUsd: Number.POSITIVE_INFINITY,
+  maxP95WallMs: Number.POSITIVE_INFINITY,
+  requireAsiForFailures: true,
+  failureScoreThreshold: 0.5
+};
+function releaseTraceEvidenceFromMultiShotTrials(trials) {
+  return trials.map((trial) => ({
+    scenarioId: trial.scenarioId,
+    candidateId: trial.variantId,
+    split: trial.split === "holdout" ? "holdout" : trial.split === "dev" ? "dev" : "search",
+    score: trial.score,
+    ok: trial.ok,
+    turnCount: Array.isArray(trial.trace?.turns) ? trial.trace.turns.length : void 0,
+    costUsd: trial.cost,
+    durationMs: trial.durationMs,
+    failureMode: trial.error ? "runtime_error" : void 0,
+    asi: trial.asi,
+    metadata: trial.metadata
+  }));
+}
+function evaluateReleaseConfidence(input) {
+  const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds };
+  const candidateId = input.candidateId ?? null;
+  const runs = filterCandidate(input.runs ?? [], candidateId, input.baselineId);
+  const traces = filterTraceCandidate(input.traces ?? [], candidateId, input.baselineId);
+  const scenarios = input.scenarios ?? [];
+  const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length;
+  const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios);
+  const searchScores = scoresFor(runs, "search");
+  const holdoutScores = scoresFor(runs, "holdout");
+  const allScores = [...searchScores, ...holdoutScores];
+  const traceScores = traces.map((t) => t.score).filter(isFiniteNumber);
+  const scoreUniverse = allScores.length > 0 ? allScores : traceScores;
+  const searchRuns = runs.filter((r) => r.splitTag === "search").length;
+  const holdoutRuns = runs.filter((r) => r.splitTag === "holdout").length;
+  const searchMeanScore = mean(searchScores);
+  const holdoutMeanScore = mean(holdoutScores);
+  const metrics = {
+    scenarioCount,
+    searchRuns,
+    holdoutRuns,
+    passRate: passRate(runs, traces, thresholds.failureScoreThreshold),
+    meanScore: mean(scoreUniverse),
+    searchMeanScore,
+    holdoutMeanScore,
+    overfitGap: safeDiff(searchMeanScore, holdoutMeanScore),
+    meanCostUsd: mean([...runs.map((r) => r.costUsd), ...traces.map((t) => t.costUsd).filter(isFiniteNumber)]),
+    p95WallMs: percentile([...runs.map((r) => r.wallMs), ...traces.map((t) => t.durationMs).filter(isFiniteNumber)], 0.95),
+    failedRows: failedRows(runs, traces, thresholds.failureScoreThreshold).length,
+    failuresWithAsi: failedRows(runs, traces, thresholds.failureScoreThreshold).filter((row) => row.hasAsi).length,
+    singleShotTraces: traces.filter((t) => t.turnCount === 1).length,
+    multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,
+    splitCounts,
+    domainCounts: countDomains(scenarios),
+    failureModeCounts: countFailureModes(runs, traces, thresholds.failureScoreThreshold),
+    responsibleSurfaceCounts: countResponsibleSurfaces(traces)
+  };
+  const issues = [];
+  checkCorpus(input, thresholds, metrics, issues);
+  checkQuality(thresholds, metrics, issues);
+  checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues);
+  checkDiagnostics(thresholds, metrics, issues);
+  checkEfficiency(thresholds, metrics, issues);
+  const axes = buildAxes(metrics, thresholds, input.gateDecision ?? null, issues);
+  const status = issues.some((i) => i.severity === "critical") ? "fail" : issues.length > 0 ? "warn" : "pass";
+  return {
+    target: input.target,
+    candidateId,
+    baselineId: input.baselineId ?? null,
+    status,
+    promote: status === "pass" && (input.gateDecision ? input.gateDecision.promote : true),
+    axes,
+    issues,
+    metrics,
+    dataset: input.dataset ?? null,
+    gateDecision: input.gateDecision ?? null,
+    summary: renderSummary(input.target, status, metrics, issues)
+  };
+}
+function assertReleaseConfidence(input) {
+  const scorecard = evaluateReleaseConfidence(input);
+  if (scorecard.status === "fail") {
+    throw new Error(scorecard.summary);
+  }
+  return scorecard;
+}
+function filterCandidate(runs, candidateId, baselineId) {
+  if (candidateId) return runs.filter((r) => r.candidateId === candidateId);
+  if (baselineId) return runs.filter((r) => r.candidateId !== baselineId);
+  return [...runs];
+}
+function filterTraceCandidate(traces, candidateId, baselineId) {
+  if (candidateId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId === candidateId);
+  if (baselineId) return traces.filter((t) => t.candidateId === void 0 || t.candidateId !== baselineId);
+  return [...traces];
+}
+function checkCorpus(input, thresholds, metrics, issues) {
+  if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) {
+    issues.push({ axis: "corpus", severity: "critical", code: "missing_corpus", detail: "No Dataset manifest or scenarios supplied." });
+  }
+  if (metrics.scenarioCount < thresholds.minScenarioCount) {
+    issues.push({ axis: "corpus", severity: "critical", code: "few_scenarios", detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.` });
+  }
+  if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) {
+    issues.push({ axis: "corpus", severity: "critical", code: "missing_holdout_split", detail: "Corpus has no holdout scenarios." });
+  }
+}
+function checkQuality(thresholds, metrics, issues) {
+  if (metrics.searchRuns < thresholds.minSearchRuns) {
+    issues.push({ axis: "quality", severity: "critical", code: "few_search_runs", detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.` });
+  }
+  if (metrics.passRate < thresholds.minPassRate) {
+    issues.push({ axis: "quality", severity: "critical", code: "low_pass_rate", detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.` });
+  }
+  if (metrics.meanScore < thresholds.minMeanScore) {
+    issues.push({ axis: "quality", severity: "critical", code: "low_mean_score", detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.` });
+  }
+}
+function checkGeneralization(gateDecision, thresholds, metrics, issues) {
+  if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {
+    issues.push({ axis: "generalization", severity: "critical", code: "few_holdout_runs", detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.` });
+  }
+  if (Number.isFinite(metrics.overfitGap) && metrics.overfitGap > thresholds.maxOverfitGap) {
+    issues.push({ axis: "generalization", severity: "critical", code: "overfit_gap", detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.` });
+  }
+  if (gateDecision && !gateDecision.promote) {
+    issues.push({ axis: "generalization", severity: "critical", code: `gate_${gateDecision.rejectionCode ?? "reject"}`, detail: gateDecision.reason });
+  }
+}
+function checkDiagnostics(thresholds, metrics, issues) {
+  if (!thresholds.requireAsiForFailures) return;
+  if (metrics.failedRows > metrics.failuresWithAsi) {
+    issues.push({
+      axis: "diagnostics",
+      severity: "critical",
+      code: "missing_failure_asi",
+      detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`
+    });
+  }
+}
+function checkEfficiency(thresholds, metrics, issues) {
+  if (metrics.meanCostUsd > thresholds.maxMeanCostUsd) {
+    issues.push({ axis: "efficiency", severity: "critical", code: "cost_budget", detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.` });
+  }
+  if (metrics.p95WallMs > thresholds.maxP95WallMs) {
+    issues.push({ axis: "efficiency", severity: "critical", code: "latency_budget", detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.` });
+  }
+}
+function buildAxes(metrics, thresholds, gateDecision, issues) {
+  return [
+    axis("corpus", issues, bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)), `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`),
+    axis("quality", issues, Math.min(metrics.passRate, metrics.meanScore), `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`),
+    axis("generalization", issues, gateDecision && !gateDecision.promote ? 0 : gapScore(metrics.overfitGap, thresholds.maxOverfitGap), `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`),
+    axis("diagnostics", issues, metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows, `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`),
+    axis("efficiency", issues, efficiencyScore(metrics, thresholds), `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`)
+  ];
+}
+function axis(name, issues, score, detail) {
+  const own = issues.filter((i) => i.axis === name);
+  const status = own.some((i) => i.severity === "critical") ? "fail" : own.length > 0 ? "warn" : "pass";
+  return { name, status, score: bounded(score), detail };
+}
+function countScenarioSplits(scenarios) {
+  const counts = { train: 0, dev: 0, test: 0, holdout: 0 };
+  for (const scenario of scenarios) counts[scenario.split ?? "train"]++;
+  return counts;
+}
+function countDomains(scenarios) {
+  const out = {};
+  for (const scenario of scenarios) {
+    const domain = scenario.tags?.domain ?? scenario.tags?.category ?? "uncategorized";
+    out[domain] = (out[domain] ?? 0) + 1;
+  }
+  return out;
+}
+function countFailureModes(runs, traces, threshold) {
+  const out = {};
+  for (const run of runs) {
+    const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+    if (run.failureMode || score !== void 0 && score < threshold) {
+      const mode = run.failureMode ?? "low_score";
+      out[mode] = (out[mode] ?? 0) + 1;
+    }
+  }
+  for (const trace of traces) {
+    if (trace.failureMode || trace.ok === false || trace.score !== void 0 && trace.score < threshold) {
+      const mode = trace.failureMode ?? (trace.ok === false ? "not_ok" : "low_score");
+      out[mode] = (out[mode] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function countResponsibleSurfaces(traces) {
+  const out = {};
+  for (const trace of traces) {
+    for (const asi of trace.asi ?? []) {
+      const surface = asi.responsibleSurface ?? "unknown";
+      out[surface] = (out[surface] ?? 0) + 1;
+    }
+  }
+  return out;
+}
+function failedRows(runs, traces, threshold) {
+  const out = [];
+  for (const run of runs) {
+    const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+    if (run.failureMode || score !== void 0 && score < threshold) {
+      const asiMetric = run.outcome.raw.asi;
+      out.push({ hasAsi: typeof asiMetric === "number" && asiMetric > 0 });
+    }
+  }
+  for (const trace of traces) {
+    if (trace.failureMode || trace.ok === false || trace.score !== void 0 && trace.score < threshold) {
+      out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 });
+    }
+  }
+  return out;
+}
+function passRate(runs, traces, threshold) {
+  const outcomes = [
+    ...runs.map((run) => {
+      const score = run.outcome.holdoutScore ?? run.outcome.searchScore;
+      return !run.failureMode && score !== void 0 && score >= threshold;
+    }),
+    ...traces.map((trace) => trace.ok !== false && (trace.score === void 0 || trace.score >= threshold))
+  ];
+  if (outcomes.length === 0) return 0;
+  return outcomes.filter(Boolean).length / outcomes.length;
+}
+function scoresFor(runs, split) {
+  return runs.filter((run) => run.splitTag === split).map((run) => split === "holdout" ? run.outcome.holdoutScore : run.outcome.searchScore).filter(isFiniteNumber);
+}
+function mean(xs) {
+  if (xs.length === 0) return Number.NaN;
+  return xs.reduce((sum, x) => sum + x, 0) / xs.length;
+}
+function percentile(xs, p) {
+  if (xs.length === 0) return Number.NaN;
+  const sorted = [...xs].sort((a, b) => a - b);
+  return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
+}
+function isFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+function safeDiff(a, b) {
+  if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN;
+  return a - b;
+}
+function gapScore(gap, maxGap) {
+  if (!Number.isFinite(gap)) return 0;
+  if (maxGap <= 0) return gap <= 0 ? 1 : 0;
+  return bounded(1 - Math.max(0, gap) / maxGap);
+}
+function efficiencyScore(metrics, thresholds) {
+  const cost = Number.isFinite(thresholds.maxMeanCostUsd) && Number.isFinite(metrics.meanCostUsd) ? bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12)) : 1;
+  const latency = Number.isFinite(thresholds.maxP95WallMs) && Number.isFinite(metrics.p95WallMs) ? bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12)) : 1;
+  return Math.min(cost, latency);
+}
+function bounded(x) {
+  if (!Number.isFinite(x)) return 0;
+  return Math.max(0, Math.min(1, x));
+}
+function renderSummary(target, status, metrics, issues) {
+  const prefix = `release confidence ${status}: ${target}`;
+  const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`;
+  if (issues.length === 0) return `${prefix}; ${metricText}`;
+  return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(",")}`;
+}
+function fmt(x) {
+  if (!Number.isFinite(x)) return String(x);
+  return x.toFixed(4);
+}
+
+// src/release-report.ts
+function renderReleaseReport(scorecard, options = {}) {
+  const title = options.title ?? `Release Report: ${scorecard.target}`;
+  const lines = [];
+  lines.push(`# ${title}`);
+  lines.push("");
+  lines.push(`Status: **${scorecard.status.toUpperCase()}**`);
+  lines.push(`Promote: **${scorecard.promote ? "yes" : "no"}**`);
+  if (scorecard.candidateId) lines.push(`Candidate: \`${scorecard.candidateId}\``);
+  if (scorecard.baselineId) lines.push(`Baseline: \`${scorecard.baselineId}\``);
+  lines.push("");
+  lines.push(scorecard.summary);
+  lines.push("");
+  lines.push("## Metrics");
+  lines.push("");
+  lines.push("| Metric | Value |");
+  lines.push("|---|---:|");
+  lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`);
+  lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`);
+  lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`);
+  lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`);
+  lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`);
+  lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`);
+  lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`);
+  lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`);
+  lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`);
+  lines.push(`| p95 wall time | ${Math.round(scorecard.metrics.p95WallMs)} ms |`);
+  lines.push("");
+  if (scorecard.issues.length > 0) {
+    lines.push("## Issues");
+    lines.push("");
+    for (const issue of scorecard.issues) {
+      lines.push(`- **${issue.severity}** \`${issue.code}\` (${issue.axis}): ${issue.detail}`);
+    }
+    lines.push("");
+  }
+  const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts);
+  if (surfaces.length > 0) {
+    lines.push("## Responsible Surfaces");
+    lines.push("");
+    for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`);
+    lines.push("");
+  }
+  const failures = entries(scorecard.metrics.failureModeCounts);
+  if (failures.length > 0) {
+    lines.push("## Failure Modes");
+    lines.push("");
+    for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`);
+    lines.push("");
+  }
+  if (options.runs && options.runs.length > 0) {
+    lines.push("## Run Summary");
+    lines.push("");
+    lines.push(summaryTable([...options.runs], {
+      comparator: options.comparator ?? scorecard.baselineId ?? void 0,
+      split: "holdout"
+    }).markdown);
+    lines.push("");
+  }
+  if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {
+    lines.push("## TraceAnalyst Findings");
+    lines.push("");
+    for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`);
+    lines.push("");
+  }
+  const nextActions = options.nextActions ?? defaultNextActions(scorecard);
+  if (nextActions.length > 0) {
+    lines.push("## Next Actions");
+    lines.push("");
+    for (const action of nextActions) lines.push(`- ${action}`);
+    lines.push("");
+  }
+  return lines.join("\n").trimEnd() + "\n";
+}
+function defaultNextActions(scorecard) {
+  if (scorecard.promote) return ["Promote the candidate and keep canaries enabled."];
+  return scorecard.issues.filter((issue) => issue.severity === "critical").map((issue) => `Resolve ${issue.code}: ${issue.detail}`);
+}
+function entries(values) {
+  return Object.entries(values).filter(([, count]) => count > 0).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+}
+function pct(value) {
+  return Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : "n/a";
+}
+function num(value) {
+  return Number.isFinite(value) ? value.toFixed(3) : "n/a";
+}
+
+// src/promotion-gate.ts
+function bootstrapCi(baseline, candidate, options = {}) {
+  const alpha = options.alpha ?? 0.05;
+  const iterations = options.iterations ?? 1e3;
+  const minTotal = options.minTotalSamples ?? 6;
+  const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate));
+  const baselineMean = mean2(baseline);
+  const candidateMean = mean2(candidate);
+  const delta = candidateMean - baselineMean;
+  if (baseline.length + candidate.length < minTotal || baseline.length === 0 || candidate.length === 0) {
+    return {
+      baselineMean,
+      candidateMean,
+      delta,
+      ciLower: -Infinity,
+      ciUpper: Infinity,
+      iterations: 0,
+      alpha,
+      verdict: "INCONCLUSIVE"
+    };
+  }
+  const deltas = new Array(iterations);
+  for (let i = 0; i < iterations; i++) {
+    const bResample = resample(baseline, rng);
+    const cResample = resample(candidate, rng);
+    deltas[i] = mean2(cResample) - mean2(bResample);
+  }
+  deltas.sort((a, b) => a - b);
+  const lowerIdx = Math.floor(alpha / 2 * iterations);
+  const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1;
+  const ciLower = deltas[Math.max(0, lowerIdx)];
+  const ciUpper = deltas[Math.min(iterations - 1, upperIdx)];
+  let verdict;
+  if (ciLower > 0) verdict = "ADVANCE";
+  else if (ciUpper < 0) verdict = "REVERT";
+  else if (delta >= 0) verdict = "KEEP";
+  else verdict = "INCONCLUSIVE";
+  return {
+    baselineMean,
+    candidateMean,
+    delta,
+    ciLower,
+    ciUpper,
+    iterations,
+    alpha,
+    verdict
+  };
+}
+function mean2(xs) {
+  if (xs.length === 0) return 0;
+  let s = 0;
+  for (const x of xs) s += x;
+  return s / xs.length;
+}
+function resample(xs, rng) {
+  const out = new Array(xs.length);
+  for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)];
+  return out;
+}
+function mulberry32(seed) {
+  let t = seed >>> 0;
+  return () => {
+    t += 1831565813;
+    let r = t;
+    r = Math.imul(r ^ r >>> 15, r | 1);
+    r ^= r + Math.imul(r ^ r >>> 7, r | 61);
+    return ((r ^ r >>> 14) >>> 0) / 4294967296;
+  };
+}
+function hashSeed(a, b) {
+  let h = 2166136261;
+  for (const x of [...a, ...b]) {
+    const view = new Float64Array([x]);
+    const bytes = new Uint8Array(view.buffer);
+    for (const byte of bytes) {
+      h ^= byte;
+      h = Math.imul(h, 16777619);
+    }
+  }
+  return h >>> 0;
+}
+async function judgeReplayGate(args) {
+  const concurrency = args.judgeConcurrency ?? 4;
+  const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency);
+  const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency);
+  const ci = bootstrapCi(baselineScores, candidateScores, {
+    ...args.alpha !== void 0 ? { alpha: args.alpha } : {},
+    ...args.iterations !== void 0 ? { iterations: args.iterations } : {},
+    ...args.seed !== void 0 ? { seed: args.seed } : {}
+  });
+  return {
+    ...ci,
+    baselineSamples: baselineScores.length,
+    candidateSamples: candidateScores.length
+  };
+}
+async function scoreAll(outputs, judge, concurrency) {
+  const results = new Array(outputs.length);
+  let next = 0;
+  async function worker() {
+    while (true) {
+      const i = next++;
+      if (i >= outputs.length) return;
+      const v = await judge(outputs[i]);
+      results[i] = Number.isFinite(v) ? v : 0;
+    }
+  }
+  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
+  return results;
+}
+
+export {
+  releaseTraceEvidenceFromMultiShotTrials,
+  evaluateReleaseConfidence,
+  assertReleaseConfidence,
+  renderReleaseReport,
+  bootstrapCi,
+  judgeReplayGate
+};
+//# sourceMappingURL=chunk-7EAUOUQS.js.map

package/dist/chunk-7EAUOUQS.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/release-confidence.ts","../src/release-report.ts","../src/promotion-gate.ts"],"sourcesContent":["/**\n * Release confidence gate.\n *\n * This is the production-facing composition layer over the lower-level\n * primitives:\n *   - Dataset manifests prove corpus/version coverage.\n *   - RunRecord rows prove reproducible search/holdout outcomes.\n *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.\n *   - HeldOutGate decisions remain the paired promotion authority.\n *\n * The gate is intentionally pure and conservative. Missing declared evidence\n * fails closed instead of being treated as a neutral zero.\n */\n\nimport type { DatasetManifest, DatasetScenario, DatasetSplit } from './dataset'\nimport type { GateDecision } from './held-out-gate'\nimport type { ActionableSideInfo, MultiShotTrialResult } from './multi-shot-optimization'\nimport type { RunRecord, RunSplitTag } from './run-record'\n\nexport type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail'\nexport type ReleaseConfidenceAxisName =\n  | 'corpus'\n  | 'quality'\n  | 'generalization'\n  | 'diagnostics'\n  | 'efficiency'\n\nexport interface ReleaseTraceEvidence {\n  scenarioId: string\n  candidateId?: string\n  split?: RunSplitTag\n  score?: number\n  ok?: boolean\n  turnCount?: number\n  costUsd?: number\n  durationMs?: number\n  failureMode?: string\n  asi?: ActionableSideInfo[]\n  metadata?: Record<string, unknown>\n}\n\nexport interface ReleaseConfidenceThresholds {\n  /** Require a Dataset manifest or explicit scenarios. Default true. */\n  requireCorpus?: boolean\n  minScenarioCount?: number\n  minSearchRuns?: number\n  minHoldoutRuns?: number\n  /** Require at least one holdout scenario/run. Default true. */\n  requireHoldout?: boolean\n  minPassRate?: number\n  minMeanScore?: number\n  /** Search mean may exceed holdout mean by at most this much. */\n  maxOverfitGap?: number\n  maxMeanCostUsd?: number\n  maxP95WallMs?: number\n  /** Low-score/failed rows must carry ASI. Default true. */\n  requireAsiForFailures?: boolean\n  /** Score below this is considered a failure for ASI coverage. Default 0.5. */\n  failureScoreThreshold?: number\n}\n\nexport interface ReleaseConfidenceInput {\n  target: string\n  candidateId?: string\n  baselineId?: string\n  dataset?: DatasetManifest\n  scenarios?: readonly DatasetScenario[]\n  runs?: readonly RunRecord[]\n  traces?: readonly ReleaseTraceEvidence[]\n  gateDecision?: GateDecision | null\n  thresholds?: ReleaseConfidenceThresholds\n}\n\nexport interface ReleaseConfidenceAxis {\n  name: ReleaseConfidenceAxisName\n  status: ReleaseConfidenceStatus\n  score: number\n  detail: string\n}\n\nexport interface ReleaseConfidenceIssue {\n  axis: ReleaseConfidenceAxisName\n  severity: 'critical' | 'warning'\n  code: string\n  detail: string\n}\n\nexport interface ReleaseConfidenceMetrics {\n  scenarioCount: number\n  searchRuns: number\n  holdoutRuns: number\n  passRate: number\n  meanScore: number\n  searchMeanScore: number\n  holdoutMeanScore: number\n  overfitGap: number\n  meanCostUsd: number\n  p95WallMs: number\n  failedRows: number\n  failuresWithAsi: number\n  singleShotTraces: number\n  multiShotTraces: number\n  splitCounts: Record<DatasetSplit, number>\n  domainCounts: Record<string, number>\n  failureModeCounts: Record<string, number>\n  responsibleSurfaceCounts: Record<string, number>\n}\n\nexport interface ReleaseConfidenceScorecard {\n  target: string\n  candidateId: string | null\n  baselineId: string | null\n  status: ReleaseConfidenceStatus\n  promote: boolean\n  axes: ReleaseConfidenceAxis[]\n  issues: ReleaseConfidenceIssue[]\n  metrics: ReleaseConfidenceMetrics\n  dataset: DatasetManifest | null\n  gateDecision: GateDecision | null\n  summary: string\n}\n\nconst DEFAULT_THRESHOLDS: Required<ReleaseConfidenceThresholds> = {\n  requireCorpus: true,\n  minScenarioCount: 1,\n  minSearchRuns: 1,\n  minHoldoutRuns: 1,\n  requireHoldout: true,\n  minPassRate: 0.8,\n  minMeanScore: 0.7,\n  maxOverfitGap: 0.15,\n  maxMeanCostUsd: Number.POSITIVE_INFINITY,\n  maxP95WallMs: Number.POSITIVE_INFINITY,\n  requireAsiForFailures: true,\n  failureScoreThreshold: 0.5,\n}\n\nexport function releaseTraceEvidenceFromMultiShotTrials(\n  trials: readonly MultiShotTrialResult[],\n): ReleaseTraceEvidence[] {\n  return trials.map((trial) => ({\n    scenarioId: trial.scenarioId,\n    candidateId: trial.variantId,\n    split: trial.split === 'holdout' ? 'holdout' : trial.split === 'dev' ? 'dev' : 'search',\n    score: trial.score,\n    ok: trial.ok,\n    turnCount: Array.isArray(trial.trace?.turns) ? trial.trace.turns.length : undefined,\n    costUsd: trial.cost,\n    durationMs: trial.durationMs,\n    failureMode: trial.error ? 'runtime_error' : undefined,\n    asi: trial.asi,\n    metadata: trial.metadata,\n  }))\n}\n\nexport function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard {\n  const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds }\n  const candidateId = input.candidateId ?? null\n  const runs = filterCandidate(input.runs ?? [], candidateId, input.baselineId)\n  const traces = filterTraceCandidate(input.traces ?? [], candidateId, input.baselineId)\n  const scenarios = input.scenarios ?? []\n  const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length\n  const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios)\n  const searchScores = scoresFor(runs, 'search')\n  const holdoutScores = scoresFor(runs, 'holdout')\n  const allScores = [...searchScores, ...holdoutScores]\n  const traceScores = traces.map((t) => t.score).filter(isFiniteNumber)\n  const scoreUniverse = allScores.length > 0 ? allScores : traceScores\n  const searchRuns = runs.filter((r) => r.splitTag === 'search').length\n  const holdoutRuns = runs.filter((r) => r.splitTag === 'holdout').length\n  const searchMeanScore = mean(searchScores)\n  const holdoutMeanScore = mean(holdoutScores)\n  const metrics: ReleaseConfidenceMetrics = {\n    scenarioCount,\n    searchRuns,\n    holdoutRuns,\n    passRate: passRate(runs, traces, thresholds.failureScoreThreshold),\n    meanScore: mean(scoreUniverse),\n    searchMeanScore,\n    holdoutMeanScore,\n    overfitGap: safeDiff(searchMeanScore, holdoutMeanScore),\n    meanCostUsd: mean([...runs.map((r) => r.costUsd), ...traces.map((t) => t.costUsd).filter(isFiniteNumber)]),\n    p95WallMs: percentile([...runs.map((r) => r.wallMs), ...traces.map((t) => t.durationMs).filter(isFiniteNumber)], 0.95),\n    failedRows: failedRows(runs, traces, thresholds.failureScoreThreshold).length,\n    failuresWithAsi: failedRows(runs, traces, thresholds.failureScoreThreshold).filter((row) => row.hasAsi).length,\n    singleShotTraces: traces.filter((t) => t.turnCount === 1).length,\n    multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,\n    splitCounts,\n    domainCounts: countDomains(scenarios),\n    failureModeCounts: countFailureModes(runs, traces, thresholds.failureScoreThreshold),\n    responsibleSurfaceCounts: countResponsibleSurfaces(traces),\n  }\n\n  const issues: ReleaseConfidenceIssue[] = []\n  checkCorpus(input, thresholds, metrics, issues)\n  checkQuality(thresholds, metrics, issues)\n  checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues)\n  checkDiagnostics(thresholds, metrics, issues)\n  checkEfficiency(thresholds, metrics, issues)\n\n  const axes = buildAxes(metrics, thresholds, input.gateDecision ?? null, issues)\n  const status = issues.some((i) => i.severity === 'critical') ? 'fail'\n    : issues.length > 0 ? 'warn'\n    : 'pass'\n\n  return {\n    target: input.target,\n    candidateId,\n    baselineId: input.baselineId ?? null,\n    status,\n    promote: status === 'pass' && (input.gateDecision ? input.gateDecision.promote : true),\n    axes,\n    issues,\n    metrics,\n    dataset: input.dataset ?? null,\n    gateDecision: input.gateDecision ?? null,\n    summary: renderSummary(input.target, status, metrics, issues),\n  }\n}\n\nexport function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard {\n  const scorecard = evaluateReleaseConfidence(input)\n  if (scorecard.status === 'fail') {\n    throw new Error(scorecard.summary)\n  }\n  return scorecard\n}\n\nfunction filterCandidate(\n  runs: readonly RunRecord[],\n  candidateId: string | null,\n  baselineId?: string,\n): RunRecord[] {\n  if (candidateId) return runs.filter((r) => r.candidateId === candidateId)\n  if (baselineId) return runs.filter((r) => r.candidateId !== baselineId)\n  return [...runs]\n}\n\nfunction filterTraceCandidate(\n  traces: readonly ReleaseTraceEvidence[],\n  candidateId: string | null,\n  baselineId?: string,\n): ReleaseTraceEvidence[] {\n  if (candidateId) return traces.filter((t) => t.candidateId === undefined || t.candidateId === candidateId)\n  if (baselineId) return traces.filter((t) => t.candidateId === undefined || t.candidateId !== baselineId)\n  return [...traces]\n}\n\nfunction checkCorpus(\n  input: ReleaseConfidenceInput,\n  thresholds: Required<ReleaseConfidenceThresholds>,\n  metrics: ReleaseConfidenceMetrics,\n  issues: ReleaseConfidenceIssue[],\n): void {\n  if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) {\n    issues.push({ axis: 'corpus', severity: 'critical', code: 'missing_corpus', detail: 'No Dataset manifest or scenarios supplied.' })\n  }\n  if (metrics.scenarioCount < thresholds.minScenarioCount) {\n    issues.push({ axis: 'corpus', severity: 'critical', code: 'few_scenarios', detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.` })\n  }\n  if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) {\n    issues.push({ axis: 'corpus', severity: 'critical', code: 'missing_holdout_split', detail: 'Corpus has no holdout scenarios.' })\n  }\n}\n\nfunction checkQuality(\n  thresholds: Required<ReleaseConfidenceThresholds>,\n  metrics: ReleaseConfidenceMetrics,\n  issues: ReleaseConfidenceIssue[],\n): void {\n  if (metrics.searchRuns < thresholds.minSearchRuns) {\n    issues.push({ axis: 'quality', severity: 'critical', code: 'few_search_runs', detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.` })\n  }\n  if (metrics.passRate < thresholds.minPassRate) {\n    issues.push({ axis: 'quality', severity: 'critical', code: 'low_pass_rate', detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.` })\n  }\n  if (metrics.meanScore < thresholds.minMeanScore) {\n    issues.push({ axis: 'quality', severity: 'critical', code: 'low_mean_score', detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.` })\n  }\n}\n\nfunction checkGeneralization(\n  gateDecision: GateDecision | null,\n  thresholds: Required<ReleaseConfidenceThresholds>,\n  metrics: ReleaseConfidenceMetrics,\n  issues: ReleaseConfidenceIssue[],\n): void {\n  if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {\n    issues.push({ axis: 'generalization', severity: 'critical', code: 'few_holdout_runs', detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.` })\n  }\n  if (Number.isFinite(metrics.overfitGap) && metrics.overfitGap > thresholds.maxOverfitGap) {\n    issues.push({ axis: 'generalization', severity: 'critical', code: 'overfit_gap', detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.` })\n  }\n  if (gateDecision && !gateDecision.promote) {\n    issues.push({ axis: 'generalization', severity: 'critical', code: `gate_${gateDecision.rejectionCode ?? 'reject'}`, detail: gateDecision.reason })\n  }\n}\n\nfunction checkDiagnostics(\n  thresholds: Required<ReleaseConfidenceThresholds>,\n  metrics: ReleaseConfidenceMetrics,\n  issues: ReleaseConfidenceIssue[],\n): void {\n  if (!thresholds.requireAsiForFailures) return\n  if (metrics.failedRows > metrics.failuresWithAsi) {\n    issues.push({\n      axis: 'diagnostics',\n      severity: 'critical',\n      code: 'missing_failure_asi',\n      detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`,\n    })\n  }\n}\n\nfunction checkEfficiency(\n  thresholds: Required<ReleaseConfidenceThresholds>,\n  metrics: ReleaseConfidenceMetrics,\n  issues: ReleaseConfidenceIssue[],\n): void {\n  if (metrics.meanCostUsd > thresholds.maxMeanCostUsd) {\n    issues.push({ axis: 'efficiency', severity: 'critical', code: 'cost_budget', detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.` })\n  }\n  if (metrics.p95WallMs > thresholds.maxP95WallMs) {\n    issues.push({ axis: 'efficiency', severity: 'critical', code: 'latency_budget', detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.` })\n  }\n}\n\nfunction buildAxes(\n  metrics: ReleaseConfidenceMetrics,\n  thresholds: Required<ReleaseConfidenceThresholds>,\n  gateDecision: GateDecision | null,\n  issues: ReleaseConfidenceIssue[],\n): ReleaseConfidenceAxis[] {\n  return [\n    axis('corpus', issues, bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)), `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`),\n    axis('quality', issues, Math.min(metrics.passRate, metrics.meanScore), `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`),\n    axis('generalization', issues, gateDecision && !gateDecision.promote ? 0 : gapScore(metrics.overfitGap, thresholds.maxOverfitGap), `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`),\n    axis('diagnostics', issues, metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows, `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`),\n    axis('efficiency', issues, efficiencyScore(metrics, thresholds), `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`),\n  ]\n}\n\nfunction axis(\n  name: ReleaseConfidenceAxisName,\n  issues: ReleaseConfidenceIssue[],\n  score: number,\n  detail: string,\n): ReleaseConfidenceAxis {\n  const own = issues.filter((i) => i.axis === name)\n  const status = own.some((i) => i.severity === 'critical') ? 'fail'\n    : own.length > 0 ? 'warn'\n    : 'pass'\n  return { name, status, score: bounded(score), detail }\n}\n\nfunction countScenarioSplits(scenarios: readonly DatasetScenario[]): Record<DatasetSplit, number> {\n  const counts: Record<DatasetSplit, number> = { train: 0, dev: 0, test: 0, holdout: 0 }\n  for (const scenario of scenarios) counts[scenario.split ?? 'train']++\n  return counts\n}\n\nfunction countDomains(scenarios: readonly DatasetScenario[]): Record<string, number> {\n  const out: Record<string, number> = {}\n  for (const scenario of scenarios) {\n    const domain = scenario.tags?.domain ?? scenario.tags?.category ?? 'uncategorized'\n    out[domain] = (out[domain] ?? 0) + 1\n  }\n  return out\n}\n\nfunction countFailureModes(\n  runs: readonly RunRecord[],\n  traces: readonly ReleaseTraceEvidence[],\n  threshold: number,\n): Record<string, number> {\n  const out: Record<string, number> = {}\n  for (const run of runs) {\n    const score = run.outcome.holdoutScore ?? run.outcome.searchScore\n    if (run.failureMode || (score !== undefined && score < threshold)) {\n      const mode = run.failureMode ?? 'low_score'\n      out[mode] = (out[mode] ?? 0) + 1\n    }\n  }\n  for (const trace of traces) {\n    if (trace.failureMode || trace.ok === false || (trace.score !== undefined && trace.score < threshold)) {\n      const mode = trace.failureMode ?? (trace.ok === false ? 'not_ok' : 'low_score')\n      out[mode] = (out[mode] ?? 0) + 1\n    }\n  }\n  return out\n}\n\nfunction countResponsibleSurfaces(traces: readonly ReleaseTraceEvidence[]): Record<string, number> {\n  const out: Record<string, number> = {}\n  for (const trace of traces) {\n    for (const asi of trace.asi ?? []) {\n      const surface = asi.responsibleSurface ?? 'unknown'\n      out[surface] = (out[surface] ?? 0) + 1\n    }\n  }\n  return out\n}\n\nfunction failedRows(\n  runs: readonly RunRecord[],\n  traces: readonly ReleaseTraceEvidence[],\n  threshold: number,\n): Array<{ hasAsi: boolean }> {\n  const out: Array<{ hasAsi: boolean }> = []\n  for (const run of runs) {\n    const score = run.outcome.holdoutScore ?? run.outcome.searchScore\n    if (run.failureMode || (score !== undefined && score < threshold)) {\n      const asiMetric = run.outcome.raw.asi\n      out.push({ hasAsi: typeof asiMetric === 'number' && asiMetric > 0 })\n    }\n  }\n  for (const trace of traces) {\n    if (trace.failureMode || trace.ok === false || (trace.score !== undefined && trace.score < threshold)) {\n      out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 })\n    }\n  }\n  return out\n}\n\nfunction passRate(\n  runs: readonly RunRecord[],\n  traces: readonly ReleaseTraceEvidence[],\n  threshold: number,\n): number {\n  const outcomes = [\n    ...runs.map((run) => {\n      const score = run.outcome.holdoutScore ?? run.outcome.searchScore\n      return !run.failureMode && score !== undefined && score >= threshold\n    }),\n    ...traces.map((trace) => trace.ok !== false && (trace.score === undefined || trace.score >= threshold)),\n  ]\n  if (outcomes.length === 0) return 0\n  return outcomes.filter(Boolean).length / outcomes.length\n}\n\nfunction scoresFor(runs: readonly RunRecord[], split: RunSplitTag): number[] {\n  return runs\n    .filter((run) => run.splitTag === split)\n    .map((run) => split === 'holdout' ? run.outcome.holdoutScore : run.outcome.searchScore)\n    .filter(isFiniteNumber)\n}\n\nfunction mean(xs: readonly number[]): number {\n  if (xs.length === 0) return Number.NaN\n  return xs.reduce((sum, x) => sum + x, 0) / xs.length\n}\n\nfunction percentile(xs: readonly number[], p: number): number {\n  if (xs.length === 0) return Number.NaN\n  const sorted = [...xs].sort((a, b) => a - b)\n  return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))]!\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value)\n}\n\nfunction safeDiff(a: number, b: number): number {\n  if (!Number.isFinite(a) || !Number.isFinite(b)) return Number.NaN\n  return a - b\n}\n\nfunction gapScore(gap: number, maxGap: number): number {\n  if (!Number.isFinite(gap)) return 0\n  if (maxGap <= 0) return gap <= 0 ? 1 : 0\n  return bounded(1 - Math.max(0, gap) / maxGap)\n}\n\nfunction efficiencyScore(\n  metrics: ReleaseConfidenceMetrics,\n  thresholds: Required<ReleaseConfidenceThresholds>,\n): number {\n  const cost = Number.isFinite(thresholds.maxMeanCostUsd) && Number.isFinite(metrics.meanCostUsd)\n    ? bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12))\n    : 1\n  const latency = Number.isFinite(thresholds.maxP95WallMs) && Number.isFinite(metrics.p95WallMs)\n    ? bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12))\n    : 1\n  return Math.min(cost, latency)\n}\n\nfunction bounded(x: number): number {\n  if (!Number.isFinite(x)) return 0\n  return Math.max(0, Math.min(1, x))\n}\n\nfunction renderSummary(\n  target: string,\n  status: ReleaseConfidenceStatus,\n  metrics: ReleaseConfidenceMetrics,\n  issues: ReleaseConfidenceIssue[],\n): string {\n  const prefix = `release confidence ${status}: ${target}`\n  const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`\n  if (issues.length === 0) return `${prefix}; ${metricText}`\n  return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(',')}`\n}\n\nfunction fmt(x: number): string {\n  if (!Number.isFinite(x)) return String(x)\n  return x.toFixed(4)\n}\n","import type { ReleaseConfidenceScorecard } from './release-confidence'\nimport { summaryTable } from './summary-report'\nimport type { RunRecord } from './run-record'\n\nexport interface RenderReleaseReportOptions {\n  title?: string\n  runs?: readonly RunRecord[]\n  comparator?: string\n  traceAnalystFindings?: readonly string[]\n  nextActions?: readonly string[]\n}\n\nexport function renderReleaseReport(\n  scorecard: ReleaseConfidenceScorecard,\n  options: RenderReleaseReportOptions = {},\n): string {\n  const title = options.title ?? `Release Report: ${scorecard.target}`\n  const lines: string[] = []\n  lines.push(`# ${title}`)\n  lines.push('')\n  lines.push(`Status: **${scorecard.status.toUpperCase()}**`)\n  lines.push(`Promote: **${scorecard.promote ? 'yes' : 'no'}**`)\n  if (scorecard.candidateId) lines.push(`Candidate: \\`${scorecard.candidateId}\\``)\n  if (scorecard.baselineId) lines.push(`Baseline: \\`${scorecard.baselineId}\\``)\n  lines.push('')\n  lines.push(scorecard.summary)\n  lines.push('')\n\n  lines.push('## Metrics')\n  lines.push('')\n  lines.push('| Metric | Value |')\n  lines.push('|---|---:|')\n  lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`)\n  lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`)\n  lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`)\n  lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`)\n  lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`)\n  lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`)\n  lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`)\n  lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`)\n  lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`)\n  lines.push(`| p95 wall time | ${Math.round(scorecard.metrics.p95WallMs)} ms |`)\n  lines.push('')\n\n  if (scorecard.issues.length > 0) {\n    lines.push('## Issues')\n    lines.push('')\n    for (const issue of scorecard.issues) {\n      lines.push(`- **${issue.severity}** \\`${issue.code}\\` (${issue.axis}): ${issue.detail}`)\n    }\n    lines.push('')\n  }\n\n  const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts)\n  if (surfaces.length > 0) {\n    lines.push('## Responsible Surfaces')\n    lines.push('')\n    for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`)\n    lines.push('')\n  }\n\n  const failures = entries(scorecard.metrics.failureModeCounts)\n  if (failures.length > 0) {\n    lines.push('## Failure Modes')\n    lines.push('')\n    for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`)\n    lines.push('')\n  }\n\n  if (options.runs && options.runs.length > 0) {\n    lines.push('## Run Summary')\n    lines.push('')\n    lines.push(summaryTable([...options.runs], {\n      comparator: options.comparator ?? scorecard.baselineId ?? undefined,\n      split: 'holdout',\n    }).markdown)\n    lines.push('')\n  }\n\n  if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {\n    lines.push('## TraceAnalyst Findings')\n    lines.push('')\n    for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`)\n    lines.push('')\n  }\n\n  const nextActions = options.nextActions ?? defaultNextActions(scorecard)\n  if (nextActions.length > 0) {\n    lines.push('## Next Actions')\n    lines.push('')\n    for (const action of nextActions) lines.push(`- ${action}`)\n    lines.push('')\n  }\n\n  return lines.join('\\n').trimEnd() + '\\n'\n}\n\nfunction defaultNextActions(scorecard: ReleaseConfidenceScorecard): string[] {\n  if (scorecard.promote) return ['Promote the candidate and keep canaries enabled.']\n  return scorecard.issues\n    .filter((issue) => issue.severity === 'critical')\n    .map((issue) => `Resolve ${issue.code}: ${issue.detail}`)\n}\n\nfunction entries(values: Record<string, number>): Array<[string, number]> {\n  return Object.entries(values)\n    .filter(([, count]) => count > 0)\n    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))\n}\n\nfunction pct(value: number): string {\n  return Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : 'n/a'\n}\n\nfunction num(value: number): string {\n  return Number.isFinite(value) ? value.toFixed(3) : 'n/a'\n}\n","/**\n * Bootstrap-CI promotion gate.\n *\n * In any iterative-improvement loop (GEPA, prompt evolution, dataset\n * curation), the question is \"did this generation actually improve, or are\n * we celebrating noise?\". With small N and noisy outcomes, point-estimate\n * deltas lie. Bootstrap confidence intervals tell the operator whether the\n * delta is real before code or prompts get promoted.\n *\n * This module is pure functions — no I/O, no model calls. Easy to unit-test\n * and to compose into any verdict gate.\n *\n * Default gate:\n *   - Bootstrap mean baseline vs candidate (1k resamples).\n *   - Compute the delta distribution; pass if the lower CI bound > 0.\n *   - Tunable confidence (default 95%) and resample count.\n *\n * Verdict semantics intentionally match the existing `experiments.jsonl`\n * vocabulary:\n *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)\n *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)\n *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)\n *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal\n */\n\nexport type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE'\n\nexport interface BootstrapResult {\n  baselineMean: number\n  candidateMean: number\n  /** candidateMean - baselineMean, point estimate. */\n  delta: number\n  /** Lower bound of the (1 - alpha) CI on the delta. */\n  ciLower: number\n  /** Upper bound of the (1 - alpha) CI on the delta. */\n  ciUpper: number\n  /** Number of bootstrap resamples used. */\n  iterations: number\n  alpha: number\n  verdict: Verdict\n}\n\nexport interface BootstrapOptions {\n  /** Confidence level alpha (default 0.05 → 95% CI). */\n  alpha?: number\n  /** Number of resamples (default 1000). */\n  iterations?: number\n  /**\n   * Minimum total samples (baseline + candidate) below which we always\n   * return INCONCLUSIVE — bootstrap with too few samples is meaningless.\n   * Default 6 (combined).\n   */\n  minTotalSamples?: number\n  /** RNG seed for reproducibility. Default: Math.random. */\n  seed?: number\n}\n\n/**\n * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.\n *\n * Uses simple percentile bootstrap on the difference of resampled means.\n * That's the standard non-parametric primitive — no distributional\n * assumptions, robust to skew, easy to reason about.\n */\nexport function bootstrapCi(\n  baseline: number[],\n  candidate: number[],\n  options: BootstrapOptions = {},\n): BootstrapResult {\n  const alpha = options.alpha ?? 0.05\n  const iterations = options.iterations ?? 1000\n  const minTotal = options.minTotalSamples ?? 6\n  const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate))\n\n  const baselineMean = mean(baseline)\n  const candidateMean = mean(candidate)\n  const delta = candidateMean - baselineMean\n\n  if (baseline.length + candidate.length < minTotal || baseline.length === 0 || candidate.length === 0) {\n    return {\n      baselineMean,\n      candidateMean,\n      delta,\n      ciLower: -Infinity,\n      ciUpper: Infinity,\n      iterations: 0,\n      alpha,\n      verdict: 'INCONCLUSIVE',\n    }\n  }\n\n  const deltas: number[] = new Array(iterations)\n  for (let i = 0; i < iterations; i++) {\n    const bResample = resample(baseline, rng)\n    const cResample = resample(candidate, rng)\n    deltas[i] = mean(cResample) - mean(bResample)\n  }\n  deltas.sort((a, b) => a - b)\n  const lowerIdx = Math.floor((alpha / 2) * iterations)\n  const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1\n  const ciLower = deltas[Math.max(0, lowerIdx)]!\n  const ciUpper = deltas[Math.min(iterations - 1, upperIdx)]!\n\n  let verdict: Verdict\n  if (ciLower > 0) verdict = 'ADVANCE'\n  else if (ciUpper < 0) verdict = 'REVERT'\n  else if (delta >= 0) verdict = 'KEEP'\n  else verdict = 'INCONCLUSIVE'\n\n  return {\n    baselineMean,\n    candidateMean,\n    delta,\n    ciLower,\n    ciUpper,\n    iterations,\n    alpha,\n    verdict,\n  }\n}\n\nfunction mean(xs: number[]): number {\n  if (xs.length === 0) return 0\n  let s = 0\n  for (const x of xs) s += x\n  return s / xs.length\n}\n\nfunction resample(xs: number[], rng: () => number): number[] {\n  const out = new Array(xs.length)\n  for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)]\n  return out\n}\n\n/** Mulberry32 — fast deterministic PRNG. Stable across runs given the same seed. */\nfunction mulberry32(seed: number): () => number {\n  let t = seed >>> 0\n  return () => {\n    t += 0x6d2b79f5\n    let r = t\n    r = Math.imul(r ^ (r >>> 15), r | 1)\n    r ^= r + Math.imul(r ^ (r >>> 7), r | 61)\n    return ((r ^ (r >>> 14)) >>> 0) / 4294967296\n  }\n}\n\n/** Stable seed derived from the inputs — same data → same CI bounds. */\nfunction hashSeed(a: number[], b: number[]): number {\n  let h = 2166136261\n  for (const x of [...a, ...b]) {\n    const view = new Float64Array([x])\n    const bytes = new Uint8Array(view.buffer)\n    for (const byte of bytes) {\n      h ^= byte\n      h = Math.imul(h, 16777619)\n    }\n  }\n  return h >>> 0\n}\n\n/**\n * Judge-replay promotion gate.\n *\n * The cheap inner-loop judge that drives an evolution run is by definition\n * fast and noisy. When you're about to promote a winning variant to the\n * canonical default, you want a STRONGER judge (a more expensive model, a\n * human grader, a separately-trained reward model) to confirm the win\n * generalises beyond the inner loop.\n *\n * This helper takes raw winner + baseline outputs, scores both through the\n * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger\n * judge agrees the winner is real with the configured confidence. Doesn't\n * matter what shape your \"output\" is — pass a string, an object, anything\n * the judge can read.\n */\nexport interface JudgeReplayGateArgs<TOutput> {\n  baselineOutputs: TOutput[]\n  candidateOutputs: TOutput[]\n  /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */\n  judge: (output: TOutput) => Promise<number> | number\n  alpha?: number\n  iterations?: number\n  /** RNG seed for reproducibility. */\n  seed?: number\n  /** Maximum concurrent judge calls. Default 4. */\n  judgeConcurrency?: number\n}\n\nexport async function judgeReplayGate<TOutput>(\n  args: JudgeReplayGateArgs<TOutput>,\n): Promise<BootstrapResult & { baselineSamples: number; candidateSamples: number }> {\n  const concurrency = args.judgeConcurrency ?? 4\n  const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency)\n  const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency)\n  const ci = bootstrapCi(baselineScores, candidateScores, {\n    ...(args.alpha !== undefined ? { alpha: args.alpha } : {}),\n    ...(args.iterations !== undefined ? { iterations: args.iterations } : {}),\n    ...(args.seed !== undefined ? { seed: args.seed } : {}),\n  })\n  return {\n    ...ci,\n    baselineSamples: baselineScores.length,\n    candidateSamples: candidateScores.length,\n  }\n}\n\nasync function scoreAll<TOutput>(\n  outputs: TOutput[],\n  judge: (output: TOutput) => Promise<number> | number,\n  concurrency: number,\n): Promise<number[]> {\n  const results: number[] = new Array(outputs.length)\n  let next = 0\n  async function worker(): Promise<void> {\n    while (true) {\n      const i = next++\n      if (i >= outputs.length) return\n      const v = await judge(outputs[i]!)\n      results[i] = Number.isFinite(v) ? v : 0\n    }\n  }\n  await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()))\n  return results\n}\n"],"mappings":";;;;;AA0HA,IAAM,qBAA4D;AAAA,EAChE,eAAe;AAAA,EACf,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,gBAAgB;AAAA,EAChB,gBAAgB;AAAA,EAChB,aAAa;AAAA,EACb,cAAc;AAAA,EACd,eAAe;AAAA,EACf,gBAAgB,OAAO;AAAA,EACvB,cAAc,OAAO;AAAA,EACrB,uBAAuB;AAAA,EACvB,uBAAuB;AACzB;AAEO,SAAS,wCACd,QACwB;AACxB,SAAO,OAAO,IAAI,CAAC,WAAW;AAAA,IAC5B,YAAY,MAAM;AAAA,IAClB,aAAa,MAAM;AAAA,IACnB,OAAO,MAAM,UAAU,YAAY,YAAY,MAAM,UAAU,QAAQ,QAAQ;AAAA,IAC/E,OAAO,MAAM;AAAA,IACb,IAAI,MAAM;AAAA,IACV,WAAW,MAAM,QAAQ,MAAM,OAAO,KAAK,IAAI,MAAM,MAAM,MAAM,SAAS;AAAA,IAC1E,SAAS,MAAM;AAAA,IACf,YAAY,MAAM;AAAA,IAClB,aAAa,MAAM,QAAQ,kBAAkB;AAAA,IAC7C,KAAK,MAAM;AAAA,IACX,UAAU,MAAM;AAAA,EAClB,EAAE;AACJ;AAEO,SAAS,0BAA0B,OAA2D;AACnG,QAAM,aAAa,EAAE,GAAG,oBAAoB,GAAG,MAAM,WAAW;AAChE,QAAM,cAAc,MAAM,eAAe;AACzC,QAAM,OAAO,gBAAgB,MAAM,QAAQ,CAAC,GAAG,aAAa,MAAM,UAAU;AAC5E,QAAM,SAAS,qBAAqB,MAAM,UAAU,CAAC,GAAG,aAAa,MAAM,UAAU;AACrF,QAAM,YAAY,MAAM,aAAa,CAAC;AACtC,QAAM,gBAAgB,MAAM,SAAS,iBAAiB,UAAU;AAChE,QAAM,cAAc,MAAM,SAAS,eAAe,oBAAoB,SAAS;AAC/E,QAAM,eAAe,UAAU,MAAM,QAAQ;AAC7C,QAAM,gBAAgB,UAAU,MAAM,SAAS;AAC/C,QAAM,YAAY,CAAC,GAAG,cAAc,GAAG,aAAa;AACpD,QAAM,cAAc,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,cAAc;AACpE,QAAM,gBAAgB,UAAU,SAAS,IAAI,YAAY;AACzD,QAAM,aAAa,KAAK,OAAO,CAAC,MAAM,EAAE,aAAa,QAAQ,EAAE;AAC/D,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,aAAa,SAAS,EAAE;AACjE,QAAM,kBAAkB,KAAK,YAAY;AACzC,QAAM,mBAAmB,KAAK,aAAa;AAC3C,QAAM,UAAoC;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,SAAS,MAAM,QAAQ,WAAW,qBAAqB;AAAA,IACjE,WAAW,KAAK,aAAa;AAAA,IAC7B;AAAA,IACA;AAAA,IACA,YAAY,SAAS,iBAAiB,gBAAgB;AAAA,IACtD,aAAa,KAAK,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,OAAO,GAAG,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,cAAc,CAAC,CAAC;AAAA,IACzG,WAAW,WAAW,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,cAAc,CAAC,GAAG,IAAI;AAAA,IACrH,YAAY,WAAW,MAAM,QAAQ,WAAW,qBAAqB,EAAE;AAAA,IACvE,iBAAiB,WAAW,MAAM,QAAQ,WAAW,qBAAqB,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE;AAAA,IACxG,kBAAkB,OAAO,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE;AAAA,IAC1D,iBAAiB,OAAO,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,CAAC,EAAE;AAAA,IAC9D;AAAA,IACA,cAAc,aAAa,SAAS;AAAA,IACpC,mBAAmB,kBAAkB,MAAM,QAAQ,WAAW,qBAAqB;AAAA,IACnF,0BAA0B,yBAAyB,MAAM;AAAA,EAC3D;AAEA,QAAM,SAAmC,CAAC;AAC1C,cAAY,OAAO,YAAY,SAAS,MAAM;AAC9C,eAAa,YAAY,SAAS,MAAM;AACxC,sBAAoB,MAAM,gBAAgB,MAAM,YAAY,SAAS,MAAM;AAC3E,mBAAiB,YAAY,SAAS,MAAM;AAC5C,kBAAgB,YAAY,SAAS,MAAM;AAE3C,QAAM,OAAO,UAAU,SAAS,YAAY,MAAM,gBAAgB,MAAM,MAAM;AAC9E,QAAM,SAAS,OAAO,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU,IAAI,SAC3D,OAAO,SAAS,IAAI,SACpB;AAEJ,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd;AAAA,IACA,YAAY,MAAM,cAAc;AAAA,IAChC;AAAA,IACA,SAAS,WAAW,WAAW,MAAM,eAAe,MAAM,aAAa,UAAU;AAAA,IACjF;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,MAAM,WAAW;AAAA,IAC1B,cAAc,MAAM,gBAAgB;AAAA,IACpC,SAAS,cAAc,MAAM,QAAQ,QAAQ,SAAS,MAAM;AAAA,EAC9D;AACF;AAEO,SAAS,wBAAwB,OAA2D;AACjG,QAAM,YAAY,0BAA0B,KAAK;AACjD,MAAI,UAAU,WAAW,QAAQ;AAC/B,UAAM,IAAI,MAAM,UAAU,OAAO;AAAA,EACnC;AACA,SAAO;AACT;AAEA,SAAS,gBACP,MACA,aACA,YACa;AACb,MAAI,YAAa,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,WAAW;AACxE,MAAI,WAAY,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAU;AACtE,SAAO,CAAC,GAAG,IAAI;AACjB;AAEA,SAAS,qBACP,QACA,aACA,YACwB;AACxB,MAAI,YAAa,QAAO,OAAO,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAa,EAAE,gBAAgB,WAAW;AACzG,MAAI,WAAY,QAAO,OAAO,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAa,EAAE,gBAAgB,UAAU;AACvG,SAAO,CAAC,GAAG,MAAM;AACnB;AAEA,SAAS,YACP,OACA,YACA,SACA,QACM;AACN,MAAI,WAAW,iBAAiB,CAAC,MAAM,YAAY,MAAM,WAAW,UAAU,OAAO,GAAG;AACtF,WAAO,KAAK,EAAE,MAAM,UAAU,UAAU,YAAY,MAAM,kBAAkB,QAAQ,6CAA6C,CAAC;AAAA,EACpI;AACA,MAAI,QAAQ,gBAAgB,WAAW,kBAAkB;AACvD,WAAO,KAAK,EAAE,MAAM,UAAU,UAAU,YAAY,MAAM,iBAAiB,QAAQ,GAAG,QAAQ,aAAa,sBAAsB,WAAW,gBAAgB,IAAI,CAAC;AAAA,EACnK;AACA,MAAI,WAAW,kBAAkB,QAAQ,YAAY,YAAY,GAAG;AAClE,WAAO,KAAK,EAAE,MAAM,UAAU,UAAU,YAAY,MAAM,yBAAyB,QAAQ,mCAAmC,CAAC;AAAA,EACjI;AACF;AAEA,SAAS,aACP,YACA,SACA,QACM;AACN,MAAI,QAAQ,aAAa,WAAW,eAAe;AACjD,WAAO,KAAK,EAAE,MAAM,WAAW,UAAU,YAAY,MAAM,mBAAmB,QAAQ,GAAG,QAAQ,UAAU,wBAAwB,WAAW,aAAa,IAAI,CAAC;AAAA,EAClK;AACA,MAAI,QAAQ,WAAW,WAAW,aAAa;AAC7C,WAAO,KAAK,EAAE,MAAM,WAAW,UAAU,YAAY,MAAM,iBAAiB,QAAQ,YAAY,IAAI,QAAQ,QAAQ,CAAC,MAAM,IAAI,WAAW,WAAW,CAAC,IAAI,CAAC;AAAA,EAC7J;AACA,MAAI,QAAQ,YAAY,WAAW,cAAc;AAC/C,WAAO,KAAK,EAAE,MAAM,WAAW,UAAU,YAAY,MAAM,kBAAkB,QAAQ,aAAa,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,WAAW,YAAY,CAAC,IAAI,CAAC;AAAA,EACjK;AACF;AAEA,SAAS,oBACP,cACA,YACA,SACA,QACM;AACN,MAAI,WAAW,kBAAkB,QAAQ,cAAc,WAAW,gBAAgB;AAChF,WAAO,KAAK,EAAE,MAAM,kBAAkB,UAAU,YAAY,MAAM,oBAAoB,QAAQ,GAAG,QAAQ,WAAW,yBAAyB,WAAW,cAAc,IAAI,CAAC;AAAA,EAC7K;AACA,MAAI,OAAO,SAAS,QAAQ,UAAU,KAAK,QAAQ,aAAa,WAAW,eAAe;AACxF,WAAO,KAAK,EAAE,MAAM,kBAAkB,UAAU,YAAY,MAAM,eAAe,QAAQ,sBAAsB,IAAI,QAAQ,UAAU,CAAC,MAAM,IAAI,WAAW,aAAa,CAAC,IAAI,CAAC;AAAA,EAChL;AACA,MAAI,gBAAgB,CAAC,aAAa,SAAS;AACzC,WAAO,KAAK,EAAE,MAAM,kBAAkB,UAAU,YAAY,MAAM,QAAQ,aAAa,iBAAiB,QAAQ,IAAI,QAAQ,aAAa,OAAO,CAAC;AAAA,EACnJ;AACF;AAEA,SAAS,iBACP,YACA,SACA,QACM;AACN,MAAI,CAAC,WAAW,sBAAuB;AACvC,MAAI,QAAQ,aAAa,QAAQ,iBAAiB;AAChD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,aAAa,QAAQ,eAAe;AAAA,IACzD,CAAC;AAAA,EACH;AACF;AAEA,SAAS,gBACP,YACA,SACA,QACM;AACN,MAAI,QAAQ,cAAc,WAAW,gBAAgB;AACnD,WAAO,KAAK,EAAE,MAAM,cAAc,UAAU,YAAY,MAAM,eAAe,QAAQ,eAAe,IAAI,QAAQ,WAAW,CAAC,MAAM,IAAI,WAAW,cAAc,CAAC,IAAI,CAAC;AAAA,EACvK;AACA,MAAI,QAAQ,YAAY,WAAW,cAAc;AAC/C,WAAO,KAAK,EAAE,MAAM,cAAc,UAAU,YAAY,MAAM,kBAAkB,QAAQ,aAAa,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,WAAW,YAAY,CAAC,IAAI,CAAC;AAAA,EACpK;AACF;AAEA,SAAS,UACP,SACA,YACA,cACA,QACyB;AACzB,SAAO;AAAA,IACL,KAAK,UAAU,QAAQ,QAAQ,QAAQ,gBAAgB,KAAK,IAAI,GAAG,WAAW,gBAAgB,CAAC,GAAG,GAAG,QAAQ,aAAa,uBAAuB,QAAQ,YAAY,OAAO,EAAE;AAAA,IAC9K,KAAK,WAAW,QAAQ,KAAK,IAAI,QAAQ,UAAU,QAAQ,SAAS,GAAG,YAAY,IAAI,QAAQ,QAAQ,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC,EAAE;AAAA,IAC9I,KAAK,kBAAkB,QAAQ,gBAAgB,CAAC,aAAa,UAAU,IAAI,SAAS,QAAQ,YAAY,WAAW,aAAa,GAAG,eAAe,QAAQ,WAAW,eAAe,IAAI,QAAQ,UAAU,CAAC,EAAE;AAAA,IAC7M,KAAK,eAAe,QAAQ,QAAQ,eAAe,IAAI,IAAI,QAAQ,kBAAkB,QAAQ,YAAY,mBAAmB,QAAQ,eAAe,IAAI,QAAQ,UAAU,EAAE;AAAA,IAC3K,KAAK,cAAc,QAAQ,gBAAgB,SAAS,UAAU,GAAG,eAAe,IAAI,QAAQ,WAAW,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC,EAAE;AAAA,EAChJ;AACF;AAEA,SAAS,KACP,MACA,QACA,OACA,QACuB;AACvB,QAAM,MAAM,OAAO,OAAO,CAAC,MAAM,EAAE,SAAS,IAAI;AAChD,QAAM,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU,IAAI,SACxD,IAAI,SAAS,IAAI,SACjB;AACJ,SAAO,EAAE,MAAM,QAAQ,OAAO,QAAQ,KAAK,GAAG,OAAO;AACvD;AAEA,SAAS,oBAAoB,WAAqE;AAChG,QAAM,SAAuC,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,EAAE;AACrF,aAAW,YAAY,UAAW,QAAO,SAAS,SAAS,OAAO;AAClE,SAAO;AACT;AAEA,SAAS,aAAa,WAA+D;AACnF,QAAM,MAA8B,CAAC;AACrC,aAAW,YAAY,WAAW;AAChC,UAAM,SAAS,SAAS,MAAM,UAAU,SAAS,MAAM,YAAY;AACnE,QAAI,MAAM,KAAK,IAAI,MAAM,KAAK,KAAK;AAAA,EACrC;AACA,SAAO;AACT;AAEA,SAAS,kBACP,MACA,QACA,WACwB;AACxB,QAAM,MAA8B,CAAC;AACrC,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,IAAI,QAAQ,gBAAgB,IAAI,QAAQ;AACtD,QAAI,IAAI,eAAgB,UAAU,UAAa,QAAQ,WAAY;AACjE,YAAM,OAAO,IAAI,eAAe;AAChC,UAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK;AAAA,IACjC;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,QAAI,MAAM,eAAe,MAAM,OAAO,SAAU,MAAM,UAAU,UAAa,MAAM,QAAQ,WAAY;AACrG,YAAM,OAAO,MAAM,gBAAgB,MAAM,OAAO,QAAQ,WAAW;AACnE,UAAI,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK;AAAA,IACjC;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,yBAAyB,QAAiE;AACjG,QAAM,MAA8B,CAAC;AACrC,aAAW,SAAS,QAAQ;AAC1B,eAAW,OAAO,MAAM,OAAO,CAAC,GAAG;AACjC,YAAM,UAAU,IAAI,sBAAsB;AAC1C,UAAI,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK;AAAA,IACvC;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,WACP,MACA,QACA,WAC4B;AAC5B,QAAM,MAAkC,CAAC;AACzC,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,IAAI,QAAQ,gBAAgB,IAAI,QAAQ;AACtD,QAAI,IAAI,eAAgB,UAAU,UAAa,QAAQ,WAAY;AACjE,YAAM,YAAY,IAAI,QAAQ,IAAI;AAClC,UAAI,KAAK,EAAE,QAAQ,OAAO,cAAc,YAAY,YAAY,EAAE,CAAC;AAAA,IACrE;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,QAAI,MAAM,eAAe,MAAM,OAAO,SAAU,MAAM,UAAU,UAAa,MAAM,QAAQ,WAAY;AACrG,UAAI,KAAK,EAAE,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE,CAAC;AAAA,IACnD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,SACP,MACA,QACA,WACQ;AACR,QAAM,WAAW;AAAA,IACf,GAAG,KAAK,IAAI,CAAC,QAAQ;AACnB,YAAM,QAAQ,IAAI,QAAQ,gBAAgB,IAAI,QAAQ;AACtD,aAAO,CAAC,IAAI,eAAe,UAAU,UAAa,SAAS;AAAA,IAC7D,CAAC;AAAA,IACD,GAAG,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO,UAAU,MAAM,UAAU,UAAa,MAAM,SAAS,UAAU;AAAA,EACxG;AACA,MAAI,SAAS,WAAW,EAAG,QAAO;AAClC,SAAO,SAAS,OAAO,OAAO,EAAE,SAAS,SAAS;AACpD;AAEA,SAAS,UAAU,MAA4B,OAA8B;AAC3E,SAAO,KACJ,OAAO,CAAC,QAAQ,IAAI,aAAa,KAAK,EACtC,IAAI,CAAC,QAAQ,UAAU,YAAY,IAAI,QAAQ,eAAe,IAAI,QAAQ,WAAW,EACrF,OAAO,cAAc;AAC1B;AAEA,SAAS,KAAK,IAA+B;AAC3C,MAAI,GAAG,WAAW,EAAG,QAAO,OAAO;AACnC,SAAO,GAAG,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG;AAChD;AAEA,SAAS,WAAW,IAAuB,GAAmB;AAC5D,MAAI,GAAG,WAAW,EAAG,QAAO,OAAO;AACnC,QAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,SAAO,OAAO,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,OAAO,MAAM,IAAI,CAAC,CAAC,CAAC;AAC1F;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,SAAS,GAAW,GAAmB;AAC9C,MAAI,CAAC,OAAO,SAAS,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO,OAAO;AAC9D,SAAO,IAAI;AACb;AAEA,SAAS,SAAS,KAAa,QAAwB;AACrD,MAAI,CAAC,OAAO,SAAS,GAAG,EAAG,QAAO;AAClC,MAAI,UAAU,EAAG,QAAO,OAAO,IAAI,IAAI;AACvC,SAAO,QAAQ,IAAI,KAAK,IAAI,GAAG,GAAG,IAAI,MAAM;AAC9C;AAEA,SAAS,gBACP,SACA,YACQ;AACR,QAAM,OAAO,OAAO,SAAS,WAAW,cAAc,KAAK,OAAO,SAAS,QAAQ,WAAW,IAC1F,QAAQ,WAAW,iBAAiB,KAAK,IAAI,QAAQ,aAAa,KAAK,CAAC,IACxE;AACJ,QAAM,UAAU,OAAO,SAAS,WAAW,YAAY,KAAK,OAAO,SAAS,QAAQ,SAAS,IACzF,QAAQ,WAAW,eAAe,KAAK,IAAI,QAAQ,WAAW,KAAK,CAAC,IACpE;AACJ,SAAO,KAAK,IAAI,MAAM,OAAO;AAC/B;AAEA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,cACP,QACA,QACA,SACA,QACQ;AACR,QAAM,SAAS,sBAAsB,MAAM,KAAK,MAAM;AACtD,QAAM,aAAa,aAAa,QAAQ,aAAa,eAAe,QAAQ,UAAU,gBAAgB,QAAQ,WAAW,aAAa,IAAI,QAAQ,QAAQ,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAC/L,MAAI,OAAO,WAAW,EAAG,QAAO,GAAG,MAAM,KAAK,UAAU;AACxD,SAAO,GAAG,MAAM,KAAK,UAAU,YAAY,OAAO,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC;AAChF;AAEA,SAAS,IAAI,GAAmB;AAC9B,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO,OAAO,CAAC;AACxC,SAAO,EAAE,QAAQ,CAAC;AACpB;;;AC9eO,SAAS,oBACd,WACA,UAAsC,CAAC,GAC/B;AACR,QAAM,QAAQ,QAAQ,SAAS,mBAAmB,UAAU,MAAM;AAClE,QAAM,QAAkB,CAAC;AACzB,QAAM,KAAK,KAAK,KAAK,EAAE;AACvB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,aAAa,UAAU,OAAO,YAAY,CAAC,IAAI;AAC1D,QAAM,KAAK,cAAc,UAAU,UAAU,QAAQ,IAAI,IAAI;AAC7D,MAAI,UAAU,YAAa,OAAM,KAAK,gBAAgB,UAAU,WAAW,IAAI;AAC/E,MAAI,UAAU,WAAY,OAAM,KAAK,eAAe,UAAU,UAAU,IAAI;AAC5E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,UAAU,OAAO;AAC5B,QAAM,KAAK,EAAE;AAEb,QAAM,KAAK,YAAY;AACvB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,oBAAoB;AAC/B,QAAM,KAAK,YAAY;AACvB,QAAM,KAAK,iBAAiB,UAAU,QAAQ,aAAa,IAAI;AAC/D,QAAM,KAAK,mBAAmB,UAAU,QAAQ,UAAU,IAAI;AAC9D,QAAM,KAAK,oBAAoB,UAAU,QAAQ,WAAW,IAAI;AAChE,QAAM,KAAK,iBAAiB,IAAI,UAAU,QAAQ,QAAQ,CAAC,IAAI;AAC/D,QAAM,KAAK,kBAAkB,IAAI,UAAU,QAAQ,SAAS,CAAC,IAAI;AACjE,QAAM,KAAK,mBAAmB,IAAI,UAAU,QAAQ,eAAe,CAAC,IAAI;AACxE,QAAM,KAAK,oBAAoB,IAAI,UAAU,QAAQ,gBAAgB,CAAC,IAAI;AAC1E,QAAM,KAAK,mBAAmB,IAAI,UAAU,QAAQ,UAAU,CAAC,IAAI;AACnE,QAAM,KAAK,kBAAkB,IAAI,UAAU,QAAQ,WAAW,CAAC,IAAI;AACnE,QAAM,KAAK,qBAAqB,KAAK,MAAM,UAAU,QAAQ,SAAS,CAAC,OAAO;AAC9E,QAAM,KAAK,EAAE;AAEb,MAAI,UAAU,OAAO,SAAS,GAAG;AAC/B,UAAM,KAAK,WAAW;AACtB,UAAM,KAAK,EAAE;AACb,eAAW,SAAS,UAAU,QAAQ;AACpC,YAAM,KAAK,OAAO,MAAM,QAAQ,QAAQ,MAAM,IAAI,OAAO,MAAM,IAAI,MAAM,MAAM,MAAM,EAAE;AAAA,IACzF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,WAAW,QAAQ,UAAU,QAAQ,wBAAwB;AACnE,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,KAAK,yBAAyB;AACpC,UAAM,KAAK,EAAE;AACb,eAAW,CAAC,SAAS,KAAK,KAAK,SAAU,OAAM,KAAK,KAAK,OAAO,KAAK,KAAK,EAAE;AAC5E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,WAAW,QAAQ,UAAU,QAAQ,iBAAiB;AAC5D,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,KAAK,kBAAkB;AAC7B,UAAM,KAAK,EAAE;AACb,eAAW,CAAC,MAAM,KAAK,KAAK,SAAU,OAAM,KAAK,KAAK,IAAI,KAAK,KAAK,EAAE;AACtE,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,QAAQ,QAAQ,QAAQ,KAAK,SAAS,GAAG;AAC3C,UAAM,KAAK,gBAAgB;AAC3B,UAAM,KAAK,EAAE;AACb,UAAM,KAAK,aAAa,CAAC,GAAG,QAAQ,IAAI,GAAG;AAAA,MACzC,YAAY,QAAQ,cAAc,UAAU,cAAc;AAAA,MAC1D,OAAO;AAAA,IACT,CAAC,EAAE,QAAQ;AACX,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,QAAQ,wBAAwB,QAAQ,qBAAqB,SAAS,GAAG;AAC3E,UAAM,KAAK,0BAA0B;AACrC,UAAM,KAAK,EAAE;AACb,eAAW,WAAW,QAAQ,qBAAsB,OAAM,KAAK,KAAK,OAAO,EAAE;AAC7E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,cAAc,QAAQ,eAAe,mBAAmB,SAAS;AACvE,MAAI,YAAY,SAAS,GAAG;AAC1B,UAAM,KAAK,iBAAiB;AAC5B,UAAM,KAAK,EAAE;AACb,eAAW,UAAU,YAAa,OAAM,KAAK,KAAK,MAAM,EAAE;AAC1D,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,SAAO,MAAM,KAAK,IAAI,EAAE,QAAQ,IAAI;AACtC;AAEA,SAAS,mBAAmB,WAAiD;AAC3E,MAAI,UAAU,QAAS,QAAO,CAAC,kDAAkD;AACjF,SAAO,UAAU,OACd,OAAO,CAAC,UAAU,MAAM,aAAa,UAAU,EAC/C,IAAI,CAAC,UAAU,WAAW,MAAM,IAAI,KAAK,MAAM,MAAM,EAAE;AAC5D;AAEA,SAAS,QAAQ,QAAyD;AACxE,SAAO,OAAO,QAAQ,MAAM,EACzB,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,QAAQ,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;AAC3D;AAEA,SAAS,IAAI,OAAuB;AAClC,SAAO,OAAO,SAAS,KAAK,IAAI,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,MAAM;AACnE;AAEA,SAAS,IAAI,OAAuB;AAClC,SAAO,OAAO,SAAS,KAAK,IAAI,MAAM,QAAQ,CAAC,IAAI;AACrD;;;ACpDO,SAAS,YACd,UACA,WACA,UAA4B,CAAC,GACZ;AACjB,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,WAAW,QAAQ,mBAAmB;AAC5C,QAAM,MAAM,WAAW,QAAQ,QAAQ,SAAS,UAAU,SAAS,CAAC;AAEpE,QAAM,eAAeA,MAAK,QAAQ;AAClC,QAAM,gBAAgBA,MAAK,SAAS;AACpC,QAAM,QAAQ,gBAAgB;AAE9B,MAAI,SAAS,SAAS,UAAU,SAAS,YAAY,SAAS,WAAW,KAAK,UAAU,WAAW,GAAG;AACpG,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,SAAS;AAAA,MACT,YAAY;AAAA,MACZ;AAAA,MACA,SAAS;AAAA,IACX;AAAA,EACF;AAEA,QAAM,SAAmB,IAAI,MAAM,UAAU;AAC7C,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,UAAM,YAAY,SAAS,UAAU,GAAG;AACxC,UAAM,YAAY,SAAS,WAAW,GAAG;AACzC,WAAO,CAAC,IAAIA,MAAK,SAAS,IAAIA,MAAK,SAAS;AAAA,EAC9C;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAC5D,QAAM,UAAU,OAAO,KAAK,IAAI,GAAG,QAAQ,CAAC;AAC5C,QAAM,UAAU,OAAO,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC;AAEzD,MAAI;AACJ,MAAI,UAAU,EAAG,WAAU;AAAA,WAClB,UAAU,EAAG,WAAU;AAAA,WACvB,SAAS,EAAG,WAAU;AAAA,MAC1B,WAAU;AAEf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAASA,MAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,MAAI,IAAI;AACR,aAAW,KAAK,GAAI,MAAK;AACzB,SAAO,IAAI,GAAG;AAChB;AAEA,SAAS,SAAS,IAAc,KAA6B;AAC3D,QAAM,MAAM,IAAI,MAAM,GAAG,MAAM;AAC/B,WAAS,IAAI,GAAG,IAAI,GAAG,QAAQ,IAAK,KAAI,CAAC,IAAI,GAAG,KAAK,MAAM,IAAI,IAAI,GAAG,MAAM,CAAC;AAC7E,SAAO;AACT;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,IAAI,SAAS;AACjB,SAAO,MAAM;AACX,SAAK;AACL,QAAI,IAAI;AACR,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AACF;AAGA,SAAS,SAAS,GAAa,GAAqB;AAClD,MAAI,IAAI;AACR,aAAW,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG;AAC5B,UAAM,OAAO,IAAI,aAAa,CAAC,CAAC,CAAC;AACjC,UAAM,QAAQ,IAAI,WAAW,KAAK,MAAM;AACxC,eAAW,QAAQ,OAAO;AACxB,WAAK;AACL,UAAI,KAAK,KAAK,GAAG,QAAQ;AAAA,IAC3B;AAAA,EACF;AACA,SAAO,MAAM;AACf;AA8BA,eAAsB,gBACpB,MACkF;AAClF,QAAM,cAAc,KAAK,oBAAoB;AAC7C,QAAM,iBAAiB,MAAM,SAAS,KAAK,iBAAiB,KAAK,OAAO,WAAW;AACnF,QAAM,kBAAkB,MAAM,SAAS,KAAK,kBAAkB,KAAK,OAAO,WAAW;AACrF,QAAM,KAAK,YAAY,gBAAgB,iBAAiB;AAAA,IACtD,GAAI,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,IACxD,GAAI,KAAK,eAAe,SAAY,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;AAAA,IACvE,GAAI,KAAK,SAAS,SAAY,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC;AAAA,EACvD,CAAC;AACD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,iBAAiB,eAAe;AAAA,IAChC,kBAAkB,gBAAgB;AAAA,EACpC;AACF;AAEA,eAAe,SACb,SACA,OACA,aACmB;AACnB,QAAM,UAAoB,IAAI,MAAM,QAAQ,MAAM;AAClD,MAAI,OAAO;AACX,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,QAAQ,OAAQ;AACzB,YAAM,IAAI,MAAM,MAAM,QAAQ,CAAC,CAAE;AACjC,cAAQ,CAAC,IAAI,OAAO,SAAS,CAAC,IAAI,IAAI;AAAA,IACxC;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAClF,SAAO;AACT;","names":["mean"]}