@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/prm/index.js

@@ -0,0 +1,222 @@
+import {
+  exportTrainingData,
+  toNdjson
+} from "../chunk-KMPRBJK4.js";
+import {
+  buildTrajectory
+} from "../chunk-RZTMDUO7.js";
+import "../chunk-5BKGXME7.js";
+import {
+  TraceEmitter
+} from "../chunk-TVVP3ZZQ.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/prm/builtin-rubrics.ts
+function outputLengthRubric(args = {}) {
+  const min = args.minChars ?? 20;
+  const max = args.maxChars ?? 8e3;
+  return {
+    id: "output-length",
+    kinds: ["llm"],
+    weight: args.weight ?? 0.5,
+    async grade({ step }) {
+      const llm = step.span;
+      const len = (llm.output ?? "").length;
+      if (len === 0) return { score: 0, rationale: "empty output" };
+      if (len < min)
+        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` };
+      if (len > max)
+        return {
+          score: Math.max(0, 1 - (len - max) / max),
+          rationale: `above max (${len} > ${max})`
+        };
+      return { score: 1, rationale: `${len} chars in bounds` };
+    }
+  };
+}
+function toolSuccessRubric(args = {}) {
+  return {
+    id: "tool-success",
+    kinds: ["tool"],
+    weight: args.weight ?? 1,
+    async grade({ step }) {
+      const tool = step.span;
+      if (tool.status === "error")
+        return { score: 0, rationale: `error: ${tool.error ?? "unknown"}` };
+      const r = tool.result;
+      if (r === null || r === void 0) return { score: 0.3, rationale: "empty result" };
+      const asText = typeof r === "string" ? r : JSON.stringify(r);
+      if (asText.length < 4) return { score: 0.5, rationale: "tiny result" };
+      return { score: 1, rationale: `${tool.toolName} ok` };
+    }
+  };
+}
+function toolNonRedundantRubric(args = {}) {
+  const weight = args.weight ?? 0.5;
+  return {
+    id: "tool-non-redundant",
+    kinds: ["tool"],
+    weight,
+    async grade({ step, prior }) {
+      const tool = step.span;
+      const priorMatches = prior.filter((p) => {
+        if (p.span.kind !== "tool") return false;
+        const pt = p.span;
+        return pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args);
+      });
+      if (priorMatches.length === 0) return { score: 1, rationale: "novel call" };
+      return {
+        score: Math.max(0, 1 - priorMatches.length * 0.5),
+        rationale: `${priorMatches.length} duplicate(s)`
+      };
+    }
+  };
+}
+function nonRefusalRubric(args = {}) {
+  const weight = args.weight ?? 1;
+  const markers = args.markers ?? [
+    /\bi\s+(?:can(?:not|'t)|won't|will\s+not)\b/i,
+    /\b(?:as\s+an?\s+)?ai\b.*?\b(?:can't|cannot)\b/i
+  ];
+  return {
+    id: "non-refusal",
+    kinds: ["llm"],
+    weight,
+    async grade({ step }) {
+      const llm = step.span;
+      const out = llm.output ?? "";
+      const refused = markers.some((re) => re.test(out));
+      return refused ? { score: 0, rationale: "refusal marker present" } : { score: 1, rationale: "no refusal" };
+    }
+  };
+}
+function toolIntentAlignmentRubric(args = {}) {
+  return {
+    id: "tool-intent-alignment",
+    kinds: ["llm"],
+    weight: args.weight ?? 0.5,
+    async grade({ step, next }) {
+      const llm = step.span;
+      const nextTool = next.find((s) => s.span.kind === "tool");
+      if (!nextTool) return null;
+      const toolName = nextTool.span.toolName;
+      const out = (llm.output ?? "").toLowerCase();
+      const mentioned = out.includes(toolName.toLowerCase());
+      return mentioned ? { score: 1, rationale: `mentioned "${toolName}" before calling it` } : { score: 0.5, rationale: `called "${toolName}" without announcing it` };
+    }
+  };
+}
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+  const keys = Object.keys(value).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(",")}}`;
+}
+
+// src/prm/inference.ts
+async function prmBestOfN(store, grader, runIds) {
+  if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
+  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
+  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
+  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
+  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length;
+  return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
+}
+async function prmEnsembleBestOfN(store, graders, runIds) {
+  if (graders.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 grader");
+  const perGrader = await Promise.all(
+    graders.map(async (g) => {
+      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)));
+      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore);
+    })
+  );
+  const bordaScores = /* @__PURE__ */ new Map();
+  for (const ranking of perGrader) {
+    ranking.forEach((g, rank) => {
+      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank));
+    });
+  }
+  const canonical = perGrader[0];
+  const byRun = new Map(canonical.map((g) => [g.runId, g]));
+  const ranked = [...byRun.values()].sort(
+    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
+  );
+  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
+  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length;
+  return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
+}
+
+// src/prm/rubric.ts
+var PrmGrader = class {
+  constructor(rubrics) {
+    this.rubrics = rubrics;
+    if (rubrics.length === 0) throw new Error("PrmGrader: at least 1 rubric required");
+  }
+  rubrics;
+  /**
+   * Grade every eligible span in a run. Emits a JudgeVerdict span for each
+   * (rubric × span) verdict so the result is visible to downstream pipelines
+   * (judgeAgreementView, etc.) — PRM is just "a judge that runs per span."
+   */
+  async grade(store, runId) {
+    const trajectory = await buildTrajectory(store, runId);
+    const emitter = new TraceEmitter(store, { runId });
+    const steps = [];
+    let ungraded = 0;
+    for (let i = 0; i < trajectory.steps.length; i++) {
+      const step = trajectory.steps[i];
+      const ctx = {
+        trajectory,
+        step,
+        prior: trajectory.steps.slice(0, i),
+        next: trajectory.steps.slice(i + 1)
+      };
+      let gradedThis = false;
+      for (const rubric of this.rubrics) {
+        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue;
+        const verdict = await rubric.grade(ctx);
+        if (verdict === null) continue;
+        const weight = rubric.weight ?? 1;
+        steps.push({
+          spanId: step.span.spanId,
+          rubricId: rubric.id,
+          score: verdict.score,
+          weight,
+          rationale: verdict.rationale,
+          evidence: verdict.evidence
+        });
+        gradedThis = true;
+        await emitter.recordJudge({
+          judgeId: `prm:${rubric.id}`,
+          targetSpanId: step.span.spanId,
+          dimension: "step_quality",
+          score: verdict.score,
+          rationale: verdict.rationale,
+          evidence: verdict.evidence,
+          name: `prm:${rubric.id}`
+        });
+      }
+      if (!gradedThis) ungraded++;
+    }
+    const totalWeight = steps.reduce((a, s) => a + s.weight, 0);
+    const aggregateScore = totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight;
+    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded };
+  }
+};
+function isPrmVerdict(verdict) {
+  return verdict.judgeId.startsWith("prm:");
+}
+export {
+  PrmGrader,
+  exportTrainingData,
+  isPrmVerdict,
+  nonRefusalRubric,
+  outputLengthRubric,
+  prmBestOfN,
+  prmEnsembleBestOfN,
+  toNdjson,
+  toolIntentAlignmentRubric,
+  toolNonRedundantRubric,
+  toolSuccessRubric
+};
+//# sourceMappingURL=index.js.map

package/dist/prm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/prm/builtin-rubrics.ts","../../src/prm/inference.ts","../../src/prm/rubric.ts"],"sourcesContent":["/**\n * Built-in reference rubrics. Consumers combine these with domain\n * rubrics. All are deterministic, rule-based — cheap to run + easy\n * to unit-test. LLM-based rubrics are trivially authored by\n * following the StepRubric contract.\n */\n\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { StepRubric } from './rubric'\n\n/** Penalize very short or very long assistant outputs. */\nexport function outputLengthRubric(\n  args: { minChars?: number; maxChars?: number; weight?: number } = {},\n): StepRubric {\n  const min = args.minChars ?? 20\n  const max = args.maxChars ?? 8000\n  return {\n    id: 'output-length',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const len = (llm.output ?? '').length\n      if (len === 0) return { score: 0, rationale: 'empty output' }\n      if (len < min)\n        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` }\n      if (len > max)\n        return {\n          score: Math.max(0, 1 - (len - max) / max),\n          rationale: `above max (${len} > ${max})`,\n        }\n      return { score: 1, rationale: `${len} chars in bounds` }\n    },\n  }\n}\n\n/** Reward tool calls that succeeded (status='ok') with an informative result. */\nexport function toolSuccessRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-success',\n    kinds: ['tool'],\n    weight: args.weight ?? 1,\n    async grade({ step }) {\n      const tool = step.span as ToolSpan\n      if (tool.status === 'error')\n        return { score: 0, rationale: `error: ${tool.error ?? 'unknown'}` }\n      const r = tool.result\n      if (r === null || r === undefined) return { score: 0.3, rationale: 'empty result' }\n      const asText = typeof r === 'string' ? r : JSON.stringify(r)\n      if (asText.length < 4) return { score: 0.5, rationale: 'tiny result' }\n      return { score: 1, rationale: `${tool.toolName} ok` }\n    },\n  }\n}\n\n/** Penalize tool calls that duplicate a prior call with identical args. */\nexport function toolNonRedundantRubric(args: { weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 0.5\n  return {\n    id: 'tool-non-redundant',\n    kinds: ['tool'],\n    weight,\n    async grade({ step, prior }) {\n      const tool = step.span as ToolSpan\n      const priorMatches = prior.filter((p) => {\n        if (p.span.kind !== 'tool') return false\n        const pt = p.span as ToolSpan\n        return (\n          pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args)\n        )\n      })\n      if (priorMatches.length === 0) return { score: 1, rationale: 'novel call' }\n      return {\n        score: Math.max(0, 1 - priorMatches.length * 0.5),\n        rationale: `${priorMatches.length} duplicate(s)`,\n      }\n    },\n  }\n}\n\n/** Penalize LLM outputs that contain common refusal markers when a refusal\n *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */\nexport function nonRefusalRubric(args: { markers?: RegExp[]; weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 1\n  const markers = args.markers ?? [\n    /\\bi\\s+(?:can(?:not|'t)|won't|will\\s+not)\\b/i,\n    /\\b(?:as\\s+an?\\s+)?ai\\b.*?\\b(?:can't|cannot)\\b/i,\n  ]\n  return {\n    id: 'non-refusal',\n    kinds: ['llm'],\n    weight,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const out = llm.output ?? ''\n      const refused = markers.some((re) => re.test(out))\n      return refused\n        ? { score: 0, rationale: 'refusal marker present' }\n        : { score: 1, rationale: 'no refusal' }\n    },\n  }\n}\n\n/** Reward outputs that invoke the next-step tool the trajectory actually uses\n *  (i.e. the LLM span announced \"I will call X\" and the following tool span IS X). */\nexport function toolIntentAlignmentRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-intent-alignment',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step, next }) {\n      const llm = step.span as LlmSpan\n      const nextTool = next.find((s) => s.span.kind === 'tool')\n      if (!nextTool) return null\n      const toolName = (nextTool.span as ToolSpan).toolName\n      const out = (llm.output ?? '').toLowerCase()\n      const mentioned = out.includes(toolName.toLowerCase())\n      return mentioned\n        ? { score: 1, rationale: `mentioned \"${toolName}\" before calling it` }\n        : { score: 0.5, rationale: `called \"${toolName}\" without announcing it` }\n    },\n  }\n}\n\nfunction stableStringify(value: unknown): string {\n  if (value === null || typeof value !== 'object') return JSON.stringify(value)\n  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n  const keys = Object.keys(value as Record<string, unknown>).sort()\n  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record<string, unknown>)[k])}`).join(',')}}`\n}\n","/**\n * Inference-time PRM scoring — pick the best of N candidate trajectories\n * using a trained reward model (or a rule-based PRM as a proxy).\n *\n * The canonical Best-of-N pattern: generate N completions, score each\n * with a PRM, pick the winner. Here the scoring loop is framework-agnostic\n * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport type { PrmGradedTrace, PrmGrader } from './rubric'\n\nexport interface BestOfNResult {\n  winner: PrmGradedTrace\n  ranked: PrmGradedTrace[]\n  /** Standard deviation of aggregate scores — small = candidates were homogenous. */\n  stdDev: number\n}\n\nexport async function prmBestOfN(\n  store: TraceStore,\n  grader: PrmGrader,\n  runIds: string[],\n): Promise<BestOfNResult> {\n  if (runIds.length === 0) throw new Error('prmBestOfN: at least 1 candidate required')\n  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)))\n  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore)\n  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length\n  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n\n/**\n * Weighted vote across multiple graders — use when you want a PRM ensemble\n * (e.g. rule-based + LLM-based + trained model). Each grader produces its\n * own ranking; we aggregate via rank-sum (Borda count) so no single grader\n * dominates via a different score scale.\n */\nexport async function prmEnsembleBestOfN(\n  store: TraceStore,\n  graders: PrmGrader[],\n  runIds: string[],\n): Promise<BestOfNResult> {\n  if (graders.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 grader')\n  const perGrader = await Promise.all(\n    graders.map(async (g) => {\n      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)))\n      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore)\n    }),\n  )\n  // Borda: rank-sum across graders.\n  const bordaScores = new Map<string, number>()\n  for (const ranking of perGrader) {\n    ranking.forEach((g, rank) => {\n      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank))\n    })\n  }\n  // Return a synthesized ranking using the first grader's graded traces\n  // ordered by Borda score. aggregateScore field kept for UX.\n  const canonical = perGrader[0]!\n  const byRun = new Map(canonical.map((g) => [g.runId, g]))\n  const ranked = [...byRun.values()].sort(\n    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0),\n  )\n  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length\n  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n","/**\n * Process Reward Modeling — per-step rubric grading.\n *\n * A StepRubric inspects one span and returns a score + rationale.\n * PrmGrader applies an array of rubrics to every LLM span in a\n * trajectory (consumers can broaden to tool/retrieval spans via the\n * `kind` filter on each rubric).\n *\n * Why this matters: outcome-only eval (did the final artifact work?)\n * gives sparse reward — most agent turns are unattributable. PRMs\n * densify the signal so optimizers and RL fine-tuning can assign\n * credit per turn.\n */\n\nimport { TraceEmitter } from '../trace/emitter'\nimport type { JudgeSpan, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface StepContext {\n  trajectory: Trajectory\n  step: TrajectoryStep\n  /** Steps preceding `step` in trajectory order. */\n  prior: TrajectoryStep[]\n  /** Steps following `step`. */\n  next: TrajectoryStep[]\n}\n\nexport interface StepRubric {\n  id: string\n  /** Only grade spans of these kinds (default: all). */\n  kinds?: Array<Span['kind']>\n  /** Weight in the aggregate score. Default 1. */\n  weight?: number\n  /** Returns score in 0..1 + optional rationale/evidence. Return `null` to\n   *  skip grading (rubric doesn't apply to this step). */\n  grade: (\n    ctx: StepContext,\n  ) => Promise<{ score: number; rationale?: string; evidence?: string } | null>\n}\n\nexport interface GradedStep {\n  spanId: string\n  rubricId: string\n  score: number\n  weight: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface PrmGradedTrace {\n  runId: string\n  steps: GradedStep[]\n  /** Weighted mean of all graded steps; 0..1. */\n  aggregateScore: number\n  /** Number of spans graded — useful for sanity-checking coverage. */\n  gradedCount: number\n  /** Number of spans in the trajectory that no rubric matched. */\n  ungradedCount: number\n}\n\nexport class PrmGrader {\n  constructor(private rubrics: StepRubric[]) {\n    if (rubrics.length === 0) throw new Error('PrmGrader: at least 1 rubric required')\n  }\n\n  /**\n   * Grade every eligible span in a run. Emits a JudgeVerdict span for each\n   * (rubric × span) verdict so the result is visible to downstream pipelines\n   * (judgeAgreementView, etc.) — PRM is just \"a judge that runs per span.\"\n   */\n  async grade(store: TraceStore, runId: string): Promise<PrmGradedTrace> {\n    const trajectory = await buildTrajectory(store, runId)\n    const emitter = new TraceEmitter(store, { runId })\n    const steps: GradedStep[] = []\n    let ungraded = 0\n    for (let i = 0; i < trajectory.steps.length; i++) {\n      const step = trajectory.steps[i]!\n      const ctx: StepContext = {\n        trajectory,\n        step,\n        prior: trajectory.steps.slice(0, i),\n        next: trajectory.steps.slice(i + 1),\n      }\n      let gradedThis = false\n      for (const rubric of this.rubrics) {\n        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue\n        const verdict = await rubric.grade(ctx)\n        if (verdict === null) continue\n        const weight = rubric.weight ?? 1\n        steps.push({\n          spanId: step.span.spanId,\n          rubricId: rubric.id,\n          score: verdict.score,\n          weight,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n        })\n        gradedThis = true\n        // Persist the verdict as a JudgeSpan so the query pipelines see it\n        await emitter.recordJudge({\n          judgeId: `prm:${rubric.id}`,\n          targetSpanId: step.span.spanId,\n          dimension: 'step_quality',\n          score: verdict.score,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n          name: `prm:${rubric.id}`,\n        })\n      }\n      if (!gradedThis) ungraded++\n    }\n\n    const totalWeight = steps.reduce((a, s) => a + s.weight, 0)\n    const aggregateScore =\n      totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight\n\n    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded }\n  }\n}\n\n/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines\n *  can distinguish PRM verdicts from human or top-level LLM judges. */\nexport function isPrmVerdict(verdict: JudgeSpan): boolean {\n  return verdict.judgeId.startsWith('prm:')\n}\n"],"mappings":";;;;;;;;;;;;;;AAWO,SAAS,mBACd,OAAkE,CAAC,GACvD;AACZ,QAAM,MAAM,KAAK,YAAY;AAC7B,QAAM,MAAM,KAAK,YAAY;AAC7B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,OAAO,IAAI,UAAU,IAAI;AAC/B,UAAI,QAAQ,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,eAAe;AAC5D,UAAI,MAAM;AACR,eAAO,EAAE,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,WAAW,cAAc,GAAG,MAAM,GAAG,IAAI;AACnF,UAAI,MAAM;AACR,eAAO;AAAA,UACL,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,GAAG;AAAA,UACxC,WAAW,cAAc,GAAG,MAAM,GAAG;AAAA,QACvC;AACF,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,GAAG,mBAAmB;AAAA,IACzD;AAAA,EACF;AACF;AAGO,SAAS,kBAAkB,OAA4B,CAAC,GAAe;AAC5E,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,OAAO,KAAK;AAClB,UAAI,KAAK,WAAW;AAClB,eAAO,EAAE,OAAO,GAAG,WAAW,UAAU,KAAK,SAAS,SAAS,GAAG;AACpE,YAAM,IAAI,KAAK;AACf,UAAI,MAAM,QAAQ,MAAM,OAAW,QAAO,EAAE,OAAO,KAAK,WAAW,eAAe;AAClF,YAAM,SAAS,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC;AAC3D,UAAI,OAAO,SAAS,EAAG,QAAO,EAAE,OAAO,KAAK,WAAW,cAAc;AACrE,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,KAAK,QAAQ,MAAM;AAAA,IACtD;AAAA,EACF;AACF;AAGO,SAAS,uBAAuB,OAA4B,CAAC,GAAe;AACjF,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd;AAAA,IACA,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAO,KAAK;AAClB,YAAM,eAAe,MAAM,OAAO,CAAC,MAAM;AACvC,YAAI,EAAE,KAAK,SAAS,OAAQ,QAAO;AACnC,cAAM,KAAK,EAAE;AACb,eACE,GAAG,aAAa,KAAK,YAAY,gBAAgB,GAAG,IAAI,MAAM,gBAAgB,KAAK,IAAI;AAAA,MAE3F,CAAC;AACD,UAAI,aAAa,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,aAAa;AAC1E,aAAO;AAAA,QACL,OAAO,KAAK,IAAI,GAAG,IAAI,aAAa,SAAS,GAAG;AAAA,QAChD,WAAW,GAAG,aAAa,MAAM;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AACF;AAIO,SAAS,iBAAiB,OAAgD,CAAC,GAAe;AAC/F,QAAM,SAAS,KAAK,UAAU;AAC9B,QAAM,UAAU,KAAK,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb;AAAA,IACA,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,MAAM,IAAI,UAAU;AAC1B,YAAM,UAAU,QAAQ,KAAK,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC;AACjD,aAAO,UACH,EAAE,OAAO,GAAG,WAAW,yBAAyB,IAChD,EAAE,OAAO,GAAG,WAAW,aAAa;AAAA,IAC1C;AAAA,EACF;AACF;AAIO,SAAS,0BAA0B,OAA4B,CAAC,GAAe;AACpF,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,MAAM,KAAK,GAAG;AAC1B,YAAM,MAAM,KAAK;AACjB,YAAM,WAAW,KAAK,KAAK,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM;AACxD,UAAI,CAAC,SAAU,QAAO;AACtB,YAAM,WAAY,SAAS,KAAkB;AAC7C,YAAM,OAAO,IAAI,UAAU,IAAI,YAAY;AAC3C,YAAM,YAAY,IAAI,SAAS,SAAS,YAAY,CAAC;AACrD,aAAO,YACH,EAAE,OAAO,GAAG,WAAW,cAAc,QAAQ,sBAAsB,IACnE,EAAE,OAAO,KAAK,WAAW,WAAW,QAAQ,0BAA0B;AAAA,IAC5E;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC5E,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AACzE,QAAM,OAAO,OAAO,KAAK,KAAgC,EAAE,KAAK;AAChE,SAAO,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAiB,MAAkC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC;AACxH;;;AC9GA,eAAsB,WACpB,OACA,QACA,QACwB;AACxB,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,2CAA2C;AACpF,QAAM,SAAS,MAAM,QAAQ,IAAI,OAAO,IAAI,CAAC,OAAO,OAAO,MAAM,OAAO,EAAE,CAAC,CAAC;AAC5E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAC7E,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;AAQA,eAAsB,mBACpB,OACA,SACA,QACwB;AACxB,MAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AACjF,QAAM,YAAY,MAAM,QAAQ;AAAA,IAC9B,QAAQ,IAAI,OAAO,MAAM;AACvB,YAAM,SAAS,MAAM,QAAQ,IAAI,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;AACvE,aAAO,OAAO,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAAA,IAClE,CAAC;AAAA,EACH;AAEA,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,WAAW,WAAW;AAC/B,YAAQ,QAAQ,CAAC,GAAG,SAAS;AAC3B,kBAAY,IAAI,EAAE,QAAQ,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,QAAQ,SAAS,KAAK;AAAA,IACpF,CAAC;AAAA,EACH;AAGA,QAAM,YAAY,UAAU,CAAC;AAC7B,QAAM,QAAQ,IAAI,IAAI,UAAU,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AACxD,QAAM,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE;AAAA,IACjC,CAAC,GAAG,OAAO,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,YAAY,IAAI,EAAE,KAAK,KAAK;AAAA,EAC3E;AACA,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;;;ACNO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAAoB,SAAuB;AAAvB;AAClB,QAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AAAA,EACnF;AAAA,EAFoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASpB,MAAM,MAAM,OAAmB,OAAwC;AACrE,UAAM,aAAa,MAAM,gBAAgB,OAAO,KAAK;AACrD,UAAM,UAAU,IAAI,aAAa,OAAO,EAAE,MAAM,CAAC;AACjD,UAAM,QAAsB,CAAC;AAC7B,QAAI,WAAW;AACf,aAAS,IAAI,GAAG,IAAI,WAAW,MAAM,QAAQ,KAAK;AAChD,YAAM,OAAO,WAAW,MAAM,CAAC;AAC/B,YAAM,MAAmB;AAAA,QACvB;AAAA,QACA;AAAA,QACA,OAAO,WAAW,MAAM,MAAM,GAAG,CAAC;AAAA,QAClC,MAAM,WAAW,MAAM,MAAM,IAAI,CAAC;AAAA,MACpC;AACA,UAAI,aAAa;AACjB,iBAAW,UAAU,KAAK,SAAS;AACjC,YAAI,OAAO,SAAS,CAAC,OAAO,MAAM,SAAS,KAAK,KAAK,IAAI,EAAG;AAC5D,cAAM,UAAU,MAAM,OAAO,MAAM,GAAG;AACtC,YAAI,YAAY,KAAM;AACtB,cAAM,SAAS,OAAO,UAAU;AAChC,cAAM,KAAK;AAAA,UACT,QAAQ,KAAK,KAAK;AAAA,UAClB,UAAU,OAAO;AAAA,UACjB,OAAO,QAAQ;AAAA,UACf;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,QACpB,CAAC;AACD,qBAAa;AAEb,cAAM,QAAQ,YAAY;AAAA,UACxB,SAAS,OAAO,OAAO,EAAE;AAAA,UACzB,cAAc,KAAK,KAAK;AAAA,UACxB,WAAW;AAAA,UACX,OAAO,QAAQ;AAAA,UACf,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,UAClB,MAAM,OAAO,OAAO,EAAE;AAAA,QACxB,CAAC;AAAA,MACH;AACA,UAAI,CAAC,WAAY;AAAA,IACnB;AAEA,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC1D,UAAM,iBACJ,gBAAgB,IAAI,IAAI,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI;AAE9E,WAAO,EAAE,OAAO,OAAO,gBAAgB,aAAa,MAAM,QAAQ,eAAe,SAAS;AAAA,EAC5F;AACF;AAIO,SAAS,aAAa,SAA6B;AACxD,SAAO,QAAQ,QAAQ,WAAW,MAAM;AAC1C;","names":[]}

package/dist/query-DODUYdPg.d.ts

@@ -0,0 +1,30 @@
+import { L as LlmSpan, T as TraceStore, J as JudgeSpan, R as Run, F as FailureClass, a as ToolSpan } from './store-Db2Bv8Cf.js';
+
+/**
+ * Typed query helpers over TraceStore.
+ *
+ * Not a full SQL engine — a minimal, composable set of operators that
+ * cover the canned-pipeline use cases. For ad-hoc analytics, persist to
+ * NDJSON and point DuckDB at it; the schema is stable so external SQL
+ * tooling works out of the box.
+ */
+
+declare function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]>;
+declare function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]>;
+declare function toolSpans(store: TraceStore, runId?: string, toolName?: string): Promise<ToolSpan[]>;
+declare function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]>;
+/** Group spans by any key selector. */
+declare function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]>;
+/** Hash tool arguments to an orderless-key-stable string for de-duplication. */
+declare function argHash(args: unknown): string;
+/** Sum an LLM-span array into aggregate token + cost. */
+declare function aggregateLlm(spans: LlmSpan[]): {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+};
+/** Pick the outcome's failure class when present, else derive 'success' from run status. */
+declare function runFailureClass(run: Run): FailureClass;
+
+export { aggregateLlm as a, argHash as b, runsForScenario as c, groupBy as g, judgeSpans as j, llmSpans as l, runFailureClass as r, toolSpans as t };

package/dist/release-report-TDPn1cxq.d.ts

@@ -0,0 +1,292 @@
+import { D as DatasetSplit, b as DatasetManifest, a as DatasetScenario } from './dataset-CiK_3LDr.js';
+import { a3 as GateDecision, A as ActionableSideInfo, m as MultiShotTrialResult } from './summary-report-BXGs_9V0.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-CqzahIbx.js';
+
+/**
+ * Release confidence gate.
+ *
+ * This is the production-facing composition layer over the lower-level
+ * primitives:
+ *   - Dataset manifests prove corpus/version coverage.
+ *   - RunRecord rows prove reproducible search/holdout outcomes.
+ *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
+ *   - HeldOutGate decisions remain the paired promotion authority.
+ *
+ * The gate is intentionally pure and conservative. Missing declared evidence
+ * fails closed instead of being treated as a neutral zero.
+ */
+
+type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
+type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+interface ReleaseTraceEvidence {
+    scenarioId: string;
+    candidateId?: string;
+    split?: RunSplitTag;
+    score?: number;
+    ok?: boolean;
+    turnCount?: number;
+    costUsd?: number;
+    durationMs?: number;
+    failureMode?: string;
+    asi?: ActionableSideInfo[];
+    metadata?: Record<string, unknown>;
+}
+interface ReleaseConfidenceThresholds {
+    /** Require a Dataset manifest or explicit scenarios. Default true. */
+    requireCorpus?: boolean;
+    minScenarioCount?: number;
+    minSearchRuns?: number;
+    minHoldoutRuns?: number;
+    /** Require at least one holdout scenario/run. Default true. */
+    requireHoldout?: boolean;
+    minPassRate?: number;
+    minMeanScore?: number;
+    /** Search mean may exceed holdout mean by at most this much. */
+    maxOverfitGap?: number;
+    maxMeanCostUsd?: number;
+    maxP95WallMs?: number;
+    /** Low-score/failed rows must carry ASI. Default true. */
+    requireAsiForFailures?: boolean;
+    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
+    failureScoreThreshold?: number;
+}
+interface ReleaseConfidenceInput {
+    target: string;
+    candidateId?: string;
+    baselineId?: string;
+    dataset?: DatasetManifest;
+    scenarios?: readonly DatasetScenario[];
+    runs?: readonly RunRecord[];
+    traces?: readonly ReleaseTraceEvidence[];
+    gateDecision?: GateDecision | null;
+    thresholds?: ReleaseConfidenceThresholds;
+}
+interface ReleaseConfidenceAxis {
+    name: ReleaseConfidenceAxisName;
+    status: ReleaseConfidenceStatus;
+    score: number;
+    detail: string;
+}
+interface ReleaseConfidenceIssue {
+    axis: ReleaseConfidenceAxisName;
+    severity: 'critical' | 'warning';
+    code: string;
+    detail: string;
+}
+interface ReleaseConfidenceMetrics {
+    scenarioCount: number;
+    searchRuns: number;
+    holdoutRuns: number;
+    passRate: number;
+    meanScore: number;
+    searchMeanScore: number;
+    holdoutMeanScore: number;
+    overfitGap: number;
+    meanCostUsd: number;
+    p95WallMs: number;
+    failedRows: number;
+    failuresWithAsi: number;
+    singleShotTraces: number;
+    multiShotTraces: number;
+    splitCounts: Record<DatasetSplit, number>;
+    domainCounts: Record<string, number>;
+    failureModeCounts: Record<string, number>;
+    responsibleSurfaceCounts: Record<string, number>;
+}
+interface ReleaseConfidenceScorecard {
+    target: string;
+    candidateId: string | null;
+    baselineId: string | null;
+    status: ReleaseConfidenceStatus;
+    promote: boolean;
+    axes: ReleaseConfidenceAxis[];
+    issues: ReleaseConfidenceIssue[];
+    metrics: ReleaseConfidenceMetrics;
+    dataset: DatasetManifest | null;
+    gateDecision: GateDecision | null;
+    summary: string;
+}
+declare function releaseTraceEvidenceFromMultiShotTrials(trials: readonly MultiShotTrialResult[]): ReleaseTraceEvidence[];
+declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+
+/**
+ * Paper-grade paired statistics for held-out promotion gates.
+ *
+ * The promotion gate (`HeldOutGate`) needs three things:
+ *
+ *   1. A bootstrap confidence interval on the per-item paired delta
+ *      (`pairedBootstrap`). Median delta is the headline number; the
+ *      CI lower bound is what the gate checks against `pairedDeltaThreshold`.
+ *   2. A non-parametric significance test on the paired deltas
+ *      (`pairedWilcoxon` — re-export of `wilcoxonSignedRank` under the
+ *      paper-style name).
+ *   3. False-discovery-rate correction across simultaneously-tested
+ *      candidate variants (`bhAdjust` — re-export of `benjaminiHochberg`).
+ *
+ * Why a separate file: every existing primitive lives in `statistics.ts`
+ * (general) or `power-analysis.ts` (correction). Paired-bootstrap is
+ * paired-only, paper-grade, and load-bearing for the promotion gate.
+ * Putting it next to `statistics.ts` would require editing that file;
+ * the brief forbids that. New file, new exports, no surface change.
+ */
+interface PairedBootstrapResult {
+    /** Number of paired observations (after dropping unequal lengths is rejected). */
+    n: number;
+    /** Median of paired deltas (after − before). */
+    median: number;
+    /** Mean of paired deltas. */
+    mean: number;
+    /** Lower bound of the bootstrap CI on the median delta. */
+    low: number;
+    /** Upper bound of the bootstrap CI on the median delta. */
+    high: number;
+    /** Confidence level used (e.g. 0.95). */
+    confidence: number;
+    /** Number of bootstrap resamples used. */
+    resamples: number;
+}
+interface PairedBootstrapOptions {
+    /** Confidence level. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resample count. Default 2000. */
+    resamples?: number;
+    /** Statistic to bootstrap. Default 'median'. */
+    statistic?: 'median' | 'mean';
+    /** Deterministic seed. If omitted, uses Math.random(). */
+    seed?: number;
+}
+/**
+ * Paired bootstrap on (after - before) deltas. Returns a CI on the
+ * chosen statistic (median by default). Pairs are resampled with
+ * replacement. The lower bound is what the promotion gate checks: if
+ * `low > pairedDeltaThreshold`, the gain is real at the chosen
+ * confidence level.
+ *
+ * Throws on unequal sample sizes — caller must align pairs upstream.
+ */
+declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
+/**
+ * Paper-style alias for `wilcoxonSignedRank`. The signed-rank test on
+ * paired deltas is the standard non-parametric significance test for
+ * "candidate beats baseline on matched items." Use alongside the
+ * bootstrap CI: bootstrap gives effect size, Wilcoxon gives p.
+ */
+declare function pairedWilcoxon(before: number[], after: number[]): {
+    w: number;
+    p: number;
+};
+/**
+ * Paper-style alias for `benjaminiHochberg`. Use to correct p-values
+ * across multiple candidate-vs-baseline comparisons run in the same
+ * promotion sweep. Returns BH-adjusted q-values and significance at
+ * the requested FDR (default 0.05).
+ */
+declare function bhAdjust(pValues: number[], fdr?: number): {
+    qValues: number[];
+    significant: boolean[];
+};
+
+/**
+ * Bootstrap-CI promotion gate.
+ *
+ * In any iterative-improvement loop (GEPA, prompt evolution, dataset
+ * curation), the question is "did this generation actually improve, or are
+ * we celebrating noise?". With small N and noisy outcomes, point-estimate
+ * deltas lie. Bootstrap confidence intervals tell the operator whether the
+ * delta is real before code or prompts get promoted.
+ *
+ * This module is pure functions — no I/O, no model calls. Easy to unit-test
+ * and to compose into any verdict gate.
+ *
+ * Default gate:
+ *   - Bootstrap mean baseline vs candidate (1k resamples).
+ *   - Compute the delta distribution; pass if the lower CI bound > 0.
+ *   - Tunable confidence (default 95%) and resample count.
+ *
+ * Verdict semantics intentionally match the existing `experiments.jsonl`
+ * vocabulary:
+ *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
+ *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
+ *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
+ *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
+ */
+type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
+interface BootstrapResult {
+    baselineMean: number;
+    candidateMean: number;
+    /** candidateMean - baselineMean, point estimate. */
+    delta: number;
+    /** Lower bound of the (1 - alpha) CI on the delta. */
+    ciLower: number;
+    /** Upper bound of the (1 - alpha) CI on the delta. */
+    ciUpper: number;
+    /** Number of bootstrap resamples used. */
+    iterations: number;
+    alpha: number;
+    verdict: Verdict;
+}
+interface BootstrapOptions {
+    /** Confidence level alpha (default 0.05 → 95% CI). */
+    alpha?: number;
+    /** Number of resamples (default 1000). */
+    iterations?: number;
+    /**
+     * Minimum total samples (baseline + candidate) below which we always
+     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
+     * Default 6 (combined).
+     */
+    minTotalSamples?: number;
+    /** RNG seed for reproducibility. Default: Math.random. */
+    seed?: number;
+}
+/**
+ * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+ *
+ * Uses simple percentile bootstrap on the difference of resampled means.
+ * That's the standard non-parametric primitive — no distributional
+ * assumptions, robust to skew, easy to reason about.
+ */
+declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
+/**
+ * Judge-replay promotion gate.
+ *
+ * The cheap inner-loop judge that drives an evolution run is by definition
+ * fast and noisy. When you're about to promote a winning variant to the
+ * canonical default, you want a STRONGER judge (a more expensive model, a
+ * human grader, a separately-trained reward model) to confirm the win
+ * generalises beyond the inner loop.
+ *
+ * This helper takes raw winner + baseline outputs, scores both through the
+ * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
+ * judge agrees the winner is real with the configured confidence. Doesn't
+ * matter what shape your "output" is — pass a string, an object, anything
+ * the judge can read.
+ */
+interface JudgeReplayGateArgs<TOutput> {
+    baselineOutputs: TOutput[];
+    candidateOutputs: TOutput[];
+    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
+    judge: (output: TOutput) => Promise<number> | number;
+    alpha?: number;
+    iterations?: number;
+    /** RNG seed for reproducibility. */
+    seed?: number;
+    /** Maximum concurrent judge calls. Default 4. */
+    judgeConcurrency?: number;
+}
+declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
+    baselineSamples: number;
+    candidateSamples: number;
+}>;
+
+interface RenderReleaseReportOptions {
+    title?: string;
+    runs?: readonly RunRecord[];
+    comparator?: string;
+    traceAnalystFindings?: readonly string[];
+    nextActions?: readonly string[];
+}
+declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
+
+export { type BootstrapOptions as B, type JudgeReplayGateArgs as J, type PairedBootstrapOptions as P, type ReleaseConfidenceAxis as R, type Verdict as V, type BootstrapResult as a, type PairedBootstrapResult as b, type ReleaseConfidenceAxisName as c, type ReleaseConfidenceInput as d, type ReleaseConfidenceIssue as e, type ReleaseConfidenceMetrics as f, type ReleaseConfidenceScorecard as g, type ReleaseConfidenceStatus as h, type ReleaseConfidenceThresholds as i, type ReleaseTraceEvidence as j, type RenderReleaseReportOptions as k, assertReleaseConfidence as l, bhAdjust as m, bootstrapCi as n, evaluateReleaseConfidence as o, judgeReplayGate as p, pairedBootstrap as q, pairedWilcoxon as r, releaseTraceEvidenceFromMultiShotTrials as s, renderReleaseReport as t };