@tangle-network/agent-eval 0.65.0 → 0.67.0

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.65.0",
+    "version": "0.67.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/provenance-CChUqexv.d.ts

@@ -0,0 +1,314 @@
+import { o as Mutator, I as ImprovementDriver, S as Scenario, G as Gate, f as CampaignResult, M as MutableSurface, k as GateResult, j as GateDecision } from './types-c2R2kfmv.js';
+import { R as RedTeamCase } from './red-team-DW9Ca_tj.js';
+import { R as RunRecord } from './run-record-BgTFzO2r.js';
+import { a as RunCampaignOptions, C as CampaignStorage } from './run-improvement-loop-BKpM5T4t.js';
+import { H as HostedClient, T as TraceSpanEvent } from './index-DSEHMwvS.js';
+
+/**
+ * @experimental
+ *
+ * `evolutionaryDriver` — adapts a stateless `Mutator` (population mutation:
+ * GEPA / AxGEPA / reflective-mutation) into an `ImprovementDriver`. This is
+ * the evolutionary strategy: each generation, mutate the current best surface
+ * into N candidates, measure, select. No generation memory beyond the current
+ * surface; the loop body handles ranking + promotion.
+ *
+ * The reflective alternative is agent-runtime's `improvementDriver` with a
+ * `reflectiveGenerator` / `agenticGenerator`: it reasons over the report +
+ * trace findings to propose targeted edits rather than blind mutations. Both
+ * conform to `ImprovementDriver`; the improvement loop is identical regardless
+ * of which drives it.
+ */
+
+interface EvolutionaryDriverOptions<TFindings = unknown> {
+    mutator: Mutator<TFindings>;
+    /** External findings fed to the mutator each generation. Default: []. */
+    findings?: TFindings[];
+}
+declare function evolutionaryDriver<TFindings = unknown>(opts: EvolutionaryDriverOptions<TFindings>): ImprovementDriver<TFindings>;
+
+/**
+ * @experimental
+ *
+ * Compose multiple `Gate` implementations — every gate must pass for the
+ * composite to ship. Closes the alignment reviewer's "default-only
+ * heldOutGate + costGate would happily promote a reward-hacked prompt"
+ * concern by making safety gates first-class composable defaults.
+ */
+
+/** Compose gates — all must `ship` for the composite to `ship`. First
+ *  non-ship verdict short-circuits the composite verdict, but ALL gates run
+ *  (so the result records every gate's reason — useful for diagnostics). */
+declare function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
+
+/**
+ * @experimental
+ *
+ * `defaultProductionGate` — composes the substrate's existing safety
+ * primitives (red-team / reward-hacking / canary / heldout) into a single
+ * Gate.decide shape. Closes the alignment + Anthropic-SI reviewers' "safety
+ * primitives are off the critical path" blocker.
+ *
+ * The composition is opinionated — when consumers wire `runImprovementLoop`,
+ * THIS gate is the default. Consumers can still pass a custom gate to
+ * override; the recommended pattern is to compose THIS gate with whatever
+ * extra domain-specific gates they need (`composeGate(defaultProductionGate(...), customGate)`).
+ */
+
+interface DefaultProductionGateOptions {
+    /** Required: scenarios held out from training; substrate compares
+     *  candidate-on-holdout vs baseline-on-holdout. */
+    holdoutScenarios: Scenario[];
+    /** Minimum held-out lift the **paired-bootstrap CI lower bound** must clear
+     *  to ship — NOT a point estimate. Default 0 ⇒ "confidently positive at the
+     *  confidence level". Interpreted in the judge's native composite scale (set
+     *  e.g. 2 for a 0-100 rubric to require a ≥2-point significant gain). */
+    deltaThreshold?: number;
+    /** Confidence level for the held-out + dimension bootstraps. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resamples. Default 2000. */
+    bootstrapResamples?: number;
+    /** Fixed bootstrap seed for a deterministic verdict. Default 1337. */
+    bootstrapSeed?: number;
+    /** Minimum paired holdout observations (scenarios × reps) before a
+     *  significance claim is allowed; below it the gate HOLDS with `few_runs`
+     *  rather than reading a degenerate CI. Default 3. */
+    minProductiveRuns?: number;
+    /** Critical judge dimensions that must NOT significantly regress even when
+     *  the net composite rises (anti-Goodhart). The gate HOLDS if any listed
+     *  dimension's paired-delta CI lower bound < −`regressionTolerance`. E.g.
+     *  `['hallucination_free']` for a legal agent. */
+    criticalDimensions?: string[];
+    /** Tolerance for the per-dimension regression guard, in the dimension's
+     *  native scale. When omitted it auto-scales off observed magnitudes:
+     *  0.05 on [0,1], 5 on 0-100. */
+    regressionTolerance?: number;
+    /** Total $ budget for ALL cells in this campaign — including baseline + candidate.
+     *  Composite verdict refuses to ship when spend exceeded budget. */
+    budgetUsd?: number;
+    /** Red-team cases to probe candidate outputs against. When omitted the
+     *  substrate uses `DEFAULT_RED_TEAM_CORPUS`. Provide a domain-specific
+     *  battery for tighter coverage. */
+    redTeamBattery?: RedTeamCase[];
+    /** Run records (oldest-first) needed for the reward-hacking detector.
+     *  Substrate populates from prior production-loop generations. */
+    recentRuns?: RunRecord[];
+    /** When true, the gate refuses to ship if the reward-hacking detector
+     *  fires at the `gaming` severity. Default true. */
+    blockOnRewardHackingGaming?: boolean;
+}
+declare function defaultProductionGate<TArtifact, TScenario extends Scenario>(options: DefaultProductionGateOptions): Gate<TArtifact, TScenario>;
+
+/**
+ * @experimental
+ *
+ * Thin Gate adapter — exposes delta-threshold-on-holdout as a composable
+ * `Gate`. Use when you want held-out as one of N composed gates instead of
+ * the full `defaultProductionGate` stack.
+ */
+
+interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
+    scenarios: TScenario[];
+    deltaThreshold?: number;
+}
+declare function heldOutGate<TArtifact, TScenario extends Scenario>(options: HeldOutGateOptions<TScenario>): Gate<TArtifact, TScenario>;
+
+/**
+ * @experimental
+ *
+ * `runEval` — the simplest preset over `runCampaign`. No optimizer, no
+ * gate, no auto-PR. Just: run scenarios through dispatch, score with
+ * judges, return CampaignResult.
+ *
+ * The 80% case for consumers who want a scorecard, not an improvement loop.
+ */
+
+interface RunEvalOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'runDir'> {
+    runDir: string;
+}
+declare function runEval<TScenario extends Scenario, TArtifact>(opts: RunEvalOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
+
+/**
+ * @experimental
+ *
+ * Loop provenance — the durable, queryable record of WHAT a self-improvement
+ * loop did and WHY, plus the OTel spans that let an OTLP collector pivot from
+ * an eval-run to the underlying candidate→cell→gate→promote chain.
+ *
+ * Two artifacts, one source of truth:
+ *
+ *   1. `LoopProvenanceRecord` — a structured JSON record capturing every
+ *      candidate (surfaceHash + label + rationale), its measured composite,
+ *      the gate decision + reasons + delta, the held-out lift, the explicit
+ *      baseline→candidate diff, and BACKEND PROVENANCE (the
+ *      `assertRealBackend` verdict + worker call count + model). This is the
+ *      ingestable audit artifact: the +lift recomputes from it, the "because
+ *      Z" rationale survives in it, and a stub backend is detectable from it.
+ *
+ *   2. `loopProvenanceSpans()` — the same chain emitted as OTLP-ingestable
+ *      `TraceSpanEvent`s, pivoted on the substrate's standard
+ *      `tangle.runId` / `tangle.scenarioId` / `tangle.cellId` /
+ *      `tangle.generation` attributes (the same pivots `/adapters/otel`
+ *      reads). The hosted `/v1/ingest/traces` endpoint receives the FULL loop,
+ *      not just the `cost.*` spans `runCampaign` already emits per cell.
+ *
+ * The record is built from the substrate's own loop result + the per-call
+ * `RunRecord`s the worker emitted — no new measurement, no recomputation that
+ * could drift from what the gate actually saw.
+ */
+
+/** Stable sha256 (full hex) of a surface's effective text. Code surfaces hash
+ *  their worktree+base identity since the content lives in git. Distinct from
+ *  `surfaceHash` (16-char content fingerprint used as a loop identity key);
+ *  this is the byte-identical-verifiable content hash the provenance record +
+ *  `RunRecord.promptHash` carry. */
+declare function surfaceContentHash(surface: MutableSurface): string;
+interface LoopProvenanceCandidate {
+    /** Generation index this candidate was proposed in. */
+    generation: number;
+    /** 16-char loop-identity fingerprint (matches `GenerationCandidate.surfaceHash`). */
+    surfaceHash: string;
+    /** Full sha256 content hash — byte-identical-verifiable. */
+    contentHash: string;
+    /** Driver label, when the driver returned a `ProposedCandidate`. */
+    label?: string;
+    /** Driver rationale — the "because Z". When the driver returned a bare
+     *  surface (blind mutator) this is absent. */
+    rationale?: string;
+    /** Mean composite this candidate scored on the search split. */
+    composite: number;
+    /** Whether this candidate was promoted out of its generation. */
+    promoted: boolean;
+}
+interface LoopProvenanceBackend {
+    /** `assertRealBackend`-grade verdict over the worker call records. */
+    verdict: 'real' | 'mixed' | 'stub';
+    /** Number of worker LLM calls captured (the audit's "worker call count"). */
+    workerCallCount: number;
+    /** Distinct model ids observed across worker calls. */
+    models: string[];
+    totalInputTokens: number;
+    totalOutputTokens: number;
+    totalCostUsd: number;
+}
+/**
+ * The durable provenance record. Aligns to the hosted `EvalRunEvent` path but
+ * ADDS the rationale + the explicit baseline→candidate diff (both omitted from
+ * the bare hosted event) + backend provenance.
+ */
+interface LoopProvenanceRecord {
+    schema: 'tangle.loop-provenance.v1';
+    runId: string;
+    runDir: string;
+    timestamp: string;
+    /** Baseline + winner surface content hashes — distinguishable, byte-verifiable. */
+    baselineContentHash: string;
+    winnerContentHash: string;
+    /** Driver label/rationale for the promoted change. Absent ⇒ winner == baseline. */
+    winnerLabel?: string;
+    winnerRationale?: string;
+    /** The explicit baseline→winner unified diff the gate decided on. */
+    diff: string;
+    /** Every candidate across every generation, each carrying its rationale. */
+    candidates: LoopProvenanceCandidate[];
+    /** The gate verdict — decision + reasons + contributing gates + delta. */
+    gate: {
+        decision: GateDecision;
+        reasons: string[];
+        delta?: number;
+        contributingGates: Array<{
+            name: string;
+            passed: boolean;
+        }>;
+    };
+    /** baseline-on-holdout composite mean. */
+    baselineHoldoutComposite: number;
+    /** winner-on-holdout composite mean. */
+    winnerHoldoutComposite: number;
+    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. */
+    heldOutLift: number;
+    /** Backend provenance: stub-vs-real verdict + worker call count + models. */
+    backend: LoopProvenanceBackend;
+    totalCostUsd: number;
+    totalDurationMs: number;
+}
+interface BuildLoopProvenanceArgs<TArtifact, TScenario extends Scenario> {
+    runId: string;
+    runDir: string;
+    timestamp: string;
+    baselineSurface: MutableSurface;
+    winnerSurface: MutableSurface;
+    winnerLabel?: string;
+    winnerRationale?: string;
+    diff: string;
+    /** Per-generation candidate records straight off the loop result. */
+    generations: Array<{
+        generationIndex: number;
+        candidates: Array<{
+            surfaceHash: string;
+            composite: number;
+            label?: string;
+            rationale?: string;
+        }>;
+        promoted: string[];
+        /** Surfaces measured this generation, keyed positionally to candidates so
+         *  the content hash can be computed from the real surface text. */
+        surfaces: Array<{
+            surfaceHash: string;
+            surface: MutableSurface;
+        }>;
+    }>;
+    gate: GateResult;
+    baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
+    winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
+    /** Worker call records — the source for backend provenance. */
+    workerRecords: ReadonlyArray<RunRecord>;
+    totalCostUsd: number;
+    totalDurationMs: number;
+}
+/** Build the durable provenance record from a completed loop result. */
+declare function buildLoopProvenanceRecord<TArtifact, TScenario extends Scenario>(args: BuildLoopProvenanceArgs<TArtifact, TScenario>): LoopProvenanceRecord;
+/**
+ * Build the loop's OTLP-ingestable spans from a provenance record. One root
+ * span per loop (`tangle.runId`), one span per generation, one span per
+ * candidate (carrying its surfaceHash + label), and one span for the gate
+ * decision (carrying reasons + delta + lift). Candidate + gate spans pivot on
+ * the same `tangle.runId` / `tangle.generation` attributes `/adapters/otel`
+ * reads, so the hosted collector reconstructs the full tree.
+ *
+ * Times are synthesized monotonically off a single base so the span tree is
+ * orderable; the substrate does not retain per-candidate wall-clock starts.
+ */
+declare function loopProvenanceSpans(record: LoopProvenanceRecord, opts?: {
+    baseTimeMs?: number;
+}): TraceSpanEvent[];
+/** Canonical durable paths under the run dir. */
+declare function provenanceRecordPath(runDir: string): string;
+declare function provenanceSpansPath(runDir: string): string;
+interface EmitLoopProvenanceResult {
+    record: LoopProvenanceRecord;
+    spans: TraceSpanEvent[];
+    /** Absolute paths the record + spans were written to, when storage persists. */
+    recordPath: string;
+    spansPath: string;
+}
+interface EmitLoopProvenanceArgs<TArtifact, TScenario extends Scenario> extends BuildLoopProvenanceArgs<TArtifact, TScenario> {
+    /** Storage the record + spans are written through. */
+    storage: CampaignStorage;
+    /** When set, the spans are also shipped to the hosted `/v1/ingest/traces`
+     *  endpoint so the collector receives the full loop, not just `cost.*`. */
+    hostedClient?: HostedClient;
+}
+/**
+ * Build the provenance record + OTel spans and persist them durably under the
+ * run dir (and ship spans to a hosted collector when one is wired). Returns
+ * both artifacts so the caller can assert on / re-derive from them.
+ *
+ * Fail-loud: the durable write throws on storage failure (a swallowed write is
+ * exactly the "emitted but lost" failure this closes). The hosted span ship is
+ * the one best-effort leg — its failure is logged, not thrown, so an offline
+ * collector never fails the loop (the durable artifact is the source of truth).
+ */
+declare function emitLoopProvenance<TArtifact, TScenario extends Scenario>(args: EmitLoopProvenanceArgs<TArtifact, TScenario>): Promise<EmitLoopProvenanceResult>;
+
+export { type BuildLoopProvenanceArgs as B, type DefaultProductionGateOptions as D, type EvolutionaryDriverOptions as E, type HeldOutGateOptions as H, type LoopProvenanceRecord as L, type RunEvalOptions as R, type EmitLoopProvenanceArgs as a, type EmitLoopProvenanceResult as b, composeGate as c, defaultProductionGate as d, evolutionaryDriver as e, type LoopProvenanceBackend as f, type LoopProvenanceCandidate as g, heldOutGate as h, buildLoopProvenanceRecord as i, emitLoopProvenance as j, provenanceSpansPath as k, loopProvenanceSpans as l, provenanceRecordPath as p, runEval as r, surfaceContentHash as s };

package/dist/{registry-DPly4_hZ.d.ts → registry-BGKyX6bw.d.ts}

@@ -1,7 +1,7 @@
 import { b as LlmCallRequest, c as LlmCallResult } from './llm-client-DbjLfz-K.js';
 import { R as RunRecord } from './run-record-BgTFzO2r.js';
 import { T as TraceAnalysisStore } from './store-jzKpMl16.js';
-import { J as JudgeInput } from './types-DhqpAi_z.js';
+import { a as JudgeInput } from './types-Croy5h7V.js';
 
 /**
  * ChatClient — the single LLM abstraction analysts call.
@@ -454,4 +454,4 @@ declare class AnalystRegistry {
     private routeInput;
 }
 
-export { AnalystRegistry as A, type BudgetPolicy as B, type ChatCallOpts as C, type DirectProviderTransportOpts as D, type EvidenceRef as E, type MockTransportOpts as M, type RegistryRunOpts as R, type SandboxSdkTransportOpts as S, type Analyst as a, type AnalystSeverity as b, type AnalystFinding as c, type AnalystCost as d, type AnalystContext as e, type AnalystHooks as f, type AnalystInputKind as g, type AnalystRegistryOptions as h, type AnalystRequirements as i, type AnalystRunEvent as j, type AnalystRunInputs as k, type AnalystRunResult as l, type AnalystRunSummary as m, type ChatClient as n, type ChatRequest as o, type ChatResponse as p, type ChatTransport as q, type CliBridgeTransportOpts as r, type CreateChatClientOpts as s, type RouterTransportOpts as t, computeFindingId as u, createChatClient as v, makeFinding as w };
+export { AnalystRegistry as A, type BudgetPolicy as B, type ChatRequest as C, type DirectProviderTransportOpts as D, type EvidenceRef as E, type MockTransportOpts as M, type RegistryRunOpts as R, type SandboxSdkTransportOpts as S, type Analyst as a, type AnalystSeverity as b, type AnalystFinding as c, type AnalystCost as d, type AnalystContext as e, type CreateChatClientOpts as f, type AnalystHooks as g, type AnalystInputKind as h, type AnalystRegistryOptions as i, type AnalystRequirements as j, type AnalystRunEvent as k, type AnalystRunInputs as l, type AnalystRunResult as m, type AnalystRunSummary as n, type ChatCallOpts as o, type ChatClient as p, type ChatResponse as q, type ChatTransport as r, type CliBridgeTransportOpts as s, type RouterTransportOpts as t, computeFindingId as u, createChatClient as v, makeFinding as w };

package/dist/release-report-CN8hJlhk.d.ts

@@ -0,0 +1,233 @@
+import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-B2kL-fSM.js';
+import { m as GateDecision } from './summary-report-ByiOUrHj.js';
+import { R as RunRecord, b as RunSplitTag } from './run-record-BgTFzO2r.js';
+
+/**
+ * Release confidence gate.
+ *
+ * This is the production-facing composition layer over the lower-level
+ * primitives:
+ *   - Dataset manifests prove corpus/version coverage.
+ *   - RunRecord rows prove reproducible search/holdout outcomes.
+ *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
+ *   - HeldOutGate decisions remain the paired promotion authority.
+ *
+ * The gate is intentionally pure and conservative. Missing declared evidence
+ * fails closed instead of being treated as a neutral zero.
+ */
+
+/** Severity of an actionable finding attached to a run/trace. */
+type AsiSeverity = 'info' | 'warning' | 'error' | 'critical';
+/** Actionable side-info — a diagnosed finding the loop can act on. */
+interface ActionableSideInfo {
+    /** Stable expectation/check id when available. */
+    expectationId?: string;
+    /** Human-readable diagnosis of what happened. */
+    message: string;
+    severity?: AsiSeverity;
+    /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */
+    evidence?: string;
+    /** Prompt/tool/context surface likely responsible. */
+    responsibleSurface?: string;
+    /** Suggested fix in natural language. */
+    suggestion?: string;
+    /** Whether this expectation was satisfied. Defaults to false for ASI rows. */
+    matched?: boolean;
+    metadata?: Record<string, unknown>;
+}
+type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
+type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+interface ReleaseTraceEvidence {
+    scenarioId: string;
+    candidateId?: string;
+    split?: RunSplitTag;
+    score?: number;
+    ok?: boolean;
+    turnCount?: number;
+    costUsd?: number;
+    durationMs?: number;
+    failureMode?: string;
+    asi?: ActionableSideInfo[];
+    metadata?: Record<string, unknown>;
+}
+interface ReleaseConfidenceThresholds {
+    /** Require a Dataset manifest or explicit scenarios. Default true. */
+    requireCorpus?: boolean;
+    minScenarioCount?: number;
+    minSearchRuns?: number;
+    minHoldoutRuns?: number;
+    /** Require at least one holdout scenario/run. Default true. */
+    requireHoldout?: boolean;
+    minPassRate?: number;
+    minMeanScore?: number;
+    /** Search mean may exceed holdout mean by at most this much. */
+    maxOverfitGap?: number;
+    maxMeanCostUsd?: number;
+    maxP95WallMs?: number;
+    /** Low-score/failed rows must carry ASI. Default true. */
+    requireAsiForFailures?: boolean;
+    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
+    failureScoreThreshold?: number;
+}
+interface ReleaseConfidenceInput {
+    target: string;
+    candidateId?: string;
+    baselineId?: string;
+    dataset?: DatasetManifest;
+    scenarios?: readonly DatasetScenario[];
+    runs?: readonly RunRecord[];
+    traces?: readonly ReleaseTraceEvidence[];
+    gateDecision?: GateDecision | null;
+    thresholds?: ReleaseConfidenceThresholds;
+}
+interface ReleaseConfidenceAxis {
+    name: ReleaseConfidenceAxisName;
+    status: ReleaseConfidenceStatus;
+    score: number;
+    detail: string;
+}
+interface ReleaseConfidenceIssue {
+    axis: ReleaseConfidenceAxisName;
+    severity: 'critical' | 'warning';
+    code: string;
+    detail: string;
+}
+interface ReleaseConfidenceMetrics {
+    scenarioCount: number;
+    searchRuns: number;
+    holdoutRuns: number;
+    passRate: number;
+    meanScore: number;
+    searchMeanScore: number;
+    holdoutMeanScore: number;
+    overfitGap: number;
+    meanCostUsd: number;
+    p95WallMs: number;
+    failedRows: number;
+    failuresWithAsi: number;
+    singleShotTraces: number;
+    multiShotTraces: number;
+    splitCounts: Record<DatasetSplit, number>;
+    domainCounts: Record<string, number>;
+    failureModeCounts: Record<string, number>;
+    responsibleSurfaceCounts: Record<string, number>;
+}
+interface ReleaseConfidenceScorecard {
+    target: string;
+    candidateId: string | null;
+    baselineId: string | null;
+    status: ReleaseConfidenceStatus;
+    promote: boolean;
+    axes: ReleaseConfidenceAxis[];
+    issues: ReleaseConfidenceIssue[];
+    metrics: ReleaseConfidenceMetrics;
+    dataset: DatasetManifest | null;
+    gateDecision: GateDecision | null;
+    summary: string;
+}
+declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+
+/**
+ * Bootstrap-CI promotion gate.
+ *
+ * In any iterative-improvement loop (GEPA, prompt evolution, dataset
+ * curation), the question is "did this generation actually improve, or are
+ * we celebrating noise?". With small N and noisy outcomes, point-estimate
+ * deltas lie. Bootstrap confidence intervals tell the operator whether the
+ * delta is real before code or prompts get promoted.
+ *
+ * This module is pure functions — no I/O, no model calls. Easy to unit-test
+ * and to compose into any verdict gate.
+ *
+ * Default gate:
+ *   - Bootstrap mean baseline vs candidate (1k resamples).
+ *   - Compute the delta distribution; pass if the lower CI bound > 0.
+ *   - Tunable confidence (default 95%) and resample count.
+ *
+ * Verdict semantics intentionally match the existing `experiments.jsonl`
+ * vocabulary:
+ *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
+ *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
+ *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
+ *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
+ */
+type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
+interface BootstrapResult {
+    baselineMean: number;
+    candidateMean: number;
+    /** candidateMean - baselineMean, point estimate. */
+    delta: number;
+    /** Lower bound of the (1 - alpha) CI on the delta. */
+    ciLower: number;
+    /** Upper bound of the (1 - alpha) CI on the delta. */
+    ciUpper: number;
+    /** Number of bootstrap resamples used. */
+    iterations: number;
+    alpha: number;
+    verdict: Verdict;
+}
+interface BootstrapOptions {
+    /** Confidence level alpha (default 0.05 → 95% CI). */
+    alpha?: number;
+    /** Number of resamples (default 1000). */
+    iterations?: number;
+    /**
+     * Minimum total samples (baseline + candidate) below which we always
+     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
+     * Default 6 (combined).
+     */
+    minTotalSamples?: number;
+    /** RNG seed for reproducibility. Default: Math.random. */
+    seed?: number;
+}
+/**
+ * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+ *
+ * Uses simple percentile bootstrap on the difference of resampled means.
+ * That's the standard non-parametric primitive — no distributional
+ * assumptions, robust to skew, easy to reason about.
+ */
+declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
+/**
+ * Judge-replay promotion gate.
+ *
+ * The cheap inner-loop judge that drives an evolution run is by definition
+ * fast and noisy. When you're about to promote a winning variant to the
+ * canonical default, you want a STRONGER judge (a more expensive model, a
+ * human grader, a separately-trained reward model) to confirm the win
+ * generalises beyond the inner loop.
+ *
+ * This helper takes raw winner + baseline outputs, scores both through the
+ * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
+ * judge agrees the winner is real with the configured confidence. Doesn't
+ * matter what shape your "output" is — pass a string, an object, anything
+ * the judge can read.
+ */
+interface JudgeReplayGateArgs<TOutput> {
+    baselineOutputs: TOutput[];
+    candidateOutputs: TOutput[];
+    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
+    judge: (output: TOutput) => Promise<number> | number;
+    alpha?: number;
+    iterations?: number;
+    /** RNG seed for reproducibility. */
+    seed?: number;
+    /** Maximum concurrent judge calls. Default 4. */
+    judgeConcurrency?: number;
+}
+declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
+    baselineSamples: number;
+    candidateSamples: number;
+}>;
+
+interface RenderReleaseReportOptions {
+    title?: string;
+    runs?: readonly RunRecord[];
+    comparator?: string;
+    traceAnalystFindings?: readonly string[];
+    nextActions?: readonly string[];
+}
+declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
+
+export { type ActionableSideInfo as A, type BootstrapOptions as B, type JudgeReplayGateArgs as J, type ReleaseConfidenceAxis as R, type Verdict as V, type BootstrapResult as a, type ReleaseConfidenceAxisName as b, type ReleaseConfidenceInput as c, type ReleaseConfidenceIssue as d, type ReleaseConfidenceMetrics as e, type ReleaseConfidenceScorecard as f, type ReleaseConfidenceStatus as g, type ReleaseConfidenceThresholds as h, type ReleaseTraceEvidence as i, type RenderReleaseReportOptions as j, assertReleaseConfidence as k, bootstrapCi as l, evaluateReleaseConfidence as m, judgeReplayGate as n, type AsiSeverity as o, renderReleaseReport as r };

package/dist/reporting.d.ts

@@ -1,14 +1,15 @@
 export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-D_4BSXGV.js';
-export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, b as PairedBootstrapResult, R as ReleaseConfidenceAxis, c as ReleaseConfidenceAxisName, d as ReleaseConfidenceInput, e as ReleaseConfidenceIssue, f as ReleaseConfidenceMetrics, g as ReleaseConfidenceScorecard, h as ReleaseConfidenceStatus, i as ReleaseConfidenceThresholds, j as ReleaseTraceEvidence, k as RenderReleaseReportOptions, V as Verdict, l as assertReleaseConfidence, m as benjaminiHochberg, n as bootstrapCi, o as evaluateReleaseConfidence, p as judgeReplayGate, q as pairedBootstrap, r as renderReleaseReport, w as wilcoxonSignedRank } from './release-report-DGoeObZT.js';
+export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-CN8hJlhk.js';
 export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
+export { P as PairedBootstrapOptions, a as PairedBootstrapResult, b as benjaminiHochberg, p as pairedBootstrap, w as wilcoxonSignedRank } from './statistics-B7yCbi9i.js';
 export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-ByiOUrHj.js';
 import './run-record-BgTFzO2r.js';
 import './errors-Dwqw-T_m.js';
 import './schema-m0gsnbt3.js';
 import './outcome-store-D6KWmYvj.js';
+import './dataset-B2kL-fSM.js';
 import './judge-calibration-DilmB3Ml.js';
-import './types-DhqpAi_z.js';
+import './types-Croy5h7V.js';
 import '@tangle-network/tcloud';
-import './dataset-B2kL-fSM.js';
 import './failure-cluster-CL7IVgkJ.js';
 import './store-CKUAgsJz.js';

package/dist/{run-campaign-5J3ED2UJ.js → run-campaign-BVY3RGAZ.js}

@@ -1,11 +1,10 @@
 import {
   runCampaign
-} from "./chunk-7TPYV2ER.js";
-import "./chunk-E22YUOAL.js";
+} from "./chunk-6XQIEUQ2.js";
 import "./chunk-ITBRCT73.js";
 import "./chunk-3BFEG2F6.js";
 import "./chunk-PZ5AY32C.js";
 export {
   runCampaign
 };
-//# sourceMappingURL=run-campaign-5J3ED2UJ.js.map
+//# sourceMappingURL=run-campaign-BVY3RGAZ.js.map