@tangle-network/agent-eval 0.20.11 → 0.21.0

package/dist/reporting-Da2ihlcM.d.ts

@@ -0,0 +1,672 @@
+import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-B9qvlm_o.js';
+import { G as GateDecision, A as ActionableSideInfo, t as MultiShotTrialResult } from './multi-shot-optimization-Bvtz294B.js';
+import { a as RunRecord, R as RunSplitTag } from './run-record-CX_jcAyr.js';
+import { a as Run, S as Span, f as TraceEvent, F as FailureClass, T as TraceStore } from './store-u47QaJ9G.js';
+
+/**
+ * Release confidence gate.
+ *
+ * This is the production-facing composition layer over the lower-level
+ * primitives:
+ *   - Dataset manifests prove corpus/version coverage.
+ *   - RunRecord rows prove reproducible search/holdout outcomes.
+ *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
+ *   - HeldOutGate decisions remain the paired promotion authority.
+ *
+ * The gate is intentionally pure and conservative. Missing declared evidence
+ * fails closed instead of being treated as a neutral zero.
+ */
+
+type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
+type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+interface ReleaseTraceEvidence {
+    scenarioId: string;
+    candidateId?: string;
+    split?: RunSplitTag;
+    score?: number;
+    ok?: boolean;
+    turnCount?: number;
+    costUsd?: number;
+    durationMs?: number;
+    failureMode?: string;
+    asi?: ActionableSideInfo[];
+    metadata?: Record<string, unknown>;
+}
+interface ReleaseConfidenceThresholds {
+    /** Require a Dataset manifest or explicit scenarios. Default true. */
+    requireCorpus?: boolean;
+    minScenarioCount?: number;
+    minSearchRuns?: number;
+    minHoldoutRuns?: number;
+    /** Require at least one holdout scenario/run. Default true. */
+    requireHoldout?: boolean;
+    minPassRate?: number;
+    minMeanScore?: number;
+    /** Search mean may exceed holdout mean by at most this much. */
+    maxOverfitGap?: number;
+    maxMeanCostUsd?: number;
+    maxP95WallMs?: number;
+    /** Low-score/failed rows must carry ASI. Default true. */
+    requireAsiForFailures?: boolean;
+    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
+    failureScoreThreshold?: number;
+}
+interface ReleaseConfidenceInput {
+    target: string;
+    candidateId?: string;
+    baselineId?: string;
+    dataset?: DatasetManifest;
+    scenarios?: readonly DatasetScenario[];
+    runs?: readonly RunRecord[];
+    traces?: readonly ReleaseTraceEvidence[];
+    gateDecision?: GateDecision | null;
+    thresholds?: ReleaseConfidenceThresholds;
+}
+interface ReleaseConfidenceAxis {
+    name: ReleaseConfidenceAxisName;
+    status: ReleaseConfidenceStatus;
+    score: number;
+    detail: string;
+}
+interface ReleaseConfidenceIssue {
+    axis: ReleaseConfidenceAxisName;
+    severity: 'critical' | 'warning';
+    code: string;
+    detail: string;
+}
+interface ReleaseConfidenceMetrics {
+    scenarioCount: number;
+    searchRuns: number;
+    holdoutRuns: number;
+    passRate: number;
+    meanScore: number;
+    searchMeanScore: number;
+    holdoutMeanScore: number;
+    overfitGap: number;
+    meanCostUsd: number;
+    p95WallMs: number;
+    failedRows: number;
+    failuresWithAsi: number;
+    singleShotTraces: number;
+    multiShotTraces: number;
+    splitCounts: Record<DatasetSplit, number>;
+    domainCounts: Record<string, number>;
+    failureModeCounts: Record<string, number>;
+    responsibleSurfaceCounts: Record<string, number>;
+}
+interface ReleaseConfidenceScorecard {
+    target: string;
+    candidateId: string | null;
+    baselineId: string | null;
+    status: ReleaseConfidenceStatus;
+    promote: boolean;
+    axes: ReleaseConfidenceAxis[];
+    issues: ReleaseConfidenceIssue[];
+    metrics: ReleaseConfidenceMetrics;
+    dataset: DatasetManifest | null;
+    gateDecision: GateDecision | null;
+    summary: string;
+}
+declare function releaseTraceEvidenceFromMultiShotTrials(trials: readonly MultiShotTrialResult[]): ReleaseTraceEvidence[];
+declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+
+/**
+ * Failure taxonomy — canonical classes + a default classifier.
+ *
+ * Every failed run should end up in a named class. The classifier here
+ * is rule-based (fast, deterministic); an LLM fallback can be added by
+ * the consumer for novel cases and trained into the rule base over time.
+ *
+ * Consumers call `classifyFailure(run, spans, events)` and persist the
+ * returned class as `Run.outcome.failureClass`.
+ */
+
+interface FailureContext {
+    run: Run;
+    spans: Span[];
+    events: TraceEvent[];
+}
+interface FailureClassification {
+    failureClass: FailureClass;
+    reason: string;
+    triggerSpanId?: string;
+    triggerEventId?: string;
+}
+/** Ordered rules — first match wins. */
+interface FailureRule {
+    id: string;
+    match: (ctx: FailureContext) => {
+        failureClass: FailureClass;
+        reason: string;
+        triggerSpanId?: string;
+        triggerEventId?: string;
+    } | null;
+}
+declare const DEFAULT_RULES: FailureRule[];
+/** Classify the failure mode of a run using an ordered rule list. */
+declare function classifyFailure(ctx: FailureContext, rules?: FailureRule[]): FailureClassification;
+
+/**
+ * FailureClusterView — groups failed runs by (failureClass, triggerTool,
+ * argHash-prefix) so weekly reviews can prioritize the top-N clusters.
+ *
+ * Each cluster includes: N runs, scenarios affected, representative
+ * error message, a proposed mitigation hint (rule → action table).
+ */
+
+interface FailureCluster {
+    failureClass: FailureClass;
+    /** Tool name when the trigger was a tool span, else undefined. */
+    toolName?: string;
+    /** First 16 chars of argHash — clusters similar args. */
+    argPrefix?: string;
+    /**
+     * Source dimension when the trigger was a judge span (e.g. `'format'`,
+     * `'safety'`, `'correctness'`). Lets cross-template aggregators
+     * group failures by the dimension that fired without overloading
+     * `argPrefix`. Optional — legacy clusters without this field
+     * deserialize cleanly.
+     */
+    dimension?: string;
+    runCount: number;
+    scenarioIds: string[];
+    exampleError?: string;
+    exampleRunId: string;
+}
+interface FailureClusterReport {
+    clusters: FailureCluster[];
+    totalFailures: number;
+    totalRuns: number;
+}
+declare function failureClusterView(store: TraceStore, options?: {
+    rules?: FailureRule[];
+    minClusterSize?: number;
+}): Promise<FailureClusterReport>;
+
+/**
+ * Paper-grade paired statistics for held-out promotion gates.
+ *
+ * The promotion gate (`HeldOutGate`) needs three things:
+ *
+ *   1. A bootstrap confidence interval on the per-item paired delta
+ *      (`pairedBootstrap`). Median delta is the headline number; the
+ *      CI lower bound is what the gate checks against `pairedDeltaThreshold`.
+ *   2. A non-parametric significance test on the paired deltas
+ *      (`pairedWilcoxon` — re-export of `wilcoxonSignedRank` under the
+ *      paper-style name).
+ *   3. False-discovery-rate correction across simultaneously-tested
+ *      candidate variants (`bhAdjust` — re-export of `benjaminiHochberg`).
+ *
+ * Why a separate file: every existing primitive lives in `statistics.ts`
+ * (general) or `power-analysis.ts` (correction). Paired-bootstrap is
+ * paired-only, paper-grade, and load-bearing for the promotion gate.
+ * Putting it next to `statistics.ts` would require editing that file;
+ * the brief forbids that. New file, new exports, no surface change.
+ */
+interface PairedBootstrapResult {
+    /** Number of paired observations (after dropping unequal lengths is rejected). */
+    n: number;
+    /** Median of paired deltas (after − before). */
+    median: number;
+    /** Mean of paired deltas. */
+    mean: number;
+    /** Lower bound of the bootstrap CI on the median delta. */
+    low: number;
+    /** Upper bound of the bootstrap CI on the median delta. */
+    high: number;
+    /** Confidence level used (e.g. 0.95). */
+    confidence: number;
+    /** Number of bootstrap resamples used. */
+    resamples: number;
+}
+interface PairedBootstrapOptions {
+    /** Confidence level. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resample count. Default 2000. */
+    resamples?: number;
+    /** Statistic to bootstrap. Default 'median'. */
+    statistic?: 'median' | 'mean';
+    /** Deterministic seed. If omitted, uses Math.random(). */
+    seed?: number;
+}
+/**
+ * Paired bootstrap on (after - before) deltas. Returns a CI on the
+ * chosen statistic (median by default). Pairs are resampled with
+ * replacement. The lower bound is what the promotion gate checks: if
+ * `low > pairedDeltaThreshold`, the gain is real at the chosen
+ * confidence level.
+ *
+ * Throws on unequal sample sizes — caller must align pairs upstream.
+ */
+declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
+/**
+ * Paper-style alias for `wilcoxonSignedRank`. The signed-rank test on
+ * paired deltas is the standard non-parametric significance test for
+ * "candidate beats baseline on matched items." Use alongside the
+ * bootstrap CI: bootstrap gives effect size, Wilcoxon gives p.
+ */
+declare function pairedWilcoxon(before: number[], after: number[]): {
+    w: number;
+    p: number;
+};
+/**
+ * Paper-style alias for `benjaminiHochberg`. Use to correct p-values
+ * across multiple candidate-vs-baseline comparisons run in the same
+ * promotion sweep. Returns BH-adjusted q-values and significance at
+ * the requested FDR (default 0.05).
+ */
+declare function bhAdjust(pValues: number[], fdr?: number): {
+    qValues: number[];
+    significant: boolean[];
+};
+
+/**
+ * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
+ * than replacing it.
+ *
+ * Three artefacts:
+ *
+ *   - `summaryTable`           Markdown table of per-candidate means,
+ *                            95% bootstrap CIs, BH-adjusted Wilcoxon
+ *                            p-values, and Cohen's d versus a
+ *                            comparator candidate.
+ *   - `paretoChart`         Abstract spec for a cost vs quality
+ *                            scatter, with gate decisions overlaid.
+ *                            Returns numbers + labels — caller
+ *                            chooses the plotting library.
+ *   - `gainHistogram`
+ *                            Per-item paired holdout deltas as a
+ *                            histogram spec (bins + counts + median +
+ *                            CI). Same "data, not images" contract.
+ *
+ * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
+ * They aren't React components and they aren't PNGs; they are
+ * what you'd hand to vega-lite, plotly, matplotlib, or your own
+ * Canvas renderer to draw the actual figure.
+ */
+
+interface SummaryTableOptions {
+    /** Comparator candidate id. Wilcoxon + Cohen's d are computed
+     *  versus this candidate. Required for paired stats columns. */
+    comparator?: string;
+    /** Which split to read scores from. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level for the bootstrap CI on the mean. Default 0.95. */
+    confidence?: number;
+    /** FDR for BH adjustment of the comparison p-values. Default 0.05. */
+    fdr?: number;
+}
+interface SummaryTableRow {
+    candidateId: string;
+    n: number;
+    mean: number;
+    ciLow: number;
+    ciHigh: number;
+    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
+    qValue: number;
+    /** Cohen's d vs comparator. NaN if no comparator. */
+    cohensD: number;
+}
+interface SummaryTable {
+    rows: SummaryTableRow[];
+    comparator: string | null;
+    split: 'search' | 'holdout';
+    /** Pre-rendered markdown — drop into a paper or PR. */
+    markdown: string;
+}
+/**
+ * Table 1 helper. Buckets runs by `candidateId`, computes mean +
+ * bootstrap CI on the chosen split, and (when a comparator is given)
+ * BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
+ */
+declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
+interface ParetoPoint {
+    candidateId: string;
+    /** Mean USD cost per run on the chosen split. */
+    cost: number;
+    /** Mean score on the chosen split. */
+    quality: number;
+    /** Number of runs that informed this point. */
+    n: number;
+    /** Whether this candidate is on the Pareto frontier — high
+     *  quality, low cost, no dominator. */
+    onFrontier: boolean;
+    /** Optional gate verdict for this candidate, if a `GateDecision`
+     *  for it was passed in. */
+    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
+}
+interface ParetoFigureSpec {
+    kind: 'pareto-cost-quality';
+    split: 'search' | 'holdout';
+    points: ParetoPoint[];
+    axes: {
+        x: 'costUsd';
+        y: 'score';
+    };
+}
+/**
+ * Cost vs quality scatter spec. `gateDecisions` is keyed by
+ * candidate id; if present, every point picks up the gate verdict
+ * for overlay.
+ */
+declare function paretoChart(runs: RunRecord[], opts?: {
+    split?: 'search' | 'holdout';
+    gateDecisions?: Record<string, GateDecision>;
+}): ParetoFigureSpec;
+interface GainDistributionBin {
+    /** Inclusive lower edge. */
+    lo: number;
+    /** Exclusive upper edge (or inclusive if it's the last bin). */
+    hi: number;
+    /** Number of pairs whose delta lands in this bin. */
+    count: number;
+}
+interface GainDistributionFigureSpec {
+    kind: 'gain-distribution';
+    candidateId: string;
+    comparator: string;
+    split: 'search' | 'holdout';
+    /** Number of pairs used. */
+    n: number;
+    bins: GainDistributionBin[];
+    median: number;
+    ci: {
+        low: number;
+        high: number;
+    };
+}
+interface GainDistributionOptions {
+    /** Number of histogram bins. Default 11 (so the centre is exact at 0). */
+    bins?: number;
+    /** Which split to use. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level for the CI. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resamples. Default 2000. */
+    resamples?: number;
+    /** Deterministic seed. */
+    seed?: number;
+}
+/**
+ * Held-out improvement distribution: per-pair delta (candidate −
+ * comparator), histogrammed. Includes the bootstrap CI on the median
+ * delta — same primitive the promotion gate uses.
+ */
+declare function gainHistogram(runs: RunRecord[], candidateId: string, comparator: string, opts?: GainDistributionOptions): GainDistributionFigureSpec;
+type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
+/**
+ * Hard floor below which a paired comparison is treated as uninformative
+ * regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank
+ * exact tables; below this the test has no power to separate effect sizes.
+ */
+declare const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
+interface ResearchReportOptions {
+    /** Human-readable report title. */
+    title?: string;
+    /** Comparator candidate id. Required for statistical decision guidance. */
+    comparator?: string;
+    /** Which split to use for the primary decision. Default 'holdout'. */
+    split?: 'search' | 'holdout';
+    /** Confidence level used by lower-level report helpers. Default 0.95. */
+    confidence?: number;
+    /** FDR threshold for q-values. Default 0.05. */
+    fdr?: number;
+    /**
+     * Soft floor on paired observations before issuing a directional
+     * promote / reject. Below this we report `needs_more_data` and surface the
+     * minimum detectable effect at the current N. Default 20 — chosen so the
+     * Wilcoxon signed-rank approximation is reasonable and so the paired
+     * bootstrap CI has non-degenerate coverage. Hard floor is enforced at
+     * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.
+     */
+    minPairs?: number;
+    /**
+     * Region of Practical Equivalence on the paired delta. When a candidate's
+     * paired-delta CI is fully contained in `[low, high]`, the decision is
+     * `equivalent` rather than `hold`. Sourced from the domain owner — there is
+     * no statistically-defensible default.
+     */
+    rope?: {
+        low: number;
+        high: number;
+    };
+    /**
+     * Power for the minimum detectable effect (MDE) reported on each candidate.
+     * Default 0.8.
+     */
+    mdePower?: number;
+    /**
+     * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE
+     * lines up with the test the report actually runs.
+     */
+    mdeAlpha?: number;
+    /** Optional held-out gate decisions keyed by candidate id. */
+    gateDecisions?: Record<string, GateDecision>;
+    /** Optional failure clusters from failureClusterView. */
+    failureClusters?: FailureClusterReport;
+    /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */
+    candidateIds?: string[];
+    /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */
+    seed?: number;
+    /** Report timestamp. Defaults to current time. */
+    generatedAt?: string;
+    /**
+     * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).
+     * Embedded verbatim in the report so the analysis can be cited as the
+     * preregistered one rather than a post-hoc fishing expedition.
+     */
+    preregistrationHash?: string;
+}
+interface ResearchReportRecommendation {
+    decision: ResearchReportDecision;
+    candidateId: string | null;
+    rationale: string[];
+    risks: string[];
+    nextActions: string[];
+}
+interface ResearchReportCandidate {
+    candidateId: string;
+    n: number;
+    mean: number;
+    ciLow: number;
+    ciHigh: number;
+    qValue: number;
+    cohensD: number;
+    meanDeltaVsComparator: number | null;
+    pairedN: number;
+    medianGain: number | null;
+    meanGain: number | null;
+    gainCi: {
+        low: number;
+        high: number;
+    } | null;
+    /**
+     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
+     * from the same resamples that produce the gain CI; interpretable as
+     * "fraction of resamples in which the candidate beats the comparator on
+     * matched pairs."
+     */
+    prGreaterThanZero: number | null;
+    prInRope: number | null;
+    /**
+     * Minimum detectable effect (in score units) at the candidate's paired N,
+     * the configured power, and the configured alpha. Standardised by the
+     * observed paired-delta SD and inverted via `requiredSampleSize`. Reported
+     * for every candidate so a `needs_more_data` verdict is actionable.
+     */
+    mde: number | null;
+    onParetoFrontier: boolean;
+    gate?: ParetoPoint['gate'];
+    decision: ResearchReportDecision;
+    decisionReason: string;
+}
+interface ResearchReportMethodology {
+    /**
+     * Plain-language assumptions the report depends on. Read these first when
+     * deciding whether the verdict is load-bearing for a launch decision.
+     */
+    assumptions: string[];
+    /** Tests and estimators the verdict was computed from. */
+    methods: string[];
+    /** Alternatives the author considered and why this report didn't take them. */
+    alternatives: string[];
+    /** Failure modes — when this report should NOT drive a decision. */
+    whenNotToApply: string[];
+    /** Citations for the methodological choices above. */
+    citations: string[];
+}
+interface ResearchReport {
+    kind: 'agent-eval-research-report';
+    title: string;
+    generatedAt: string;
+    split: 'search' | 'holdout';
+    comparator: string | null;
+    /**
+     * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples
+     * the report was computed from, plus the comparator and split. Stable across
+     * key insertion order; recomputable by the reader to verify provenance.
+     */
+    runFingerprint: string;
+    preregistrationHash: string | null;
+    rope: {
+        low: number;
+        high: number;
+    } | null;
+    executiveSummary: string[];
+    recommendation: ResearchReportRecommendation;
+    candidates: ResearchReportCandidate[];
+    summary: SummaryTable;
+    charts: {
+        pareto: ParetoFigureSpec;
+        gains: GainDistributionFigureSpec[];
+    };
+    methodology: ResearchReportMethodology;
+    failureClusters?: FailureClusterReport;
+    markdown: string;
+    html: string;
+}
+/**
+ * Executive research report for CPO / AI-lead / launch-review consumption.
+ *
+ * Composes:
+ *   - `summaryTable`         marginal stats with BH-FDR-adjusted q-values
+ *   - `paretoChart`           cost-vs-quality frontier with gate overlay
+ *   - `gainHistogram`         per-candidate paired-delta distribution
+ *   - paired posterior (this file): bootstrap CI on median, Pr(Δ>0),
+ *                              Pr(Δ∈ROPE), MDE at the configured power
+ *
+ * Decisions are made on paired evidence — never on marginal means alone —
+ * and respect any held-out gate decision the caller passes through. The
+ * report embeds a SHA-256 fingerprint of the input run set and, optionally,
+ * the hash of a preregistered protocol so a downstream reader can verify
+ * provenance and that the analysis was the preregistered one.
+ *
+ * Async because the fingerprint uses Web Crypto via `hashJson`; deterministic
+ * for any fixed `runs`, `seed`, and ROPE.
+ */
+declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
+
+interface RenderReleaseReportOptions {
+    title?: string;
+    runs?: readonly RunRecord[];
+    comparator?: string;
+    traceAnalystFindings?: readonly string[];
+    nextActions?: readonly string[];
+}
+declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
+
+/**
+ * Bootstrap-CI promotion gate.
+ *
+ * In any iterative-improvement loop (GEPA, prompt evolution, dataset
+ * curation), the question is "did this generation actually improve, or are
+ * we celebrating noise?". With small N and noisy outcomes, point-estimate
+ * deltas lie. Bootstrap confidence intervals tell the operator whether the
+ * delta is real before code or prompts get promoted.
+ *
+ * This module is pure functions — no I/O, no model calls. Easy to unit-test
+ * and to compose into any verdict gate.
+ *
+ * Default gate:
+ *   - Bootstrap mean baseline vs candidate (1k resamples).
+ *   - Compute the delta distribution; pass if the lower CI bound > 0.
+ *   - Tunable confidence (default 95%) and resample count.
+ *
+ * Verdict semantics intentionally match the existing `experiments.jsonl`
+ * vocabulary:
+ *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
+ *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
+ *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
+ *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
+ */
+type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
+interface BootstrapResult {
+    baselineMean: number;
+    candidateMean: number;
+    /** candidateMean - baselineMean, point estimate. */
+    delta: number;
+    /** Lower bound of the (1 - alpha) CI on the delta. */
+    ciLower: number;
+    /** Upper bound of the (1 - alpha) CI on the delta. */
+    ciUpper: number;
+    /** Number of bootstrap resamples used. */
+    iterations: number;
+    alpha: number;
+    verdict: Verdict;
+}
+interface BootstrapOptions {
+    /** Confidence level alpha (default 0.05 → 95% CI). */
+    alpha?: number;
+    /** Number of resamples (default 1000). */
+    iterations?: number;
+    /**
+     * Minimum total samples (baseline + candidate) below which we always
+     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
+     * Default 6 (combined).
+     */
+    minTotalSamples?: number;
+    /** RNG seed for reproducibility. Default: Math.random. */
+    seed?: number;
+}
+/**
+ * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+ *
+ * Uses simple percentile bootstrap on the difference of resampled means.
+ * That's the standard non-parametric primitive — no distributional
+ * assumptions, robust to skew, easy to reason about.
+ */
+declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
+/**
+ * Judge-replay promotion gate.
+ *
+ * The cheap inner-loop judge that drives an evolution run is by definition
+ * fast and noisy. When you're about to promote a winning variant to the
+ * canonical default, you want a STRONGER judge (a more expensive model, a
+ * human grader, a separately-trained reward model) to confirm the win
+ * generalises beyond the inner loop.
+ *
+ * This helper takes raw winner + baseline outputs, scores both through the
+ * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
+ * judge agrees the winner is real with the configured confidence. Doesn't
+ * matter what shape your "output" is — pass a string, an object, anything
+ * the judge can read.
+ */
+interface JudgeReplayGateArgs<TOutput> {
+    baselineOutputs: TOutput[];
+    candidateOutputs: TOutput[];
+    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
+    judge: (output: TOutput) => Promise<number> | number;
+    alpha?: number;
+    iterations?: number;
+    /** RNG seed for reproducibility. */
+    seed?: number;
+    /** Maximum concurrent judge calls. Default 4. */
+    judgeConcurrency?: number;
+}
+declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
+    baselineSamples: number;
+    candidateSamples: number;
+}>;
+
+export { type SummaryTableOptions as A, type BootstrapOptions as B, type SummaryTableRow as C, DEFAULT_RULES as D, assertReleaseConfidence as E, type FailureClassification as F, type GainDistributionBin as G, bhAdjust as H, bootstrapCi as I, type JudgeReplayGateArgs as J, classifyFailure as K, evaluateReleaseConfidence as L, failureClusterView as M, gainHistogram as N, judgeReplayGate as O, type PairedBootstrapOptions as P, pairedBootstrap as Q, type ReleaseConfidenceThresholds as R, type SummaryTable as S, pairedWilcoxon as T, paretoChart as U, type Verdict as V, releaseTraceEvidenceFromMultiShotTrials as W, renderReleaseReport as X, researchReport as Y, summaryTable as Z, type ReleaseConfidenceScorecard as a, type BootstrapResult as b, type FailureCluster as c, type FailureClusterReport as d, type FailureContext as e, type FailureRule as f, type GainDistributionFigureSpec as g, type GainDistributionOptions as h, type PairedBootstrapResult as i, type ParetoFigureSpec as j, type ParetoPoint as k, RESEARCH_REPORT_HARD_PAIR_FLOOR as l, type ReleaseConfidenceAxis as m, type ReleaseConfidenceAxisName as n, type ReleaseConfidenceInput as o, type ReleaseConfidenceIssue as p, type ReleaseConfidenceMetrics as q, type ReleaseConfidenceStatus as r, type ReleaseTraceEvidence as s, type RenderReleaseReportOptions as t, type ResearchReport as u, type ResearchReportCandidate as v, type ResearchReportDecision as w, type ResearchReportMethodology as x, type ResearchReportOptions as y, type ResearchReportRecommendation as z };

package/dist/reporting.d.ts

@@ -0,0 +1,5 @@
+export { B as BootstrapOptions, b as BootstrapResult, G as GainDistributionBin, g as GainDistributionFigureSpec, h as GainDistributionOptions, J as JudgeReplayGateArgs, P as PairedBootstrapOptions, i as PairedBootstrapResult, j as ParetoFigureSpec, k as ParetoPoint, l as RESEARCH_REPORT_HARD_PAIR_FLOOR, m as ReleaseConfidenceAxis, n as ReleaseConfidenceAxisName, o as ReleaseConfidenceInput, p as ReleaseConfidenceIssue, q as ReleaseConfidenceMetrics, a as ReleaseConfidenceScorecard, r as ReleaseConfidenceStatus, R as ReleaseConfidenceThresholds, s as ReleaseTraceEvidence, t as RenderReleaseReportOptions, u as ResearchReport, v as ResearchReportCandidate, w as ResearchReportDecision, x as ResearchReportMethodology, y as ResearchReportOptions, z as ResearchReportRecommendation, S as SummaryTable, A as SummaryTableOptions, C as SummaryTableRow, V as Verdict, E as assertReleaseConfidence, H as bhAdjust, I as bootstrapCi, L as evaluateReleaseConfidence, N as gainHistogram, O as judgeReplayGate, Q as pairedBootstrap, T as pairedWilcoxon, U as paretoChart, W as releaseTraceEvidenceFromMultiShotTrials, X as renderReleaseReport, Y as researchReport, Z as summaryTable } from './reporting-Da2ihlcM.js';
+import './dataset-B9qvlm_o.js';
+import './multi-shot-optimization-Bvtz294B.js';
+import './run-record-CX_jcAyr.js';
+import './store-u47QaJ9G.js';