@tangle-network/agent-eval 0.33.0 → 0.34.0

package/dist/release-report-ChfmCmLi.d.ts

@@ -0,0 +1,713 @@
+import { C as ContinuousAgreementOptions, a as ContinuousAgreement } from './judge-calibration-DilmB3Ml.js';
+import { TCloud } from '@tangle-network/tcloud';
+import { D as DatasetSplit, b as DatasetManifest, a as DatasetScenario } from './dataset-ueRVTUoY.js';
+import { a3 as GateDecision, A as ActionableSideInfo, m as MultiShotTrialResult } from './summary-report-BPJVzIeW.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-YinVdFwu.js';
+
+interface Scenario {
+    id: string;
+    persona: string;
+    label: string;
+    thesis: string;
+    dimensions: string[];
+    turns: Turn[];
+    artifactChecks: ArtifactCheck[];
+    systemPromptAppend?: string;
+}
+interface Turn {
+    user: string;
+    expectedBehaviors: string[];
+    adversarial?: boolean;
+    feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
+}
+interface ArtifactCheck {
+    type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
+    target: string;
+    contains?: string;
+    minCount?: number;
+    description: string;
+}
+interface JudgeConfig {
+    model: string;
+    temperature: number;
+    rubric: JudgeRubric;
+}
+interface JudgeRubric {
+    name: string;
+    description: string;
+    dimensions: RubricDimension[];
+}
+interface RubricDimension {
+    name: string;
+    description: string;
+    anchor_low: string;
+    anchor_high: string;
+    weight: number;
+}
+interface ScenarioResult {
+    scenarioId: string;
+    persona: string;
+    turns: TurnResult[];
+    artifactResults: ArtifactResult[];
+    judgeScores: JudgeScore[];
+    judgeErrors: number;
+    overallScore: number;
+    totalDurationMs: number;
+    artifacts: CollectedArtifacts;
+}
+interface TurnResult {
+    turnIndex: number;
+    userMessage: string;
+    agentResponse: string;
+    durationMs: number;
+    blocksExtracted: {
+        type: string;
+        title: string;
+    }[];
+    containsCode: boolean;
+    containsToolCall: boolean;
+}
+interface ArtifactResult {
+    check: ArtifactCheck;
+    passed: boolean;
+    detail?: string;
+}
+interface JudgeScore {
+    judgeName: string;
+    dimension: string;
+    score: number;
+    reasoning: string;
+    evidence?: string;
+}
+interface CollectedArtifacts {
+    vaultFiles: {
+        path: string;
+        content: string;
+    }[];
+    blocksExtracted: {
+        type: string;
+        fields: Record<string, string>;
+    }[];
+    codeBlocks: {
+        language: string;
+        code: string;
+    }[];
+    toolCalls: string[];
+}
+interface BenchmarkReport {
+    timestamp: string;
+    generation: number;
+    promptVersion: string;
+    scenarioCount: number;
+    results: ScenarioResult[];
+    summary: {
+        overallAvg: number;
+        byPersona: Record<string, {
+            avg: number;
+            passed: number;
+            total: number;
+        }>;
+        byDimension: Record<string, {
+            avg: number;
+            scores: number[];
+        }>;
+        weakest: {
+            scenario: string;
+            score: number;
+            reason: string;
+        }[];
+        strongest: {
+            scenario: string;
+            score: number;
+            reason: string;
+        }[];
+    };
+}
+interface RouteMap {
+    signup?: string;
+    login?: string;
+    workspaces?: string;
+    threads?: string;
+    chat?: string;
+    tasks?: string;
+    events?: string;
+    approvals?: string;
+    vault?: string;
+    generations?: string;
+    [key: string]: string | undefined;
+}
+interface ProductClientConfig {
+    baseUrl: string;
+    routes: RouteMap;
+}
+interface ScenarioFile {
+    id: string;
+    category: string;
+    persona: string;
+    label: string;
+    thesis: string;
+    isControl?: boolean;
+    rubric?: {
+        dimensions: {
+            name: string;
+            description: string;
+            weight: number;
+        }[];
+    };
+    turns: Turn[];
+    artifactChecks: ArtifactCheck[];
+}
+interface CompletionCriterion {
+    name: string;
+    check: (state: DriverState) => boolean;
+    progress?: (state: DriverState) => number;
+}
+interface FeedbackPattern {
+    trigger: string;
+    response: string;
+}
+/**
+ * How hard the simulated user pushes back. The driver LLM scales its tone
+ * and follow-up aggression to this:
+ *   cooperative — forgiving early adopter; accepts reasonable answers.
+ *   demanding   — experienced professional; rejects vague or hedged answers.
+ *   relentless  — senior partner reviewing for a client who will litigate;
+ *                 interrogates every claim, accepts nothing undefended.
+ */
+type PersonaRigor = 'cooperative' | 'demanding' | 'relentless';
+interface PersonaConfig {
+    id: string;
+    role: string;
+    goal: string;
+    completionCriteria: CompletionCriterion[];
+    feedbackPatterns?: FeedbackPattern[];
+    maxTurns: number;
+    driverModel?: string;
+    /** How adversarial the simulated user is. Defaults to 'demanding'. */
+    rigor?: PersonaRigor;
+    /**
+     * Domain expertise the simulated user holds — quoted into the driver
+     * prompt so it challenges the agent with authority instead of vague
+     * dissatisfaction. e.g. "a 15-year M&A partner who knows GAAP
+     * working-capital mechanics cold".
+     */
+    expertise?: string;
+    /**
+     * Substantive issues a senior professional in this role would
+     * interrogate — traps the scenario hides, claims that must be defended.
+     * The driver probes these without revealing them verbatim; the agent
+     * must surface them on its own.
+     */
+    pressurePoints?: string[];
+    /**
+     * Curveballs the driver may inject once the agent is coasting — changed
+     * facts, a hostile counterparty position, a new constraint. Forces the
+     * agent to re-derive rather than recite.
+     */
+    curveballs?: string[];
+}
+interface DriverState {
+    tasks: number;
+    events: number;
+    proposals: {
+        pending: number;
+        approved: number;
+        rejected: number;
+    };
+    vaultFiles: string[];
+    codeBlocks: number;
+    generations: number;
+}
+interface TurnMetrics {
+    turn: number;
+    timestamp: string;
+    tasks: number;
+    events: number;
+    proposals: {
+        pending: number;
+        approved: number;
+        rejected: number;
+    };
+    vaultFiles: number;
+    responseLatencyMs: number;
+    responseChars: number;
+    codeBlocksProduced: number;
+    blocksExtracted: number;
+    qualityScore?: number;
+    inputTokens: number;
+    outputTokens: number;
+    estimatedCostUsd: number;
+    totalCostUsd: number;
+    completionPercent: number;
+}
+interface DriverResult {
+    personaId: string;
+    /** True when the simulated user professionally signed off (driver said DONE). */
+    completed: boolean;
+    /** Turn at which the simulated user signed off, or null if it never did. */
+    turnsToCompletion: number | null;
+    /**
+     * Turn at which nominal completionCriteria were first all met, or null.
+     * Distinct from turnsToCompletion: criteria can be met while the
+     * simulated professional is still unsatisfied with the work's rigor.
+     */
+    criteriaMetAtTurn: number | null;
+    totalTurns: number;
+    metrics: TurnMetrics[];
+    finalState: DriverState;
+    convergenceCurve: number[];
+    totalCostUsd: number;
+    finalQualityScore: number | null;
+}
+interface BenchmarkRunnerConfig {
+    scenarios: Scenario[];
+    judges: JudgeFn[];
+    systemPrompt: string;
+    model?: string;
+    judgeModel?: string;
+    passThreshold?: number;
+    generation?: number;
+    promptVersion?: string;
+}
+interface JudgeInput {
+    scenario: Scenario;
+    turns: TurnResult[];
+    artifacts: CollectedArtifacts;
+}
+type JudgeFn = (tc: TCloud, input: JudgeInput) => Promise<JudgeScore[]>;
+
+interface TestResult {
+    name: string;
+    passed: boolean;
+    duration: number;
+    detail?: string;
+    checks: CheckResult[];
+}
+interface CheckResult {
+    name: string;
+    passed: boolean;
+    expected: string;
+    actual: string;
+}
+interface EvalResult {
+    scenario: string;
+    status: 'pass' | 'fail' | 'skip';
+    duration: number;
+    detail?: string;
+    artifact?: string;
+}
+
+/**
+ * Release confidence gate.
+ *
+ * This is the production-facing composition layer over the lower-level
+ * primitives:
+ *   - Dataset manifests prove corpus/version coverage.
+ *   - RunRecord rows prove reproducible search/holdout outcomes.
+ *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
+ *   - HeldOutGate decisions remain the paired promotion authority.
+ *
+ * The gate is intentionally pure and conservative. Missing declared evidence
+ * fails closed instead of being treated as a neutral zero.
+ */
+
+type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
+type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+interface ReleaseTraceEvidence {
+    scenarioId: string;
+    candidateId?: string;
+    split?: RunSplitTag;
+    score?: number;
+    ok?: boolean;
+    turnCount?: number;
+    costUsd?: number;
+    durationMs?: number;
+    failureMode?: string;
+    asi?: ActionableSideInfo[];
+    metadata?: Record<string, unknown>;
+}
+interface ReleaseConfidenceThresholds {
+    /** Require a Dataset manifest or explicit scenarios. Default true. */
+    requireCorpus?: boolean;
+    minScenarioCount?: number;
+    minSearchRuns?: number;
+    minHoldoutRuns?: number;
+    /** Require at least one holdout scenario/run. Default true. */
+    requireHoldout?: boolean;
+    minPassRate?: number;
+    minMeanScore?: number;
+    /** Search mean may exceed holdout mean by at most this much. */
+    maxOverfitGap?: number;
+    maxMeanCostUsd?: number;
+    maxP95WallMs?: number;
+    /** Low-score/failed rows must carry ASI. Default true. */
+    requireAsiForFailures?: boolean;
+    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
+    failureScoreThreshold?: number;
+}
+interface ReleaseConfidenceInput {
+    target: string;
+    candidateId?: string;
+    baselineId?: string;
+    dataset?: DatasetManifest;
+    scenarios?: readonly DatasetScenario[];
+    runs?: readonly RunRecord[];
+    traces?: readonly ReleaseTraceEvidence[];
+    gateDecision?: GateDecision | null;
+    thresholds?: ReleaseConfidenceThresholds;
+}
+interface ReleaseConfidenceAxis {
+    name: ReleaseConfidenceAxisName;
+    status: ReleaseConfidenceStatus;
+    score: number;
+    detail: string;
+}
+interface ReleaseConfidenceIssue {
+    axis: ReleaseConfidenceAxisName;
+    severity: 'critical' | 'warning';
+    code: string;
+    detail: string;
+}
+interface ReleaseConfidenceMetrics {
+    scenarioCount: number;
+    searchRuns: number;
+    holdoutRuns: number;
+    passRate: number;
+    meanScore: number;
+    searchMeanScore: number;
+    holdoutMeanScore: number;
+    overfitGap: number;
+    meanCostUsd: number;
+    p95WallMs: number;
+    failedRows: number;
+    failuresWithAsi: number;
+    singleShotTraces: number;
+    multiShotTraces: number;
+    splitCounts: Record<DatasetSplit, number>;
+    domainCounts: Record<string, number>;
+    failureModeCounts: Record<string, number>;
+    responsibleSurfaceCounts: Record<string, number>;
+}
+interface ReleaseConfidenceScorecard {
+    target: string;
+    candidateId: string | null;
+    baselineId: string | null;
+    status: ReleaseConfidenceStatus;
+    promote: boolean;
+    axes: ReleaseConfidenceAxis[];
+    issues: ReleaseConfidenceIssue[];
+    metrics: ReleaseConfidenceMetrics;
+    dataset: DatasetManifest | null;
+    gateDecision: GateDecision | null;
+    summary: string;
+}
+declare function releaseTraceEvidenceFromMultiShotTrials(trials: readonly MultiShotTrialResult[]): ReleaseTraceEvidence[];
+declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+
+/**
+ * Normalize scores so all dimensions follow "higher = better".
+ * Inverted dimensions (hallucination, false_confidence, worst_failure)
+ * already use inverted scoring in the prompt (10 = no hallucination),
+ * but this function ensures consistency if raw scores leak through.
+ */
+declare function normalizeScores(scores: JudgeScore[]): JudgeScore[];
+/** Weighted mean — falls back to uniform weights when omitted */
+declare function weightedMean(scores: {
+    score: number;
+    weight?: number;
+}[]): number;
+/** Bootstrap confidence interval */
+declare function confidenceInterval(scores: number[], confidence?: number): {
+    mean: number;
+    lower: number;
+    upper: number;
+};
+/**
+ * Inter-rater reliability — simplified Krippendorff's alpha.
+ *
+ * Each inner array is one judge's scores for all items.
+ * All arrays must have the same length (same items scored).
+ */
+declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
+/**
+ * Mann-Whitney U test for comparing two independent groups.
+ * Returns U statistic and approximate p-value (normal approximation).
+ */
+declare function mannWhitneyU(a: number[], b: number[]): {
+    u: number;
+    p: number;
+};
+/** Partial credit: returns 0-1 ratio of current toward target */
+declare function partialCredit(current: number, target: number): number;
+/**
+ * Paired t-test — before/after measurements on the SAME items.
+ * Pairing removes inter-item variance, giving tighter significance than
+ * an unpaired test when comparing prompt v1 vs prompt v2 on identical
+ * scenarios.
+ */
+declare function pairedTTest(before: number[], after: number[]): {
+    t: number;
+    df: number;
+    p: number;
+};
+/**
+ * Wilcoxon signed-rank test — paired non-parametric alternative.
+ * Use when the differences aren't normally distributed.
+ */
+declare function wilcoxonSignedRank(before: number[], after: number[]): {
+    w: number;
+    p: number;
+};
+/**
+ * Cohen's d — standardized effect size for two independent groups.
+ * Positive d means group b has higher mean than group a.
+ * Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
+ */
+declare function cohensD(a: number[], b: number[]): number;
+interface CorpusScoreRecord {
+    /** Stable identifier for the rated item (scenario, span, turn, …). */
+    itemId: string;
+    /** Identifier for the judge that produced this score. */
+    judgeName: string;
+    /** Dimension name (matches `JudgeScore.dimension`). */
+    dimension: string;
+    /** Numeric score; must be finite. */
+    score: number;
+}
+interface CorpusAgreementPerDimension extends ContinuousAgreement {
+    dimension: string;
+    /** Item IDs that contributed to this dimension's matrix (every judge scored them). */
+    itemIds: string[];
+    /** Judge IDs that contributed to this dimension's matrix. */
+    judgeIds: string[];
+}
+interface CorpusAgreementReport {
+    /** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
+    perDimension: CorpusAgreementPerDimension[];
+    /** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
+    overallIcc: number;
+    /** Mean weighted κ across dimensions (NaN if none finite). */
+    overallWeightedKappa: number;
+    /** Dimensions evaluated (sorted). */
+    dimensions: string[];
+    /** Judges seen across the corpus (sorted). */
+    judgeIds: string[];
+}
+interface CorpusAgreementOptions extends ContinuousAgreementOptions {
+    /**
+     * Restrict the audit to these dimensions. Default = every dimension
+     * that appears in the input. A dimension named here but absent from
+     * the input throws — silent omission would corrupt the overall metric.
+     */
+    dimensions?: string[];
+    /**
+     * Restrict the audit to these judges. Default = every judge that
+     * appears in the input. A judge named here but absent from a
+     * dimension throws (see "fail loud" below).
+     */
+    judges?: string[];
+}
+/**
+ * Corpus-wide inter-rater agreement across N items × M judges × D dimensions.
+ *
+ * For each dimension, builds the [n_items][n_judges] matrix of scores
+ * (keeping only items every judge rated on that dimension), then runs
+ * `continuousAgreement` to get ICC(2,1), κ_w, Pearson, Spearman, and
+ * bootstrap CIs. Reports a pooled mean across dimensions as a single
+ * "is this judge panel reliable on this corpus?" number.
+ *
+ * Fail-loud contract:
+ *   - Empty input throws.
+ *   - Fewer than 2 judges or fewer than 2 items per dimension throws.
+ *   - A judge present in some dimensions but with zero scored items on
+ *     another dimension throws (would silently shrink the matrix).
+ *   - Duplicate (itemId, judgeName, dimension) records throw.
+ */
+declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
+/**
+ * Convenience adapter for `JudgeScore[]` data keyed externally by item.
+ *
+ * Use when you have per-item arrays of `JudgeScore[]` (e.g. one
+ * `ScenarioResult.judgeScores` per scenario) and want corpus-wide
+ * agreement without manually flattening. `itemId` must be unique per
+ * row of `itemsScores`.
+ */
+declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
+    itemId: string;
+    scores: JudgeScore[];
+}>, opts?: CorpusAgreementOptions): CorpusAgreementReport;
+/**
+ * Required N per arm for a two-sample comparison at target effect size,
+ * alpha, and power. Normal-approximation formula:
+ *   n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
+ * where d is Cohen's d. Returns Infinity for effect ≤ 0.
+ */
+declare function requiredSampleSize(opts: {
+    effect: number;
+    alpha?: number;
+    power?: number;
+    twoSided?: boolean;
+}): number;
+/**
+ * Minimum detectable paired effect (standardised units) for a target paired
+ * sample size: d_min = (z_{1-α/2} + z_β) / sqrt(n_paired). Multiply by
+ * sd(deltas) for score units; treat as a lower bound — Wilcoxon and bootstrap
+ * have asymptotic relative efficiency below 1 vs the t-test on heavy tails.
+ */
+declare function pairedMde(opts: {
+    nPaired: number;
+    alpha?: number;
+    power?: number;
+    twoSided?: boolean;
+}): number;
+/** Bonferroni adjustment: multiply every p-value by the test count, clamp at 1. */
+declare function bonferroni(pValues: number[], alpha?: number): {
+    adjusted: number[];
+    significant: boolean[];
+};
+/**
+ * Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
+ * significance at the target FDR; handles ties and preserves q monotonicity.
+ */
+declare function benjaminiHochberg(pValues: number[], fdr?: number): {
+    qValues: number[];
+    significant: boolean[];
+};
+interface PairedBootstrapResult {
+    /** Number of paired observations. */
+    n: number;
+    /** Median of paired deltas (after − before). */
+    median: number;
+    /** Mean of paired deltas. */
+    mean: number;
+    /** Lower bound of the bootstrap CI on the chosen statistic. */
+    low: number;
+    /** Upper bound of the bootstrap CI on the chosen statistic. */
+    high: number;
+    /** Confidence level used (e.g. 0.95). */
+    confidence: number;
+    /** Number of bootstrap resamples used. */
+    resamples: number;
+}
+interface PairedBootstrapOptions {
+    /** Confidence level. Default 0.95. */
+    confidence?: number;
+    /** Bootstrap resample count. Default 2000. */
+    resamples?: number;
+    /** Statistic to bootstrap. Default 'median'. */
+    statistic?: 'median' | 'mean';
+    /** Deterministic seed. If omitted, uses Math.random(). */
+    seed?: number;
+}
+/**
+ * Paired bootstrap on (after − before) deltas. Returns a CI on the chosen
+ * statistic (median by default); pairs are resampled with replacement. The
+ * lower bound is what the promotion gate checks — `low > threshold` means the
+ * gain is real at the confidence level. Throws on unequal sample sizes.
+ */
+declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
+
+/**
+ * Bootstrap-CI promotion gate.
+ *
+ * In any iterative-improvement loop (GEPA, prompt evolution, dataset
+ * curation), the question is "did this generation actually improve, or are
+ * we celebrating noise?". With small N and noisy outcomes, point-estimate
+ * deltas lie. Bootstrap confidence intervals tell the operator whether the
+ * delta is real before code or prompts get promoted.
+ *
+ * This module is pure functions — no I/O, no model calls. Easy to unit-test
+ * and to compose into any verdict gate.
+ *
+ * Default gate:
+ *   - Bootstrap mean baseline vs candidate (1k resamples).
+ *   - Compute the delta distribution; pass if the lower CI bound > 0.
+ *   - Tunable confidence (default 95%) and resample count.
+ *
+ * Verdict semantics intentionally match the existing `experiments.jsonl`
+ * vocabulary:
+ *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
+ *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
+ *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
+ *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
+ */
+type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
+interface BootstrapResult {
+    baselineMean: number;
+    candidateMean: number;
+    /** candidateMean - baselineMean, point estimate. */
+    delta: number;
+    /** Lower bound of the (1 - alpha) CI on the delta. */
+    ciLower: number;
+    /** Upper bound of the (1 - alpha) CI on the delta. */
+    ciUpper: number;
+    /** Number of bootstrap resamples used. */
+    iterations: number;
+    alpha: number;
+    verdict: Verdict;
+}
+interface BootstrapOptions {
+    /** Confidence level alpha (default 0.05 → 95% CI). */
+    alpha?: number;
+    /** Number of resamples (default 1000). */
+    iterations?: number;
+    /**
+     * Minimum total samples (baseline + candidate) below which we always
+     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
+     * Default 6 (combined).
+     */
+    minTotalSamples?: number;
+    /** RNG seed for reproducibility. Default: Math.random. */
+    seed?: number;
+}
+/**
+ * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
+ *
+ * Uses simple percentile bootstrap on the difference of resampled means.
+ * That's the standard non-parametric primitive — no distributional
+ * assumptions, robust to skew, easy to reason about.
+ */
+declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
+/**
+ * Judge-replay promotion gate.
+ *
+ * The cheap inner-loop judge that drives an evolution run is by definition
+ * fast and noisy. When you're about to promote a winning variant to the
+ * canonical default, you want a STRONGER judge (a more expensive model, a
+ * human grader, a separately-trained reward model) to confirm the win
+ * generalises beyond the inner loop.
+ *
+ * This helper takes raw winner + baseline outputs, scores both through the
+ * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
+ * judge agrees the winner is real with the configured confidence. Doesn't
+ * matter what shape your "output" is — pass a string, an object, anything
+ * the judge can read.
+ */
+interface JudgeReplayGateArgs<TOutput> {
+    baselineOutputs: TOutput[];
+    candidateOutputs: TOutput[];
+    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
+    judge: (output: TOutput) => Promise<number> | number;
+    alpha?: number;
+    iterations?: number;
+    /** RNG seed for reproducibility. */
+    seed?: number;
+    /** Maximum concurrent judge calls. Default 4. */
+    judgeConcurrency?: number;
+}
+declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
+    baselineSamples: number;
+    candidateSamples: number;
+}>;
+
+interface RenderReleaseReportOptions {
+    title?: string;
+    runs?: readonly RunRecord[];
+    comparator?: string;
+    traceAnalystFindings?: readonly string[];
+    nextActions?: readonly string[];
+}
+declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
+
+export { type RouteMap as $, type DriverState as A, type BootstrapOptions as B, type CheckResult as C, type DriverResult as D, type CollectedArtifacts as E, type ScenarioResult as F, type TurnMetrics as G, type ScenarioFile as H, type CompletionCriterion as I, type JudgeReplayGateArgs as J, type ArtifactCheck as K, type ArtifactResult as L, type CorpusAgreementOptions as M, type CorpusAgreementPerDimension as N, type CorpusAgreementReport as O, type PairedBootstrapOptions as P, type CorpusScoreRecord as Q, type ReleaseConfidenceAxis as R, type Scenario as S, type TestResult as T, type EvalResult as U, type Verdict as V, type FeedbackPattern as W, type JudgeConfig as X, type JudgeRubric as Y, type JudgeScore as Z, type PersonaRigor as _, type BootstrapResult as a, type RubricDimension as a0, type Turn as a1, type TurnResult as a2, bonferroni as a3, cohensD as a4, confidenceInterval as a5, corpusInterRaterAgreement as a6, corpusInterRaterAgreementFromJudgeScores as a7, interRaterReliability as a8, mannWhitneyU as a9, normalizeScores as aa, pairedMde as ab, pairedTTest as ac, partialCredit as ad, requiredSampleSize as ae, weightedMean as af, type PairedBootstrapResult as b, type ReleaseConfidenceAxisName as c, type ReleaseConfidenceInput as d, type ReleaseConfidenceIssue as e, type ReleaseConfidenceMetrics as f, type ReleaseConfidenceScorecard as g, type ReleaseConfidenceStatus as h, type ReleaseConfidenceThresholds as i, type ReleaseTraceEvidence as j, type RenderReleaseReportOptions as k, assertReleaseConfidence as l, benjaminiHochberg as m, bootstrapCi as n, evaluateReleaseConfidence as o, judgeReplayGate as p, pairedBootstrap as q, releaseTraceEvidenceFromMultiShotTrials as r, renderReleaseReport as s, type JudgeInput as t, type JudgeFn as u, type BenchmarkRunnerConfig as v, wilcoxonSignedRank as w, type BenchmarkReport as x, type ProductClientConfig as y, type PersonaConfig as z };