@tangle-network/agent-eval 0.22.0 → 0.23.0

package/dist/reporting-B82RSv9C.d.ts

@@ -1,593 +0,0 @@
-import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-B9qvlm_o.js';
-import { k as GateDecision, A as ActionableSideInfo, C as MultiShotTrialResult } from './summary-report-D4p7RlDu.js';
-import { a as RunRecord, R as RunSplitTag } from './run-record-CX_jcAyr.js';
-
-/**
- * Release confidence gate.
- *
- * This is the production-facing composition layer over the lower-level
- * primitives:
- *   - Dataset manifests prove corpus/version coverage.
- *   - RunRecord rows prove reproducible search/holdout outcomes.
- *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
- *   - HeldOutGate decisions remain the paired promotion authority.
- *
- * The gate is intentionally pure and conservative. Missing declared evidence
- * fails closed instead of being treated as a neutral zero.
- */
-
-type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
-type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
-interface ReleaseTraceEvidence {
-    scenarioId: string;
-    candidateId?: string;
-    split?: RunSplitTag;
-    score?: number;
-    ok?: boolean;
-    turnCount?: number;
-    costUsd?: number;
-    durationMs?: number;
-    failureMode?: string;
-    asi?: ActionableSideInfo[];
-    metadata?: Record<string, unknown>;
-}
-interface ReleaseConfidenceThresholds {
-    /** Require a Dataset manifest or explicit scenarios. Default true. */
-    requireCorpus?: boolean;
-    minScenarioCount?: number;
-    minSearchRuns?: number;
-    minHoldoutRuns?: number;
-    /** Require at least one holdout scenario/run. Default true. */
-    requireHoldout?: boolean;
-    minPassRate?: number;
-    minMeanScore?: number;
-    /** Search mean may exceed holdout mean by at most this much. */
-    maxOverfitGap?: number;
-    maxMeanCostUsd?: number;
-    maxP95WallMs?: number;
-    /** Low-score/failed rows must carry ASI. Default true. */
-    requireAsiForFailures?: boolean;
-    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
-    failureScoreThreshold?: number;
-}
-interface ReleaseConfidenceInput {
-    target: string;
-    candidateId?: string;
-    baselineId?: string;
-    dataset?: DatasetManifest;
-    scenarios?: readonly DatasetScenario[];
-    runs?: readonly RunRecord[];
-    traces?: readonly ReleaseTraceEvidence[];
-    gateDecision?: GateDecision | null;
-    thresholds?: ReleaseConfidenceThresholds;
-}
-interface ReleaseConfidenceAxis {
-    name: ReleaseConfidenceAxisName;
-    status: ReleaseConfidenceStatus;
-    score: number;
-    detail: string;
-}
-interface ReleaseConfidenceIssue {
-    axis: ReleaseConfidenceAxisName;
-    severity: 'critical' | 'warning';
-    code: string;
-    detail: string;
-}
-interface ReleaseConfidenceMetrics {
-    scenarioCount: number;
-    searchRuns: number;
-    holdoutRuns: number;
-    passRate: number;
-    meanScore: number;
-    searchMeanScore: number;
-    holdoutMeanScore: number;
-    overfitGap: number;
-    meanCostUsd: number;
-    p95WallMs: number;
-    failedRows: number;
-    failuresWithAsi: number;
-    singleShotTraces: number;
-    multiShotTraces: number;
-    splitCounts: Record<DatasetSplit, number>;
-    domainCounts: Record<string, number>;
-    failureModeCounts: Record<string, number>;
-    responsibleSurfaceCounts: Record<string, number>;
-}
-interface ReleaseConfidenceScorecard {
-    target: string;
-    candidateId: string | null;
-    baselineId: string | null;
-    status: ReleaseConfidenceStatus;
-    promote: boolean;
-    axes: ReleaseConfidenceAxis[];
-    issues: ReleaseConfidenceIssue[];
-    metrics: ReleaseConfidenceMetrics;
-    dataset: DatasetManifest | null;
-    gateDecision: GateDecision | null;
-    summary: string;
-}
-declare function releaseTraceEvidenceFromMultiShotTrials(trials: readonly MultiShotTrialResult[]): ReleaseTraceEvidence[];
-declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
-declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
-
-/**
- * OutcomeStore — deployment outcomes attached to Run IDs.
- *
- * Outcomes arrive asynchronously from production telemetry after the
- * eval run completed: user ratings, retention flags, conversion events,
- * revenue, support-ticket rate, anything a product team can measure.
- * The store is a peer to TraceStore — separate lifecycle, same runId
- * foreign key.
- *
- * The whole point of this module is to make the meta-eval correlation
- * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
- */
-interface DeploymentOutcome {
-    runId: string;
-    capturedAt: number;
-    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
-    metrics: Record<string, number>;
-    /** Dimensions for stratified analysis — cohort, region, user_segment. */
-    labels?: Record<string, string>;
-    /** Free-form provenance (source system, pipeline version). */
-    source?: string;
-}
-interface OutcomeFilter {
-    runIds?: string[];
-    since?: number;
-    until?: number;
-    label?: {
-        key: string;
-        value: string;
-    };
-    source?: string;
-}
-interface OutcomeStore {
-    append(outcome: DeploymentOutcome): Promise<void>;
-    /** All outcomes attached to this run (a single run can have many — multiple
-     *  capture windows over deployment time). */
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-declare class InMemoryOutcomeStore implements OutcomeStore {
-    private items;
-    append(outcome: DeploymentOutcome): Promise<void>;
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-interface FileSystemOutcomeStoreOptions {
-    dir: string;
-    maxBytes?: number;
-}
-declare class FileSystemOutcomeStore implements OutcomeStore {
-    private dir;
-    private maxBytes;
-    private memo?;
-    private loaded;
-    constructor(options: FileSystemOutcomeStoreOptions);
-    private ensureDir;
-    append(outcome: DeploymentOutcome): Promise<void>;
-    private load;
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-
-/**
- * Rubric predictive validity — does our eval rubric predict deployment
- * outcomes?
- *
- * `correlationStudy` (already in this package) joins a `TraceStore` to an
- * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
- * (eval-metric, outcome-metric) pair. That answers "does X correlate with
- * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
- * around it: take a sequence of `RunRecord`s (the canonical campaign
- * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
- * ranked verdict on every rubric whose dimension scores were captured in
- * `outcome.raw`.
- *
- * The point — quoting the methodology doc — is that **without this loop
- * every rubric is faith-based**. Once it's wired, you know which rubrics
- * have earned their promotion power and which ones are decoration.
- *
- *   const validity = await rubricPredictiveValidity({
- *     runs: lastQuarter,
- *     outcomes: shipFlagOutcomeStore,
- *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
- *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
- *   })
- *   for (const r of validity.ranked) {
- *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
- *   }
- *
- * The function is intentionally read-only. Use the verdict to deprecate
- * decorative rubrics, re-weight composite scores, or trigger a
- * recalibration sweep when predictive validity drops below a threshold.
- */
-
-interface RubricPredictiveValidityInput {
-    /**
-     * Canonical campaign output. Each record's `outcome.raw[<rubricId>]`
-     * provides the eval score; missing keys are silently skipped per pair.
-     */
-    runs: RunRecord[];
-    outcomes: OutcomeStore;
-    /**
-     * Outcome metric names to evaluate against. Each must appear in at
-     * least one `DeploymentOutcome.metrics` keyspace; pairs with too few
-     * joined samples are excluded from the result.
-     */
-    outcomeMetrics: string[];
-    /**
-     * Rubric ids to evaluate. Must appear as keys in `RunRecord.outcome.raw`.
-     * If omitted, every numeric key in `outcome.raw` across the run set is
-     * treated as a rubric.
-     */
-    rubrics?: string[];
-    /** Minimum joined-sample count before a pair is reported. Default 8. */
-    minSamples?: number;
-    /** Bootstrap resamples for CI. Default 500. */
-    bootstrapResamples?: number;
-    /** Random seed for the bootstrap (mulberry32). Default unset (Math.random). */
-    seed?: number;
-    /**
-     * Reduction when multiple outcomes attach to one runId. Default `'latest'`
-     * (most recently captured).
-     */
-    reduction?: 'latest' | 'mean' | 'max';
-}
-interface RubricOutcomePair {
-    rubric: string;
-    outcome: string;
-    n: number;
-    pearson: number;
-    spearman: number;
-    ci95: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
-     * `decorative` < 0.4 in absolute correlation. A negative correlation
-     * with a desired outcome is also `decorative` — actively misleading
-     * is worse than uninformative.
-     */
-    verdict: 'load_bearing' | 'informative' | 'decorative';
-}
-interface RubricRanking {
-    rubric: string;
-    /** Outcome metric this rubric correlated best with. */
-    bestOutcome: string;
-    spearman: number;
-    pearson: number;
-    n: number;
-    verdict: RubricOutcomePair['verdict'];
-}
-interface RubricPredictiveValidityReport {
-    pairs: RubricOutcomePair[];
-    /** Per-rubric best pair, sorted descending by |spearman|. */
-    ranked: RubricRanking[];
-    joinedSamples: number;
-    skippedRuns: number;
-    /** Rubrics that were declared but never produced a usable score. */
-    rubricsWithoutData: string[];
-}
-declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
-
-/**
- * Paper-grade paired statistics for held-out promotion gates.
- *
- * The promotion gate (`HeldOutGate`) needs three things:
- *
- *   1. A bootstrap confidence interval on the per-item paired delta
- *      (`pairedBootstrap`). Median delta is the headline number; the
- *      CI lower bound is what the gate checks against `pairedDeltaThreshold`.
- *   2. A non-parametric significance test on the paired deltas
- *      (`pairedWilcoxon` — re-export of `wilcoxonSignedRank` under the
- *      paper-style name).
- *   3. False-discovery-rate correction across simultaneously-tested
- *      candidate variants (`bhAdjust` — re-export of `benjaminiHochberg`).
- *
- * Why a separate file: every existing primitive lives in `statistics.ts`
- * (general) or `power-analysis.ts` (correction). Paired-bootstrap is
- * paired-only, paper-grade, and load-bearing for the promotion gate.
- * Putting it next to `statistics.ts` would require editing that file;
- * the brief forbids that. New file, new exports, no surface change.
- */
-interface PairedBootstrapResult {
-    /** Number of paired observations (after dropping unequal lengths is rejected). */
-    n: number;
-    /** Median of paired deltas (after − before). */
-    median: number;
-    /** Mean of paired deltas. */
-    mean: number;
-    /** Lower bound of the bootstrap CI on the median delta. */
-    low: number;
-    /** Upper bound of the bootstrap CI on the median delta. */
-    high: number;
-    /** Confidence level used (e.g. 0.95). */
-    confidence: number;
-    /** Number of bootstrap resamples used. */
-    resamples: number;
-}
-interface PairedBootstrapOptions {
-    /** Confidence level. Default 0.95. */
-    confidence?: number;
-    /** Bootstrap resample count. Default 2000. */
-    resamples?: number;
-    /** Statistic to bootstrap. Default 'median'. */
-    statistic?: 'median' | 'mean';
-    /** Deterministic seed. If omitted, uses Math.random(). */
-    seed?: number;
-}
-/**
- * Paired bootstrap on (after - before) deltas. Returns a CI on the
- * chosen statistic (median by default). Pairs are resampled with
- * replacement. The lower bound is what the promotion gate checks: if
- * `low > pairedDeltaThreshold`, the gain is real at the chosen
- * confidence level.
- *
- * Throws on unequal sample sizes — caller must align pairs upstream.
- */
-declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
-/**
- * Paper-style alias for `wilcoxonSignedRank`. The signed-rank test on
- * paired deltas is the standard non-parametric significance test for
- * "candidate beats baseline on matched items." Use alongside the
- * bootstrap CI: bootstrap gives effect size, Wilcoxon gives p.
- */
-declare function pairedWilcoxon(before: number[], after: number[]): {
-    w: number;
-    p: number;
-};
-/**
- * Paper-style alias for `benjaminiHochberg`. Use to correct p-values
- * across multiple candidate-vs-baseline comparisons run in the same
- * promotion sweep. Returns BH-adjusted q-values and significance at
- * the requested FDR (default 0.05).
- */
-declare function bhAdjust(pValues: number[], fdr?: number): {
-    qValues: number[];
-    significant: boolean[];
-};
-
-/**
- * Always-valid sequential evaluation.
- *
- * `researchReport` (0.21+) assumes a single pre-specified analysis. Real
- * consumers run campaigns weekly / nightly / per-PR; each new run silently
- * inflates the false-discovery rate, because the BH-FDR guarantee was for
- * the *first* look, not the 47th. Without time-uniform inference,
- * launch-decision teams either (a) don't peek, which forfeits the cost
- * advantage of stop-when-decisive, or (b) peek and pretend they didn't,
- * which forfeits scientific validity.
- *
- * This module ships **e-value-based confidence sequences** for paired
- * bounded outcomes. The methodology is the predictable plug-in betting
- * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*
- * stopping time. Concretely:
- *
- *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,
- *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable
- *   plug-in), and the running e-value is
- *
- *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)
- *
- *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by
- *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null
- *   at any time without inflating the type-I error.
- *
- * Combined with `runEvalCampaign`, every consumer running rolling
- * campaigns gains the ability to ship the moment evidence is decisive,
- * stop-early on dead-on-arrival variants, and accumulate evidence across
- * partial runs without spending the FDR budget. No new sweep is wasted.
- *
- * References:
- *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
- *     Time-uniform, nonparametric, nonasymptotic confidence sequences.
- *     Annals of Statistics, 49(2), 1055–1080.
- *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded
- *     random variables by betting. JRSS B, 86(1), 1–27.
- */
-type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent';
-interface PairedEvalueOptions {
-    /**
-     * Bound on |delta|. Default 1 (matching most score scales). Must satisfy
-     * c > 0; deltas outside [-c, c] are clipped with a warning attached to
-     * the return value.
-     */
-    bound?: number;
-    /** Target Type-I error. Default 0.05. */
-    alpha?: number;
-    /**
-     * Region of Practical Equivalence on the *mean* paired delta. When
-     * supplied, the verdict can return `'equivalent'` once the running
-     * confidence sequence on the mean is fully contained in [low, high].
-     */
-    rope?: {
-        low: number;
-        high: number;
-    };
-    /** Initial bet shrinkage (0 < scale ≤ 1). Default 0.5 — empirically robust. */
-    initialBetShrinkage?: number;
-}
-interface PairedEvalueStep {
-    /** 1-indexed observation count. */
-    t: number;
-    delta: number;
-    /** Running e-value E_t = ∏ (1 + λ_i · D_i). */
-    evalue: number;
-    /** Time-uniform p-value at stopping time t. */
-    pValue: number;
-    /** Lower bound of the empirical Bernstein confidence sequence at level 1-α. */
-    csLow: number;
-    csHigh: number;
-    /** Verdict at this stopping time. */
-    decision: SequentialDecision;
-}
-interface PairedEvalueSequence {
-    steps: PairedEvalueStep[];
-    /** The decision at the final step. */
-    finalDecision: SequentialDecision;
-    /** Index (1-based) at which a non-`continue` decision first fired, or null. */
-    decisionFiredAt: number | null;
-    /** True if any deltas were clipped to [-bound, bound]. */
-    clipped: boolean;
-}
-/**
- * Run the paired e-value sequence over an in-order delta stream.
- *
- * Use for *streaming* / interim analyses: pass the deltas you have so
- * far, get the verdict at every prefix length. The decision is
- * monotone-stable in the sense that once `'reject_now'` or `'promote_now'`
- * fires, the verdict at later steps remains decisive (the e-value is a
- * non-negative martingale; once it crosses the threshold, it's crossed).
- */
-declare function pairedEvalueSequence(deltas: number[], opts?: PairedEvalueOptions): PairedEvalueSequence;
-interface InterimReleaseConfidenceInput {
-    /**
-     * One delta series per candidate (paired deltas vs comparator). Order
-     * within a series is the order the campaigns were run.
-     */
-    deltaSeries: Array<{
-        candidateId: string;
-        deltas: number[];
-    }>;
-    alpha?: number;
-    bound?: number;
-    rope?: {
-        low: number;
-        high: number;
-    };
-}
-interface InterimReleaseConfidence {
-    candidates: Array<{
-        candidateId: string;
-        decision: SequentialDecision;
-        decisionFiredAt: number | null;
-        finalEvalue: number;
-        finalPValue: number;
-        pairs: number;
-        csLow: number;
-        csHigh: number;
-    }>;
-    /**
-     * Campaign-level recommendation: pick the strongest 'promote_now', else
-     * 'continue' if any candidate is still live, else 'reject_now' if every
-     * candidate is dead, else 'equivalent'.
-     */
-    recommendation: {
-        decision: SequentialDecision;
-        candidateId: string | null;
-    };
-}
-/**
- * Run interim sequential analyses across many candidates at once,
- * preserving the time-uniform α guarantee for each candidate's series and
- * synthesising a campaign-level recommendation. Designed to be called on
- * every campaign tick — the recommendation is anytime-valid.
- */
-declare function evaluateInterimReleaseConfidence(input: InterimReleaseConfidenceInput): InterimReleaseConfidence;
-
-interface RenderReleaseReportOptions {
-    title?: string;
-    runs?: readonly RunRecord[];
-    comparator?: string;
-    traceAnalystFindings?: readonly string[];
-    nextActions?: readonly string[];
-}
-declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
-
-/**
- * Bootstrap-CI promotion gate.
- *
- * In any iterative-improvement loop (GEPA, prompt evolution, dataset
- * curation), the question is "did this generation actually improve, or are
- * we celebrating noise?". With small N and noisy outcomes, point-estimate
- * deltas lie. Bootstrap confidence intervals tell the operator whether the
- * delta is real before code or prompts get promoted.
- *
- * This module is pure functions — no I/O, no model calls. Easy to unit-test
- * and to compose into any verdict gate.
- *
- * Default gate:
- *   - Bootstrap mean baseline vs candidate (1k resamples).
- *   - Compute the delta distribution; pass if the lower CI bound > 0.
- *   - Tunable confidence (default 95%) and resample count.
- *
- * Verdict semantics intentionally match the existing `experiments.jsonl`
- * vocabulary:
- *   - ADVANCE: candidate's CI lower bound > baseline mean (real win)
- *   - KEEP:    overlap, but candidate point estimate >= baseline (neutral)
- *   - REVERT:  candidate's CI upper bound < baseline mean (real regression)
- *   - INCONCLUSIVE: not enough samples or CI straddles zero with no signal
- */
-type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE';
-interface BootstrapResult {
-    baselineMean: number;
-    candidateMean: number;
-    /** candidateMean - baselineMean, point estimate. */
-    delta: number;
-    /** Lower bound of the (1 - alpha) CI on the delta. */
-    ciLower: number;
-    /** Upper bound of the (1 - alpha) CI on the delta. */
-    ciUpper: number;
-    /** Number of bootstrap resamples used. */
-    iterations: number;
-    alpha: number;
-    verdict: Verdict;
-}
-interface BootstrapOptions {
-    /** Confidence level alpha (default 0.05 → 95% CI). */
-    alpha?: number;
-    /** Number of resamples (default 1000). */
-    iterations?: number;
-    /**
-     * Minimum total samples (baseline + candidate) below which we always
-     * return INCONCLUSIVE — bootstrap with too few samples is meaningless.
-     * Default 6 (combined).
-     */
-    minTotalSamples?: number;
-    /** RNG seed for reproducibility. Default: Math.random. */
-    seed?: number;
-}
-/**
- * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.
- *
- * Uses simple percentile bootstrap on the difference of resampled means.
- * That's the standard non-parametric primitive — no distributional
- * assumptions, robust to skew, easy to reason about.
- */
-declare function bootstrapCi(baseline: number[], candidate: number[], options?: BootstrapOptions): BootstrapResult;
-/**
- * Judge-replay promotion gate.
- *
- * The cheap inner-loop judge that drives an evolution run is by definition
- * fast and noisy. When you're about to promote a winning variant to the
- * canonical default, you want a STRONGER judge (a more expensive model, a
- * human grader, a separately-trained reward model) to confirm the win
- * generalises beyond the inner loop.
- *
- * This helper takes raw winner + baseline outputs, scores both through the
- * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger
- * judge agrees the winner is real with the configured confidence. Doesn't
- * matter what shape your "output" is — pass a string, an object, anything
- * the judge can read.
- */
-interface JudgeReplayGateArgs<TOutput> {
-    baselineOutputs: TOutput[];
-    candidateOutputs: TOutput[];
-    /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */
-    judge: (output: TOutput) => Promise<number> | number;
-    alpha?: number;
-    iterations?: number;
-    /** RNG seed for reproducibility. */
-    seed?: number;
-    /** Maximum concurrent judge calls. Default 4. */
-    judgeConcurrency?: number;
-}
-declare function judgeReplayGate<TOutput>(args: JudgeReplayGateArgs<TOutput>): Promise<BootstrapResult & {
-    baselineSamples: number;
-    candidateSamples: number;
-}>;
-
-export { evaluateReleaseConfidence as A, type BootstrapOptions as B, judgeReplayGate as C, type DeploymentOutcome as D, pairedBootstrap as E, FileSystemOutcomeStore as F, pairedEvalueSequence as G, pairedWilcoxon as H, InMemoryOutcomeStore as I, type JudgeReplayGateArgs as J, releaseTraceEvidenceFromMultiShotTrials as K, renderReleaseReport as L, rubricPredictiveValidity as M, type OutcomeFilter as O, type PairedBootstrapOptions as P, type ReleaseConfidenceThresholds as R, type SequentialDecision as S, type Verdict as V, type ReleaseConfidenceScorecard as a, type OutcomeStore as b, type BootstrapResult as c, type FileSystemOutcomeStoreOptions as d, type InterimReleaseConfidence as e, type InterimReleaseConfidenceInput as f, type PairedBootstrapResult as g, type PairedEvalueOptions as h, type PairedEvalueSequence as i, type PairedEvalueStep as j, type ReleaseConfidenceAxis as k, type ReleaseConfidenceAxisName as l, type ReleaseConfidenceInput as m, type ReleaseConfidenceIssue as n, type ReleaseConfidenceMetrics as o, type ReleaseConfidenceStatus as p, type ReleaseTraceEvidence as q, type RenderReleaseReportOptions as r, type RubricOutcomePair as s, type RubricPredictiveValidityInput as t, type RubricPredictiveValidityReport as u, type RubricRanking as v, assertReleaseConfidence as w, bhAdjust as x, bootstrapCi as y, evaluateInterimReleaseConfidence as z };