@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/sequential-DgU2mFsE.d.ts

@@ -1,304 +0,0 @@
-import { R as RunRecord } from './run-record-DNiOMBrZ.js';
-
-/**
- * OutcomeStore — deployment outcomes attached to Run IDs.
- *
- * Outcomes arrive asynchronously from production telemetry after the
- * eval run completed: user ratings, retention flags, conversion events,
- * revenue, support-ticket rate, anything a product team can measure.
- * The store is a peer to TraceStore — separate lifecycle, same runId
- * foreign key.
- *
- * The whole point of this module is to make the meta-eval correlation
- * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
- */
-interface DeploymentOutcome {
-    runId: string;
-    capturedAt: number;
-    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
-    metrics: Record<string, number>;
-    /** Dimensions for stratified analysis — cohort, region, user_segment. */
-    labels?: Record<string, string>;
-    /** Free-form provenance (source system, pipeline version). */
-    source?: string;
-}
-interface OutcomeFilter {
-    runIds?: string[];
-    since?: number;
-    until?: number;
-    label?: {
-        key: string;
-        value: string;
-    };
-    source?: string;
-}
-interface OutcomeStore {
-    append(outcome: DeploymentOutcome): Promise<void>;
-    /** All outcomes attached to this run (a single run can have many — multiple
-     *  capture windows over deployment time). */
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-declare class InMemoryOutcomeStore implements OutcomeStore {
-    private items;
-    append(outcome: DeploymentOutcome): Promise<void>;
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-interface FileSystemOutcomeStoreOptions {
-    dir: string;
-    maxBytes?: number;
-}
-declare class FileSystemOutcomeStore implements OutcomeStore {
-    private dir;
-    private maxBytes;
-    private memo?;
-    private loaded;
-    constructor(options: FileSystemOutcomeStoreOptions);
-    private ensureDir;
-    append(outcome: DeploymentOutcome): Promise<void>;
-    private load;
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-
-/**
- * Rubric predictive validity — does our eval rubric predict deployment
- * outcomes?
- *
- * `correlationStudy` (already in this package) joins a `TraceStore` to an
- * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
- * (eval-metric, outcome-metric) pair. That answers "does X correlate with
- * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
- * around it: take a sequence of `RunRecord`s (the canonical campaign
- * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
- * ranked verdict on every rubric whose dimension scores were captured in
- * `outcome.raw`.
- *
- * The point — quoting the methodology doc — is that **without this loop
- * every rubric is faith-based**. Once it's wired, you know which rubrics
- * have earned their promotion power and which ones are decoration.
- *
- *   const validity = await rubricPredictiveValidity({
- *     runs: lastQuarter,
- *     outcomes: shipFlagOutcomeStore,
- *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
- *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
- *   })
- *   for (const r of validity.ranked) {
- *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
- *   }
- *
- * The function is intentionally read-only. Use the verdict to deprecate
- * decorative rubrics, re-weight composite scores, or trigger a
- * recalibration sweep when predictive validity drops below a threshold.
- */
-
-interface RubricPredictiveValidityInput {
-    /**
-     * Canonical campaign output. Each record's `outcome.raw[<rubricId>]`
-     * provides the eval score; missing keys are silently skipped per pair.
-     */
-    runs: RunRecord[];
-    outcomes: OutcomeStore;
-    /**
-     * Outcome metric names to evaluate against. Each must appear in at
-     * least one `DeploymentOutcome.metrics` keyspace; pairs with too few
-     * joined samples are excluded from the result.
-     */
-    outcomeMetrics: string[];
-    /**
-     * Rubric ids to evaluate. Must appear as keys in `RunRecord.outcome.raw`.
-     * If omitted, every numeric key in `outcome.raw` across the run set is
-     * treated as a rubric.
-     */
-    rubrics?: string[];
-    /** Minimum joined-sample count before a pair is reported. Default 8. */
-    minSamples?: number;
-    /** Bootstrap resamples for CI. Default 500. */
-    bootstrapResamples?: number;
-    /** Random seed for the bootstrap (mulberry32). Default unset (Math.random). */
-    seed?: number;
-    /**
-     * Reduction when multiple outcomes attach to one runId. Default `'latest'`
-     * (most recently captured).
-     */
-    reduction?: 'latest' | 'mean' | 'max';
-}
-interface RubricOutcomePair {
-    rubric: string;
-    outcome: string;
-    n: number;
-    pearson: number;
-    spearman: number;
-    ci95: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
-     * `decorative` < 0.4 in absolute correlation. A negative correlation
-     * with a desired outcome is also `decorative` — actively misleading
-     * is worse than uninformative.
-     */
-    verdict: 'load_bearing' | 'informative' | 'decorative';
-}
-interface RubricRanking {
-    rubric: string;
-    /** Outcome metric this rubric correlated best with. */
-    bestOutcome: string;
-    spearman: number;
-    pearson: number;
-    n: number;
-    verdict: RubricOutcomePair['verdict'];
-}
-interface RubricPredictiveValidityReport {
-    pairs: RubricOutcomePair[];
-    /** Per-rubric best pair, sorted descending by |spearman|. */
-    ranked: RubricRanking[];
-    joinedSamples: number;
-    skippedRuns: number;
-    /** Rubrics that were declared but never produced a usable score. */
-    rubricsWithoutData: string[];
-}
-declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
-
-/**
- * Always-valid sequential evaluation.
- *
- * `researchReport` (0.21+) assumes a single pre-specified analysis. Real
- * consumers run campaigns weekly / nightly / per-PR; each new run silently
- * inflates the false-discovery rate, because the BH-FDR guarantee was for
- * the *first* look, not the 47th. Without time-uniform inference,
- * launch-decision teams either (a) don't peek, which forfeits the cost
- * advantage of stop-when-decisive, or (b) peek and pretend they didn't,
- * which forfeits scientific validity.
- *
- * This module ships **e-value-based confidence sequences** for paired
- * bounded outcomes. The methodology is the predictable plug-in betting
- * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*
- * stopping time. Concretely:
- *
- *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,
- *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable
- *   plug-in), and the running e-value is
- *
- *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)
- *
- *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by
- *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null
- *   at any time without inflating the type-I error.
- *
- * Combined with `runEvalCampaign`, every consumer running rolling
- * campaigns gains the ability to ship the moment evidence is decisive,
- * stop-early on dead-on-arrival variants, and accumulate evidence across
- * partial runs without spending the FDR budget. No new sweep is wasted.
- *
- * References:
- *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
- *     Time-uniform, nonparametric, nonasymptotic confidence sequences.
- *     Annals of Statistics, 49(2), 1055–1080.
- *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded
- *     random variables by betting. JRSS B, 86(1), 1–27.
- */
-type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent';
-interface PairedEvalueOptions {
-    /**
-     * Bound on |delta|. Default 1 (matching most score scales). Must satisfy
-     * c > 0; deltas outside [-c, c] are clipped with a warning attached to
-     * the return value.
-     */
-    bound?: number;
-    /** Target Type-I error. Default 0.05. */
-    alpha?: number;
-    /**
-     * Region of Practical Equivalence on the *mean* paired delta. When
-     * supplied, the verdict can return `'equivalent'` once the running
-     * confidence sequence on the mean is fully contained in [low, high].
-     */
-    rope?: {
-        low: number;
-        high: number;
-    };
-    /** Initial bet shrinkage (0 < scale ≤ 1). Default 0.5 — empirically robust. */
-    initialBetShrinkage?: number;
-}
-interface PairedEvalueStep {
-    /** 1-indexed observation count. */
-    t: number;
-    delta: number;
-    /** Running e-value E_t = ∏ (1 + λ_i · D_i). */
-    evalue: number;
-    /** Time-uniform p-value at stopping time t. */
-    pValue: number;
-    /** Lower bound of the empirical Bernstein confidence sequence at level 1-α. */
-    csLow: number;
-    csHigh: number;
-    /** Verdict at this stopping time. */
-    decision: SequentialDecision;
-}
-interface PairedEvalueSequence {
-    steps: PairedEvalueStep[];
-    /** The decision at the final step. */
-    finalDecision: SequentialDecision;
-    /** Index (1-based) at which a non-`continue` decision first fired, or null. */
-    decisionFiredAt: number | null;
-    /** True if any deltas were clipped to [-bound, bound]. */
-    clipped: boolean;
-}
-/**
- * Run the paired e-value sequence over an in-order delta stream.
- *
- * Use for *streaming* / interim analyses: pass the deltas you have so
- * far, get the verdict at every prefix length. The decision is
- * monotone-stable in the sense that once `'reject_now'` or `'promote_now'`
- * fires, the verdict at later steps remains decisive (the e-value is a
- * non-negative martingale; once it crosses the threshold, it's crossed).
- */
-declare function pairedEvalueSequence(deltas: number[], opts?: PairedEvalueOptions): PairedEvalueSequence;
-interface InterimReleaseConfidenceInput {
-    /**
-     * One delta series per candidate (paired deltas vs comparator). Order
-     * within a series is the order the campaigns were run.
-     */
-    deltaSeries: Array<{
-        candidateId: string;
-        deltas: number[];
-    }>;
-    alpha?: number;
-    bound?: number;
-    rope?: {
-        low: number;
-        high: number;
-    };
-}
-interface InterimReleaseConfidence {
-    candidates: Array<{
-        candidateId: string;
-        decision: SequentialDecision;
-        decisionFiredAt: number | null;
-        finalEvalue: number;
-        finalPValue: number;
-        pairs: number;
-        csLow: number;
-        csHigh: number;
-    }>;
-    /**
-     * Campaign-level recommendation: pick the strongest 'promote_now', else
-     * 'continue' if any candidate is still live, else 'reject_now' if every
-     * candidate is dead, else 'equivalent'.
-     */
-    recommendation: {
-        decision: SequentialDecision;
-        candidateId: string | null;
-    };
-}
-/**
- * Run interim sequential analyses across many candidates at once,
- * preserving the time-uniform α guarantee for each candidate's series and
- * synthesising a campaign-level recommendation. Designed to be called on
- * every campaign tick — the recommendation is anytime-valid.
- */
-declare function evaluateInterimReleaseConfidence(input: InterimReleaseConfidenceInput): InterimReleaseConfidence;
-
-export { type DeploymentOutcome as D, FileSystemOutcomeStore as F, InMemoryOutcomeStore as I, type OutcomeFilter as O, type PairedEvalueOptions as P, type RubricOutcomePair as R, type SequentialDecision as S, type OutcomeStore as a, type FileSystemOutcomeStoreOptions as b, type InterimReleaseConfidence as c, type InterimReleaseConfidenceInput as d, type PairedEvalueSequence as e, type PairedEvalueStep as f, type RubricPredictiveValidityInput as g, type RubricPredictiveValidityReport as h, type RubricRanking as i, evaluateInterimReleaseConfidence as j, pairedEvalueSequence as p, rubricPredictiveValidity as r };