@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/rl.d.ts

@@ -1 +1,1193 @@
-export * from './rl/index'
+import { R as RunRecord, a as RunSplitTag } from './run-record-CP2ObebC.js';
+export { A as AdversarialMutation, a as AdversarialScenario, b as AdversarialSearchOptions, c as AdversarialSearchReport, d as adversarialScenarioSearch } from './adversarial-DIVcDoI_.js';
+import { P as PreferenceExtractionReport, D as DpoExportRow, G as GrpoExportRow, S as SftExportRow, E as ExtractPreferencesOptions, a as DpoLookups, b as GrpoLookups, c as SftLookups } from './corpus-D4YW9UoJ.js';
+export { d as CorpusAppendResult, C as CorpusRecord, e as DatasetFormat, f as ExtractStepRewardsOptions, H as HarvestOptions, g as PreferenceStrategy, h as PreferenceTriple, i as PrmExportRow, j as PrmLookups, k as PrmTrainingTriple, R as RewardKind, l as RewardStats, m as RlDatasetBundle, n as RlDatasetConfig, o as RlDatasetManifest, p as RlDatasetStats, q as RunwiseStepSummary, r as StepReward, s as StepRewardJsonlRow, t as StepScorer, u as appendToCorpus, v as buildDatasetFromCorpus, w as buildRlDataset, x as datasheetToMarkdown, y as extractPreferences, z as extractStepRewards, A as prmTrainingPairs, B as readCorpus, F as runwiseStepRewardSummary, I as stepRewardsToJsonl, J as toAnthropicFormat, K as toDpoJsonl, L as toDpoRows, M as toGrpoJsonl, N as toGrpoRows, O as toPrmJsonl, Q as toPrmRows, T as toSftJsonl, U as toSftRows, V as toTRLFormat } from './corpus-D4YW9UoJ.js';
+export { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory, d as doublyRobust, i as inverseProbabilityWeighting, o as offPolicyEstimateAll, s as selfNormalizedImportanceWeighting } from './off-policy-DiwuKKg7.js';
+import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
+export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from './outcome-store-rnXLEqSn.js';
+import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-C2hDKM8Z.js';
+import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-Jr8ME1dZ.js';
+export { r as runEvalCampaign } from './researcher-Jr8ME1dZ.js';
+import { a as VerificationReport } from './multi-layer-verifier-DUZXrPDA.js';
+import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
+import { C as CampaignResult } from './types-DQRY8ZT-.js';
+import '@tangle-network/agent-interface';
+import './errors-CzMUYo7b.js';
+import './schema-m0gsnbt3.js';
+import './store-BcFXE6LG.js';
+import './llm-client-Bj7g0rqu.js';
+import './raw-provider-sink-C46HDghv.js';
+import './summary-report-CInXwsza.js';
+import './failure-cluster-DH9Flgcf.js';
+import './emitter-C2rqGH_l.js';
+import './integrity-D2t12mMw.js';
+import './verdict-C9MlYujm.js';
+
+/**
+ * Adaptive curriculum / active scenario selection.
+ *
+ * Fixed scenario sets waste sample budget on cells the policy already
+ * passes (no information left) and cells the policy never passes (no
+ * gradient available either). Active learning over scenarios fixes this
+ * by allocating the next sample budget to cells where the policy's
+ * outcome is *uncertain* — those carry the most decision-relevant signal.
+ *
+ * This module ships two complementary strategies:
+ *
+ *   1. **Variance-based** — score each (variant, scenario) cell by the
+ *      empirical variance of past observations. Allocate next-round budget
+ *      proportional to variance. Standard active-learning-by-uncertainty
+ *      heuristic; works well when the policy is non-deterministic and
+ *      cells differ in observation noise.
+ *
+ *   2. **Bandit-based (Thompson sampling)** — model each (variant,
+ *      scenario) cell as a Beta-Bernoulli arm; sample a posterior; pick
+ *      cells whose posterior mean is closest to the per-scenario decision
+ *      threshold. The right primitive when scenarios are
+ *      "pass/fail" rather than continuous, and when promotion gates fire
+ *      at a known threshold (e.g., 0.5).
+ *
+ * The output is a *next-round budget allocation* — a list of (variant,
+ * scenario, count) triples. The consumer's matrix runner consumes the
+ * allocation, runs those cells, feeds the new observations back. Loop.
+ *
+ * Out of scope (deliberate): scenario *generation* — that's the
+ * adversarial primitive's job. This module allocates over an existing
+ * scenario pool.
+ */
+
+interface CellObservation {
+    variantId: string;
+    scenarioId: string;
+    /** Observed score in [0, 1]. */
+    score: number;
+    /** For Bernoulli arms — derive from the score with a threshold if needed. */
+    pass?: boolean;
+}
+interface CurriculumAllocation {
+    variantId: string;
+    scenarioId: string;
+    /** How many additional reps to run on this cell. */
+    count: number;
+    /** Strategy-specific reason for the allocation. */
+    reason: string;
+}
+interface VarianceCurriculumOptions {
+    /** Total reps to allocate across all cells. */
+    budget: number;
+    /**
+     * Smoothing prior on variance — keeps the allocator from concentrating
+     * on a cell with one observation just because its 1-sample variance is
+     * 0. Default 0.05.
+     */
+    variancePrior?: number;
+    /**
+     * Minimum reps per cell — even when the variance estimate is low, give
+     * every cell at least this many. Default 1.
+     */
+    floorPerCell?: number;
+}
+/**
+ * Variance-proportional allocation. For each cell, estimate variance from
+ * past observations + a prior, then allocate the budget proportional to
+ * (sqrt(variance) + 1/sqrt(n)) — a classical optimal-allocation rule
+ * (Neyman 1934) that balances "explore noisy cells" with "explore
+ * under-sampled cells."
+ */
+declare function varianceBasedCurriculum(observations: CellObservation[], candidateCells: Array<{
+    variantId: string;
+    scenarioId: string;
+}>, opts: VarianceCurriculumOptions): CurriculumAllocation[];
+interface ThompsonCurriculumOptions {
+    budget: number;
+    /**
+     * The per-scenario decision threshold. Cells whose posterior mean is
+     * closest to this get the most budget — that's where the next observation
+     * has the highest information value for the gate decision. Default 0.5.
+     */
+    decisionThreshold?: number;
+    /** Beta prior parameters. Default α=β=1 (uniform). */
+    priorAlpha?: number;
+    priorBeta?: number;
+    /** Seed the Thompson sampler. Default unset (Math.random). */
+    seed?: number;
+}
+/**
+ * Thompson-sampling-style allocation for pass/fail cells. For each cell:
+ *
+ *   - Maintain Beta(α + passes, β + failures) posterior on pass-rate
+ *   - Allocation weight ∝ exp(-((sampledMean - threshold) / σ)^2):
+ *     cells whose sampled posterior straddles the decision boundary get
+ *     the most weight; cells already clearly above or below get less.
+ *
+ * This is the right primitive when promotion gates fire at a known
+ * threshold and you want to sharpen the posterior near the boundary.
+ */
+declare function thompsonCurriculum(observations: CellObservation[], candidateCells: Array<{
+    variantId: string;
+    scenarioId: string;
+}>, opts: ThompsonCurriculumOptions): CurriculumAllocation[];
+/** Convenience: extract `CellObservation[]` directly from `RunRecord[]`. */
+declare function observationsFromRunRecords(runs: RunRecord[], opts?: {
+    passThreshold?: number;
+    useHoldout?: boolean;
+}): CellObservation[];
+
+/**
+ * Sample-efficient adaptation evaluation.
+ *
+ * For foundation-model-based agents, the load-bearing capability isn't
+ * raw end-state performance — it's *how fast the agent reaches that
+ * performance from cold start*. The same model with a worse prompt that
+ * adapts in 5 demonstrations beats the same model with a better prompt
+ * that needs 50. Standard meta-learning eval (Finn et al., MAML, RL² lit)
+ * reports an *adaptation curve*: score after k=0, 1, 2, 4, 8, 16, …
+ * in-context examples or fine-tune steps.
+ *
+ * This module ships:
+ *
+ *   1. `runAdaptationCurve` — given a runner that takes k demonstrations
+ *      and returns a score, produce the (k, score) curve.
+ *   2. `compareAdaptationCurves` — paired comparison across two policies.
+ *      Returns per-k delta with bootstrap CIs and an "area-under-curve"
+ *      summary statistic.
+ *   3. `firstPassK` — for pass/fail evaluation, the minimum k at which
+ *      the policy reliably passes (≥ pass-rate threshold over reps).
+ *
+ * Use cases:
+ *   - Compare two prompt designs that have similar end-state performance
+ *     but different in-context efficiency.
+ *   - Decide between fine-tuning and prompting based on adaptation cost.
+ *   - Detect when a policy "memorizes" k=0 inputs vs. genuinely adapts.
+ */
+interface AdaptationRunner<S> {
+    /**
+     * Runs the policy on `scenario` with `k` demonstrations. Returns a
+     * scalar score in [0, 1]. The runner is responsible for any caching;
+     * the harness calls it once per (scenario, k, rep) cell.
+     */
+    run(args: {
+        scenario: S;
+        k: number;
+        rep: number;
+    }): Promise<number>;
+}
+interface RunAdaptationCurveOptions<S> {
+    scenarios: S[];
+    /** Number-of-shots to evaluate at. Default `[0, 1, 2, 4, 8, 16]`. */
+    ks?: number[];
+    /** Reps per (scenario, k) cell. Default 3. */
+    reps?: number;
+    runner: AdaptationRunner<S>;
+    /** Pass-rate threshold for `firstPassK` reporting. Default 0.5. */
+    passThreshold?: number;
+}
+interface AdaptationPoint {
+    k: number;
+    meanScore: number;
+    passRate: number;
+    std: number;
+    n: number;
+    /** Per-scenario means at this k. */
+    perScenario: Array<{
+        scenarioId: string;
+        meanScore: number;
+        passes: number;
+        total: number;
+    }>;
+}
+interface AdaptationCurve {
+    points: AdaptationPoint[];
+    /**
+     * Smallest `k` at which `passRate ≥ passThreshold`. `null` if no `k`
+     * tested reaches it.
+     */
+    firstPassK: number | null;
+    /**
+     * Area under the (k, meanScore) curve, normalized by max-k. A
+     * single-number summary of "how well does this policy adapt from
+     * cold-start to fully-conditioned." Higher = better adapter.
+     */
+    adaptationArea: number;
+}
+declare function runAdaptationCurve<S extends {
+    scenarioId?: string;
+}>(opts: RunAdaptationCurveOptions<S>): Promise<AdaptationCurve>;
+interface CompareCurvesResult {
+    perK: Array<{
+        k: number;
+        deltaMean: number;
+        aLow: number;
+        aHigh: number;
+        bLow: number;
+        bHigh: number;
+    }>;
+    areaDelta: number;
+    firstPassKDelta: number | null;
+    /** Verdict: 'a_better' | 'b_better' | 'similar'. */
+    verdict: 'a_better' | 'b_better' | 'similar';
+    /** Rationale, ready to render. */
+    rationale: string;
+}
+/**
+ * Paired comparison of two adaptation curves. Per-k deltas with 95%
+ * bootstrap CIs (constructed from each curve's `perScenario` per-k means
+ * — the bootstrap unit is the scenario, not the rep).
+ */
+declare function compareAdaptationCurves(a: AdaptationCurve, b: AdaptationCurve, opts?: {
+    confidence?: number;
+    bootstrapResamples?: number;
+    seed?: number;
+}): CompareCurvesResult;
+/** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
+declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
+
+/**
+ * Test-time compute scaling curves.
+ *
+ * The test-time-compute frontier paper (Snell et al. 2024) and the
+ * subsequent o1-style scaling work both show that LLM-agent capability
+ * is a function of the compute budget at inference, not just of the
+ * training run. The right way to characterize a candidate is therefore
+ * a *curve* — score at compute budgets {1×, 4×, 16×, …} — not a single
+ * point.
+ *
+ * This module ships:
+ *
+ *   1. The compute-curve harness — `runComputeCurve(runner, budgets)` —
+ *      that evaluates one candidate at a sequence of compute budgets
+ *      and returns the (compute, score) curve.
+ *   2. A best-of-N evaluator — `bestOfN(runner, n, scoreFn)` — the
+ *      simplest test-time-compute scaling primitive: sample N
+ *      independent rollouts, return the best.
+ *   3. A self-consistency evaluator — `selfConsistency(runner, n)` —
+ *      the majority-vote variant of best-of-N for tasks with a small
+ *      categorical answer space.
+ *   4. Pareto-frontier extraction over multiple candidates — given
+ *      (candidate, compute, score) tuples, return the set of
+ *      candidate-compute combinations that aren't dominated.
+ *
+ * Caveat: "compute" here is the caller's notion of a compute unit. For
+ * agent eval that's typically wall-time × parallelism, or token budget,
+ * or LLM-call count. We accept whatever the caller provides; the curve
+ * is on whatever axis they pick.
+ */
+interface ComputeCurveBudget {
+    /** Identifier — for the report. Common: '1x', '4x', '16x'. */
+    id: string;
+    /** Numeric value on the chosen axis (tokens, calls, USD, ms — caller picks). */
+    cost: number;
+    /** Free-form metadata (the caller can carry per-budget config). */
+    meta?: Record<string, unknown>;
+}
+interface ComputeCurvePoint {
+    budgetId: string;
+    cost: number;
+    score: number;
+    /** Number of underlying samples used at this budget. */
+    samples: number;
+    /** Optional spread / variance information. */
+    std?: number;
+    /** Any extra metrics the runner returned. */
+    metrics?: Record<string, number>;
+}
+interface ComputeCurve {
+    candidateId: string;
+    points: ComputeCurvePoint[];
+    /** Rough exponent fit: score ≈ a + b * log(cost). Useful for "how steep is the curve?" */
+    logSlope: number | null;
+    /** Best (highest-score) point on the curve. */
+    best: ComputeCurvePoint;
+}
+interface RunComputeCurveOptions {
+    candidateId: string;
+    budgets: ComputeCurveBudget[];
+    /**
+     * Run the candidate at one budget. Returns the realized score plus
+     * optional spread + extra metrics.
+     */
+    runAtBudget: (budget: ComputeCurveBudget) => Promise<{
+        score: number;
+        samples: number;
+        std?: number;
+        metrics?: Record<string, number>;
+    }>;
+}
+declare function runComputeCurve(opts: RunComputeCurveOptions): Promise<ComputeCurve>;
+interface ComputeBestOfNOptions<O> {
+    /** Number of independent samples to draw. */
+    n: number;
+    /** Sampler — produces one rollout. */
+    sample: (sampleIdx: number) => Promise<O>;
+    /** Score one rollout. */
+    scoreFn: (rollout: O) => Promise<number> | number;
+}
+interface ComputeBestOfNResult<O> {
+    best: O;
+    bestScore: number;
+    scores: number[];
+    meanScore: number;
+    /** Index of the best rollout, for diagnostics. */
+    bestIndex: number;
+}
+/** The simplest test-time scaling primitive. */
+declare function bestOfN<O>(opts: ComputeBestOfNOptions<O>): Promise<ComputeBestOfNResult<O>>;
+interface SelfConsistencyOptions<O> {
+    n: number;
+    sample: (sampleIdx: number) => Promise<O>;
+    /** Extract the canonical answer key (string) from a rollout. */
+    answerKey: (rollout: O) => string;
+}
+interface SelfConsistencyResult<O> {
+    /** Modal answer (the majority vote). */
+    answer: string;
+    /** Fraction of samples voting for the modal answer in [0, 1]. */
+    agreement: number;
+    /** Histogram of all answers. */
+    histogram: Record<string, number>;
+    /** A representative rollout that voted for the modal answer. */
+    representative: O;
+    /** All rollouts. */
+    rollouts: O[];
+}
+/**
+ * Self-consistency / majority-vote test-time scaling. For tasks with a
+ * small categorical answer space (math problems, multiple choice).
+ */
+declare function selfConsistency<O>(opts: SelfConsistencyOptions<O>): Promise<SelfConsistencyResult<O>>;
+/**
+ * Pareto frontier over (candidate, compute, score) tuples. A point is on
+ * the frontier iff no other point dominates it in both score (higher
+ * better) and cost (lower better). Returns the frontier sorted ascending
+ * by cost.
+ */
+interface ParetoPointInput {
+    candidateId: string;
+    budgetId: string;
+    cost: number;
+    score: number;
+}
+declare function paretoFrontier(points: ParetoPointInput[]): ParetoPointInput[];
+
+/**
+ * Contamination probe — held-out perturbation tests.
+ *
+ * The bug class: once a benchmark scenario set is published, models train
+ * on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and
+ * MMLU-Pro all exist because their predecessors got contaminated within
+ * months. The right defense is to keep a held-out *perturbed* version of
+ * every scenario — same task, slightly different surface — and check
+ * whether scores diverge significantly. Genuine capability transfers; rote
+ * memorization doesn't.
+ *
+ * This module ships the probe contract:
+ *
+ *   1. A `ScenarioPerturbation` strategy type — function that produces a
+ *      perturbed scenario from an original.
+ *   2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs
+ *      both halves and reports per-scenario score divergence + a global
+ *      contamination verdict via paired Wilcoxon.
+ *   3. Several stock perturbations: `renameVariables`, `shuffleOrder`,
+ *      `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the
+ *      task's structural difficulty while breaking surface memorization.
+ *
+ * The verdict is conservative: if the perturbed-vs-original score
+ * difference is statistically significant (BH-adjusted p < 0.05) AND
+ * the median drop is > 5 percentage points, we flag *contamination
+ * suspected*. False positives are possible (the perturbation might
+ * actually be harder); the default is to flag for review, not to
+ * autoreject.
+ */
+type ScenarioPerturbationKind = 'rename_variables' | 'shuffle_order' | 'paraphrase' | 'inject_irrelevant_clause' | 'custom';
+interface ScenarioPerturbation<S> {
+    kind: ScenarioPerturbationKind;
+    /** Apply to one scenario, return its perturbed sibling. */
+    apply: (scenario: S) => Promise<S> | S;
+    /** Optional id — for the report. */
+    id?: string;
+}
+interface ContaminationProbeInput<S> {
+    /** Identity of every scenario. The probe's `runFingerprint` keys on these. */
+    scenarioId: (s: S) => string;
+    /** Original scenarios. */
+    originals: S[];
+    /**
+     * Either pre-computed perturbations (one per original, same order) OR a
+     * `perturbation` strategy that synthesizes them on the fly.
+     */
+    perturbed?: S[];
+    perturbation?: ScenarioPerturbation<S>;
+    /**
+     * Run the policy/agent against one scenario and return a scalar score
+     * in [0, 1]. The probe doesn't care what the policy is — that's the
+     * caller's contract.
+     */
+    scoreFn: (s: S) => Promise<number>;
+}
+interface ContaminationProbeOptions {
+    /** Drop scores below this from the probe; treats partial failures separately. Default 0. */
+    scoreFloor?: number;
+    /**
+     * BH-FDR threshold for declaring contamination on each per-scenario
+     * delta. Default 0.05.
+     */
+    fdr?: number;
+    /**
+     * Minimum median per-scenario drop to flag global contamination. Default
+     * 0.05 (5 percentage points). Smaller drops may be noise.
+     */
+    minMedianDrop?: number;
+}
+interface ContaminationProbeReport {
+    perScenario: Array<{
+        scenarioId: string;
+        originalScore: number;
+        perturbedScore: number;
+        delta: number;
+        /** Per-scenario q-value (single-test BH for a single scenario). Mainly for display. */
+        qValue: number;
+    }>;
+    /** Wilcoxon paired-test on the deltas. */
+    pairedTest: {
+        w: number;
+        p: number;
+    };
+    medianDelta: number;
+    meanDelta: number;
+    contaminationSuspected: boolean;
+    reason: string;
+    /** Number of scenarios processed. */
+    n: number;
+}
+declare function runContaminationProbe<S>(input: ContaminationProbeInput<S>, opts?: ContaminationProbeOptions): Promise<ContaminationProbeReport>;
+/**
+ * Identifier-rename perturbation for code/text scenarios. Replaces every
+ * occurrence of the listed identifiers with synthesized aliases. Use when
+ * the scenario's structural difficulty is independent of variable names
+ * (e.g. SWE-Bench-style coding tasks).
+ */
+declare function renameVariables<S extends {
+    prompt: string;
+}>(identifiers: string[], rename?: (name: string, idx: number) => string): ScenarioPerturbation<S>;
+/**
+ * Order-shuffle perturbation. Reshuffles a list-shaped section of the
+ * prompt (for QA scenarios that present options A/B/C/D — answer depends
+ * on the option labels, not order). Caller provides the section extractor.
+ */
+declare function shuffleOrder<S extends {
+    prompt: string;
+}>(shuffleSection: (prompt: string, rng: () => number) => string, seed: number): ScenarioPerturbation<S>;
+/**
+ * Inject-irrelevant-clause perturbation. Adds a benign sentence that
+ * shouldn't change the answer. Tests for "did the model just memorize
+ * the input string."
+ */
+declare function injectIrrelevantClause<S extends {
+    prompt: string;
+}>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
+
+/**
+ * `PredictiveValidityResearcher` — concrete `Researcher` implementation
+ * that drives selection from outcome-anchored predictive validity.
+ *
+ * Each method:
+ *
+ *   - `inspectFailures(runs)` — synthesizes failure modes from the
+ *     bottom-quartile of `RunRecord`s on the configured proxy reward.
+ *   - `proposeChange(failures)` — proposes steering changes that target
+ *     the rubrics with the lowest predictive validity (decorative ones).
+ *     Either reduce their weight in the composite, or recalibrate them.
+ *   - `applyChange(changes, baseline)` — merges the proposed steering
+ *     into the experiment plan.
+ *   - `evaluateChange(plan)` — re-runs the predictive-validity check on
+ *     the post-change runs and reports the delta.
+ *
+ * The result is a closed loop: the rubric weights drift toward the ones
+ * that actually predict deployment outcomes, automatically. Pair with
+ * `runRLCampaign` for the full auto-research story.
+ */
+
+interface PredictiveValidityResearcherOptions {
+    outcomes: OutcomeStore;
+    outcomeMetrics: string[];
+    /** Score threshold below which a run counts as a "failure." Default 0.5. */
+    failureThreshold?: number;
+    /** Spearman bucket below which a rubric is "decorative." Default 0.4. */
+    decorativeThreshold?: number;
+    /** Optional steering-namespace prefix for proposed changes. Default `'rubric_weight'`. */
+    steeringNamespace?: string;
+    /** Override the rubric set the researcher inspects. Default: every numeric `outcome.raw` key seen. */
+    rubrics?: string[];
+    /**
+     * Snapshot stash hook — called with the most recent predictive-validity
+     * report. Useful when a downstream system wants to log rubric drift over
+     * time. Default no-op.
+     */
+    onReport?: (report: RubricPredictiveValidityReport) => void | Promise<void>;
+}
+/**
+ * Concrete `Researcher` driven by `rubricPredictiveValidity`. The brain:
+ * rubrics that don't predict deployment outcomes don't earn weight.
+ */
+declare class PredictiveValidityResearcher implements Researcher {
+    private opts;
+    private lastReport;
+    constructor(opts: PredictiveValidityResearcherOptions);
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+    /**
+     * Run the predictive-validity check explicitly against a fresh RunRecord
+     * set. Updates the researcher's cached report so subsequent
+     * `proposeChange` calls have evidence to draw from.
+     */
+    runValidityCheck(runs: RunRecord[]): Promise<RubricPredictiveValidityReport>;
+    /**
+     * Force-feed a predictive-validity report into the researcher state —
+     * useful when the consumer ran the report out-of-band and wants the
+     * researcher's later proposals informed by it.
+     */
+    setReport(report: RubricPredictiveValidityReport): void;
+    getLastReport(): RubricPredictiveValidityReport | null;
+}
+
+/**
+ * Verifiable reward channel.
+ *
+ * For RL on coding / math / theorem-proving / structured-output tasks, the
+ * reward signal is *decidable* — a test passes or fails, a proof checks or
+ * doesn't, an output validates against a schema or doesn't. These rewards
+ * are dramatically more useful for RL training than LLM-judge scores
+ * because they don't drift, can't be Goodhart-gamed by the policy in the
+ * same way, and don't require a separate calibration loop.
+ *
+ * The `MultiLayerVerifier` already produces this signal — it just doesn't
+ * surface it in a shape that's clean enough for RL training. This module
+ * wraps the verifier output so consumers can:
+ *
+ *   1. Extract a clean `VerifiableReward` from a `VerificationReport`
+ *   2. Distinguish *deterministic* rewards (compile, test, schema) from
+ *      *probabilistic* rewards (judge) so they can be weighted differently
+ *      in the RL training step
+ *   3. Filter `RunRecord[]` to only those with a verifiable reward,
+ *      producing the clean training set that DeepSeek-R1-style GRPO and
+ *      AlphaProof-style search both depend on
+ *
+ * Why this matters: every credible 2025-2026 frontier RL result on coding
+ * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,
+ * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing
+ * judge scores into the reward signal poisons the gradient. This module
+ * is the seam.
+ */
+
+type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
+interface VerifiableReward {
+    /** Scalar in [0, 1]. The RL training signal. */
+    value: number;
+    /** What produced the reward — different sources have different determinism. */
+    source: VerifiableRewardSource;
+    /**
+     * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte
+     * given the same inputs (compile, test, schema validation, sandbox exit code).
+     * `'probabilistic'` rewards depend on a stochastic component (LLM judge).
+     * Mixing these in the same training batch without separation is a known
+     * footgun in production RLHF pipelines.
+     */
+    determinism: 'deterministic' | 'probabilistic';
+    /**
+     * Confidence in the reward value. For deterministic sources this is 1.0
+     * (the bit either flipped or didn't). For judge sources this is the
+     * judge-reported confidence or — when missing — a calibrated prior.
+     */
+    confidence: number;
+    /** The layer / judge id that produced the signal, for provenance. */
+    origin: string;
+    /**
+     * Per-source contribution to `value`, keyed by layer/judge id. Single-source
+     * rewards carry one entry (`{ [origin]: value }`); composite rewards carry
+     * every contributing layer's score — the anti-scalar-collapse surface RL
+     * consumers weight per-source instead of trusting one blended number.
+     */
+    components: Record<string, number>;
+    /**
+     * @deprecated Read `components` for per-source reward values. Kept for
+     * published-API compatibility: single-source rewards carry the layer's
+     * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry
+     * the same per-layer scores `components` now holds.
+     */
+    breakdown?: Record<string, number>;
+}
+interface VerifiableRewardExtractionOptions {
+    /**
+     * Which layers count as deterministic-reward sources. The verifier doesn't
+     * tag layers as "this is verifiable"; the caller declares it via this list
+     * (or via the layer name → source mapping). Default treats common names
+     * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
+     * `sandbox`) as deterministic.
+     */
+    deterministicLayers?: string[];
+    /**
+     * Map layer name → reward source. Defaults to a sensible string-match.
+     */
+    sourceFor?: (layerName: string) => VerifiableRewardSource;
+    /**
+     * Whether to fall back to a probabilistic (judge) reward when no
+     * deterministic layer produced a numeric score. Default `true`. Set to
+     * `false` for "deterministic-only" training pipelines that should
+     * discard runs without a verifiable signal.
+     */
+    fallbackToJudge?: boolean;
+    /**
+     * Default confidence for probabilistic (judge) rewards when the judge
+     * doesn't report one. Default `0.7`.
+     */
+    judgeConfidenceFloor?: number;
+}
+/**
+ * Extract a `VerifiableReward` from a `VerificationReport`.
+ *
+ * Strategy: prefer the deterministic layers (in order: test → compile →
+ * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is
+ * true, return `null` if no signal qualifies. When multiple deterministic
+ * layers contribute, return a `'composite'` source with a weighted blend.
+ */
+declare function extractVerifiableReward(report: VerificationReport, opts?: VerifiableRewardExtractionOptions): VerifiableReward | null;
+/**
+ * Extract verifiable rewards from `RunRecord[]` produced via the
+ * `verificationReportToRunRecord` adapter (which encodes per-layer scores
+ * in `outcome.raw['layer.<name>']`). For records that don't carry layer
+ * scores, returns `null` for that record.
+ *
+ * This is the canonical bridge from "campaign-shaped artifacts" to
+ * "RL-training-ready reward signals": every record that has a clean
+ * verifiable reward becomes a training datum, every record that doesn't
+ * gets filtered out (or kept with `'probabilistic'` determinism for
+ * separate downstream handling).
+ */
+declare function extractVerifiableRewardsFromRecords(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+    runId: string;
+    reward: VerifiableReward | null;
+}>;
+/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */
+declare function filterDeterministicallyRewarded(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+    run: RunRecord;
+    reward: VerifiableReward;
+}>;
+
+/**
+ * Reward hacking / Goodhart detection.
+ *
+ * Goodhart's Law says: when a measure becomes a target, it ceases to be
+ * a good measure. In RLHF and agentic-RL settings this is the dominant
+ * failure mode — the policy learns to produce outputs that score well on
+ * the proxy reward (judge, rubric, test pass-rate) without producing
+ * the underlying capability the proxy was meant to track.
+ *
+ * Krakovna et al. (2020, "Specification Gaming Examples in AI") and the
+ * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.
+ * 2023) converge on a few diagnostic signatures:
+ *
+ *   1. **Reward divergence:** the proxy reward grows while the held-out
+ *      ground-truth signal stagnates or drops. Predictive validity over
+ *      time captures this.
+ *   2. **Distributional shift in outputs:** after RL, the policy produces
+ *      outputs that no longer match the reference distribution — usually
+ *      because it found a high-reward attractor that's degenerate (e.g.
+ *      one-token responses, repetition, formatting tricks).
+ *   3. **Disagreement between independent rewards:** if you train on
+ *      reward A and a held-out independent reward B drops sharply, you're
+ *      probably hacking A.
+ *   4. **Calibration drift:** the verifiable / deterministic component of
+ *      the reward is stable; the probabilistic / judge component drifts up
+ *      while the deterministic component doesn't. The judge is being
+ *      gamed.
+ *
+ * This module ships explicit detectors for all four signatures, plus a
+ * combined verdict. The output is diagnostic — actionable signals,
+ * not autoreject — because each signature has known false positives
+ * (e.g., a policy that genuinely improves can show distributional shift).
+ *
+ * Differs from `rubricPredictiveValidity` (which is a *standing* check on
+ * whether rubrics correlate with deployment outcomes) — this is a
+ * *temporal* check on whether the reward-vs-truth gap is *widening over
+ * time during a training run*.
+ */
+
+type RewardHackingSignal = 'reward_divergence' | 'distribution_shift' | 'reward_disagreement' | 'judge_drift';
+interface RewardHackingFinding {
+    signal: RewardHackingSignal;
+    /** Severity in [0, 1]. >0.5 = strong signal. */
+    severity: number;
+    message: string;
+    /** Numeric evidence the consumer can render. */
+    detail: Record<string, number>;
+}
+interface RewardHackingReport {
+    findings: RewardHackingFinding[];
+    /**
+     * Composite verdict. `'clean'` if every signal severity < 0.3;
+     * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.
+     */
+    verdict: 'clean' | 'suspect' | 'gaming';
+    /** Rationale for the verdict, ready to paste into an audit log. */
+    rationale: string[];
+    /** Number of paired (proxy, truth) data points the report saw. */
+    n: number;
+}
+interface DetectRewardHackingInput {
+    /**
+     * Run records ordered by recency (oldest first). The detector segments
+     * them into prefix/suffix windows to compute "did the gap widen."
+     */
+    runs: RunRecord[];
+    /**
+     * The metric the policy was trained to optimize. Should be present on
+     * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
+     */
+    proxyOf?: (run: RunRecord) => number | null;
+    /**
+     * The held-out ground-truth metric. For RL on coding, this is typically
+     * test pass-rate. For RLHF, it's downstream task performance or human
+     * preference. For knowledge tasks, it's an independently-graded score.
+     */
+    truthOf?: (run: RunRecord) => number | null;
+    /**
+     * Independent secondary reward. Used for the `reward_disagreement`
+     * signal. Default uses the verifiable reward extractor (deterministic
+     * sources only).
+     */
+    secondaryRewardOf?: (run: RunRecord) => number | null;
+    /**
+     * Window size — how many of the most recent runs count as the "after"
+     * cohort. Default min(50, half the runs).
+     */
+    windowSize?: number;
+    /**
+     * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
+     * (gaming).
+     */
+    thresholds?: {
+        suspect?: number;
+        gaming?: number;
+    };
+    /**
+     * Verifiable-reward options used for the secondary-reward fallback.
+     */
+    verifiableRewardOptions?: VerifiableRewardExtractionOptions;
+}
+declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
+
+/**
+ * `runRLCampaign` — top-level orchestrator that runs the matrix and
+ * produces every RL-ready artifact in one call.
+ *
+ * Wires:
+ *   1. `runEvalCampaign` for the matrix run (capture, integrity, hooks)
+ *   2. `extractVerifiableReward` over each run, separating deterministic
+ *      from probabilistic reward sources for the trainer
+ *   3. `extractPreferences` to produce DPO/PPO/KTO triples
+ *   4. `evaluateInterimReleaseConfidence` over paired deltas (anytime-valid)
+ *   5. `rubricPredictiveValidity` against an outcome store, when provided
+ *   6. `detectRewardHacking` as a standing hygiene check
+ *   7. Trainer-format export rows ready for prime-rl / TRL / verl
+ *
+ * The output `RLCampaignResult` is a single, audit-ready artifact: every
+ * stage's output is in there. The consumer's downstream fits in a single
+ * line: pass `result.preferences` to their DPO trainer, `result.grpoRows`
+ * to GRPO, `result.runs` plus `result.rewardSignals` to a custom RL loop.
+ */
+
+interface RunRLCampaignOptions<V> extends EvalCampaignOptions<V> {
+    /** Preference-extraction options. Default uses paired-by-scenario-and-seed with min-margin 0.05. */
+    preferences?: ExtractPreferencesOptions;
+    /** Verifiable-reward extraction options. */
+    verifiableReward?: VerifiableRewardExtractionOptions;
+    /** Outcome store + metric names — when supplied, runs `rubricPredictiveValidity` post-campaign. */
+    outcomeStore?: OutcomeStore;
+    outcomeMetrics?: string[];
+    /** Anytime-valid sequential evaluation options. */
+    sequential?: {
+        alpha?: number;
+        bound?: number;
+        rope?: {
+            low: number;
+            high: number;
+        };
+    };
+    /** Trainer-format export lookups. When provided, the orchestrator builds the corresponding rows. */
+    trainerExport?: {
+        dpo?: DpoLookups;
+        grpo?: GrpoLookups;
+        sft?: SftLookups;
+    };
+}
+interface RLCampaignResult<V> {
+    campaign: EvalCampaignResult;
+    /** Per-run verifiable reward (deterministic when available, probabilistic fallback otherwise). */
+    rewardSignals: Array<{
+        runId: string;
+        reward: VerifiableReward | null;
+    }>;
+    /** Preference extraction report. */
+    preferences: PreferenceExtractionReport;
+    /** Anytime-valid interim verdict over the paired deltas (vs comparator). */
+    interimConfidence: InterimReleaseConfidence | null;
+    /** Standing reward-hacking hygiene check. */
+    rewardHacking: RewardHackingReport;
+    /** Predictive validity, when an outcome store was supplied. */
+    predictiveValidity: RubricPredictiveValidityReport | null;
+    /** Trainer-export rows, populated only for the formats the caller requested via `trainerExport`. */
+    trainerRows: {
+        dpo?: DpoExportRow[];
+        grpo?: GrpoExportRow[];
+        sft?: SftExportRow[];
+    };
+    /**
+     * One-line top-level summary the consumer can log.
+     */
+    summary: string;
+    /**
+     * Convenience type-tag — consumers can branch on `result.kind`.
+     */
+    kind: 'agent-eval-rl-campaign';
+    unusedVariant?: V;
+}
+declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult<V>>;
+
+/**
+ * Adapters: convert measurement outputs into the canonical `RunRecord[]`
+ * artifact that `replayCache`, `pairedEvalueSequence`, and
+ * `rubricPredictiveValidity` consume. Two sources:
+ *   - `campaignToRunRecords` — the campaign substrate's per-cell results
+ *     (the modern path: `runCampaign` / `runImprovementLoop` → records).
+ *   - `verificationReportToRunRecord` — a `MultiLayerVerifier` report.
+ *
+ * Adapters are thin and explicit — every mandatory `RunRecord` field comes
+ * from a caller-supplied context (`commitSha`, `model`, `promptHash`,
+ * `configHash`) plus the cell's runtime data. The validator still rejects
+ * bare-alias model strings — the caller snapshot-pins.
+ */
+
+interface AdapterContext {
+    /** Logical experiment id — typically the campaign or sweep identifier. */
+    experimentId: string;
+    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
+    model: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** Hash of the effective prompt sent to the model. */
+    promptHash: string;
+    /** Hash of the effective config (model, temperature, tools, judges, splits). */
+    configHash: string;
+    /** Default split tag. Default `'search'`. */
+    splitTag?: RunSplitTag;
+    /** Default cost in USD when the source doesn't record one. Default `0`. */
+    defaultCostUsd?: number;
+}
+/**
+ * Convert a `CampaignResult` into canonical `RunRecord[]` — one record per
+ * scored cell. The cell's mean judge composite becomes the split score; every
+ * judge dimension is carried through to `outcome.raw`. A cell that errored
+ * becomes a record with `failureMode: 'cell_error'` (kept, not dropped — an
+ * unscored cell is signal). `candidateId` identifies the measured surface
+ * (defaults to the campaign manifest hash).
+ */
+declare function campaignToRunRecords(campaign: CampaignResult, ctx: AdapterContext & {
+    candidateId?: string;
+}): RunRecord[];
+/**
+ * Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
+ * `outcome.searchScore` (or `holdoutScore`) is `report.blendedScore`;
+ * `outcome.raw` carries every layer's score + a pass indicator; `failureMode`
+ * is the first failing layer's reason.
+ */
+declare function verificationReportToRunRecord(report: VerificationReport, ctx: AdapterContext & {
+    candidateId: string;
+    scenarioId?: string;
+}, opts?: {
+    runId?: string;
+}): RunRecord;
+
+/**
+ * Simulator fidelity — score a user SIMULATOR's realism against real-user
+ * trace distributions.
+ *
+ * Synthetic-persona evals (`PersonaConfig`-driven canonical evals, fuzz
+ * user-simulator objectives) stand in for real users in most of the numbers
+ * we publish. The standing threat is the Sim2Real gap: a simulator that is
+ * distributionally unlike production creates "easy mode" and silently
+ * inflates every score built on it. This module measures that gap from the
+ * SAME artifact both sides already produce — `RunRecord`s — so no new
+ * capture pipeline is needed:
+ *
+ *   - `simFidelityReport` — per-feature Jensen-Shannon divergence between
+ *     simulated and production record distributions, collapsed into a
+ *     fidelity coefficient in [0,1].
+ *   - `easyModeCheck` — the headline academic failure mode (sim inflates
+ *     pass-rate over production) as its own named artifact.
+ *
+ * Every synthetic-persona eval result should publish its fidelity
+ * coefficient alongside the score — a number from an unrepresentative
+ * simulator is an unlabeled estimate. Wire-in points:
+ *
+ *   - canonical persona evals: pass the campaign's `RunRecord`s as
+ *     `simulated` and intake-adapter output (`contract/intake`: OTel spans,
+ *     feedback tables, coding-agent sessions) as `production`
+ *   - the fuzz user-sim objective: use `1 - report.fidelity` as a realism
+ *     penalty when searching over generated personas
+ *   - the durable corpus (`./corpus`): both sides read straight from
+ *     `readCorpus` — tag sim vs production by `experimentId`
+ */
+
+/** Extracts a flat behavioral feature map from one record. `string` values
+ *  are categorical, `number` values are quantile-bucketed over the union of
+ *  both sides, `null` means the feature is absent on this record and is
+ *  counted explicitly as its own category (never silently dropped). */
+type BehaviorFeatures = (record: RunRecord) => Record<string, string | number | null>;
+/** Reserved histogram category for `null` feature values. A capture-rate
+ *  difference (one side instruments a signal, the other does not) registers
+ *  as divergence by design: a simulator that produces no tool traces is not
+ *  representative of production that does. */
+declare const ABSENT_CATEGORY = "(absent)";
+/** Minimum non-null observations PER SIDE for a feature to enter the
+ *  fidelity mean. Below this the JSD estimate is sampling noise. */
+declare const DEFAULT_MIN_N_PER_FEATURE = 20;
+/** Quantile buckets used to discretize numeric features. Quartiles balance
+ *  resolution against per-bucket sample size at the default minN. */
+declare const DEFAULT_QUANTILE_BUCKETS = 4;
+/** Fidelity at or above this → 'representative'; below → 'skewed'.
+ *  1 − 0.8 = mean JSD 0.2 ≈ distributions that mostly overlap with one
+ *  clearly shifted mode — the point where per-feature shifts start changing
+ *  which failure classes an eval can even observe. */
+declare const REPRESENTATIVE_MIN_FIDELITY = 0.8;
+/**
+ * Default feature set — ONLY fields verified present on both simulated and
+ * production records:
+ *
+ *   - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
+ *     validator (non-finite values read as absent rather than poisoning a
+ *     bucket).
+ *   - `failure_class` — optional taxonomy field; absent counted explicitly.
+ *   - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
+ *     `outcome.raw` counters the intake adapters and eval harnesses write
+ *     (`turns_completed`, `assistant_messages`, `tool_errors`,
+ *     `turns_aborted`); absent on records whose producer did not capture
+ *     them, counted explicitly.
+ *   - `completion_length` — from the optional `CorpusRecord` trajectory
+ *     text; the message-length proxy when records come from the corpus.
+ *
+ * `RunRecord` carries event COUNTS, not event ordering, so
+ * `tool_error_recovery` is a counts-only derivation: errors occurred and the
+ * run still completed cleanly ('recovered') vs aborted or classified as a
+ * failure ('unrecovered') — not a literal error→retry sequence check.
+ */
+declare const defaultBehaviorFeatures: BehaviorFeatures;
+/**
+ * Jensen-Shannon divergence between two categorical histograms (raw counts;
+ * normalized internally). Log base 2 → bounded [0,1]: 0 = identical
+ * distributions, 1 = disjoint support. Symmetric, defined even where the
+ * supports differ — exactly the regime sim-vs-production comparison lives in.
+ * Throws on zero-mass or negative/non-finite counts: an empty histogram has
+ * no distribution and a silent 0 would read as "perfectly representative".
+ */
+declare function jsDivergence(p: Record<string, number>, q: Record<string, number>): number;
+/**
+ * Deterministic quantile edges over a value set (the UNION of both sides, so
+ * sim and production land in the same buckets). Linear interpolation between
+ * order statistics; duplicate edges from heavy ties collapse into fewer,
+ * wider buckets. Returns `bucketCount - 1` edges before deduplication.
+ */
+declare function quantileEdges(values: number[], bucketCount?: number): number[];
+/** Stable half-open bucket label for a value against quantile edges:
+ *  `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
+declare function bucketLabel(value: number, edges: number[]): string;
+interface FeatureShift {
+    /** Category label (a string value, a numeric bucket, or `ABSENT_CATEGORY`). */
+    value: string;
+    /** Probability of this category among ALL simulated records (nulls included
+     *  via `ABSENT_CATEGORY`, so each side's shifts sum to 1). */
+    pSim: number;
+    /** Probability among ALL production records. */
+    pProd: number;
+}
+interface FeatureDivergence {
+    feature: string;
+    /** Jensen-Shannon divergence in [0,1] for this feature. */
+    divergence: number;
+    /** Largest |pSim − pProd| categories, descending — where the sim deviates. */
+    topShifts: FeatureShift[];
+    /** Non-null observations on the simulated side. */
+    nSim: number;
+    /** Non-null observations on the production side. */
+    nProd: number;
+}
+type FidelityVerdict = 'representative' | 'skewed' | 'insufficient-data';
+interface FidelityReport {
+    perDimension: FeatureDivergence[];
+    /** 1 − mean divergence over features with sufficient data. NaN when the
+     *  verdict is 'insufficient-data' — a 0 would read as "maximally skewed"
+     *  and silently poison downstream aggregation; check `verdict` first. */
+    fidelity: number;
+    /** Features excluded because either side had fewer than `minNPerFeature`
+     *  non-null observations. Named, never silently dropped. */
+    insufficientData: string[];
+    /** 'representative' when fidelity >= REPRESENTATIVE_MIN_FIDELITY (0.8),
+     *  'skewed' below, 'insufficient-data' when no feature met minN. */
+    verdict: FidelityVerdict;
+}
+interface SimFidelityOptions {
+    /** Feature extractor. Defaults to `defaultBehaviorFeatures`. */
+    features?: BehaviorFeatures;
+    /** Minimum non-null observations per side per feature. Default 20. */
+    minNPerFeature?: number;
+}
+/**
+ * Compare a simulator's RunRecords against production RunRecords, feature by
+ * feature. Numeric features are bucketed by deterministic quantiles of the
+ * union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
+ * inputs — "no records" is a wiring error, not a distribution.
+ */
+declare function simFidelityReport(simulated: RunRecord[], production: RunRecord[], opts?: SimFidelityOptions): FidelityReport;
+interface EasyModeOptions {
+    /** A run passes when its score (holdout, else search) >= this. Default 0.5
+     *  — matches the pass-threshold convention across the rl/ primitives. */
+    passThreshold?: number;
+    /** Pass-rate gap above which the sim is flagged inflated. Default 0.1 —
+     *  a 10-point inflation is enough to flip most promotion gates. */
+    inflationTolerance?: number;
+}
+interface EasyModeReport {
+    simPassRate: number;
+    prodPassRate: number;
+    /** simPassRate − prodPassRate. Positive = the simulator is easier than reality. */
+    gap: number;
+    /** True when gap > inflationTolerance: numbers measured against this
+     *  simulator overstate production performance. */
+    inflated: boolean;
+}
+/**
+ * The headline simulator failure mode as its own named artifact: a simulator
+ * that creates "easy mode" inflates pass-rate relative to production, and
+ * every score measured against it overstates reality. Throws on empty inputs
+ * and on records carrying neither score — a silently-skipped record would
+ * bias the very rate this check exists to keep honest.
+ */
+declare function easyModeCheck(simulated: RunRecord[], production: RunRecord[], opts?: EasyModeOptions): EasyModeReport;
+
+/**
+ * Bradley-Terry / Elo tournament evaluation.
+ *
+ * For multi-candidate sweeps, comparing every candidate's score against
+ * a fixed comparator wastes information — the comparator becomes a high-
+ * variance reference and rank flips between near-tied middle-rank
+ * candidates are dominated by noise. Pairwise tournaments fix this:
+ * every (i, j) pair contributes a comparison to a Bradley-Terry MLE that
+ * estimates each candidate's strength on a unified scale.
+ *
+ * For online updating (rolling campaigns where new candidates arrive
+ * over time), we also ship classical Elo with configurable K-factor.
+ *
+ * References:
+ *   - Bradley, R. A., Terry, M. E. (1952). Rank analysis of incomplete
+ *     block designs. Biometrika, 39(3/4), 324–345.
+ *   - Hunter, D. R. (2004). MM algorithms for generalized Bradley-Terry
+ *     models. Annals of Statistics, 32(1), 384–406. (The MLE algorithm
+ *     used here.)
+ *   - Elo, A. E. (1978). The Rating of Chess Players, Past and Present.
+ *
+ * This is a useful primitive because most LLM-eval communities (Chatbot
+ * Arena, AlpacaEval, ELO-style ablation) have converged on pairwise
+ * tournament eval as the most sample-efficient and most rank-stable
+ * method when you have many candidates.
+ */
+interface PairwiseOutcome {
+    /** Winner candidate id. */
+    winner: string;
+    /** Loser candidate id. */
+    loser: string;
+    /**
+     * Optional draw flag. When true, both candidates get half-credit
+     * (Bradley-Terry handles draws as half-wins for each side).
+     */
+    draw?: boolean;
+    /**
+     * Optional weight — useful if some pairwise comparisons are stronger
+     * signals than others (e.g. a paired test with a wider score gap is
+     * a more confident comparison). Default 1.
+     */
+    weight?: number;
+}
+interface BradleyTerryRating {
+    candidateId: string;
+    /** Latent strength θ ≥ 0 from the BT MLE. */
+    strength: number;
+    /** Log-strength = log(θ) — interpretable on a linear scale. */
+    logStrength: number;
+    /** Number of pairwise comparisons this candidate appears in. */
+    n: number;
+    /** Win count (+ 0.5 per draw). */
+    wins: number;
+}
+interface BradleyTerryFit {
+    ratings: BradleyTerryRating[];
+    /** Iterations of the MM algorithm before convergence. */
+    iterations: number;
+    /** Final maximum |θ_new - θ_old| / θ_old. */
+    finalDelta: number;
+    converged: boolean;
+}
+/**
+ * Bradley-Terry MLE via Hunter's MM algorithm.
+ *
+ * Iteration: θ_i^new = W_i / Σ_{j ≠ i} N_ij / (θ_i + θ_j)
+ *   where W_i = wins by i (+ 0.5 per draw), N_ij = total comparisons.
+ *
+ * Returns log-strengths normalized so the smallest is 0 (any constant
+ * offset is unobservable in BT — only differences are identified).
+ */
+declare function fitBradleyTerry(outcomes: PairwiseOutcome[], opts?: {
+    tolerance?: number;
+    maxIterations?: number;
+    smoothing?: number;
+}): BradleyTerryFit;
+/**
+ * Online Elo updates. Use when comparisons arrive over time and you want
+ * a running rating without re-fitting the full BT MLE on every update.
+ *
+ * Initialize ratings to `defaultRating` (1500 by default). Each call to
+ * `applyEloUpdate` mutates the map in place and returns the deltas so
+ * the caller can log per-comparison rating changes.
+ */
+interface EloOptions {
+    /** Default rating for unseen candidates. Default 1500. */
+    defaultRating?: number;
+    /** K-factor controls the step size. Default 32 (FIDE-ish). */
+    kFactor?: number;
+}
+declare function applyEloUpdate(ratings: Map<string, number>, outcome: PairwiseOutcome, opts?: EloOptions): {
+    winnerDelta: number;
+    loserDelta: number;
+};
+/**
+ * Build pairwise outcomes from the campaign artifact: for every scenario
+ * shared by two candidates, the higher-scoring run wins. Useful when you
+ * want a tournament view of an existing campaign without an additional
+ * pairwise judge call.
+ */
+interface BuildPairwiseFromCampaignInput {
+    runs: Array<{
+        candidateId: string;
+        /** Stable identifier for the matching unit (typically scenarioId). */
+        matchKey: string;
+        score: number;
+    }>;
+    /**
+     * Tied-score margin. Below this, the comparison is a draw. Default 0
+     * (no ties).
+     */
+    drawMargin?: number;
+}
+declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
+
+export { ABSENT_CATEGORY, type AdaptationCurve, type AdaptationPoint, type AdaptationRunner, type AdapterContext, type BehaviorFeatures, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type CellObservation, type CompareCurvesResult, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CurriculumAllocation, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, type DetectRewardHackingInput, DpoExportRow, DpoLookups, type EasyModeOptions, type EasyModeReport, type EloOptions, ExtractPreferencesOptions, type FeatureDivergence, type FeatureShift, type FidelityReport, type FidelityVerdict, GrpoExportRow, GrpoLookups, OutcomeStore, type PairwiseOutcome, type ParetoPointInput, PredictiveValidityResearcher, type PredictiveValidityResearcherOptions, PreferenceExtractionReport, REPRESENTATIVE_MIN_FIDELITY, type RLCampaignResult, type RewardHackingFinding, type RewardHackingReport, type RewardHackingSignal, type RunAdaptationCurveOptions, type RunComputeCurveOptions, type RunRLCampaignOptions, type ScenarioPerturbation, type ScenarioPerturbationKind, type SelfConsistencyOptions, type SelfConsistencyResult, SftExportRow, SftLookups, type SimFidelityOptions, type ThompsonCurriculumOptions, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, applyEloUpdate, bestOfN, bucketLabel, buildPairwiseFromCampaign, campaignToRunRecords, compareAdaptationCurves, defaultBehaviorFeatures, detectRewardHacking, easyModeCheck, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, jsDivergence, observationsFromRunRecords, paretoFrontier, quantileEdges, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runRLCampaign, selfConsistency, shuffleOrder, simFidelityReport, thompsonCurriculum, varianceBasedCurriculum, verificationReportToRunRecord };