@tangle-network/agent-eval 0.21.0 → 0.23.0

package/dist/index-ekBXweiQ.d.ts

@@ -0,0 +1,1894 @@
+import { t as TrialResult, V as VariantAggregate, q as PromptEvolutionResult, e as MultiShotOptimizationResult } from './summary-report-Ce1r4EYo.js';
+import { a as RunSplitTag, R as RunRecord } from './run-record-DNiOMBrZ.js';
+import { S as Span, T as TraceStore } from './store-u47QaJ9G.js';
+import { i as EvalCampaignResult, E as EvalCampaignOptions, R as Researcher, l as FailureMode, S as SteeringChange, j as ExperimentPlan, k as ExperimentResult } from './eval-campaign-Ds5QljIh.js';
+import { c as InterimReleaseConfidence, h as RubricPredictiveValidityReport, a as OutcomeStore } from './sequential-DgU2mFsE.js';
+
+/**
+ * Multi-layer verifier — ordered pipeline of verification layers.
+ *
+ * Different contract from {@link JudgeRunner} (which runs parallel
+ * specs against a sandbox). MultiLayerVerifier is a DAG of layers
+ * (install → typecheck → build → lint → serve → semantic → …) with
+ * dependency-based skip, per-layer findings, soft-fail semantics, and
+ * an aggregated `blendedScore` across all passed layers.
+ *
+ * Use when you want:
+ *   - ordered stages where a failing upstream stage skips downstream ones
+ *   - each stage produces rich `findings` (severity + message + evidence)
+ *   - a single composite score across stages with per-stage weights
+ *   - soft-fail stages whose failure doesn't abort the pipeline
+ *
+ * Use {@link JudgeRunner} when you want:
+ *   - N independent judges running in parallel against the same artifact
+ *   - no inter-judge dependencies
+ *   - boolean `passed` per judge + overall
+ *
+ * Both primitives compose — JudgeRunner can be invoked as a single
+ * layer inside a MultiLayerVerifier if that suits the caller.
+ */
+type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
+type Severity = 'critical' | 'major' | 'minor' | 'info';
+interface Finding {
+    severity: Severity;
+    message: string;
+    evidence?: string;
+    /** Optional layer name the finding belongs to (set by the verifier if omitted). */
+    layer?: string;
+    /**
+     * Free-form structured payload — used by `multiToolchainLayer` to attach
+     * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc.
+     * Renderers MAY interrogate; agent-eval primitives never assume shape.
+     */
+    detail?: Record<string, unknown>;
+}
+interface LayerResult {
+    layer: string;
+    status: LayerStatus;
+    /** 0..1 score, optional — layers that don't produce a numeric score omit. */
+    score?: number;
+    durationMs: number;
+    findings: Finding[];
+    /** Short human-readable summary (one line). */
+    reason?: string;
+    /**
+     * Numeric layer-level diagnostics: error counts, warning counts,
+     * cyclomatic complexity, total adapter wall-time, etc. Keyed by
+     * diagnostic name; null = "diagnostic not applicable / not measured."
+     * Renderers that know the keys can display them; ones that don't,
+     * ignore. Free-form on purpose — consumers type the value shape in
+     * their own namespace. Added in 0.10.
+     */
+    diagnostics?: Record<string, number | null>;
+    /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
+    detail?: Record<string, unknown>;
+}
+interface VerifyContext<Env = unknown> {
+    /** Per-run opaque context the caller provides. Layers destructure what they need. */
+    env: Env;
+    /** Previously-computed results from layers that already ran. */
+    prior: Record<string, LayerResult>;
+    /** Signal — if aborted, layers MUST bail within reasonable wall. */
+    signal: AbortSignal;
+}
+interface Layer<Env = unknown> {
+    name: string;
+    /** Stages that must have `status: 'pass'` before this layer runs. */
+    dependsOn?: string[];
+    /**
+     * Weight in the composite `blendedScore`. Default 1.0. Layers with weight 0
+     * contribute findings but not score.
+     */
+    weight?: number;
+    /**
+     * If true, a `fail` status contributes to `blendedScore` (as 0) instead of
+     * being dropped — use for layers whose failure is a real signal. Default:
+     * fail drops from numerator + denominator, matching VB's existing semantics.
+     */
+    failContributesToScore?: boolean;
+    /** Optional per-layer wall-cap in ms. Honored by the verifier (AbortSignal). */
+    capMs?: number;
+    run: (ctx: VerifyContext<Env>) => Promise<LayerResult> | LayerResult;
+}
+interface VerifyOptions<Env = unknown> {
+    env: Env;
+    /**
+     * Overall wall cap. Default: sum of layer capMs, or Infinity if any layer
+     * omits a cap. The verifier short-circuits remaining layers on overall cap.
+     */
+    overallCapMs?: number;
+    /** Called with each layer result as it completes. */
+    onLayer?: (result: LayerResult) => void;
+}
+interface VerificationReport {
+    layers: LayerResult[];
+    passCount: number;
+    failCount: number;
+    skippedCount: number;
+    errorCount: number;
+    /** True iff at least one scored layer ran AND every scored layer passed. */
+    allPass: boolean;
+    /**
+     * Weighted mean of `score` across contributing layers. 0 when no layers
+     * contributed. See {@link Layer.failContributesToScore} for fail semantics.
+     */
+    blendedScore: number;
+    durationMs: number;
+    startedAt: string;
+    finishedAt: string;
+}
+/**
+ * Grade a semantic-concept-style judge result into a single layer status.
+ *
+ * Pass when overall score >= threshold AND no critical-severity concept gap.
+ * Fail otherwise. Use inside a `Layer.run` when wrapping a concept judge.
+ *
+ * Generalized from VerticalBench H3 fix: `failingConcepts.length === 0` was
+ * too strict — a single concept at 6/10 failed the entire layer despite
+ * overall score being >= 0.7. Now we trust the judge's own `severity` field:
+ * `critical` findings veto; `major`/`minor` reduce the score but don't veto.
+ */
+declare function gradeSemanticStatus(input: {
+    score: number;
+    findings: Array<{
+        severity: Severity;
+        present?: boolean;
+        score?: number;
+    }>;
+    available: boolean;
+    threshold?: number;
+}): LayerStatus;
+declare class MultiLayerVerifier<Env = unknown> {
+    private readonly layers;
+    constructor(layers: Layer<Env>[]);
+    run(opts: VerifyOptions<Env>): Promise<VerificationReport>;
+}
+
+/**
+ * Adapters: convert legacy optimization outputs into the canonical
+ * `RunRecord[]` artifact that 0.22+ primitives consume.
+ *
+ * The 0.22 release standardized the campaign artifact: every cell of an
+ * eval matrix produces one `RunRecord`. The pre-0.22 optimization
+ * primitives (`runMultiShotOptimization`, `runPromptEvolution`) produce
+ * `TrialResult[]` with a different shape. This file bridges the two so
+ * the new primitives (`replayCache`, `pairedEvalueSequence`,
+ * `rubricPredictiveValidity`) compose cleanly with the existing RL stack.
+ *
+ * The adapters are thin and explicit — every mandatory `RunRecord` field
+ * comes from a caller-supplied context (`commitSha`, `model`,
+ * `promptHash`, `configHash`) plus the trial's runtime data. Defaults
+ * exist for fields the trial doesn't carry (`tokenUsage`, `costUsd`),
+ * but the validator still rejects records with bare-alias model strings
+ * — the caller is responsible for snapshot-pinning.
+ */
+
+interface AdapterContext {
+    /** Logical experiment id — typically the campaign or sweep identifier. */
+    experimentId: string;
+    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
+    model: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** Hash of the effective prompt sent to the model. */
+    promptHash: string | ((t: TrialResult) => string);
+    /** Hash of the effective config (model, temperature, tools, judges, splits). */
+    configHash: string | ((t: TrialResult) => string);
+    /** Default split tag. Default `'search'` — optimization sweeps run on the search split. */
+    splitTag?: RunSplitTag;
+    /** Default cost in USD when the trial doesn't record one. Default `0`. */
+    defaultCostUsd?: number;
+}
+/**
+ * Convert one `TrialResult` (from `runPromptEvolution` or
+ * `runMultiShotOptimization`) into a canonical `RunRecord`.
+ *
+ * The conversion is **not lossy** — every `TrialResult.metrics` field is
+ * carried through to `outcome.raw`, plus a synthetic
+ * `raw.cost_unknown = 1` flag when the trial omits cost (so downstream
+ * filters can distinguish "free" from "untracked"). This preserves the
+ * paper-grade contract: a record without a cost number is unbounded by
+ * definition, but we don't drop the record.
+ */
+declare function trialToRunRecord(trial: TrialResult, ctx: AdapterContext, opts?: {
+    runId?: string;
+    experimentIdPerTrial?: (t: TrialResult) => string;
+}): RunRecord;
+/** Convenience: convert an array of `TrialResult` in one go. */
+declare function trialsToRunRecords(trials: TrialResult[], ctx: AdapterContext): RunRecord[];
+/**
+ * Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
+ *
+ * The verifier produces per-layer results; we synthesize one canonical
+ * record where:
+ *   - `outcome.searchScore` (or `holdoutScore`) is `report.blendedScore`
+ *   - `outcome.raw` carries every layer's score keyed `layer.<name>`
+ *     plus a `layer_<name>_pass` 1/0 indicator
+ *   - `failureMode` is taken from the first failing layer's `reason`
+ *   - `wallMs` is `report.durationMs`
+ */
+declare function verificationReportToRunRecord(report: VerificationReport, ctx: AdapterContext & {
+    candidateId: string;
+    scenarioId?: string;
+}, opts?: {
+    runId?: string;
+}): RunRecord;
+/**
+ * Convert a `VariantAggregate` (per-variant rollup from `prompt-evolution`)
+ * into a synthetic `RunRecord` representing the aggregate. Useful when the
+ * downstream consumer wants per-variant entries for a `researchReport`
+ * rather than per-(variant, scenario, rep) trial entries.
+ */
+declare function variantAggregateToRunRecord(agg: VariantAggregate, ctx: AdapterContext, opts?: {
+    runId?: string;
+}): RunRecord;
+
+/**
+ * Verifiable reward channel.
+ *
+ * For RL on coding / math / theorem-proving / structured-output tasks, the
+ * reward signal is *decidable* — a test passes or fails, a proof checks or
+ * doesn't, an output validates against a schema or doesn't. These rewards
+ * are dramatically more useful for RL training than LLM-judge scores
+ * because they don't drift, can't be Goodhart-gamed by the policy in the
+ * same way, and don't require a separate calibration loop.
+ *
+ * The `MultiLayerVerifier` already produces this signal — it just doesn't
+ * surface it in a shape that's clean enough for RL training. This module
+ * wraps the verifier output so consumers can:
+ *
+ *   1. Extract a clean `VerifiableReward` from a `VerificationReport`
+ *   2. Distinguish *deterministic* rewards (compile, test, schema) from
+ *      *probabilistic* rewards (judge) so they can be weighted differently
+ *      in the RL training step
+ *   3. Filter `RunRecord[]` to only those with a verifiable reward,
+ *      producing the clean training set that DeepSeek-R1-style GRPO and
+ *      AlphaProof-style search both depend on
+ *
+ * Why this matters: every credible 2025-2026 frontier RL result on coding
+ * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,
+ * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing
+ * judge scores into the reward signal poisons the gradient. This module
+ * is the seam.
+ */
+
+type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
+interface VerifiableReward {
+    /** Scalar in [0, 1]. The RL training signal. */
+    value: number;
+    /** What produced the reward — different sources have different determinism. */
+    source: VerifiableRewardSource;
+    /**
+     * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte
+     * given the same inputs (compile, test, schema validation, sandbox exit code).
+     * `'probabilistic'` rewards depend on a stochastic component (LLM judge).
+     * Mixing these in the same training batch without separation is a known
+     * footgun in production RLHF pipelines.
+     */
+    determinism: 'deterministic' | 'probabilistic';
+    /**
+     * Confidence in the reward value. For deterministic sources this is 1.0
+     * (the bit either flipped or didn't). For judge sources this is the
+     * judge-reported confidence or — when missing — a calibrated prior.
+     */
+    confidence: number;
+    /** The layer / judge id that produced the signal, for provenance. */
+    origin: string;
+    /**
+     * Any per-source breakdown the consumer might want — e.g. `{ tests_passed: 7, tests_total: 10 }`.
+     */
+    breakdown?: Record<string, number>;
+}
+interface VerifiableRewardExtractionOptions {
+    /**
+     * Which layers count as deterministic-reward sources. The verifier doesn't
+     * tag layers as "this is verifiable"; the caller declares it via this list
+     * (or via the layer name → source mapping). Default treats common names
+     * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
+     * `sandbox`) as deterministic.
+     */
+    deterministicLayers?: string[];
+    /**
+     * Map layer name → reward source. Defaults to a sensible string-match.
+     */
+    sourceFor?: (layerName: string) => VerifiableRewardSource;
+    /**
+     * Whether to fall back to a probabilistic (judge) reward when no
+     * deterministic layer produced a numeric score. Default `true`. Set to
+     * `false` for "deterministic-only" training pipelines that should
+     * discard runs without a verifiable signal.
+     */
+    fallbackToJudge?: boolean;
+    /**
+     * Default confidence for probabilistic (judge) rewards when the judge
+     * doesn't report one. Default `0.7`.
+     */
+    judgeConfidenceFloor?: number;
+}
+/**
+ * Extract a `VerifiableReward` from a `VerificationReport`.
+ *
+ * Strategy: prefer the deterministic layers (in order: test → compile →
+ * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is
+ * true, return `null` if no signal qualifies. When multiple deterministic
+ * layers contribute, return a `'composite'` source with a weighted blend.
+ */
+declare function extractVerifiableReward(report: VerificationReport, opts?: VerifiableRewardExtractionOptions): VerifiableReward | null;
+/**
+ * Extract verifiable rewards from `RunRecord[]` produced via the
+ * `verificationReportToRunRecord` adapter (which encodes per-layer scores
+ * in `outcome.raw['layer.<name>']`). For records that don't carry layer
+ * scores, returns `null` for that record.
+ *
+ * This is the canonical bridge from "campaign-shaped artifacts" to
+ * "RL-training-ready reward signals": every record that has a clean
+ * verifiable reward becomes a training datum, every record that doesn't
+ * gets filtered out (or kept with `'probabilistic'` determinism for
+ * separate downstream handling).
+ */
+declare function extractVerifiableRewardsFromRecords(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+    runId: string;
+    reward: VerifiableReward | null;
+}>;
+/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */
+declare function filterDeterministicallyRewarded(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+    run: RunRecord;
+    reward: VerifiableReward;
+}>;
+
+/**
+ * Preference dataset extraction — bridge from `RunRecord[]` to RL training.
+ *
+ * Production RLHF / DPO / KTO / SimPO pipelines need preference triples:
+ * `(prompt, chosen, rejected)`. The campaign artifact already contains the
+ * ingredients — every (variantId, scenarioId, seed) cell is a candidate
+ * that ran the same prompt against the same scenario, scored by the same
+ * judge — but turning that into a clean preference dataset requires
+ * deciding *what counts as a preference*.
+ *
+ * This module ships three preference-extraction strategies with explicit
+ * tradeoffs, plus a unified output type compatible with HuggingFace TRL,
+ * Anthropic finetuning JSONL, and OpenAI fine-tuning APIs. The strategies
+ * are deliberately not auto-magical — picking the wrong one corrupts the
+ * gradient.
+ *
+ * Strategies:
+ *
+ *   1. **`paired-by-scenario-and-seed`** — exact-match comparisons. For
+ *      each scenario × seed pair, compare every (variantA, variantB) on
+ *      that exact (scenario, seed). Matches scenarios so the comparison
+ *      isolates variant effects. Highest signal-to-noise; smallest
+ *      dataset (only matched pairs count).
+ *
+ *   2. **`paired-by-scenario`** — looser matching. For each scenario,
+ *      compare every (variantA, variantB) where both have ≥ 1 run on the
+ *      same scenario. Aggregates across seeds to compute mean scores per
+ *      (variant, scenario), then forms preferences from the means. More
+ *      data, lower per-pair signal.
+ *
+ *   3. **`top-vs-bottom`** — coarsest. Within each scenario, the highest-
+ *      scoring run is `chosen`, the lowest is `rejected`. Smallest dataset
+ *      per scenario but biggest score gap per pair. Useful for early
+ *      bootstrapping when you have few variants.
+ *
+ * The output `PreferenceTriple` is *agent-eval-canonical* but trivially
+ * mappable to TRL's `DPODataset` shape (`prompt`, `chosen`, `rejected`)
+ * via the `toTRLFormat` helper.
+ */
+
+type PreferenceStrategy = 'paired-by-scenario-and-seed' | 'paired-by-scenario' | 'top-vs-bottom';
+interface PreferenceTriple {
+    /** The scenario (input) the variants were run against. */
+    scenarioId: string;
+    /** RunRecord ids on each side, for traceability. */
+    chosenRunId: string;
+    rejectedRunId: string;
+    /** Variant ids — load-bearing for the RL update. */
+    chosenVariantId: string;
+    rejectedVariantId: string;
+    /** The score gap between chosen and rejected. Larger = stronger signal. */
+    marginScore: number;
+    /**
+     * Optional `(chosen_score, rejected_score)` pair for soft-margin DPO
+     * variants. Omitted for `top-vs-bottom` runs that don't carry meaningful
+     * scalar gaps.
+     */
+    scores?: {
+        chosen: number;
+        rejected: number;
+    };
+    /** Tie-breaker — when multiple seeds match this scenario, the one used. */
+    seed?: number;
+    /**
+     * Free-form metadata propagated from the run records — e.g. original
+     * prompt-hash, model, etc. Lets the RL trainer reconstruct the prompt.
+     */
+    meta: {
+        chosenPromptHash: string;
+        rejectedPromptHash: string;
+        chosenConfigHash: string;
+        rejectedConfigHash: string;
+        chosenModel: string;
+        rejectedModel: string;
+    };
+}
+interface ExtractPreferencesOptions {
+    strategy?: PreferenceStrategy;
+    /**
+     * Minimum score gap required to admit a pair. Pairs below this are
+     * dropped — they're noise, not signal. Default 0.05 (5% of [0,1]).
+     */
+    minMargin?: number;
+    /**
+     * Optional split tag filter — restrict to runs from one split. Default
+     * `'holdout'` (the canonical "real" signal).
+     */
+    splitTag?: RunRecord['splitTag'];
+    /**
+     * Optional reward extractor that overrides `outcome.holdoutScore` /
+     * `outcome.searchScore`. Use to drive preferences off a verifiable
+     * reward instead of the headline score.
+     */
+    rewardOf?: (run: RunRecord) => number | null;
+}
+interface PreferenceExtractionReport {
+    pairs: PreferenceTriple[];
+    /** Number of (scenario, seed) cells inspected. */
+    cellsInspected: number;
+    /** Number of pairs filtered by `minMargin`. */
+    pairsBelowMargin: number;
+    /** Number of cells with only one variant (no comparison possible). */
+    cellsSingleton: number;
+    /** Strategy used. */
+    strategy: PreferenceStrategy;
+}
+/**
+ * Convert `RunRecord[]` to preference triples for RL training.
+ *
+ * Returns a structured report so callers can see how much data was
+ * dropped and why (low-margin pairs, singleton cells). For production
+ * pipelines, you usually want to:
+ *
+ *   1. Run a campaign producing 5–10 variants × 50–200 scenarios × 3 seeds
+ *   2. Call this with `strategy: 'paired-by-scenario-and-seed'` and a
+ *      verifiable-reward extractor as `rewardOf`
+ *   3. Pass `report.pairs` to `toTRLFormat` and pipe to your DPO trainer
+ */
+declare function extractPreferences(runs: RunRecord[], opts?: ExtractPreferencesOptions): PreferenceExtractionReport;
+/**
+ * TRL-compatible export. TRL's `DPODataset` is `{ prompt, chosen, rejected }`
+ * but the prompt isn't stored on the RunRecord — only its hash. The caller
+ * passes a `promptOf(promptHash)` lookup that the TRL trainer can use.
+ */
+declare function toTRLFormat(triples: PreferenceTriple[], promptOf: (hash: string) => string): Array<{
+    prompt: string;
+    chosen: string;
+    rejected: string;
+}>;
+/**
+ * Anthropic finetuning JSONL export — `{ system, user, assistant_chosen, assistant_rejected }`
+ * shape. Same caveat as TRL: prompt + outputs are content the caller has
+ * to map back from the run record / raw event log.
+ */
+declare function toAnthropicFormat(triples: PreferenceTriple[]): Array<{
+    scenarioId: string;
+    chosenRunId: string;
+    rejectedRunId: string;
+    margin: number;
+}>;
+
+/**
+ * Off-policy evaluation primitives.
+ *
+ * Standard inverse-probability-weighted (IPS), self-normalized
+ * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
+ * value of a *target* policy given trajectories collected under a
+ * *behavior* policy. This is the canonical RL eval task: "we have last
+ * week's runs, we changed the policy — how would the new one do without
+ * re-running?"
+ *
+ * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
+ * & Joachims 2015 for SNIPS) but the *application* to LLM-agent
+ * evaluation needs care:
+ *
+ *   - The "policy" is the (prompt, tool config, model snapshot) triple.
+ *     Two policies have the same probability over an action *iff* their
+ *     LLM call would emit the same token with the same probability —
+ *     which is generally unknowable without the model log-probs.
+ *   - For LLM agents, propensity scores must be supplied by the caller
+ *     (logged in the trace, recovered from token log-probs, or estimated
+ *     via a learned propensity model). We do NOT estimate propensity here.
+ *   - Doubly-robust requires a Q-function (model-based reward predictor).
+ *     We accept any callable; consumers pass either a tabular average,
+ *     a regression fit, or a learned reward model.
+ *
+ * Bias / variance tradeoffs:
+ *   - IPS: unbiased; high variance for small overlap, infinite variance
+ *     when target has support outside behavior.
+ *   - SNIPS: lower variance, slight bias; usually preferred in practice.
+ *   - DR: doubly-robust — unbiased if either propensity OR Q-function is
+ *     correct. Lowest practical variance when Q is decent. Use this.
+ *
+ * Caveat the panel will land: on the LLM-agent setting, propensity scores
+ * recovered from token log-probs are noisy, the action space is enormous,
+ * and overlap is often poor. These estimators are useful but not magic;
+ * complement with `replayCampaign` (exact replay where the request hashes
+ * match) for high-confidence answers and OPE for the gap.
+ */
+interface OffPolicyTrajectory {
+    /** Stable id, for traceability through the dataset. */
+    runId: string;
+    /** Reward observed under the behavior policy (the realized outcome). */
+    reward: number;
+    /**
+     * Behavior-policy probability of the action that was taken. For LLM
+     * agents this is typically `exp(sum(token_log_probs))` over the chosen
+     * trajectory. Must be in (0, 1].
+     */
+    behaviorProb: number;
+    /**
+     * Target-policy probability of the same action. For replay-style
+     * counterfactual evaluation this is what the *new* policy would have
+     * assigned to the *old* trajectory. Must be in [0, 1].
+     */
+    targetProb: number;
+    /**
+     * Optional model-based reward prediction at the same context. Used by
+     * `doublyRobust`. Set to `null` for IPS-only evaluation.
+     */
+    qHat?: number | null;
+}
+interface OffPolicyEstimate {
+    /** Estimated value of the target policy. */
+    value: number;
+    /** Standard error of the estimate. */
+    standardError: number;
+    /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */
+    effectiveSampleSize: number;
+    /** Number of trajectories used. */
+    n: number;
+    /**
+     * Diagnostic: maximum importance weight observed. Large values (>>10x
+     * mean) are a red flag — variance is dominated by a few outliers.
+     */
+    maxImportanceWeight: number;
+}
+interface OffPolicyOptions {
+    /**
+     * Cap importance weights at this value (Ionides 2008 truncated IS) to
+     * trade unbiasedness for variance reduction. Default `Infinity` (no cap).
+     * Set e.g. `10` for stable estimates when the policies are close.
+     */
+    weightCap?: number;
+    /** Reward clipping range. Default `[0, 1]`. */
+    rewardClip?: {
+        low: number;
+        high: number;
+    };
+}
+/**
+ * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
+ * of E[reward under target policy]. Variance scales with the spread of
+ * target/behavior ratios.
+ */
+declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+/**
+ * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
+ * the cost of small bias (vanishing as N grows). The right default for
+ * LLM-agent evaluation where overlap is often poor.
+ */
+declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+/**
+ * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
+ *
+ *     V_DR = (1/N) * sum_i [ q_hat_i + (target_prob_i / behavior_prob_i) * (r_i - q_hat_i) ]
+ *
+ * Unbiased if EITHER:
+ *   - the importance ratios are correct (IPS-style validity), OR
+ *   - the Q-hat function is correct (model-based validity).
+ *
+ * In practice both are imperfect, but the residual bias is the *product*
+ * of both errors — much smaller than either alone. This is why DR is the
+ * default in production OPE pipelines.
+ *
+ * Requires `qHat` on every trajectory. If any are `null`, the estimator
+ * falls back to SNIPS for those entries (loud-fallback behavior; the
+ * report's `n` reflects the full set but `effectiveSampleSize` accounts
+ * for the lost variance reduction).
+ */
+declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
+/**
+ * Convenience: run all three estimators and return them side-by-side.
+ * The recommended diagnostic — agreement across estimators is a much
+ * stronger signal than any single one.
+ */
+declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
+    ips: OffPolicyEstimate;
+    snips: OffPolicyEstimate;
+    dr: OffPolicyEstimate;
+};
+
+/**
+ * Process reward extraction — step-level credit assignment from trace spans.
+ *
+ * RL on long-horizon agents needs *step-level* rewards, not run-level
+ * ones. The classic credit-assignment problem (Sutton & Barto) requires
+ * knowing which sub-decisions in a trajectory contributed to the
+ * outcome. Modern systems (DeepSeek-R1, OpenAI o-series, Lightman et al.
+ * "Let's Verify Step by Step" 2023) train *process reward models* (PRMs)
+ * that score every step, then do RL with the PRM as the reward signal.
+ *
+ * This module extracts `StepReward[]` from trace spans — one per
+ * meaningful step — and ships:
+ *
+ *   1. `extractStepRewards(store, runId, opts)` — span → step-reward
+ *      conversion using configurable per-span scorers (LLM judge over the
+ *      span output, deterministic checkers, or a learned PRM).
+ *   2. `runwiseStepRewardSummary(stepRewards)` — aggregate the per-step
+ *      signal into a credit-assignment-aware run-level score.
+ *   3. `prmTrainingPairs(stepRewards, options)` — produce the
+ *      `(prefix, suffix_chosen, suffix_rejected)` triples that PRM
+ *      training pipelines consume.
+ *
+ * What we ship: the *extraction* and *aggregation* infrastructure plus
+ * the data shape PRM training expects. We do NOT ship the actual PRM
+ * training (gradient descent over a transformer is out of scope for a
+ * TS package). The interface is the contract; downstream consumers wire
+ * their preferred trainer.
+ *
+ * Caveat the panel will land: this is descriptive credit assignment
+ * (which steps correlate with outcome), not causal credit assignment
+ * (which steps caused outcome). For causal claims you need
+ * counterfactual rollouts or a learned dynamics model. Future work; the
+ * descriptive version is what production PRM training actually uses.
+ */
+
+interface StepReward {
+    /** Trace span this reward attaches to. */
+    spanId: string;
+    runId: string;
+    /** Index in the trajectory (0-based, in started-at order). */
+    stepIndex: number;
+    /** Span kind (typically 'tool', 'llm', 'judge'). */
+    kind: Span['kind'];
+    /** Span name — for the consumer's downstream filtering. */
+    name: string;
+    /** Step-level reward in [0, 1]. */
+    reward: number;
+    /**
+     * Determinism class. Mirrors the verifiable-reward distinction:
+     * deterministic = test/compile/schema check; probabilistic = LLM judge.
+     */
+    determinism: 'deterministic' | 'probabilistic';
+    /** Optional rationale / evidence — the trainer typically discards. */
+    rationale?: string;
+    /** Optional weight — how much this step contributes to credit assignment. */
+    weight?: number;
+}
+interface StepScorer {
+    /** Span kinds this scorer applies to. */
+    appliesTo: Span['kind'][];
+    /** Returns null to skip the span; returns a `StepReward` shape (without index/runId/spanId, which are filled in). */
+    score(span: Span): Promise<Omit<StepReward, 'spanId' | 'runId' | 'stepIndex'>> | null | undefined;
+}
+interface ExtractStepRewardsOptions {
+    /**
+     * Ordered list of scorers. Each span runs through scorers in order;
+     * the first non-null result wins. If no scorer applies, the span is
+     * skipped (not all spans are training-worthy).
+     */
+    scorers: StepScorer[];
+    /** Optional filter — return null to drop the span entirely before scoring. */
+    preFilter?: (span: Span) => boolean;
+}
+declare function extractStepRewards(store: TraceStore, runId: string, opts: ExtractStepRewardsOptions): Promise<StepReward[]>;
+interface RunwiseStepSummary {
+    runId: string;
+    totalSteps: number;
+    meanReward: number;
+    /** Sum-of-rewards (weighted by `weight ?? 1`). Use as the run-level proxy. */
+    sumWeightedReward: number;
+    /** Fraction of steps where reward < 0.5 — proxy for "where the policy was wrong." */
+    failureFraction: number;
+    /** Maximum drop in reward between consecutive steps — diagnoses a step where things went sideways. */
+    worstStepDelta: number;
+    worstStepIndex: number | null;
+}
+declare function runwiseStepRewardSummary(stepRewards: StepReward[]): RunwiseStepSummary;
+interface PrmTrainingTriple {
+    /** Prefix run-id (or composite key) — the trajectory up to step k-1. */
+    prefixRunId: string;
+    prefixStepIndex: number;
+    /** The step that came next on a high-reward trajectory. */
+    chosenSpanId: string;
+    chosenReward: number;
+    /** A step from a divergent low-reward trajectory at the same prefix length. */
+    rejectedSpanId: string;
+    rejectedReward: number;
+    /** The prefix run came from this run; the rejected step came from `rejectedRunId`. */
+    rejectedRunId: string;
+    marginScore: number;
+}
+/**
+ * Build PRM training triples. The shape: pair runs that share an early
+ * prefix (same scenario, same first N steps) and diverge later — at the
+ * point of divergence, the high-reward run's next step is `chosen`, the
+ * low-reward run's next step is `rejected`. This is the canonical PRM
+ * training data shape from Lightman et al. and DeepSeek-R1 process
+ * supervision.
+ *
+ * Implementation note: we don't have a way to detect "same prefix" in
+ * the general agent setting (token-level prefixes require hashing model
+ * outputs). The current heuristic groups by `(scenarioId, prefixSpanName
+ * sequence)` — runs are paired when their first K span names match. For
+ * production use this should be replaced with a proper trajectory-prefix
+ * hash; the heuristic is good enough for early-stage scaffolding.
+ */
+declare function prmTrainingPairs(stepRewardsByRun: Map<string, StepReward[]>, opts?: {
+    minMargin?: number;
+    minPrefixLength?: number;
+}): PrmTrainingTriple[];
+
+/**
+ * Contamination probe — held-out perturbation tests.
+ *
+ * The bug class: once a benchmark scenario set is published, models train
+ * on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and
+ * MMLU-Pro all exist because their predecessors got contaminated within
+ * months. The right defense is to keep a held-out *perturbed* version of
+ * every scenario — same task, slightly different surface — and check
+ * whether scores diverge significantly. Genuine capability transfers; rote
+ * memorization doesn't.
+ *
+ * This module ships the probe contract:
+ *
+ *   1. A `ScenarioPerturbation` strategy type — function that produces a
+ *      perturbed scenario from an original.
+ *   2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs
+ *      both halves and reports per-scenario score divergence + a global
+ *      contamination verdict via paired Wilcoxon.
+ *   3. Several stock perturbations: `renameVariables`, `shuffleOrder`,
+ *      `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the
+ *      task's structural difficulty while breaking surface memorization.
+ *
+ * The verdict is conservative: if the perturbed-vs-original score
+ * difference is statistically significant (BH-adjusted p < 0.05) AND
+ * the median drop is > 5 percentage points, we flag *contamination
+ * suspected*. False positives are possible (the perturbation might
+ * actually be harder); the default is to flag for review, not to
+ * autoreject.
+ */
+type ScenarioPerturbationKind = 'rename_variables' | 'shuffle_order' | 'paraphrase' | 'inject_irrelevant_clause' | 'custom';
+interface ScenarioPerturbation<S> {
+    kind: ScenarioPerturbationKind;
+    /** Apply to one scenario, return its perturbed sibling. */
+    apply: (scenario: S) => Promise<S> | S;
+    /** Optional id — for the report. */
+    id?: string;
+}
+interface ContaminationProbeInput<S> {
+    /** Identity of every scenario. The probe's `runFingerprint` keys on these. */
+    scenarioId: (s: S) => string;
+    /** Original scenarios. */
+    originals: S[];
+    /**
+     * Either pre-computed perturbations (one per original, same order) OR a
+     * `perturbation` strategy that synthesizes them on the fly.
+     */
+    perturbed?: S[];
+    perturbation?: ScenarioPerturbation<S>;
+    /**
+     * Run the policy/agent against one scenario and return a scalar score
+     * in [0, 1]. The probe doesn't care what the policy is — that's the
+     * caller's contract.
+     */
+    scoreFn: (s: S) => Promise<number>;
+}
+interface ContaminationProbeOptions {
+    /** Drop scores below this from the probe; treats partial failures separately. Default 0. */
+    scoreFloor?: number;
+    /**
+     * BH-FDR threshold for declaring contamination on each per-scenario
+     * delta. Default 0.05.
+     */
+    fdr?: number;
+    /**
+     * Minimum median per-scenario drop to flag global contamination. Default
+     * 0.05 (5 percentage points). Smaller drops may be noise.
+     */
+    minMedianDrop?: number;
+}
+interface ContaminationProbeReport {
+    perScenario: Array<{
+        scenarioId: string;
+        originalScore: number;
+        perturbedScore: number;
+        delta: number;
+        /** Per-scenario q-value (single-test BH for a single scenario). Mainly for display. */
+        qValue: number;
+    }>;
+    /** Wilcoxon paired-test on the deltas. */
+    pairedTest: {
+        w: number;
+        p: number;
+    };
+    medianDelta: number;
+    meanDelta: number;
+    contaminationSuspected: boolean;
+    reason: string;
+    /** Number of scenarios processed. */
+    n: number;
+}
+declare function runContaminationProbe<S>(input: ContaminationProbeInput<S>, opts?: ContaminationProbeOptions): Promise<ContaminationProbeReport>;
+/**
+ * Identifier-rename perturbation for code/text scenarios. Replaces every
+ * occurrence of the listed identifiers with synthesized aliases. Use when
+ * the scenario's structural difficulty is independent of variable names
+ * (e.g. SWE-Bench-style coding tasks).
+ */
+declare function renameVariables<S extends {
+    prompt: string;
+}>(identifiers: string[], rename?: (name: string, idx: number) => string): ScenarioPerturbation<S>;
+/**
+ * Order-shuffle perturbation. Reshuffles a list-shaped section of the
+ * prompt (for QA scenarios that present options A/B/C/D — answer depends
+ * on the option labels, not order). Caller provides the section extractor.
+ */
+declare function shuffleOrder<S extends {
+    prompt: string;
+}>(shuffleSection: (prompt: string, rng: () => number) => string, seed: number): ScenarioPerturbation<S>;
+/**
+ * Inject-irrelevant-clause perturbation. Adds a benign sentence that
+ * shouldn't change the answer. Tests for "did the model just memorize
+ * the input string."
+ */
+declare function injectIrrelevantClause<S extends {
+    prompt: string;
+}>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
+
+/**
+ * Bradley-Terry / Elo tournament evaluation.
+ *
+ * For multi-candidate sweeps, comparing every candidate's score against
+ * a fixed comparator wastes information — the comparator becomes a high-
+ * variance reference and rank flips between near-tied middle-rank
+ * candidates are dominated by noise. Pairwise tournaments fix this:
+ * every (i, j) pair contributes a comparison to a Bradley-Terry MLE that
+ * estimates each candidate's strength on a unified scale.
+ *
+ * For online updating (rolling campaigns where new candidates arrive
+ * over time), we also ship classical Elo with configurable K-factor.
+ *
+ * References:
+ *   - Bradley, R. A., Terry, M. E. (1952). Rank analysis of incomplete
+ *     block designs. Biometrika, 39(3/4), 324–345.
+ *   - Hunter, D. R. (2004). MM algorithms for generalized Bradley-Terry
+ *     models. Annals of Statistics, 32(1), 384–406. (The MLE algorithm
+ *     used here.)
+ *   - Elo, A. E. (1978). The Rating of Chess Players, Past and Present.
+ *
+ * This is a useful primitive because most LLM-eval communities (Chatbot
+ * Arena, AlpacaEval, ELO-style ablation) have converged on pairwise
+ * tournament eval as the most sample-efficient and most rank-stable
+ * method when you have many candidates.
+ */
+interface PairwiseOutcome {
+    /** Winner candidate id. */
+    winner: string;
+    /** Loser candidate id. */
+    loser: string;
+    /**
+     * Optional draw flag. When true, both candidates get half-credit
+     * (Bradley-Terry handles draws as half-wins for each side).
+     */
+    draw?: boolean;
+    /**
+     * Optional weight — useful if some pairwise comparisons are stronger
+     * signals than others (e.g. a paired test with a wider score gap is
+     * a more confident comparison). Default 1.
+     */
+    weight?: number;
+}
+interface BradleyTerryRating {
+    candidateId: string;
+    /** Latent strength θ ≥ 0 from the BT MLE. */
+    strength: number;
+    /** Log-strength = log(θ) — interpretable on a linear scale. */
+    logStrength: number;
+    /** Number of pairwise comparisons this candidate appears in. */
+    n: number;
+    /** Win count (+ 0.5 per draw). */
+    wins: number;
+}
+interface BradleyTerryFit {
+    ratings: BradleyTerryRating[];
+    /** Iterations of the MM algorithm before convergence. */
+    iterations: number;
+    /** Final maximum |θ_new - θ_old| / θ_old. */
+    finalDelta: number;
+    converged: boolean;
+}
+/**
+ * Bradley-Terry MLE via Hunter's MM algorithm.
+ *
+ * Iteration: θ_i^new = W_i / Σ_{j ≠ i} N_ij / (θ_i + θ_j)
+ *   where W_i = wins by i (+ 0.5 per draw), N_ij = total comparisons.
+ *
+ * Returns log-strengths normalized so the smallest is 0 (any constant
+ * offset is unobservable in BT — only differences are identified).
+ */
+declare function fitBradleyTerry(outcomes: PairwiseOutcome[], opts?: {
+    tolerance?: number;
+    maxIterations?: number;
+    smoothing?: number;
+}): BradleyTerryFit;
+/**
+ * Online Elo updates. Use when comparisons arrive over time and you want
+ * a running rating without re-fitting the full BT MLE on every update.
+ *
+ * Initialize ratings to `defaultRating` (1500 by default). Each call to
+ * `applyEloUpdate` mutates the map in place and returns the deltas so
+ * the caller can log per-comparison rating changes.
+ */
+interface EloOptions {
+    /** Default rating for unseen candidates. Default 1500. */
+    defaultRating?: number;
+    /** K-factor controls the step size. Default 32 (FIDE-ish). */
+    kFactor?: number;
+}
+declare function applyEloUpdate(ratings: Map<string, number>, outcome: PairwiseOutcome, opts?: EloOptions): {
+    winnerDelta: number;
+    loserDelta: number;
+};
+/**
+ * Build pairwise outcomes from the campaign artifact: for every scenario
+ * shared by two candidates, the higher-scoring run wins. Useful when you
+ * want a tournament view of an existing campaign without an additional
+ * pairwise judge call.
+ */
+interface BuildPairwiseFromCampaignInput {
+    runs: Array<{
+        candidateId: string;
+        /** Stable identifier for the matching unit (typically scenarioId). */
+        matchKey: string;
+        score: number;
+    }>;
+    /**
+     * Tied-score margin. Below this, the comparison is a draw. Default 0
+     * (no ties).
+     */
+    drawMargin?: number;
+}
+declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
+
+/**
+ * Adversarial scenario search.
+ *
+ * Capability evaluation on a fixed scenario set measures performance on
+ * the distribution someone curated. Production failure modes live in the
+ * tail — inputs the curator didn't think of, or actively avoided. The
+ * adversarial-search primitive actively looks for them: starting from a
+ * pool of scenarios where the policy already passes, it mutates them
+ * (paraphrase, edge-case substitution, compositional combination) and
+ * keeps the mutations that *break* the policy.
+ *
+ * This is not magic. It's the simplest version of the loop that AdA
+ * (Open-Ended Adaptation, DeepMind 2023), POET, and Anthropic's
+ * auto-jailbreak rigs all run: hill-climb against a failure indicator,
+ * keep the survivors, repeat. We ship the harness; consumers supply the
+ * mutation strategies and the failure detector.
+ *
+ * Why ship this in agent-eval and not as a separate red-team tool: every
+ * piece of the standard adversarial loop is already in this package
+ * (`runEvalCampaign` for the matrix run, `RawProviderSink` for capture,
+ * `assertRunCaptured` for integrity, `pairedEvalueSequence` for stop
+ * criteria). The adversarial primitive is just the *scenario-mutation
+ * meta-loop* on top of that machinery.
+ */
+interface AdversarialScenario<S> {
+    /** Stable id — used for deduplication and lineage tracking. */
+    id: string;
+    /** Generation index — 0 for seeds, 1 for first round of mutations, etc. */
+    generation: number;
+    /** Lineage — id of the parent scenario this was mutated from, if any. */
+    parentId: string | null;
+    scenario: S;
+    /** Score on the policy under test. Lower = adversarial signal. */
+    score: number | null;
+    /** Strategy that produced this mutation, for diagnostics. */
+    mutationStrategy: string | null;
+}
+interface AdversarialMutation<S> {
+    id: string;
+    /**
+     * Mutate one scenario. Return null to skip; return one or more new
+     * scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
+     */
+    mutate(parent: S, rng: () => number): Promise<S[]> | S[];
+}
+interface AdversarialSearchOptions<S> {
+    /** Initial scenarios — typically those the policy currently passes. */
+    seeds: S[];
+    /** Stable identifier extraction. */
+    mutateScenarioId: (s: S) => string;
+    /** Mutation strategies. */
+    mutations: AdversarialMutation<S>[];
+    /**
+     * Run the policy under test against one scenario, return a scalar score
+     * in [0, 1]. Lower = adversarial signal.
+     */
+    scoreFn: (s: S) => Promise<number>;
+    /**
+     * Threshold below which a scenario counts as a "failure" worth keeping.
+     * Default 0.5.
+     */
+    failureThreshold?: number;
+    /** Number of mutation rounds. Default 3. */
+    rounds?: number;
+    /** Children per parent per round. Default 4. */
+    childrenPerParent?: number;
+    /** Maximum total scenarios examined. Default Infinity. */
+    budget?: number;
+    /** Seed for the deterministic RNG. Default 1. */
+    seed?: number;
+}
+interface AdversarialSearchReport<S> {
+    scenarios: AdversarialScenario<S>[];
+    /** Discovered failures sorted by score ascending. */
+    failures: AdversarialScenario<S>[];
+    /** Round-by-round counts. */
+    byGeneration: Array<{
+        generation: number;
+        total: number;
+        failures: number;
+        meanScore: number;
+    }>;
+    /** Total scoreFn invocations consumed. */
+    scoreCalls: number;
+}
+declare function adversarialScenarioSearch<S>(opts: AdversarialSearchOptions<S>): Promise<AdversarialSearchReport<S>>;
+
+/**
+ * Test-time compute scaling curves.
+ *
+ * The test-time-compute frontier paper (Snell et al. 2024) and the
+ * subsequent o1-style scaling work both show that LLM-agent capability
+ * is a function of the compute budget at inference, not just of the
+ * training run. The right way to characterize a candidate is therefore
+ * a *curve* — score at compute budgets {1×, 4×, 16×, …} — not a single
+ * point.
+ *
+ * This module ships:
+ *
+ *   1. The compute-curve harness — `runComputeCurve(runner, budgets)` —
+ *      that evaluates one candidate at a sequence of compute budgets
+ *      and returns the (compute, score) curve.
+ *   2. A best-of-N evaluator — `bestOfN(runner, n, scoreFn)` — the
+ *      simplest test-time-compute scaling primitive: sample N
+ *      independent rollouts, return the best.
+ *   3. A self-consistency evaluator — `selfConsistency(runner, n)` —
+ *      the majority-vote variant of best-of-N for tasks with a small
+ *      categorical answer space.
+ *   4. Pareto-frontier extraction over multiple candidates — given
+ *      (candidate, compute, score) tuples, return the set of
+ *      candidate-compute combinations that aren't dominated.
+ *
+ * Caveat: "compute" here is the caller's notion of a compute unit. For
+ * agent eval that's typically wall-time × parallelism, or token budget,
+ * or LLM-call count. We accept whatever the caller provides; the curve
+ * is on whatever axis they pick.
+ */
+interface ComputeCurveBudget {
+    /** Identifier — for the report. Common: '1x', '4x', '16x'. */
+    id: string;
+    /** Numeric value on the chosen axis (tokens, calls, USD, ms — caller picks). */
+    cost: number;
+    /** Free-form metadata (the caller can carry per-budget config). */
+    meta?: Record<string, unknown>;
+}
+interface ComputeCurvePoint {
+    budgetId: string;
+    cost: number;
+    score: number;
+    /** Number of underlying samples used at this budget. */
+    samples: number;
+    /** Optional spread / variance information. */
+    std?: number;
+    /** Any extra metrics the runner returned. */
+    metrics?: Record<string, number>;
+}
+interface ComputeCurve {
+    candidateId: string;
+    points: ComputeCurvePoint[];
+    /** Rough exponent fit: score ≈ a + b * log(cost). Useful for "how steep is the curve?" */
+    logSlope: number | null;
+    /** Best (highest-score) point on the curve. */
+    best: ComputeCurvePoint;
+}
+interface RunComputeCurveOptions {
+    candidateId: string;
+    budgets: ComputeCurveBudget[];
+    /**
+     * Run the candidate at one budget. Returns the realized score plus
+     * optional spread + extra metrics.
+     */
+    runAtBudget: (budget: ComputeCurveBudget) => Promise<{
+        score: number;
+        samples: number;
+        std?: number;
+        metrics?: Record<string, number>;
+    }>;
+}
+declare function runComputeCurve(opts: RunComputeCurveOptions): Promise<ComputeCurve>;
+interface ComputeBestOfNOptions<O> {
+    /** Number of independent samples to draw. */
+    n: number;
+    /** Sampler — produces one rollout. */
+    sample: (sampleIdx: number) => Promise<O>;
+    /** Score one rollout. */
+    scoreFn: (rollout: O) => Promise<number> | number;
+}
+interface ComputeBestOfNResult<O> {
+    best: O;
+    bestScore: number;
+    scores: number[];
+    meanScore: number;
+    /** Index of the best rollout, for diagnostics. */
+    bestIndex: number;
+}
+/** The simplest test-time scaling primitive. */
+declare function bestOfN<O>(opts: ComputeBestOfNOptions<O>): Promise<ComputeBestOfNResult<O>>;
+interface SelfConsistencyOptions<O> {
+    n: number;
+    sample: (sampleIdx: number) => Promise<O>;
+    /** Extract the canonical answer key (string) from a rollout. */
+    answerKey: (rollout: O) => string;
+}
+interface SelfConsistencyResult<O> {
+    /** Modal answer (the majority vote). */
+    answer: string;
+    /** Fraction of samples voting for the modal answer in [0, 1]. */
+    agreement: number;
+    /** Histogram of all answers. */
+    histogram: Record<string, number>;
+    /** A representative rollout that voted for the modal answer. */
+    representative: O;
+    /** All rollouts. */
+    rollouts: O[];
+}
+/**
+ * Self-consistency / majority-vote test-time scaling. For tasks with a
+ * small categorical answer space (math problems, multiple choice).
+ */
+declare function selfConsistency<O>(opts: SelfConsistencyOptions<O>): Promise<SelfConsistencyResult<O>>;
+/**
+ * Pareto frontier over (candidate, compute, score) tuples. A point is on
+ * the frontier iff no other point dominates it in both score (higher
+ * better) and cost (lower better). Returns the frontier sorted ascending
+ * by cost.
+ */
+interface ParetoPointInput {
+    candidateId: string;
+    budgetId: string;
+    cost: number;
+    score: number;
+}
+declare function paretoFrontier(points: ParetoPointInput[]): ParetoPointInput[];
+
+/**
+ * Adaptive curriculum / active scenario selection.
+ *
+ * Fixed scenario sets waste sample budget on cells the policy already
+ * passes (no information left) and cells the policy never passes (no
+ * gradient available either). Active learning over scenarios fixes this
+ * by allocating the next sample budget to cells where the policy's
+ * outcome is *uncertain* — those carry the most decision-relevant signal.
+ *
+ * This module ships two complementary strategies:
+ *
+ *   1. **Variance-based** — score each (variant, scenario) cell by the
+ *      empirical variance of past observations. Allocate next-round budget
+ *      proportional to variance. Standard active-learning-by-uncertainty
+ *      heuristic; works well when the policy is non-deterministic and
+ *      cells differ in observation noise.
+ *
+ *   2. **Bandit-based (Thompson sampling)** — model each (variant,
+ *      scenario) cell as a Beta-Bernoulli arm; sample a posterior; pick
+ *      cells whose posterior mean is closest to the per-scenario decision
+ *      threshold. The right primitive when scenarios are
+ *      "pass/fail" rather than continuous, and when promotion gates fire
+ *      at a known threshold (e.g., 0.5).
+ *
+ * The output is a *next-round budget allocation* — a list of (variant,
+ * scenario, count) triples. The consumer's matrix runner consumes the
+ * allocation, runs those cells, feeds the new observations back. Loop.
+ *
+ * Out of scope (deliberate): scenario *generation* — that's the
+ * adversarial primitive's job. This module allocates over an existing
+ * scenario pool.
+ */
+
+interface CellObservation {
+    variantId: string;
+    scenarioId: string;
+    /** Observed score in [0, 1]. */
+    score: number;
+    /** For Bernoulli arms — derive from the score with a threshold if needed. */
+    pass?: boolean;
+}
+interface CurriculumAllocation {
+    variantId: string;
+    scenarioId: string;
+    /** How many additional reps to run on this cell. */
+    count: number;
+    /** Strategy-specific reason for the allocation. */
+    reason: string;
+}
+interface VarianceCurriculumOptions {
+    /** Total reps to allocate across all cells. */
+    budget: number;
+    /**
+     * Smoothing prior on variance — keeps the allocator from concentrating
+     * on a cell with one observation just because its 1-sample variance is
+     * 0. Default 0.05.
+     */
+    variancePrior?: number;
+    /**
+     * Minimum reps per cell — even when the variance estimate is low, give
+     * every cell at least this many. Default 1.
+     */
+    floorPerCell?: number;
+}
+/**
+ * Variance-proportional allocation. For each cell, estimate variance from
+ * past observations + a prior, then allocate the budget proportional to
+ * (sqrt(variance) + 1/sqrt(n)) — a classical optimal-allocation rule
+ * (Neyman 1934) that balances "explore noisy cells" with "explore
+ * under-sampled cells."
+ */
+declare function varianceBasedCurriculum(observations: CellObservation[], candidateCells: Array<{
+    variantId: string;
+    scenarioId: string;
+}>, opts: VarianceCurriculumOptions): CurriculumAllocation[];
+interface ThompsonCurriculumOptions {
+    budget: number;
+    /**
+     * The per-scenario decision threshold. Cells whose posterior mean is
+     * closest to this get the most budget — that's where the next observation
+     * has the highest information value for the gate decision. Default 0.5.
+     */
+    decisionThreshold?: number;
+    /** Beta prior parameters. Default α=β=1 (uniform). */
+    priorAlpha?: number;
+    priorBeta?: number;
+    /** Seed the Thompson sampler. Default unset (Math.random). */
+    seed?: number;
+}
+/**
+ * Thompson-sampling-style allocation for pass/fail cells. For each cell:
+ *
+ *   - Maintain Beta(α + passes, β + failures) posterior on pass-rate
+ *   - Allocation weight ∝ exp(-((sampledMean - threshold) / σ)^2):
+ *     cells whose sampled posterior straddles the decision boundary get
+ *     the most weight; cells already clearly above or below get less.
+ *
+ * This is the right primitive when promotion gates fire at a known
+ * threshold and you want to sharpen the posterior near the boundary.
+ */
+declare function thompsonCurriculum(observations: CellObservation[], candidateCells: Array<{
+    variantId: string;
+    scenarioId: string;
+}>, opts: ThompsonCurriculumOptions): CurriculumAllocation[];
+/** Convenience: extract `CellObservation[]` directly from `RunRecord[]`. */
+declare function observationsFromRunRecords(runs: RunRecord[], opts?: {
+    passThreshold?: number;
+    useHoldout?: boolean;
+}): CellObservation[];
+
+/**
+ * Reward hacking / Goodhart detection.
+ *
+ * Goodhart's Law says: when a measure becomes a target, it ceases to be
+ * a good measure. In RLHF and agentic-RL settings this is the dominant
+ * failure mode — the policy learns to produce outputs that score well on
+ * the proxy reward (judge, rubric, test pass-rate) without producing
+ * the underlying capability the proxy was meant to track.
+ *
+ * Krakovna et al. (2020, "Specification Gaming Examples in AI") and the
+ * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.
+ * 2023) converge on a few diagnostic signatures:
+ *
+ *   1. **Reward divergence:** the proxy reward grows while the held-out
+ *      ground-truth signal stagnates or drops. Predictive validity over
+ *      time captures this.
+ *   2. **Distributional shift in outputs:** after RL, the policy produces
+ *      outputs that no longer match the reference distribution — usually
+ *      because it found a high-reward attractor that's degenerate (e.g.
+ *      one-token responses, repetition, formatting tricks).
+ *   3. **Disagreement between independent rewards:** if you train on
+ *      reward A and a held-out independent reward B drops sharply, you're
+ *      probably hacking A.
+ *   4. **Calibration drift:** the verifiable / deterministic component of
+ *      the reward is stable; the probabilistic / judge component drifts up
+ *      while the deterministic component doesn't. The judge is being
+ *      gamed.
+ *
+ * This module ships explicit detectors for all four signatures, plus a
+ * combined verdict. The output is diagnostic — actionable signals,
+ * not autoreject — because each signature has known false positives
+ * (e.g., a policy that genuinely improves can show distributional shift).
+ *
+ * Differs from `rubricPredictiveValidity` (which is a *standing* check on
+ * whether rubrics correlate with deployment outcomes) — this is a
+ * *temporal* check on whether the reward-vs-truth gap is *widening over
+ * time during a training run*.
+ */
+
+type RewardHackingSignal = 'reward_divergence' | 'distribution_shift' | 'reward_disagreement' | 'judge_drift';
+interface RewardHackingFinding {
+    signal: RewardHackingSignal;
+    /** Severity in [0, 1]. >0.5 = strong signal. */
+    severity: number;
+    message: string;
+    /** Numeric evidence the consumer can render. */
+    detail: Record<string, number>;
+}
+interface RewardHackingReport {
+    findings: RewardHackingFinding[];
+    /**
+     * Composite verdict. `'clean'` if every signal severity < 0.3;
+     * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.
+     */
+    verdict: 'clean' | 'suspect' | 'gaming';
+    /** Rationale for the verdict, ready to paste into an audit log. */
+    rationale: string[];
+    /** Number of paired (proxy, truth) data points the report saw. */
+    n: number;
+}
+interface DetectRewardHackingInput {
+    /**
+     * Run records ordered by recency (oldest first). The detector segments
+     * them into prefix/suffix windows to compute "did the gap widen."
+     */
+    runs: RunRecord[];
+    /**
+     * The metric the policy was trained to optimize. Should be present on
+     * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
+     */
+    proxyOf?: (run: RunRecord) => number | null;
+    /**
+     * The held-out ground-truth metric. For RL on coding, this is typically
+     * test pass-rate. For RLHF, it's downstream task performance or human
+     * preference. For knowledge tasks, it's an independently-graded score.
+     */
+    truthOf?: (run: RunRecord) => number | null;
+    /**
+     * Independent secondary reward. Used for the `reward_disagreement`
+     * signal. Default uses the verifiable reward extractor (deterministic
+     * sources only).
+     */
+    secondaryRewardOf?: (run: RunRecord) => number | null;
+    /**
+     * Window size — how many of the most recent runs count as the "after"
+     * cohort. Default min(50, half the runs).
+     */
+    windowSize?: number;
+    /**
+     * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
+     * (gaming).
+     */
+    thresholds?: {
+        suspect?: number;
+        gaming?: number;
+    };
+    /**
+     * Verifiable-reward options used for the secondary-reward fallback.
+     */
+    verifiableRewardOptions?: VerifiableRewardExtractionOptions;
+}
+declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
+
+/**
+ * Sample-efficient adaptation evaluation.
+ *
+ * For foundation-model-based agents, the load-bearing capability isn't
+ * raw end-state performance — it's *how fast the agent reaches that
+ * performance from cold start*. The same model with a worse prompt that
+ * adapts in 5 demonstrations beats the same model with a better prompt
+ * that needs 50. Standard meta-learning eval (Finn et al., MAML, RL² lit)
+ * reports an *adaptation curve*: score after k=0, 1, 2, 4, 8, 16, …
+ * in-context examples or fine-tune steps.
+ *
+ * This module ships:
+ *
+ *   1. `runAdaptationCurve` — given a runner that takes k demonstrations
+ *      and returns a score, produce the (k, score) curve.
+ *   2. `compareAdaptationCurves` — paired comparison across two policies.
+ *      Returns per-k delta with bootstrap CIs and an "area-under-curve"
+ *      summary statistic.
+ *   3. `firstPassK` — for pass/fail evaluation, the minimum k at which
+ *      the policy reliably passes (≥ pass-rate threshold over reps).
+ *
+ * Use cases:
+ *   - Compare two prompt designs that have similar end-state performance
+ *     but different in-context efficiency.
+ *   - Decide between fine-tuning and prompting based on adaptation cost.
+ *   - Detect when a policy "memorizes" k=0 inputs vs. genuinely adapts.
+ */
+interface AdaptationRunner<S> {
+    /**
+     * Runs the policy on `scenario` with `k` demonstrations. Returns a
+     * scalar score in [0, 1]. The runner is responsible for any caching;
+     * the harness calls it once per (scenario, k, rep) cell.
+     */
+    run(args: {
+        scenario: S;
+        k: number;
+        rep: number;
+    }): Promise<number>;
+}
+interface RunAdaptationCurveOptions<S> {
+    scenarios: S[];
+    /** Number-of-shots to evaluate at. Default `[0, 1, 2, 4, 8, 16]`. */
+    ks?: number[];
+    /** Reps per (scenario, k) cell. Default 3. */
+    reps?: number;
+    runner: AdaptationRunner<S>;
+    /** Pass-rate threshold for `firstPassK` reporting. Default 0.5. */
+    passThreshold?: number;
+}
+interface AdaptationPoint {
+    k: number;
+    meanScore: number;
+    passRate: number;
+    std: number;
+    n: number;
+    /** Per-scenario means at this k. */
+    perScenario: Array<{
+        scenarioId: string;
+        meanScore: number;
+        passes: number;
+        total: number;
+    }>;
+}
+interface AdaptationCurve {
+    points: AdaptationPoint[];
+    /**
+     * Smallest `k` at which `passRate ≥ passThreshold`. `null` if no `k`
+     * tested reaches it.
+     */
+    firstPassK: number | null;
+    /**
+     * Area under the (k, meanScore) curve, normalized by max-k. A
+     * single-number summary of "how well does this policy adapt from
+     * cold-start to fully-conditioned." Higher = better adapter.
+     */
+    adaptationArea: number;
+}
+declare function runAdaptationCurve<S extends {
+    scenarioId?: string;
+}>(opts: RunAdaptationCurveOptions<S>): Promise<AdaptationCurve>;
+interface CompareCurvesResult {
+    perK: Array<{
+        k: number;
+        deltaMean: number;
+        aLow: number;
+        aHigh: number;
+        bLow: number;
+        bHigh: number;
+    }>;
+    areaDelta: number;
+    firstPassKDelta: number | null;
+    /** Verdict: 'a_better' | 'b_better' | 'similar'. */
+    verdict: 'a_better' | 'b_better' | 'similar';
+    /** Rationale, ready to render. */
+    rationale: string;
+}
+/**
+ * Paired comparison of two adaptation curves. Per-k deltas with 95%
+ * bootstrap CIs (constructed from each curve's `perScenario` per-k means
+ * — the bootstrap unit is the scenario, not the rep).
+ */
+declare function compareAdaptationCurves(a: AdaptationCurve, b: AdaptationCurve, opts?: {
+    confidence?: number;
+    bootstrapResamples?: number;
+    seed?: number;
+}): CompareCurvesResult;
+/** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
+declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
+
+/**
+ * Trainer-format exporters.
+ *
+ * agent-eval produces canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`,
+ * `StepReward[]`, `PrmTrainingTriple[]`). RL training pipelines consume
+ * different shapes — Hugging Face TRL, Prime Intellect's prime-rl, OpenAI
+ * fine-tuning, Anthropic finetuning, OpenRLHF, verl. Each has its own
+ * JSONL conventions. Rather than ship N adapters, this module ships the
+ * canonical formats most production pipelines accept and ergonomic helpers
+ * for the rest.
+ *
+ * Shapes:
+ *   - **DPO / IPO / KTO** — `{prompt, chosen, rejected}` JSONL. Consumed
+ *     by HuggingFace TRL, prime-rl's offline DPO, OpenRLHF.
+ *   - **GRPO offline** — `{prompt, completions[], rewards[]}` JSONL.
+ *     Consumed by prime-rl GRPO, verl, OpenRLHF.
+ *   - **SFT** — `{messages[]}` JSONL with chosen completion as the final
+ *     assistant turn. Consumed by HF SFT trainers, OpenAI fine-tuning,
+ *     Anthropic finetuning.
+ *   - **PRM** — `{prompt, prefix_steps[], chosen_step, rejected_step}` JSONL.
+ *     Consumed by Lightman-style PRM trainers and prime-rl's PRM mode.
+ *
+ * Why ship this in agent-eval rather than a separate adapter package: the
+ * canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`, etc.) are
+ * agent-eval's contract; without first-party exporters consumers reverse-
+ * engineer the mapping every release. The exporters codify it.
+ *
+ * The exporters take callbacks for any field that isn't on the canonical
+ * artifact (specifically: prompt + completion text, since the package
+ * stores only their hashes by design — full text is the consumer's
+ * trace store / raw event log).
+ */
+
+interface DpoLookups {
+    /** Resolve the prompt text for a run (typically from a trace store / raw event sink). */
+    promptOf: (runId: string) => string | Promise<string>;
+    /** Resolve the assistant completion text for a run. */
+    completionOf: (runId: string) => string | Promise<string>;
+}
+interface DpoExportRow {
+    prompt: string;
+    chosen: string;
+    rejected: string;
+    /** Carried-through margin. Some KTO / IPO variants use this. */
+    margin?: number;
+    /** Free-form metadata for downstream filtering / sharding. */
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert preference triples to TRL-compatible DPO rows. The shape
+ * `{prompt, chosen, rejected}` is the canonical HuggingFace DPODataset
+ * entry; every major DPO trainer accepts it.
+ */
+declare function toDpoRows(triples: PreferenceTriple[], lookups: DpoLookups): Promise<DpoExportRow[]>;
+/** Serialize DPO rows as JSONL. One line per row. */
+declare function toDpoJsonl(rows: DpoExportRow[]): string;
+interface GrpoLookups {
+    promptOf: (runId: string) => string | Promise<string>;
+    completionOf: (runId: string) => string | Promise<string>;
+    /** Optional: derive a custom reward from the run. Defaults to score. */
+    rewardOf?: (run: RunRecord) => number | null;
+}
+interface GrpoExportRow {
+    prompt: string;
+    completions: string[];
+    rewards: number[];
+    /** runIds in the same order as `completions[]` for traceability. */
+    runIds: string[];
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert RunRecord[] grouped by `(scenarioId)` into GRPO offline rows —
+ * one row per scenario, with one completion per run on that scenario.
+ *
+ * GRPO (Shao et al. 2024 / DeepSeek-R1) trains on relative advantages
+ * within a group of completions for the same prompt; this is the
+ * canonical input format.
+ */
+declare function toGrpoRows(runs: RunRecord[], lookups: GrpoLookups): Promise<GrpoExportRow[]>;
+declare function toGrpoJsonl(rows: GrpoExportRow[]): string;
+interface SftLookups {
+    promptOf: (runId: string) => string | Promise<string>;
+    completionOf: (runId: string) => string | Promise<string>;
+    /** Optional system message. Default omits. */
+    systemOf?: (run: RunRecord) => string | null | undefined;
+    /** Filter — return false to skip the run (e.g., low score, failed cases). */
+    include?: (run: RunRecord) => boolean;
+}
+interface SftExportRow {
+    messages: Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert RunRecord[] into Hugging Face / OpenAI / Anthropic-style
+ * conversational SFT rows. By default every record becomes one row;
+ * pass `include` to filter (e.g., keep only `score >= 0.8` for
+ * rejection-sampling SFT).
+ */
+declare function toSftRows(runs: RunRecord[], lookups: SftLookups): Promise<SftExportRow[]>;
+declare function toSftJsonl(rows: SftExportRow[]): string;
+interface PrmLookups {
+    /** Resolve the prompt text for a run. */
+    promptOf: (runId: string) => string | Promise<string>;
+    /** Resolve the trajectory step text for a (runId, spanId) pair. */
+    stepTextOf: (runId: string, spanId: string) => string | Promise<string>;
+    /** Optional: sequence of prefix span ids leading up to the divergence. */
+    prefixOf?: (runId: string, prefixStepIndex: number) => string[] | Promise<string[]>;
+}
+interface PrmExportRow {
+    prompt: string;
+    /** Span ids for the steps before divergence — caller resolves text via `stepTextOf`. */
+    prefixSpanIds: string[];
+    prefixStepText: string[];
+    chosenStep: string;
+    rejectedStep: string;
+    chosenReward: number;
+    rejectedReward: number;
+    marginScore: number;
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert PRM training triples to JSONL rows. Caller's `stepTextOf`
+ * callback resolves span text from the consumer's trace store.
+ */
+declare function toPrmRows(triples: PrmTrainingTriple[], lookups: PrmLookups): Promise<PrmExportRow[]>;
+declare function toPrmJsonl(rows: PrmExportRow[]): string;
+interface StepRewardJsonlRow {
+    runId: string;
+    spanId: string;
+    stepIndex: number;
+    reward: number;
+    determinism: 'deterministic' | 'probabilistic';
+    weight: number;
+}
+declare function stepRewardsToJsonl(stepRewards: StepReward[]): string;
+
+/**
+ * `runRLCampaign` — the missing top-level orchestrator.
+ *
+ * `runEvalCampaign` runs the matrix and produces `RunRecord[]`. The 0.23
+ * RL primitives consume that artifact in different ways. Until 0.24 they
+ * had to be wired together by hand at every consumer; that defeats the
+ * cohesion the package is supposed to provide.
+ *
+ * `runRLCampaign` wires:
+ *   1. `runEvalCampaign` for the matrix run (capture, integrity, hooks)
+ *   2. `extractVerifiableReward` over each run, separating deterministic
+ *      from probabilistic reward sources for the trainer
+ *   3. `extractPreferences` to produce DPO/PPO/KTO triples
+ *   4. `evaluateInterimReleaseConfidence` over paired deltas (anytime-valid)
+ *   5. `rubricPredictiveValidity` against an outcome store, when provided
+ *   6. `detectRewardHacking` as a standing hygiene check
+ *   7. Trainer-format export rows ready for prime-rl / TRL / verl
+ *
+ * The output `RLCampaignResult` is a single, audit-ready artifact: every
+ * stage's output is in there. The consumer's downstream fits in a single
+ * line: pass `result.preferences` to their DPO trainer, `result.grpoRows`
+ * to GRPO, `result.runs` plus `result.rewardSignals` to a custom RL loop.
+ *
+ * This is what the 0.23 panel critique called the "missing top-level
+ * primitive." Now shipped.
+ */
+
+interface RunRLCampaignOptions<V> extends EvalCampaignOptions<V> {
+    /** Preference-extraction options. Default uses paired-by-scenario-and-seed with min-margin 0.05. */
+    preferences?: ExtractPreferencesOptions;
+    /** Verifiable-reward extraction options. */
+    verifiableReward?: VerifiableRewardExtractionOptions;
+    /** Outcome store + metric names — when supplied, runs `rubricPredictiveValidity` post-campaign. */
+    outcomeStore?: OutcomeStore;
+    outcomeMetrics?: string[];
+    /** Anytime-valid sequential evaluation options. */
+    sequential?: {
+        alpha?: number;
+        bound?: number;
+        rope?: {
+            low: number;
+            high: number;
+        };
+    };
+    /** Trainer-format export lookups. When provided, the orchestrator builds the corresponding rows. */
+    trainerExport?: {
+        dpo?: DpoLookups;
+        grpo?: GrpoLookups;
+        sft?: SftLookups;
+    };
+}
+interface RLCampaignResult<V> {
+    campaign: EvalCampaignResult;
+    /** Per-run verifiable reward (deterministic when available, probabilistic fallback otherwise). */
+    rewardSignals: Array<{
+        runId: string;
+        reward: VerifiableReward | null;
+    }>;
+    /** Preference extraction report. */
+    preferences: PreferenceExtractionReport;
+    /** Anytime-valid interim verdict over the paired deltas (vs comparator). */
+    interimConfidence: InterimReleaseConfidence | null;
+    /** Standing reward-hacking hygiene check. */
+    rewardHacking: RewardHackingReport;
+    /** Predictive validity, when an outcome store was supplied. */
+    predictiveValidity: RubricPredictiveValidityReport | null;
+    /** Trainer-export rows, populated only for the formats the caller requested via `trainerExport`. */
+    trainerRows: {
+        dpo?: DpoExportRow[];
+        grpo?: GrpoExportRow[];
+        sft?: SftExportRow[];
+    };
+    /**
+     * One-line top-level summary the consumer can log.
+     */
+    summary: string;
+    /**
+     * Convenience type-tag — consumers can branch on `result.kind`.
+     */
+    kind: 'agent-eval-rl-campaign';
+    unusedVariant?: V;
+}
+declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult<V>>;
+
+/**
+ * `PredictiveValidityResearcher` — concrete `Researcher` implementation
+ * that drives selection from outcome-anchored predictive validity.
+ *
+ * `Researcher` was a placeholder interface plus `NoopResearcher` until
+ * 0.23. The 0.23 panel critique called this out: shipping the interface
+ * without a default implementation that drives the loop is incomplete.
+ *
+ * This researcher answers each method:
+ *
+ *   - `inspectFailures(runs)` — synthesizes failure modes from the
+ *     bottom-quartile of `RunRecord`s on the configured proxy reward.
+ *   - `proposeChange(failures)` — proposes steering changes that target
+ *     the rubrics with the lowest predictive validity (decorative ones).
+ *     Either reduce their weight in the composite, or recalibrate them.
+ *   - `applyChange(changes, baseline)` — merges the proposed steering
+ *     into the experiment plan.
+ *   - `evaluateChange(plan)` — re-runs the predictive-validity check on
+ *     the post-change runs and reports the delta.
+ *
+ * The result is a closed loop: the rubric weights drift toward the ones
+ * that actually predict deployment outcomes, automatically. Pair with
+ * `runRLCampaign` for the full auto-research story.
+ */
+
+interface PredictiveValidityResearcherOptions {
+    outcomes: OutcomeStore;
+    outcomeMetrics: string[];
+    /** Score threshold below which a run counts as a "failure." Default 0.5. */
+    failureThreshold?: number;
+    /** Spearman bucket below which a rubric is "decorative." Default 0.4. */
+    decorativeThreshold?: number;
+    /** Optional steering-namespace prefix for proposed changes. Default `'rubric_weight'`. */
+    steeringNamespace?: string;
+    /** Override the rubric set the researcher inspects. Default: every numeric `outcome.raw` key seen. */
+    rubrics?: string[];
+    /**
+     * Snapshot stash hook — called with the most recent predictive-validity
+     * report. Useful when a downstream system wants to log rubric drift over
+     * time. Default no-op.
+     */
+    onReport?: (report: RubricPredictiveValidityReport) => void | Promise<void>;
+}
+/**
+ * Concrete `Researcher` driven by `rubricPredictiveValidity`. The brain:
+ * rubrics that don't predict deployment outcomes don't earn weight.
+ */
+declare class PredictiveValidityResearcher implements Researcher {
+    private opts;
+    private lastReport;
+    constructor(opts: PredictiveValidityResearcherOptions);
+    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+    /**
+     * Run the predictive-validity check explicitly against a fresh RunRecord
+     * set. Updates the researcher's cached report so subsequent
+     * `proposeChange` calls have evidence to draw from.
+     */
+    runValidityCheck(runs: RunRecord[]): Promise<RubricPredictiveValidityReport>;
+    /**
+     * Force-feed a predictive-validity report into the researcher state —
+     * useful when the consumer ran the report out-of-band and wants the
+     * researcher's later proposals informed by it.
+     */
+    setReport(report: RubricPredictiveValidityReport): void;
+    getLastReport(): RubricPredictiveValidityReport | null;
+}
+
+/**
+ * `analyzeOptimizationResult` — unifies the pre-0.22 auto-research stack
+ * (`runPromptEvolution`, `runMultiShotOptimization`, reflective-mutation,
+ * Ax/AxRLM trace analyst) with the 0.23 RL bridge in a single call.
+ *
+ * What this fixes: until 0.23 the optimization stack and the RL bridge
+ * lived in parallel namespaces. The optimization primitives produced
+ * `TrialResult[]`; the RL bridge consumed `RunRecord[]`. Trace-analyst
+ * was decoupled from both. `analyzeOptimizationResult` does the wiring
+ * once so consumers don't have to:
+ *
+ *    Optimization (existing primitives)           RL bridge (0.23)
+ *    ──────────────────────────────────           ────────────────
+ *    runPromptEvolution → TrialResult[]    →
+ *    runMultiShotOptimization → MSTrial[]  → analyzeOptimizationResult →
+ *    reflective-mutation → mutations.jsonl →                             ↓
+ *                                                                        │
+ *    ↓ (per-generation inputs flow back)                                 │
+ *    PredictiveValidityResearcher.proposeChange  ←─────────────────────  │
+ *                                                                        │
+ *    ↓                                                                   │
+ *    TraceAnalyst.analyze(progressLog)         ←─────────────────────────┘
+ *
+ * The output of this function is the canonical RL artifact set:
+ * `RunRecord[]` (so every other 0.22+ primitive composes), preference
+ * triples, verifiable reward signals, reward-hacking diagnosis,
+ * sequential interim verdict, and (when wired) trace-analyst summary.
+ *
+ * What this primitive does NOT do: it does not modify the optimization
+ * primitives' internals. They keep producing `TrialResult` and emitting
+ * `onProgress` events; this function bridges *after* the sweep completes.
+ * Per-step capture-integrity (raw HTTP events from inside the score
+ * adapter) requires the consumer to wire `RawProviderSink` into their
+ * own `ScoreAdapter` — that's a per-consumer integration point.
+ */
+
+interface AnalyzeOptimizationResultOptions {
+    /**
+     * The optimization output. Either a `PromptEvolutionResult` or a
+     * `MultiShotOptimizationResult`. The function detects which by
+     * structural typing and produces canonical `RunRecord[]` from either.
+     */
+    result: PromptEvolutionResult | MultiShotOptimizationResult;
+    /** Adapter context — `commitSha`, `model`, `promptHash`, `configHash`. */
+    ctx: AdapterContext;
+    /** Optional comparator candidate id for paired analyses. */
+    comparator?: string;
+    /** Verifiable-reward extraction options. */
+    verifiableReward?: VerifiableRewardExtractionOptions;
+    /** Preference extraction options. */
+    preferences?: ExtractPreferencesOptions;
+    /** Sequential interim-confidence options. */
+    sequential?: {
+        alpha?: number;
+        bound?: number;
+        rope?: {
+            low: number;
+            high: number;
+        };
+    };
+    /** Outcome calibration store + metrics. */
+    outcomes?: {
+        store: OutcomeStore;
+        metrics: string[];
+    };
+    /** Trainer-format export — DPO + GRPO lookups. */
+    trainerExport?: {
+        dpo?: DpoLookups;
+        grpo?: GrpoLookups;
+    };
+}
+interface AnalyzeOptimizationResultReport {
+    /** All trials promoted to canonical `RunRecord` shape. */
+    runs: RunRecord[];
+    /** Per-run verifiable reward signal. */
+    rewardSignals: Array<{
+        runId: string;
+        reward: VerifiableReward | null;
+    }>;
+    /** Preference triples ready for DPO/PPO/KTO training. */
+    preferences: PreferenceExtractionReport;
+    /** Anytime-valid sequential verdict, when a comparator is supplied. */
+    interimConfidence: InterimReleaseConfidence | null;
+    /** Standing reward-hacking hygiene check. */
+    rewardHacking: RewardHackingReport;
+    /** Predictive validity, when an outcome store is supplied. */
+    predictiveValidity: RubricPredictiveValidityReport | null;
+    /** Trainer-export rows, populated only for the formats requested. */
+    trainerRows: {
+        dpo?: DpoExportRow[];
+        grpo?: GrpoExportRow[];
+    };
+    /** One-line summary suitable for logs. */
+    summary: string;
+}
+/**
+ * Convert an optimization sweep output into a fully-analysed RL artifact
+ * set. Idempotent and read-only with respect to the optimization result.
+ */
+declare function analyzeOptimizationResult(opts: AnalyzeOptimizationResultOptions): Promise<AnalyzeOptimizationResultReport>;
+
+export { type RewardHackingFinding as $, type AdaptationCurve as A, type BradleyTerryFit as B, type CellObservation as C, type DetectRewardHackingInput as D, type EloOptions as E, type Finding as F, type GrpoExportRow as G, type GrpoLookups as H, type LayerStatus as I, type OffPolicyOptions as J, type OffPolicyTrajectory as K, type LayerResult as L, MultiLayerVerifier as M, type ParetoPointInput as N, type OffPolicyEstimate as O, type PairwiseOutcome as P, PredictiveValidityResearcher as Q, type PredictiveValidityResearcherOptions as R, type Severity as S, type PreferenceExtractionReport as T, type PreferenceStrategy as U, type VerifyContext as V, type PreferenceTriple as W, type PrmExportRow as X, type PrmLookups as Y, type PrmTrainingTriple as Z, type RLCampaignResult as _, type Layer as a, toTRLFormat as a$, type RewardHackingReport as a0, type RewardHackingSignal as a1, type RunAdaptationCurveOptions as a2, type RunComputeCurveOptions as a3, type RunRLCampaignOptions as a4, type RunwiseStepSummary as a5, type ScenarioPerturbation as a6, type ScenarioPerturbationKind as a7, type SelfConsistencyOptions as a8, type SelfConsistencyResult as a9, fitBradleyTerry as aA, gradeSemanticStatus as aB, injectIrrelevantClause as aC, inverseProbabilityWeighting as aD, observationsFromRunRecords as aE, offPolicyEstimateAll as aF, prmTrainingPairs as aG, renameVariables as aH, runAdaptationCurve as aI, runComputeCurve as aJ, runContaminationProbe as aK, runRLCampaign as aL, runwiseStepRewardSummary as aM, selfConsistency as aN, selfNormalizedImportanceWeighting as aO, shuffleOrder as aP, stepRewardsToJsonl as aQ, thompsonCurriculum as aR, toAnthropicFormat as aS, toDpoJsonl as aT, toDpoRows as aU, toGrpoJsonl as aV, toGrpoRows as aW, toPrmJsonl as aX, toPrmRows as aY, toSftJsonl as aZ, toSftRows as a_, type SftExportRow as aa, type SftLookups as ab, type StepReward as ac, type StepRewardJsonlRow as ad, type StepScorer as ae, type ThompsonCurriculumOptions as af, type VarianceCurriculumOptions as ag, type VerifiableReward as ah, type VerifiableRewardExtractionOptions as ai, type VerifiableRewardSource as aj, type VerificationReport as ak, type VerifyOptions as al, adversarialScenarioSearch as am, analyzeOptimizationResult as an, applyEloUpdate as ao, bestOfN as ap, buildPairwiseFromCampaign as aq, compareAdaptationCurves as ar, detectRewardHacking as as, doublyRobust as at, extractPreferences as au, extractStepRewards as av, extractVerifiableReward as aw, extractVerifiableRewardsFromRecords as ax, filterDeterministicallyRewarded as ay, firstPassK as az, type AdaptationPoint as b, trialToRunRecord as b0, trialsToRunRecords as b1, varianceBasedCurriculum as b2, variantAggregateToRunRecord as b3, verificationReportToRunRecord as b4, paretoFrontier as b5, type AdaptationRunner as c, type AdapterContext as d, type AdversarialMutation as e, type AdversarialScenario as f, type AdversarialSearchOptions as g, type AdversarialSearchReport as h, type AnalyzeOptimizationResultOptions as i, type AnalyzeOptimizationResultReport as j, type BradleyTerryRating as k, type BuildPairwiseFromCampaignInput as l, type CompareCurvesResult as m, type ComputeBestOfNOptions as n, type ComputeBestOfNResult as o, type ComputeCurve as p, type ComputeCurveBudget as q, type ComputeCurvePoint as r, type ContaminationProbeInput as s, type ContaminationProbeOptions as t, type ContaminationProbeReport as u, type CurriculumAllocation as v, type DpoExportRow as w, type DpoLookups as x, type ExtractPreferencesOptions as y, type ExtractStepRewardsOptions as z };