@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/rl.d.ts

@@ -1,4092 +0,0 @@
-type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
-type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | {
-    [key: string]: AgentProfileJson;
-};
-type AgentProfileDimensionValue = string | number | boolean | null;
-interface AgentProfileSource {
-    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
-    kind: string;
-    /** sha256 over the canonical source profile object. */
-    hash: string;
-}
-interface AgentProfileSourceInput {
-    kind: string;
-    /** Precomputed sha256 for callers that already sign their profile artifact. */
-    hash?: string;
-    /** Full canonical runtime profile; hashed and then discarded from the cell. */
-    profile?: AgentProfileJson;
-}
-interface AgentProfileHarness {
-    id: string;
-    version?: string;
-    hash?: string;
-}
-interface AgentProfileCellInput {
-    profileId: string;
-    sourceProfile: AgentProfileSourceInput;
-    harness?: AgentProfileHarness;
-    model?: string;
-    promptHash?: string;
-    dimensions?: Record<string, AgentProfileDimensionValue>;
-}
-interface AgentProfileCell {
-    schemaVersion: AgentProfileCellSchemaVersion;
-    cellId: string;
-    profileId: string;
-    sourceProfile: AgentProfileSource;
-    harness?: AgentProfileHarness;
-    model?: string;
-    promptHash?: string;
-    dimensions?: Record<string, AgentProfileDimensionValue>;
-}
-
-type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
-interface BudgetSpec {
-    tokens?: number;
-    wallMs?: number;
-    calls?: number;
-    usd?: number;
-}
-interface RunOutcome$1 {
-    score?: number;
-    pass?: boolean;
-    failureClass?: FailureClass;
-    notes?: string;
-}
-/**
- * Layer — optional classification in a nested build workflow.
- * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
- * `app-build`: sandbox harness that compiled + tested the generated scaffold.
- * `app-runtime`: a run of the generated agent against a domain scenario.
- * `meta`: any meta-eval (judge replay, correlation analysis).
- */
-type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
-interface Run {
-    runId: string;
-    /**
-     * Stable identifier of the scenario being executed.
-     *
-     * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
-     * input WITHOUT this field, substituting a sensible default
-     * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
-     * curated scenario to anchor to (runtime / operator / meta-eval runs). This
-     * keeps the persisted shape unambiguous for downstream filters + aggregations
-     * while removing the boilerplate of inventing placeholder ids at the call site.
-     */
-    scenarioId: string;
-    variantId?: string;
-    datasetVersion?: string;
-    /** Git SHA of agent code at run time. */
-    codeSha?: string;
-    /** Hash of the prompt template + any system prompt. */
-    promptSha?: string;
-    /** Model id + date + system-prompt hash, concatenated. */
-    modelFingerprint?: string;
-    seed?: number;
-    /** Arbitrary environment markers (shell, docker version, tz). */
-    envFingerprint?: Record<string, string>;
-    /** Version of the redaction rules applied to this run. */
-    redactionVersion?: string;
-    /** Parent run in a nested build workflow. A builder run's children are
-     *  app-build runs; those children are app-runtime runs. */
-    parentRunId?: string;
-    /** Stable project identifier — groups runs across chats + sessions. */
-    projectId?: string;
-    /** Chat/conversation identifier within a project. */
-    chatId?: string;
-    /** Layer classification — hint for aggregation; not enforced. */
-    layer?: RunLayer;
-    startedAt: number;
-    endedAt?: number;
-    status: RunStatus;
-    outcome?: RunOutcome$1;
-    budget?: BudgetSpec;
-    /** Free-form labels for downstream grouping. */
-    tags?: Record<string, string>;
-}
-type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
-type SpanStatus = 'ok' | 'error';
-interface SpanBase {
-    spanId: string;
-    parentSpanId?: string;
-    runId: string;
-    kind: SpanKind;
-    name: string;
-    startedAt: number;
-    endedAt?: number;
-    status?: SpanStatus;
-    error?: string;
-    /** Anything not covered by typed fields. Kept deliberately free-form. */
-    attributes?: Record<string, unknown>;
-}
-interface Message {
-    role: 'system' | 'user' | 'assistant' | 'tool';
-    content: string;
-    tokens?: number;
-    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
-    images?: Array<{
-        artifactId?: string;
-        url?: string;
-        mime?: string;
-    }>;
-}
-interface LlmSpan extends SpanBase {
-    kind: 'llm';
-    model: string;
-    messages: Message[];
-    output?: string;
-    inputTokens?: number;
-    /** All generated tokens, including the reasoning subset when present. */
-    outputTokens?: number;
-    cachedTokens?: number;
-    cacheWriteTokens?: number;
-    /** Reasoning-token subset of `outputTokens`. */
-    reasoningTokens?: number;
-    costUsd?: number;
-    finishReason?: string;
-}
-interface ToolSpan extends SpanBase {
-    kind: 'tool';
-    toolName: string;
-    args: unknown;
-    /** False when the source observed the call but did not capture its arguments. */
-    argsCaptured?: boolean;
-    result?: unknown;
-    latencyMs?: number;
-}
-interface RetrievalSpan extends SpanBase {
-    kind: 'retrieval';
-    query: string;
-    hits: Array<{
-        docId: string;
-        score: number;
-        content?: string;
-    }>;
-}
-interface JudgeSpan extends SpanBase {
-    kind: 'judge';
-    judgeId: string;
-    /** Span this judgment applies to. */
-    targetSpanId: string;
-    dimension: string;
-    /** Numeric score (free-range; interpretation up to the judge). */
-    score: number;
-    rationale?: string;
-    evidence?: string;
-}
-interface SandboxSpan extends SpanBase {
-    kind: 'sandbox';
-    image?: string;
-    command?: string;
-    exitCode?: number;
-    testsTotal?: number;
-    testsPassed?: number;
-    stdoutHash?: string;
-    stderrHash?: string;
-    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
-    wallMs?: number;
-}
-interface GenericSpan extends SpanBase {
-    kind: 'agent' | 'custom';
-}
-type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
-type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
-interface TraceEvent {
-    eventId: string;
-    runId: string;
-    spanId?: string;
-    kind: EventKind;
-    timestamp: number;
-    payload: Record<string, unknown>;
-}
-interface BudgetLedgerEntry {
-    runId: string;
-    dimension: keyof BudgetSpec;
-    limit: number;
-    consumed: number;
-    remaining: number;
-    timestamp: number;
-    breached: boolean;
-    /** Span that triggered this entry, if any. */
-    spanId?: string;
-}
-interface Artifact {
-    artifactId: string;
-    runId: string;
-    spanId?: string;
-    contentType: string;
-    sizeBytes: number;
-    /** sha256 in hex. */
-    hash: string;
-    /** External storage URL (R2, S3, filesystem path). */
-    storageUrl?: string;
-    /** Inline content for small blobs — keep under ~64KB. */
-    inlineContent?: string;
-}
-type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
-
-/**
- * Paper-grade RunRecord schema + runtime validator.
- *
- * Every run that participates in a promotion gate, paper table, or
- * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
- * fields are exactly those the paper "Two Loops, Three Roles" requires
- * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
- *
- * This is intentionally NOT a replacement for the rich `Run` /
- * `ProposeReviewReport` / `ScenarioResult` types already in the
- * package. Those are runtime structures with full provenance. A
- * `RunRecord` is the analysis-time projection — the JSON-friendly
- * row you'd put in a parquet file or paste into a notebook.
- *
- * Validate at the boundary:
- *
- *   const rec = validateRunRecord(rawJson)         // throws on missing
- *   const ok  = isRunRecord(rawJson)               // boolean check
- *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
- *
- * The validator runs in pure TS — zod is intentionally NOT a
- * dependency. Round-trip tested in `tests/run-record.test.ts`.
- */
-
-/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
- *  combined train+test pool that the optimizer is allowed to read. */
-type RunSplitTag = 'search' | 'dev' | 'holdout';
-interface RunTokenUsage {
-    input: number;
-    /** All generated tokens charged as output, including reasoning tokens. */
-    output: number;
-    /** Reasoning-token subset of `output`, when the provider reports it. */
-    reasoning?: number;
-    /** Prompt tokens served from a provider cache. */
-    cached?: number;
-    /** Prompt tokens written into a provider cache. */
-    cacheWrite?: number;
-}
-/**
- * How a run's USD amount was obtained.
- *
- * `costUsd` remains mandatory for wire compatibility. New producers should
- * always populate this discriminated union so a missing bill is never
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
- * the legacy `0` sentinel while this field carries the truthful null.
- */
-type RunCostProvenance = {
-    kind: 'observed';
-    usd: number;
-} | {
-    kind: 'estimated';
-    usd: number;
-} | {
-    kind: 'uncaptured';
-    usd: null;
-};
-interface RunJudgeMetadata {
-    model: string;
-    promptVersion: string;
-    /** [0,1] confidence the judge declared. Constant judge confidence
-     *  across many runs is a fallback signal (see `canary.ts`). */
-    confidence: number;
-    /** True if the judge degraded to a fallback path (rules-only,
-     *  prior-call cache, etc.). The canary uses this to alert. */
-    fallback: boolean;
-}
-/**
- * Per-judge / per-dimension breakdown for runs scored by an ensemble of
- * judges over a multi-dimensional rubric.
- *
- * The collapsed `outcome.searchScore` / `holdoutScore` carries the
- * composite the gate uses. The full breakdown belongs here so consumers
- * can answer "which judge disagreed?", "which dimension dragged the
- * composite down?", and "did half the panel fail?" without re-running.
- *
- * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
- * `composite` are convenience projections — derivable but precomputed so
- * downstream IRR primitives (`interRaterReliability`,
- * `corpusInterRaterAgreement`) and reporters don't pay the same
- * aggregation twice.
- *
- * Fail-loud discipline: judges that errored out land in `failedJudges`
- * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
- * run); the explicit list makes a partial-failure recorded as such.
- */
-interface JudgeScoresRecord {
-    /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
-    perJudge: Record<string, Record<string, number>>;
-    /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
-    perDimMean: Record<string, number>;
-    /** Composite mean across all dims and judges. Mirrors the score
-     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
-    composite: number;
-    /** Judges that errored or returned an unparseable verdict. Recorded
-     *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
-     *  not inferred from missing keys in `perJudge`. */
-    failedJudges?: string[];
-    /** Free-form notes the judges emitted (joined across judges or
-     *  first-judge only — consumer's choice). */
-    notes?: string;
-}
-interface RunOutcome {
-    /** Score on the search/optimization split. Optional because a
-     *  holdout-only evaluation only fills `holdoutScore`. */
-    searchScore?: number;
-    /** Score on the held-out split. Optional because a search-only run
-     *  only fills `searchScore`. At least one must be present. */
-    holdoutScore?: number;
-    /** Bag of any other metric the run produced — judge dimensions,
-     *  pass/fail counters, latency stats, etc. Numeric only — keeps
-     *  reporters honest. */
-    raw: Record<string, number>;
-    /** Per-judge / per-dim breakdown. Consumers writing ensemble
-     *  judgements populate this; substrate primitives like
-     *  `interRaterReliability` and `corpusInterRaterAgreement` accept
-     *  these records as input. Optional — single-judge or scalar-only
-     *  runs leave it unset. */
-    judgeScores?: JudgeScoresRecord;
-    /** Authenticity / realness verdict — did the run build the REAL thing on the
-     *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
-     *  with an authenticity config populate it. Carried in the corpus so the
-     *  flywheel / off-policy learning can optimize for real completion, not gamed
-     *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
-     *  must not count as a real success regardless of `score`. */
-    realness?: {
-        score: number;
-        gated: boolean;
-        reason?: string;
-    };
-}
-/**
- * Mandatory paper-grade fields for a single evaluation run. Optional
- * fields are extension points; mandatory fields throw if missing.
- *
- * Hash discipline:
- *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
- *     model (after any steering bundle merge).
- *   - `configHash` is the sha256 of the effective run config (model,
- *     temperature, tools, judges, splits). The pair (promptHash,
- *     configHash) uniquely identifies an experiment cell.
- *
- * Model snapshot discipline:
- *   - `model` MUST encode a snapshot version. Bare aliases like
- *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
- *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
- */
-interface RunRecord {
-    /** UUID for the run. */
-    runId: string;
-    /** Logical experiment grouping (a treatment vs a baseline within
-     *  the same sweep should share `experimentId`). */
-    experimentId: string;
-    /** Stable identifier for the candidate (variant) being run. The
-     *  promotion gate compares two `candidateId`s on matched items. */
-    candidateId: string;
-    /** RNG seed for the run. Always recorded — silent re-seeding is
-     *  the most common cause of non-reproducible numbers. */
-    seed: number;
-    /** Model identifier WITH snapshot version. */
-    model: string;
-    /** sha256 of the effective prompt (post-steering). */
-    promptHash: string;
-    /** sha256 of the effective config. */
-    configHash: string;
-    /** Git SHA the harness was run from. */
-    commitSha: string;
-    /** End-to-end wall-clock duration in milliseconds. */
-    wallMs: number;
-    /** Time spent queued before execution started, if known. */
-    queueMs?: number;
-    /** Total USD cost. Mandatory — runs without a cost number are
-     *  unbounded by definition and must not be admitted into the gate.
-     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
-     *  inspect `costProvenance` before treating it as observed. */
-    costUsd: number;
-    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
-     *  Optional only so existing serialized RunRecords remain valid. */
-    costProvenance?: RunCostProvenance;
-    /** Token usage breakdown. */
-    tokenUsage: RunTokenUsage;
-    /** Judge-side metadata, if a judge was used. */
-    judgeMetadata?: RunJudgeMetadata;
-    /** Per-split scores + raw bag. */
-    outcome: RunOutcome;
-    /** Canonical, cross-agent failure class drawn from the shared
-     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
-     *  "which failure dominates across the whole fleet" answerable in ONE
-     *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it via the substrate classifier; leave unset only when the failure
-     *  genuinely can't be classified. */
-    failureClass?: FailureClass;
-    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
-     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
-    failureMode?: string;
-    /** Which split this run was drawn from. */
-    splitTag: RunSplitTag;
-    /**
-     * Stable scenario identifier the run was scored against. Optional for
-     * backwards compatibility, but **strongly recommended**: every primitive
-     * that pairs runs by scenario (preferences, paired stats, BT tournament)
-     * keys on this. The campaign artifact populates it canonically; legacy
-     * runs without it fall back to inference from `outcome.raw.scenario_id`
-     * or `experimentId`.
-     */
-    scenarioId?: string;
-    /**
-     * Canonical identity for the agent profile cell that produced this row:
-     * profile artifact hash plus optional harness/model/prompt/reporting
-     * dimensions. Use `agentProfile.cellId` to group persona sweeps and
-     * longitudinal reports by the complete source profile, not by a loose
-     * candidate label or opaque config hash.
-     */
-    agentProfile?: AgentProfileCell;
-}
-
-/**
- * Adaptive curriculum / active scenario selection.
- *
- * Fixed scenario sets waste sample budget on cells the policy already
- * passes (no information left) and cells the policy never passes (no
- * gradient available either). Active learning over scenarios fixes this
- * by allocating the next sample budget to cells where the policy's
- * outcome is *uncertain* — those carry the most decision-relevant signal.
- *
- * This module ships two complementary strategies:
- *
- *   1. **Variance-based** — score each (variant, scenario) cell by the
- *      empirical variance of past observations. Allocate next-round budget
- *      proportional to variance. Standard active-learning-by-uncertainty
- *      heuristic; works well when the policy is non-deterministic and
- *      cells differ in observation noise.
- *
- *   2. **Bandit-based (Thompson sampling)** — model each (variant,
- *      scenario) cell as a Beta-Bernoulli arm; sample a posterior; pick
- *      cells whose posterior mean is closest to the per-scenario decision
- *      threshold. The right primitive when scenarios are
- *      "pass/fail" rather than continuous, and when promotion gates fire
- *      at a known threshold (e.g., 0.5).
- *
- * The output is a *next-round budget allocation* — a list of (variant,
- * scenario, count) triples. The consumer's matrix runner consumes the
- * allocation, runs those cells, feeds the new observations back. Loop.
- *
- * Out of scope (deliberate): scenario *generation* — that's the
- * adversarial primitive's job. This module allocates over an existing
- * scenario pool.
- */
-
-interface CellObservation {
-    variantId: string;
-    scenarioId: string;
-    /** Observed score in [0, 1]. */
-    score: number;
-    /** For Bernoulli arms — derive from the score with a threshold if needed. */
-    pass?: boolean;
-}
-interface CurriculumAllocation {
-    variantId: string;
-    scenarioId: string;
-    /** How many additional reps to run on this cell. */
-    count: number;
-    /** Strategy-specific reason for the allocation. */
-    reason: string;
-}
-interface VarianceCurriculumOptions {
-    /** Total reps to allocate across all cells. */
-    budget: number;
-    /**
-     * Smoothing prior on variance — keeps the allocator from concentrating
-     * on a cell with one observation just because its 1-sample variance is
-     * 0. Default 0.05.
-     */
-    variancePrior?: number;
-    /**
-     * Minimum reps per cell — even when the variance estimate is low, give
-     * every cell at least this many. Default 1.
-     */
-    floorPerCell?: number;
-}
-/**
- * Variance-proportional allocation. For each cell, estimate variance from
- * past observations + a prior, then allocate the budget proportional to
- * (sqrt(variance) + 1/sqrt(n)) — a classical optimal-allocation rule
- * (Neyman 1934) that balances "explore noisy cells" with "explore
- * under-sampled cells."
- */
-declare function varianceBasedCurriculum(observations: CellObservation[], candidateCells: Array<{
-    variantId: string;
-    scenarioId: string;
-}>, opts: VarianceCurriculumOptions): CurriculumAllocation[];
-interface ThompsonCurriculumOptions {
-    budget: number;
-    /**
-     * The per-scenario decision threshold. Cells whose posterior mean is
-     * closest to this get the most budget — that's where the next observation
-     * has the highest information value for the gate decision. Default 0.5.
-     */
-    decisionThreshold?: number;
-    /** Beta prior parameters. Default α=β=1 (uniform). */
-    priorAlpha?: number;
-    priorBeta?: number;
-    /** Seed the Thompson sampler. Default unset (Math.random). */
-    seed?: number;
-}
-/**
- * Thompson-sampling-style allocation for pass/fail cells. For each cell:
- *
- *   - Maintain Beta(α + passes, β + failures) posterior on pass-rate
- *   - Allocation weight ∝ exp(-((sampledMean - threshold) / σ)^2):
- *     cells whose sampled posterior straddles the decision boundary get
- *     the most weight; cells already clearly above or below get less.
- *
- * This is the right primitive when promotion gates fire at a known
- * threshold and you want to sharpen the posterior near the boundary.
- */
-declare function thompsonCurriculum(observations: CellObservation[], candidateCells: Array<{
-    variantId: string;
-    scenarioId: string;
-}>, opts: ThompsonCurriculumOptions): CurriculumAllocation[];
-/** Convenience: extract `CellObservation[]` directly from `RunRecord[]`. */
-declare function observationsFromRunRecords(runs: RunRecord[], opts?: {
-    passThreshold?: number;
-    useHoldout?: boolean;
-}): CellObservation[];
-
-/**
- * Sample-efficient adaptation evaluation.
- *
- * For foundation-model-based agents, the load-bearing capability isn't
- * raw end-state performance — it's *how fast the agent reaches that
- * performance from cold start*. The same model with a worse prompt that
- * adapts in 5 demonstrations beats the same model with a better prompt
- * that needs 50. Standard meta-learning eval (Finn et al., MAML, RL² lit)
- * reports an *adaptation curve*: score after k=0, 1, 2, 4, 8, 16, …
- * in-context examples or fine-tune steps.
- *
- * This module ships:
- *
- *   1. `runAdaptationCurve` — given a runner that takes k demonstrations
- *      and returns a score, produce the (k, score) curve.
- *   2. `compareAdaptationCurves` — paired comparison across two policies.
- *      Returns per-k delta with bootstrap CIs and an "area-under-curve"
- *      summary statistic.
- *   3. `firstPassK` — for pass/fail evaluation, the minimum k at which
- *      the policy reliably passes (≥ pass-rate threshold over reps).
- *
- * Use cases:
- *   - Compare two prompt designs that have similar end-state performance
- *     but different in-context efficiency.
- *   - Decide between fine-tuning and prompting based on adaptation cost.
- *   - Detect when a policy "memorizes" k=0 inputs vs. genuinely adapts.
- */
-interface AdaptationRunner<S> {
-    /**
-     * Runs the policy on `scenario` with `k` demonstrations. Returns a
-     * scalar score in [0, 1]. The runner is responsible for any caching;
-     * the harness calls it once per (scenario, k, rep) cell.
-     */
-    run(args: {
-        scenario: S;
-        k: number;
-        rep: number;
-    }): Promise<number>;
-}
-interface RunAdaptationCurveOptions<S> {
-    scenarios: S[];
-    /** Number-of-shots to evaluate at. Default `[0, 1, 2, 4, 8, 16]`. */
-    ks?: number[];
-    /** Reps per (scenario, k) cell. Default 3. */
-    reps?: number;
-    runner: AdaptationRunner<S>;
-    /** Pass-rate threshold for `firstPassK` reporting. Default 0.5. */
-    passThreshold?: number;
-}
-interface AdaptationPoint {
-    k: number;
-    meanScore: number;
-    passRate: number;
-    std: number;
-    n: number;
-    /** Per-scenario means at this k. */
-    perScenario: Array<{
-        scenarioId: string;
-        meanScore: number;
-        passes: number;
-        total: number;
-    }>;
-}
-interface AdaptationCurve {
-    points: AdaptationPoint[];
-    /**
-     * Smallest `k` at which `passRate ≥ passThreshold`. `null` if no `k`
-     * tested reaches it.
-     */
-    firstPassK: number | null;
-    /**
-     * Area under the (k, meanScore) curve, normalized by max-k. A
-     * single-number summary of "how well does this policy adapt from
-     * cold-start to fully-conditioned." Higher = better adapter.
-     */
-    adaptationArea: number;
-}
-declare function runAdaptationCurve<S extends {
-    scenarioId?: string;
-}>(opts: RunAdaptationCurveOptions<S>): Promise<AdaptationCurve>;
-interface CompareCurvesResult {
-    perK: Array<{
-        k: number;
-        deltaMean: number;
-        aLow: number;
-        aHigh: number;
-        bLow: number;
-        bHigh: number;
-    }>;
-    areaDelta: number;
-    firstPassKDelta: number | null;
-    /** Verdict: 'a_better' | 'b_better' | 'similar'. */
-    verdict: 'a_better' | 'b_better' | 'similar';
-    /** Rationale, ready to render. */
-    rationale: string;
-}
-/**
- * Paired comparison of two adaptation curves. Per-k deltas with 95%
- * bootstrap CIs (constructed from each curve's `perScenario` per-k means
- * — the bootstrap unit is the scenario, not the rep).
- */
-declare function compareAdaptationCurves(a: AdaptationCurve, b: AdaptationCurve, opts?: {
-    confidence?: number;
-    bootstrapResamples?: number;
-    seed?: number;
-}): CompareCurvesResult;
-/** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
-declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
-
-/**
- * Adversarial mutation contract.
- *
- * `AdversarialMutation<S>` is the scenario-mutation strategy the fuzz harness
- * (`fuzzAgent`, src/fuzz) drives: paraphrase, edge-case substitution, or
- * compositional combination of a scenario the policy currently passes, looking
- * for the tail inputs that break it. The harness supplies the loop; consumers
- * supply the mutations and the failure detector.
- */
-interface AdversarialMutation<S> {
-    id: string;
-    /**
-     * Mutate one scenario. Return null to skip; return one or more new
-     * scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
-     */
-    mutate(parent: S, rng: () => number): Promise<S[]> | S[];
-}
-
-/**
- * Test-time compute scaling curves.
- *
- * The test-time-compute frontier paper (Snell et al. 2024) and the
- * subsequent o1-style scaling work both show that LLM-agent capability
- * is a function of the compute budget at inference, not just of the
- * training run. The right way to characterize a candidate is therefore
- * a *curve* — score at compute budgets {1×, 4×, 16×, …} — not a single
- * point.
- *
- * This module ships:
- *
- *   1. The compute-curve harness — `runComputeCurve(runner, budgets)` —
- *      that evaluates one candidate at a sequence of compute budgets
- *      and returns the (compute, score) curve.
- *   2. A best-of-N evaluator — `bestOfN(runner, n, scoreFn)` — the
- *      simplest test-time-compute scaling primitive: sample N
- *      independent rollouts, return the best.
- *   3. A self-consistency evaluator — `selfConsistency(runner, n)` —
- *      the majority-vote variant of best-of-N for tasks with a small
- *      categorical answer space.
- *   4. Pareto-frontier extraction over multiple candidates — given
- *      (candidate, compute, score) tuples, return the set of
- *      candidate-compute combinations that aren't dominated.
- *
- * Caveat: "compute" here is the caller's notion of a compute unit. For
- * agent eval that's typically wall-time × parallelism, or token budget,
- * or LLM-call count. We accept whatever the caller provides; the curve
- * is on whatever axis they pick.
- */
-interface ComputeCurveBudget {
-    /** Identifier — for the report. Common: '1x', '4x', '16x'. */
-    id: string;
-    /** Numeric value on the chosen axis (tokens, calls, USD, ms — caller picks). */
-    cost: number;
-    /** Free-form metadata (the caller can carry per-budget config). */
-    meta?: Record<string, unknown>;
-}
-interface ComputeCurvePoint {
-    budgetId: string;
-    cost: number;
-    score: number;
-    /** Number of underlying samples used at this budget. */
-    samples: number;
-    /** Optional spread / variance information. */
-    std?: number;
-    /** Any extra metrics the runner returned. */
-    metrics?: Record<string, number>;
-}
-interface ComputeCurve {
-    candidateId: string;
-    points: ComputeCurvePoint[];
-    /** Rough exponent fit: score ≈ a + b * log(cost). Useful for "how steep is the curve?" */
-    logSlope: number | null;
-    /** Best (highest-score) point on the curve. */
-    best: ComputeCurvePoint;
-}
-interface RunComputeCurveOptions {
-    candidateId: string;
-    budgets: ComputeCurveBudget[];
-    /**
-     * Run the candidate at one budget. Returns the realized score plus
-     * optional spread + extra metrics.
-     */
-    runAtBudget: (budget: ComputeCurveBudget) => Promise<{
-        score: number;
-        samples: number;
-        std?: number;
-        metrics?: Record<string, number>;
-    }>;
-}
-declare function runComputeCurve(opts: RunComputeCurveOptions): Promise<ComputeCurve>;
-interface ComputeBestOfNOptions<O> {
-    /** Number of independent samples to draw. */
-    n: number;
-    /** Sampler — produces one rollout. */
-    sample: (sampleIdx: number) => Promise<O>;
-    /** Score one rollout. */
-    scoreFn: (rollout: O) => Promise<number> | number;
-}
-interface ComputeBestOfNResult<O> {
-    best: O;
-    bestScore: number;
-    scores: number[];
-    meanScore: number;
-    /** Index of the best rollout, for diagnostics. */
-    bestIndex: number;
-}
-/** The simplest test-time scaling primitive. */
-declare function bestOfN<O>(opts: ComputeBestOfNOptions<O>): Promise<ComputeBestOfNResult<O>>;
-interface SelfConsistencyOptions<O> {
-    n: number;
-    sample: (sampleIdx: number) => Promise<O>;
-    /** Extract the canonical answer key (string) from a rollout. */
-    answerKey: (rollout: O) => string;
-}
-interface SelfConsistencyResult<O> {
-    /** Modal answer (the majority vote). */
-    answer: string;
-    /** Fraction of samples voting for the modal answer in [0, 1]. */
-    agreement: number;
-    /** Histogram of all answers. */
-    histogram: Record<string, number>;
-    /** A representative rollout that voted for the modal answer. */
-    representative: O;
-    /** All rollouts. */
-    rollouts: O[];
-}
-/**
- * Self-consistency / majority-vote test-time scaling. For tasks with a
- * small categorical answer space (math problems, multiple choice).
- */
-declare function selfConsistency<O>(opts: SelfConsistencyOptions<O>): Promise<SelfConsistencyResult<O>>;
-/**
- * Pareto frontier over (candidate, compute, score) tuples. A point is on
- * the frontier iff no other point dominates it in both score (higher
- * better) and cost (lower better). Returns the frontier sorted ascending
- * by cost.
- */
-interface ParetoPointInput {
-    candidateId: string;
-    budgetId: string;
-    cost: number;
-    score: number;
-}
-declare function paretoFrontier(points: ParetoPointInput[]): ParetoPointInput[];
-
-/**
- * Contamination probe — held-out perturbation tests.
- *
- * The bug class: once a benchmark scenario set is published, models train
- * on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and
- * MMLU-Pro all exist because their predecessors got contaminated within
- * months. The right defense is to keep a held-out *perturbed* version of
- * every scenario — same task, slightly different surface — and check
- * whether scores diverge significantly. Genuine capability transfers; rote
- * memorization doesn't.
- *
- * This module ships the probe contract:
- *
- *   1. A `ScenarioPerturbation` strategy type — function that produces a
- *      perturbed scenario from an original.
- *   2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs
- *      both halves and reports per-scenario score divergence + a global
- *      contamination verdict via paired Wilcoxon.
- *   3. Several stock perturbations: `renameVariables`, `shuffleOrder`,
- *      `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the
- *      task's structural difficulty while breaking surface memorization.
- *
- * The verdict is conservative: if the perturbed-vs-original score
- * difference is statistically significant (BH-adjusted p < 0.05) AND
- * the median drop is > 5 percentage points, we flag *contamination
- * suspected*. False positives are possible (the perturbation might
- * actually be harder); the default is to flag for review, not to
- * autoreject.
- */
-type ScenarioPerturbationKind = 'rename_variables' | 'shuffle_order' | 'paraphrase' | 'inject_irrelevant_clause' | 'custom';
-interface ScenarioPerturbation<S> {
-    kind: ScenarioPerturbationKind;
-    /** Apply to one scenario, return its perturbed sibling. */
-    apply: (scenario: S) => Promise<S> | S;
-    /** Optional id — for the report. */
-    id?: string;
-}
-interface ContaminationProbeInput<S> {
-    /** Identity of every scenario. The probe's `runFingerprint` keys on these. */
-    scenarioId: (s: S) => string;
-    /** Original scenarios. */
-    originals: S[];
-    /**
-     * Either pre-computed perturbations (one per original, same order) OR a
-     * `perturbation` strategy that synthesizes them on the fly.
-     */
-    perturbed?: S[];
-    perturbation?: ScenarioPerturbation<S>;
-    /**
-     * Run the policy/agent against one scenario and return a scalar score
-     * in [0, 1]. The probe doesn't care what the policy is — that's the
-     * caller's contract.
-     */
-    scoreFn: (s: S) => Promise<number>;
-}
-interface ContaminationProbeOptions {
-    /** Drop scores below this from the probe; treats partial failures separately. Default 0. */
-    scoreFloor?: number;
-    /**
-     * BH-FDR threshold for declaring contamination on each per-scenario
-     * delta. Default 0.05.
-     */
-    fdr?: number;
-    /**
-     * Minimum median per-scenario drop to flag global contamination. Default
-     * 0.05 (5 percentage points). Smaller drops may be noise.
-     */
-    minMedianDrop?: number;
-}
-interface ContaminationProbeReport {
-    perScenario: Array<{
-        scenarioId: string;
-        originalScore: number;
-        perturbedScore: number;
-        delta: number;
-        /** Per-scenario q-value (single-test BH for a single scenario). Mainly for display. */
-        qValue: number;
-    }>;
-    /** Wilcoxon paired-test on the deltas. */
-    pairedTest: {
-        w: number;
-        p: number;
-    };
-    medianDelta: number;
-    meanDelta: number;
-    contaminationSuspected: boolean;
-    reason: string;
-    /** Number of scenarios processed. */
-    n: number;
-}
-declare function runContaminationProbe<S>(input: ContaminationProbeInput<S>, opts?: ContaminationProbeOptions): Promise<ContaminationProbeReport>;
-/**
- * Identifier-rename perturbation for code/text scenarios. Replaces every
- * occurrence of the listed identifiers with synthesized aliases. Use when
- * the scenario's structural difficulty is independent of variable names
- * (e.g. SWE-Bench-style coding tasks).
- */
-declare function renameVariables<S extends {
-    prompt: string;
-}>(identifiers: string[], rename?: (name: string, idx: number) => string): ScenarioPerturbation<S>;
-/**
- * Order-shuffle perturbation. Reshuffles a list-shaped section of the
- * prompt (for QA scenarios that present options A/B/C/D — answer depends
- * on the option labels, not order). Caller provides the section extractor.
- */
-declare function shuffleOrder<S extends {
-    prompt: string;
-}>(shuffleSection: (prompt: string, rng: () => number) => string, seed: number): ScenarioPerturbation<S>;
-/**
- * Inject-irrelevant-clause perturbation. Adds a benign sentence that
- * shouldn't change the answer. Tests for "did the model just memorize
- * the input string."
- */
-declare function injectIrrelevantClause<S extends {
-    prompt: string;
-}>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
-
-/**
- * Preference dataset extraction — bridge from `RunRecord[]` to RL training.
- *
- * Production RLHF / DPO / KTO / SimPO pipelines need preference triples:
- * `(prompt, chosen, rejected)`. The campaign artifact already contains the
- * ingredients — every (variantId, scenarioId, seed) cell is a candidate
- * that ran the same prompt against the same scenario, scored by the same
- * judge — but turning that into a clean preference dataset requires
- * deciding *what counts as a preference*.
- *
- * This module ships three preference-extraction strategies with explicit
- * tradeoffs, plus a unified output type compatible with HuggingFace TRL,
- * Anthropic finetuning JSONL, and OpenAI fine-tuning APIs. The strategies
- * are deliberately not auto-magical — picking the wrong one corrupts the
- * gradient.
- *
- * Strategies:
- *
- *   1. **`paired-by-scenario-and-seed`** — exact-match comparisons. For
- *      each scenario × seed pair, compare every (variantA, variantB) on
- *      that exact (scenario, seed). Matches scenarios so the comparison
- *      isolates variant effects. Highest signal-to-noise; smallest
- *      dataset (only matched pairs count).
- *
- *   2. **`paired-by-scenario`** — looser matching. For each scenario,
- *      compare every (variantA, variantB) where both have ≥ 1 run on the
- *      same scenario. Aggregates across seeds to compute mean scores per
- *      (variant, scenario), then forms preferences from the means. More
- *      data, lower per-pair signal.
- *
- *   3. **`top-vs-bottom`** — coarsest. Within each scenario, the highest-
- *      scoring run is `chosen`, the lowest is `rejected`. Smallest dataset
- *      per scenario but biggest score gap per pair. Useful for early
- *      bootstrapping when you have few variants.
- *
- * The output `PreferenceTriple` is *agent-eval-canonical* but trivially
- * mappable to TRL's `DPODataset` shape (`prompt`, `chosen`, `rejected`)
- * via the `toTRLFormat` helper.
- */
-
-type PreferenceStrategy = 'paired-by-scenario-and-seed' | 'paired-by-scenario' | 'top-vs-bottom';
-interface PreferenceTriple {
-    /** The scenario (input) the variants were run against. */
-    scenarioId: string;
-    /** RunRecord ids on each side, for traceability. */
-    chosenRunId: string;
-    rejectedRunId: string;
-    /** Variant ids — load-bearing for the RL update. */
-    chosenVariantId: string;
-    rejectedVariantId: string;
-    /** The score gap between chosen and rejected. Larger = stronger signal. */
-    marginScore: number;
-    /**
-     * Optional `(chosen_score, rejected_score)` pair for soft-margin DPO
-     * variants. Omitted for `top-vs-bottom` runs that don't carry meaningful
-     * scalar gaps.
-     */
-    scores?: {
-        chosen: number;
-        rejected: number;
-    };
-    /** Tie-breaker — when multiple seeds match this scenario, the one used. */
-    seed?: number;
-    /**
-     * Free-form metadata propagated from the run records — e.g. original
-     * prompt-hash, model, etc. Lets the RL trainer reconstruct the prompt.
-     */
-    meta: {
-        chosenPromptHash: string;
-        rejectedPromptHash: string;
-        chosenConfigHash: string;
-        rejectedConfigHash: string;
-        chosenModel: string;
-        rejectedModel: string;
-    };
-}
-interface ExtractPreferencesOptions {
-    strategy?: PreferenceStrategy;
-    /**
-     * Minimum score gap required to admit a pair. Pairs below this are
-     * dropped — they're noise, not signal. Default 0.05 (5% of [0,1]).
-     */
-    minMargin?: number;
-    /**
-     * Optional split tag filter — restrict to runs from one split. Default
-     * `'holdout'` (the canonical "real" signal).
-     */
-    splitTag?: RunRecord['splitTag'];
-    /**
-     * Optional reward extractor that overrides `outcome.holdoutScore` /
-     * `outcome.searchScore`. Use to drive preferences off a verifiable
-     * reward instead of the headline score.
-     */
-    rewardOf?: (run: RunRecord) => number | null;
-}
-interface PreferenceExtractionReport {
-    pairs: PreferenceTriple[];
-    /** Number of (scenario, seed) cells inspected. */
-    cellsInspected: number;
-    /** Number of pairs filtered by `minMargin`. */
-    pairsBelowMargin: number;
-    /** Number of cells with only one variant (no comparison possible). */
-    cellsSingleton: number;
-    /** Strategy used. */
-    strategy: PreferenceStrategy;
-}
-/**
- * Convert `RunRecord[]` to preference triples for RL training.
- *
- * Returns a structured report so callers can see how much data was
- * dropped and why (low-margin pairs, singleton cells). For production
- * pipelines, you usually want to:
- *
- *   1. Run a campaign producing 5–10 variants × 50–200 scenarios × 3 seeds
- *   2. Call this with `strategy: 'paired-by-scenario-and-seed'` and a
- *      verifiable-reward extractor as `rewardOf`
- *   3. Pass `report.pairs` to `toTRLFormat` and pipe to your DPO trainer
- */
-declare function extractPreferences(runs: RunRecord[], opts?: ExtractPreferencesOptions): PreferenceExtractionReport;
-/**
- * TRL-compatible export. TRL's `DPODataset` is `{ prompt, chosen, rejected }`
- * but the prompt isn't stored on the RunRecord — only its hash. The caller
- * passes a `promptOf(promptHash)` lookup that the TRL trainer can use.
- */
-declare function toTRLFormat(triples: PreferenceTriple[], promptOf: (hash: string) => string): Array<{
-    prompt: string;
-    chosen: string;
-    rejected: string;
-}>;
-/**
- * Anthropic finetuning JSONL export — `{ system, user, assistant_chosen, assistant_rejected }`
- * shape. Same caveat as TRL: prompt + outputs are content the caller has
- * to map back from the run record / raw event log.
- */
-declare function toAnthropicFormat(triples: PreferenceTriple[]): Array<{
-    scenarioId: string;
-    chosenRunId: string;
-    rejectedRunId: string;
-    margin: number;
-}>;
-
-interface RunFilter {
-    scenarioId?: string;
-    variantId?: string;
-    status?: RunStatus;
-    since?: number;
-    until?: number;
-    tag?: {
-        key: string;
-        value: string;
-    };
-    parentRunId?: string;
-    projectId?: string;
-    chatId?: string;
-    layer?: RunLayer;
-}
-interface SpanFilter {
-    runId?: string;
-    parentSpanId?: string;
-    kind?: SpanKind;
-    name?: string;
-    toolName?: string;
-    judgeId?: string;
-    since?: number;
-    until?: number;
-}
-interface EventFilter {
-    runId?: string;
-    spanId?: string;
-    kind?: EventKind;
-    since?: number;
-    until?: number;
-}
-interface TraceStore {
-    appendRun(run: Run): Promise<void>;
-    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
-    appendSpan(span: Span): Promise<void>;
-    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
-    appendEvent(event: TraceEvent): Promise<void>;
-    appendArtifact(artifact: Artifact): Promise<void>;
-    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
-    getRun(runId: string): Promise<Run | undefined>;
-    listRuns(filter?: RunFilter): Promise<Run[]>;
-    spans(filter?: SpanFilter): Promise<Span[]>;
-    events(filter?: EventFilter): Promise<TraceEvent[]>;
-    budget(runId: string): Promise<BudgetLedgerEntry[]>;
-    artifacts(runId: string): Promise<Artifact[]>;
-}
-
-/**
- * Process reward extraction — step-level credit assignment from trace spans.
- *
- * RL on long-horizon agents needs *step-level* rewards, not run-level
- * ones. The classic credit-assignment problem (Sutton & Barto) requires
- * knowing which sub-decisions in a trajectory contributed to the
- * outcome. Modern systems (DeepSeek-R1, OpenAI o-series, Lightman et al.
- * "Let's Verify Step by Step" 2023) train *process reward models* (PRMs)
- * that score every step, then do RL with the PRM as the reward signal.
- *
- * This module extracts `StepReward[]` from trace spans — one per
- * meaningful step — and ships:
- *
- *   1. `extractStepRewards(store, runId, opts)` — span → step-reward
- *      conversion using configurable per-span scorers (LLM judge over the
- *      span output, deterministic checkers, or a learned PRM).
- *   2. `runwiseStepRewardSummary(stepRewards)` — aggregate the per-step
- *      signal into a credit-assignment-aware run-level score.
- *   3. `prmTrainingPairs(stepRewards, options)` — produce the
- *      `(prefix, suffix_chosen, suffix_rejected)` triples that PRM
- *      training pipelines consume.
- *
- * What we ship: the *extraction* and *aggregation* infrastructure plus
- * the data shape PRM training expects. We do NOT ship the actual PRM
- * training (gradient descent over a transformer is out of scope for a
- * TS package). The interface is the contract; downstream consumers wire
- * their preferred trainer.
- *
- * Caveat the panel will land: this is descriptive credit assignment
- * (which steps correlate with outcome), not causal credit assignment
- * (which steps caused outcome). For causal claims you need
- * counterfactual rollouts or a learned dynamics model. Future work; the
- * descriptive version is what production PRM training actually uses.
- */
-
-interface StepReward {
-    /** Trace span this reward attaches to. */
-    spanId: string;
-    runId: string;
-    /** Index in the trajectory (0-based, in started-at order). */
-    stepIndex: number;
-    /** Span kind (typically 'tool', 'llm', 'judge'). */
-    kind: Span['kind'];
-    /** Span name — for the consumer's downstream filtering. */
-    name: string;
-    /** Step-level reward in [0, 1]. */
-    reward: number;
-    /**
-     * Determinism class. Mirrors the verifiable-reward distinction:
-     * deterministic = test/compile/schema check; probabilistic = LLM judge.
-     */
-    determinism: 'deterministic' | 'probabilistic';
-    /** Optional rationale / evidence — the trainer typically discards. */
-    rationale?: string;
-    /** Optional weight — how much this step contributes to credit assignment. */
-    weight?: number;
-}
-interface StepScorer {
-    /** Span kinds this scorer applies to. */
-    appliesTo: Span['kind'][];
-    /** Returns null to skip the span; returns a `StepReward` shape (without index/runId/spanId, which are filled in). */
-    score(span: Span): Promise<Omit<StepReward, 'spanId' | 'runId' | 'stepIndex'>> | null | undefined;
-}
-interface ExtractStepRewardsOptions {
-    /**
-     * Ordered list of scorers. Each span runs through scorers in order;
-     * the first non-null result wins. If no scorer applies, the span is
-     * skipped (not all spans are training-worthy).
-     */
-    scorers: StepScorer[];
-    /** Optional filter — return null to drop the span entirely before scoring. */
-    preFilter?: (span: Span) => boolean;
-}
-declare function extractStepRewards(store: TraceStore, runId: string, opts: ExtractStepRewardsOptions): Promise<StepReward[]>;
-interface RunwiseStepSummary {
-    runId: string;
-    totalSteps: number;
-    meanReward: number;
-    /** Sum-of-rewards (weighted by `weight ?? 1`). Use as the run-level proxy. */
-    sumWeightedReward: number;
-    /** Fraction of steps where reward < 0.5 — proxy for "where the policy was wrong." */
-    failureFraction: number;
-    /** Maximum drop in reward between consecutive steps — diagnoses a step where things went sideways. */
-    worstStepDelta: number;
-    worstStepIndex: number | null;
-}
-declare function runwiseStepRewardSummary(stepRewards: StepReward[]): RunwiseStepSummary;
-interface PrmTrainingTriple {
-    /** Prefix run-id (or composite key) — the trajectory up to step k-1. */
-    prefixRunId: string;
-    prefixStepIndex: number;
-    /** The step that came next on a high-reward trajectory. */
-    chosenSpanId: string;
-    chosenReward: number;
-    /** A step from a divergent low-reward trajectory at the same prefix length. */
-    rejectedSpanId: string;
-    rejectedReward: number;
-    /** The prefix run came from this run; the rejected step came from `rejectedRunId`. */
-    rejectedRunId: string;
-    marginScore: number;
-}
-/**
- * Build PRM training triples. The shape: pair runs that share an early
- * prefix (same scenario, same first N steps) and diverge later — at the
- * point of divergence, the high-reward run's next step is `chosen`, the
- * low-reward run's next step is `rejected`. This is the canonical PRM
- * training data shape from Lightman et al. and DeepSeek-R1 process
- * supervision.
- *
- * Implementation note: we don't have a way to detect "same prefix" in
- * the general agent setting (token-level prefixes require hashing model
- * outputs). The current heuristic groups by `(scenarioId, prefixSpanName
- * sequence)` — runs are paired when their first K span names match. For
- * production use this should be replaced with a proper trajectory-prefix
- * hash; the heuristic is good enough for early-stage scaffolding.
- */
-declare function prmTrainingPairs(stepRewardsByRun: Map<string, StepReward[]>, opts?: {
-    minMargin?: number;
-    minPrefixLength?: number;
-}): PrmTrainingTriple[];
-
-/**
- * Trainer-format exporters.
- *
- * agent-eval produces canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`,
- * `StepReward[]`, `PrmTrainingTriple[]`). RL training pipelines consume
- * different shapes — Hugging Face TRL, Prime Intellect's prime-rl, OpenAI
- * fine-tuning, Anthropic finetuning, OpenRLHF, verl. Each has its own
- * JSONL conventions. Rather than ship N adapters, this module ships the
- * canonical formats most production pipelines accept and ergonomic helpers
- * for the rest.
- *
- * Shapes:
- *   - **DPO / IPO / KTO** — `{prompt, chosen, rejected}` JSONL. Consumed
- *     by HuggingFace TRL, prime-rl's offline DPO, OpenRLHF.
- *   - **GRPO offline** — `{prompt, completions[], rewards[]}` JSONL.
- *     Consumed by prime-rl GRPO, verl, OpenRLHF.
- *   - **SFT** — `{messages[]}` JSONL with chosen completion as the final
- *     assistant turn. Consumed by HF SFT trainers, OpenAI fine-tuning,
- *     Anthropic finetuning.
- *   - **PRM** — `{prompt, prefix_steps[], chosen_step, rejected_step}` JSONL.
- *     Consumed by Lightman-style PRM trainers and prime-rl's PRM mode.
- *
- * Why ship this in agent-eval rather than a separate adapter package: the
- * canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`, etc.) are
- * agent-eval's contract; without first-party exporters consumers reverse-
- * engineer the mapping every release. The exporters codify it.
- *
- * The exporters take callbacks for any field that isn't on the canonical
- * artifact (specifically: prompt + completion text, since the package
- * stores only their hashes by design — full text is the consumer's
- * trace store / raw event log).
- */
-
-interface DpoLookups {
-    /** Resolve the prompt text for a run (typically from a trace store / raw event sink). */
-    promptOf: (runId: string) => string | Promise<string>;
-    /** Resolve the assistant completion text for a run. */
-    completionOf: (runId: string) => string | Promise<string>;
-}
-interface DpoExportRow {
-    prompt: string;
-    chosen: string;
-    rejected: string;
-    /** Carried-through margin. Some KTO / IPO variants use this. */
-    margin?: number;
-    /** Free-form metadata for downstream filtering / sharding. */
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert preference triples to TRL-compatible DPO rows. The shape
- * `{prompt, chosen, rejected}` is the canonical HuggingFace DPODataset
- * entry; every major DPO trainer accepts it.
- */
-declare function toDpoRows(triples: PreferenceTriple[], lookups: DpoLookups): Promise<DpoExportRow[]>;
-/** Serialize DPO rows as JSONL. One line per row. */
-declare function toDpoJsonl(rows: DpoExportRow[]): string;
-interface GrpoLookups {
-    promptOf: (runId: string) => string | Promise<string>;
-    completionOf: (runId: string) => string | Promise<string>;
-    /** Optional: derive a custom reward from the run. Defaults to score. */
-    rewardOf?: (run: RunRecord) => number | null;
-}
-interface GrpoExportRow {
-    prompt: string;
-    completions: string[];
-    rewards: number[];
-    /** runIds in the same order as `completions[]` for traceability. */
-    runIds: string[];
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert RunRecord[] grouped by `(scenarioId)` into GRPO offline rows —
- * one row per scenario, with one completion per run on that scenario.
- *
- * GRPO (Shao et al. 2024 / DeepSeek-R1) trains on relative advantages
- * within a group of completions for the same prompt; this is the
- * canonical input format.
- */
-declare function toGrpoRows(runs: RunRecord[], lookups: GrpoLookups): Promise<GrpoExportRow[]>;
-declare function toGrpoJsonl(rows: GrpoExportRow[]): string;
-interface SftLookups {
-    promptOf: (runId: string) => string | Promise<string>;
-    completionOf: (runId: string) => string | Promise<string>;
-    /** Optional system message. Default omits. */
-    systemOf?: (run: RunRecord) => string | null | undefined;
-    /** Filter — return false to skip the run (e.g., low score, failed cases). */
-    include?: (run: RunRecord) => boolean;
-}
-interface SftExportRow {
-    messages: Array<{
-        role: 'system' | 'user' | 'assistant';
-        content: string;
-    }>;
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert RunRecord[] into Hugging Face / OpenAI / Anthropic-style
- * conversational SFT rows. By default every record becomes one row;
- * pass `include` to filter (e.g., keep only `score >= 0.8` for
- * rejection-sampling SFT).
- */
-declare function toSftRows(runs: RunRecord[], lookups: SftLookups): Promise<SftExportRow[]>;
-declare function toSftJsonl(rows: SftExportRow[]): string;
-interface PrmLookups {
-    /** Resolve the prompt text for a run. */
-    promptOf: (runId: string) => string | Promise<string>;
-    /** Resolve the trajectory step text for a (runId, spanId) pair. */
-    stepTextOf: (runId: string, spanId: string) => string | Promise<string>;
-    /** Optional: sequence of prefix span ids leading up to the divergence. */
-    prefixOf?: (runId: string, prefixStepIndex: number) => string[] | Promise<string[]>;
-}
-interface PrmExportRow {
-    prompt: string;
-    /** Span ids for the steps before divergence — caller resolves text via `stepTextOf`. */
-    prefixSpanIds: string[];
-    prefixStepText: string[];
-    chosenStep: string;
-    rejectedStep: string;
-    chosenReward: number;
-    rejectedReward: number;
-    marginScore: number;
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert PRM training triples to JSONL rows. Caller's `stepTextOf`
- * callback resolves span text from the consumer's trace store.
- */
-declare function toPrmRows(triples: PrmTrainingTriple[], lookups: PrmLookups): Promise<PrmExportRow[]>;
-declare function toPrmJsonl(rows: PrmExportRow[]): string;
-interface StepRewardJsonlRow {
-    runId: string;
-    spanId: string;
-    stepIndex: number;
-    reward: number;
-    determinism: 'deterministic' | 'probabilistic';
-    weight: number;
-}
-declare function stepRewardsToJsonl(stepRewards: StepReward[]): string;
-
-/**
- * RL dataset packaging + datasheet — the publishable, sellable bundle.
- *
- * The format exporters (`toGrpoRows` / `toSftRows` / `toDpoRows`) already
- * produce trainer-ready shapes (prime-rl GRPO, TRL DPO, conversational SFT).
- * What turns that into a dataset someone can PUBLISH or BUY is the provenance
- * + a datasheet: which models produced it, which prompt/agent versions, how the
- * reward was derived (deterministic verifiable vs probabilistic judge — the
- * credibility axis a buyer checks first), the split discipline, the reward
- * distribution, the quality gates, the license, and the intended/out-of-scope
- * uses. This module computes those facts from the `RunRecord[]` and renders a
- * "Datasheet for Datasets" (Gebru et al. 2018) card alongside the format files.
- *
- * It composes the existing `rl/exporters` — it does not reimplement any trainer
- * format. The renderers token-identity step (DeepSeek/Kimi/Qwen tokenization
- * with per-token loss masks) is a downstream Python stage that consumes the
- * `messages`/`completions` this bundle emits.
- */
-
-type RewardKind = 'deterministic' | 'probabilistic' | 'mixed';
-type DatasetFormat = 'grpo' | 'sft' | 'dpo';
-/** Caller-declared context — the qualitative half of the datasheet that can't
- *  be computed from records. */
-interface RlDatasetConfig {
-    name: string;
-    version: string;
-    /** Product/task domain, e.g. 'legal-m&a', 'tax-1040'. */
-    domain: string;
-    /** SPDX id or a named commercial license. Required — an unlicensed dataset
-     *  cannot be published or sold. */
-    license: string;
-    /** How the reward was produced. `kind: 'deterministic'` (a test/schema/XPath
-     *  decided it) is the credibility signal; 'probabilistic' = LLM-judge. */
-    reward: {
-        kind: RewardKind;
-        source: string;
-        description: string;
-    };
-    intendedUse: string;
-    outOfScope: string;
-    limitations: string;
-    /** ISO timestamp — passed in (the substrate forbids Date.now()). */
-    createdAtIso: string;
-    /** Default: ['grpo', 'sft']. */
-    formats?: DatasetFormat[];
-    /** Quality gates already run, recorded on the card for the buyer. */
-    qualityGates?: {
-        contaminationProbe?: 'passed' | 'failed' | 'not-run';
-        dedup?: boolean;
-        verifiableRewardFilter?: boolean;
-    };
-}
-interface RewardStats {
-    n: number;
-    mean: number;
-    median: number;
-    min: number;
-    max: number;
-    std: number;
-}
-interface RlDatasetStats {
-    records: number;
-    /** Record count per split — a publishable dataset must declare its holdout. */
-    splits: Record<RunSplitTag, number>;
-    reward: RewardStats;
-    /** Distinct snapshot-pinned models that produced the trajectories. */
-    models: string[];
-    /** Distinct effective-prompt hashes (the agent profile/prompt versions). */
-    promptHashes: string[];
-    commitShas: string[];
-    totalTokens: {
-        input: number;
-        output: number;
-    };
-    totalCostUsd: number;
-}
-interface RlDatasetManifest extends RlDatasetConfig {
-    formats: DatasetFormat[];
-    rowCounts: Partial<Record<DatasetFormat, number>>;
-    stats: RlDatasetStats;
-}
-interface RlDatasetBundle {
-    manifest: RlDatasetManifest;
-    /** Relative filename -> contents. Write these to a directory to publish. */
-    files: Record<string, string>;
-}
-/**
- * Package graded `RunRecord[]` into a publishable RL dataset bundle: the
- * trainer-format JSONL files + a manifest + a datasheet. DPO requires
- * pre-extracted preference triples (pass `preferences`); GRPO/SFT derive from
- * the records directly via the supplied lookups. Throws on an empty corpus —
- * an empty dataset must never be published.
- */
-declare function buildRlDataset(records: RunRecord[], lookups: GrpoLookups & SftLookups, config: RlDatasetConfig, preferences?: {
-    triples: PreferenceTriple[];
-    lookups: DpoLookups;
-}): Promise<RlDatasetBundle>;
-/** Render the "Datasheet for Datasets" card — the artifact a buyer reads. */
-declare function datasheetToMarkdown(m: RlDatasetManifest): string;
-
-/**
- * RL corpus — the durable, append-only accumulation of graded RunRecords that
- * every eval run deposits BY DEFAULT.
- *
- * The dataset is the free exhaust of the normal eval process: we run evals
- * constantly to get an agent production-ready, and those runs already produce
- * graded trajectories. Instead of writing them to an ephemeral run dir and
- * throwing them away, `appendToCorpus` accumulates them into a durable corpus;
- * `buildDatasetFromCorpus` later harvests the whole corpus into a publishable
- * bundle. No separate data-collection campaign — the data accrues from work we
- * do anyway. This is the "best things for free by our process" layer.
- *
- * Trajectory text rides on the record as top-level `prompt` / `completion`
- * (what the eval harnesses capture; the RunRecord validator ignores the extra
- * keys). The harvest reads them directly — no trace store round-trip needed.
- */
-
-/** A corpus record is a RunRecord carrying the trajectory text the harness
- *  captured. `prompt`/`completion` are top-level (the validator ignores extras). */
-type CorpusRecord = RunRecord & {
-    prompt?: string;
-    completion?: string;
-};
-interface CorpusAppendResult {
-    appended: number;
-    /** Skipped because a record with the same runId was already in the corpus
-     *  (idempotent appends — NOT re-run collapsing; re-runs get fresh runIds). */
-    skipped: number;
-    total: number;
-}
-/**
- * Append graded records to the corpus (append-only JSONL). Deduplicates by
- * `runId` against what's already on disk so re-running the same harness is
- * idempotent. Creates the file and parent dir. This is the call every eval
- * harness makes by default after producing its records.
- */
-declare function appendToCorpus(records: CorpusRecord[], corpusPath: string): CorpusAppendResult;
-/** Read the full corpus. Returns [] if the corpus does not exist yet. */
-declare function readCorpus(corpusPath: string): CorpusRecord[];
-interface HarvestOptions {
-    /** Keep only records scoring >= this (rejection-sampling for SFT). */
-    minScore?: number;
-    /** Keep only these splits (e.g. ['holdout'] for an eval-only dataset). */
-    splits?: RunRecord['splitTag'][];
-}
-/**
- * Harvest the accumulated corpus into a publishable RL dataset bundle. Reads
- * trajectory text from each record's top-level `prompt`/`completion`; records
- * missing either are excluded (a graded score with no trajectory can't train).
- * Optionally filters by score / split. Throws (via buildRlDataset) if nothing
- * survives — an empty dataset must never be published.
- */
-declare function buildDatasetFromCorpus(corpusPath: string, config: RlDatasetConfig, opts?: HarvestOptions): Promise<RlDatasetBundle>;
-
-/**
- * Off-policy evaluation primitives.
- *
- * Standard inverse-probability-weighted (IPS), self-normalized
- * importance-weighted (SNIPS), and doubly-robust (DR) estimators for the
- * value of a *target* policy given trajectories collected under a
- * *behavior* policy. This is the canonical RL eval task: "we have last
- * week's runs, we changed the policy — how would the new one do without
- * re-running?"
- *
- * The math here is textbook (Dudík, Langford, Li 2011 for DR; Swaminathan
- * & Joachims 2015 for SNIPS) but the *application* to LLM-agent
- * evaluation needs care:
- *
- *   - The "policy" is the (prompt, tool config, model snapshot) triple.
- *     Two policies have the same probability over an action *iff* their
- *     LLM call would emit the same token with the same probability —
- *     which is generally unknowable without the model log-probs.
- *   - For LLM agents, propensity scores must be supplied by the caller
- *     (logged in the trace, recovered from token log-probs, or estimated
- *     via a learned propensity model). We do NOT estimate propensity here.
- *   - Doubly-robust requires a Q-function (model-based reward predictor).
- *     We accept any callable; consumers pass either a tabular average,
- *     a regression fit, or a learned reward model.
- *
- * Bias / variance tradeoffs:
- *   - IPS: unbiased; high variance for small overlap, infinite variance
- *     when target has support outside behavior.
- *   - SNIPS: lower variance, slight bias; usually preferred in practice.
- *   - DR: doubly-robust — unbiased if either propensity OR Q-function is
- *     correct. Lowest practical variance when Q is decent. Use this.
- *
- * Caveat the panel will land: on the LLM-agent setting, propensity scores
- * recovered from token log-probs are noisy, the action space is enormous,
- * and overlap is often poor. These estimators are useful but not magic;
- * complement with `replayCampaign` (exact replay where the request hashes
- * match) for high-confidence answers and OPE for the gap.
- */
-interface OffPolicyTrajectory {
-    /** Stable id, for traceability through the dataset. */
-    runId: string;
-    /** Reward observed under the behavior policy (the realized outcome). */
-    reward: number;
-    /**
-     * Behavior-policy probability of the action that was taken. For LLM
-     * agents this is typically `exp(sum(token_log_probs))` over the chosen
-     * trajectory. Must be in (0, 1].
-     */
-    behaviorProb: number;
-    /**
-     * Target-policy probability of the same action. For replay-style
-     * counterfactual evaluation this is what the *new* policy would have
-     * assigned to the *old* trajectory. Must be in [0, 1].
-     */
-    targetProb: number;
-    /**
-     * Optional model-based reward prediction at the same context. Used by
-     * `doublyRobust`. Set to `null` for IPS-only evaluation.
-     */
-    qHat?: number | null;
-}
-interface OffPolicyEstimate {
-    /** Estimated value of the target policy. */
-    value: number;
-    /** Standard error of the estimate. */
-    standardError: number;
-    /** Effective sample size (Kong 1992). Lower = more reliance on a few high-weight samples. */
-    effectiveSampleSize: number;
-    /** Number of trajectories used. */
-    n: number;
-    /**
-     * Diagnostic: maximum importance weight observed. Large values (>>10x
-     * mean) are a red flag — variance is dominated by a few outliers.
-     */
-    maxImportanceWeight: number;
-}
-interface OffPolicyOptions {
-    /**
-     * Cap importance weights at this value (Ionides 2008 truncated IS) to
-     * trade unbiasedness for variance reduction. Default `Infinity` (no cap).
-     * Set e.g. `10` for stable estimates when the policies are close.
-     */
-    weightCap?: number;
-    /** Reward clipping range. Default `[0, 1]`. */
-    rewardClip?: {
-        low: number;
-        high: number;
-    };
-}
-/**
- * Inverse Probability Weighting (Horvitz-Thompson). Unbiased estimator
- * of E[reward under target policy]. Variance scales with the spread of
- * target/behavior ratios.
- */
-declare function inverseProbabilityWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
-/**
- * Self-Normalized Importance Sampling. Lower variance than vanilla IPS at
- * the cost of small bias (vanishing as N grows). The right default for
- * LLM-agent evaluation where overlap is often poor.
- */
-declare function selfNormalizedImportanceWeighting(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
-/**
- * Doubly-robust off-policy estimator (Dudík, Langford, Li 2011).
- *
- *     V_DR = (1/N) * sum_i [ q_hat_i + (target_prob_i / behavior_prob_i) * (r_i - q_hat_i) ]
- *
- * Unbiased if EITHER:
- *   - the importance ratios are correct (IPS-style validity), OR
- *   - the Q-hat function is correct (model-based validity).
- *
- * In practice both are imperfect, but the residual bias is the *product*
- * of both errors — much smaller than either alone. This is why DR is the
- * default in production OPE pipelines.
- *
- * Requires `qHat` on every trajectory. If any are `null`, the estimator
- * falls back to SNIPS for those entries (loud-fallback behavior; the
- * report's `n` reflects the full set but `effectiveSampleSize` accounts
- * for the lost variance reduction).
- */
-declare function doublyRobust(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): OffPolicyEstimate;
-/**
- * Convenience: run all three estimators and return them side-by-side.
- * The recommended diagnostic — agreement across estimators is a much
- * stronger signal than any single one.
- */
-declare function offPolicyEstimateAll(trajectories: OffPolicyTrajectory[], opts?: OffPolicyOptions): {
-    ips: OffPolicyEstimate;
-    snips: OffPolicyEstimate;
-    dr: OffPolicyEstimate;
-};
-
-/**
- * OutcomeStore — deployment outcomes attached to Run IDs.
- *
- * Outcomes arrive asynchronously from production telemetry after the
- * eval run completed: user ratings, retention flags, conversion events,
- * revenue, support-ticket rate, anything a product team can measure.
- * The store is a peer to TraceStore — separate lifecycle, same runId
- * foreign key.
- *
- * The whole point of this module is to make the meta-eval correlation
- * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
- */
-interface DeploymentOutcome {
-    runId: string;
-    capturedAt: number;
-    /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
-    metrics: Record<string, number>;
-    /** Dimensions for stratified analysis — cohort, region, user_segment. */
-    labels?: Record<string, string>;
-    /** Free-form provenance (source system, pipeline version). */
-    source?: string;
-}
-interface OutcomeFilter {
-    runIds?: string[];
-    since?: number;
-    until?: number;
-    label?: {
-        key: string;
-        value: string;
-    };
-    source?: string;
-}
-interface OutcomeStore {
-    append(outcome: DeploymentOutcome): Promise<void>;
-    /** All outcomes attached to this run (a single run can have many — multiple
-     *  capture windows over deployment time). */
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-declare class InMemoryOutcomeStore implements OutcomeStore {
-    private items;
-    append(outcome: DeploymentOutcome): Promise<void>;
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-interface FileSystemOutcomeStoreOptions {
-    dir: string;
-    maxBytes?: number;
-}
-declare class FileSystemOutcomeStore implements OutcomeStore {
-    private dir;
-    private maxBytes;
-    private memo?;
-    private loaded;
-    constructor(options: FileSystemOutcomeStoreOptions);
-    private ensureDir;
-    append(outcome: DeploymentOutcome): Promise<void>;
-    private load;
-    forRun(runId: string): Promise<DeploymentOutcome[]>;
-    list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
-}
-
-/**
- * Rubric predictive validity — does our eval rubric predict deployment
- * outcomes?
- *
- * `correlationStudy` (already in this package) joins a `TraceStore` to an
- * `OutcomeStore` and computes Pearson + Spearman + bootstrap CI for each
- * (eval-metric, outcome-metric) pair. That answers "does X correlate with
- * Y at all." `rubricPredictiveValidity` is the campaign-shaped wrapper
- * around it: take a sequence of `RunRecord`s (the canonical campaign
- * artifact) and a `DeploymentOutcomeStore`, join on `runId`, return a
- * ranked verdict on every rubric whose dimension scores were captured in
- * `outcome.raw`.
- *
- * The point — quoting the methodology doc — is that **without this loop
- * every rubric is faith-based**. Once it's wired, you know which rubrics
- * have earned their promotion power and which ones are decoration.
- *
- *   const validity = await rubricPredictiveValidity({
- *     runs: lastQuarter,
- *     outcomes: shipFlagOutcomeStore,
- *     outcomeMetrics: ['revenue_lift', 'retention_30d', 'csat'],
- *     rubrics: ['anti_slop', 'semantic_concept', 'tool_recovery'],
- *   })
- *   for (const r of validity.ranked) {
- *     console.log(`${r.rubric} → ${r.bestOutcome}: ρ=${r.spearman.toFixed(2)}`)
- *   }
- *
- * The function is intentionally read-only. Use the verdict to deprecate
- * decorative rubrics, re-weight composite scores, or trigger a
- * recalibration sweep when predictive validity drops below a threshold.
- */
-
-interface RubricOutcomePair {
-    rubric: string;
-    outcome: string;
-    n: number;
-    pearson: number;
-    spearman: number;
-    ci95: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Verdict bucket. `load_bearing` ≥ 0.7, `informative` ≥ 0.4,
-     * `decorative` < 0.4 in absolute correlation. A negative correlation
-     * with a desired outcome is also `decorative` — actively misleading
-     * is worse than uninformative.
-     */
-    verdict: 'load_bearing' | 'informative' | 'decorative';
-}
-interface RubricRanking {
-    rubric: string;
-    /** Outcome metric this rubric correlated best with. */
-    bestOutcome: string;
-    spearman: number;
-    pearson: number;
-    n: number;
-    verdict: RubricOutcomePair['verdict'];
-}
-interface RubricPredictiveValidityReport {
-    pairs: RubricOutcomePair[];
-    /** Per-rubric best pair, sorted descending by |spearman|. */
-    ranked: RubricRanking[];
-    joinedSamples: number;
-    skippedRuns: number;
-    /** Rubrics that were declared but never produced a usable score. */
-    rubricsWithoutData: string[];
-}
-
-/**
- * HeldOutGate — first-class held-out paired-delta promotion gate.
- *
- * Encodes the "honesty override" pattern that lived inline in
- * `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
- * The optimizer's best-guess is one thing; what we should actually
- * ship is another. The gate is the line between them.
- *
- * A candidate is promoted iff ALL three pass:
- *
- *   1. **Productive runs**: the candidate has at least
- *      `minProductiveRuns` paired observations on items where BOTH
- *      candidate and baseline produced a real (non-silent) score.
- *   2. **Paired delta**: the lower bound of the bootstrap CI on the
- *      median per-item delta (candidate − baseline) on the HOLDOUT
- *      split is strictly greater than `pairedDeltaThreshold`.
- *   3. **Overfit gap**: the candidate's gap between search-split
- *      score and holdout-split score is no worse (more positive)
- *      than the baseline's gap by more than `overfitGapThreshold`.
- *      "Better on search, worse on holdout" is the canonical
- *      overfit pattern; this catches it.
- *
- * The decision carries a machine-readable `rejectionCode` plus an
- * `evidence` block with every number the gate looked at, so the
- * downstream researcher / paper / dashboard can re-derive the
- * verdict without re-running.
- *
- * See also:
- *   - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
- *   - `src/run-record.ts` for the input row schema
- *   - `src/reference-replay.ts` for the older, reference-replay-
- *     specific promotion path (still useful for replay-style evals).
- */
-
-type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
-interface GateEvidence {
-    /** Number of paired (candidate, baseline) holdout observations used. */
-    productiveRuns: number;
-    /** Median of (candidate − baseline) paired holdout deltas. */
-    medianPairedDelta: number;
-    /** Bootstrap CI on the median paired holdout delta. */
-    pairedCI: {
-        low: number;
-        high: number;
-    };
-    /** Wilcoxon signed-rank p-value on the paired holdout deltas. */
-    pairedPValue: number;
-    /** Mean candidate score on the search split (NaN if none). */
-    searchScore: number;
-    /** Mean candidate score on the holdout split (NaN if none). */
-    holdoutScore: number;
-    /** Candidate (search − holdout) gap. */
-    overfitGap: number;
-    /** Baseline (search − holdout) gap. */
-    baselineOverfitGap: number;
-    /** Median per-task USD cost across the candidate's runs. Recorded
-     *  even when no `costPerTaskCeiling` is configured so downstream
-     *  dashboards (intelligence.tangle.tools) can render \$/task per
-     *  generation regardless of gating policy. */
-    medianCandidateCost: number;
-    /** Median per-task USD cost across the baseline runs, for
-     *  symmetric reporting. */
-    medianBaselineCost: number;
-}
-interface GateDecision$1 {
-    /** Final promote/no-promote verdict. */
-    promote: boolean;
-    /** The candidate that was evaluated. */
-    candidateId: string;
-    /** The baseline it was compared against. */
-    baselineId: string;
-    /** Every number the gate looked at, for audit + paper export. */
-    evidence: GateEvidence;
-    /** Human-readable reason. */
-    reason: string;
-    /** Machine-readable rejection code, or null on promote. */
-    rejectionCode: HeldOutGateRejectionCode | null;
-}
-
-/**
- * Researcher interface — stable hook for an external autonomous-research
- * agent to drive the meta-loop.
- *
- * Implementations live downstream (typically in a private repo that
- * runs the actual LLM). This package ships only the contract + a
- * `NoopResearcher` so consumers can wire the surface without being
- * forced to implement every method up front.
- *
- * The four methods mirror the four stages of the paper "Two Loops,
- * Three Roles":
- *
- *   inspectFailures   — given the observed runs, what failure modes
- *                       are present? (data → diagnosis)
- *   proposeChange     — given diagnosed failure modes, what
- *                       structural changes should we try?
- *                       (diagnosis → plan delta)
- *   applyChange       — fold the proposed deltas into a concrete
- *                       experiment plan against an existing baseline.
- *                       (plan delta → executable plan)
- *   evaluateChange    — run the plan, return runs + the gate verdict.
- *                       (executable plan → verdict)
- *
- * Composition is the discipline: a Researcher implementation MUST
- * keep these four steps separate and inspectable. Conflating
- * "diagnose + propose + run" into a single LLM call defeats the
- * point of the framework — you can't audit which step lied.
- *
- * THIS INTERFACE IS STABLE. Breaking changes require a new module
- * (e.g. `Researcher2`) so existing implementations keep working.
- */
-
-/** A diagnosed failure mode with the run-IDs that exhibit it. */
-interface FailureMode {
-    /** Short machine-readable code. Must be stable across runs of the
-     *  same researcher to enable longitudinal tracking. */
-    code: string;
-    /** Human-readable description for the paper / dashboard. */
-    description: string;
-    evidence: {
-        /** Run IDs (from `RunRecord.runId`) where this failure mode was
-         *  observed. */
-        runIds: string[];
-        /** Number of run samples that informed the diagnosis. */
-        samples: number;
-    };
-}
-/** A single steering change the researcher wants to try. */
-interface SteeringChange {
-    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
-    /** Implementation-specific payload. Researcher implementations
-     *  define the schema — keep this `unknown` here to avoid coupling
-     *  the public interface to any one researcher's internal model. */
-    payload: unknown;
-    /** Why the researcher proposed this change. Goes into the audit
-     *  trail next to the failure-mode evidence. */
-    rationale: string;
-    /** Optional self-reported expected delta on the headline metric. */
-    expectedDelta?: number;
-}
-/** A single experiment plan, mapped onto the search/holdout splits. */
-interface ExperimentPlan {
-    baselineCandidateId: string;
-    proposedCandidateId: string;
-    changes: SteeringChange[];
-    /** USD ceiling for the entire experiment. The runner must stop
-     *  before exceeding this and report a partial result. */
-    evaluationBudgetUsd: number;
-    /** Item IDs (your dataset keys) for the search vs holdout splits. */
-    splits: {
-        search: string[];
-        holdout: string[];
-    };
-}
-/** Result of running a plan: every run, plus the gate verdict. */
-interface ExperimentResult {
-    plan: ExperimentPlan;
-    runs: RunRecord[];
-    gateDecision: GateDecision$1;
-}
-/**
- * The researcher loop. Stable, four-step, inspectable.
- *
- *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
- *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
- *   └──────────┘                   └──────────┘                └────┬─────┘
- *                                                                   │
- *                                                                   ▼
- *                              ┌────────────────┐  applyChange ┌────────┐
- *                              │ ExperimentPlan │ ◀────────────│  base  │
- *                              └────────┬───────┘              └────────┘
- *                                       │
- *                       evaluateChange  ▼
- *                              ┌────────────────┐
- *                              │ ExperimentResult│
- *                              └────────────────┘
- */
-interface Researcher {
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-
-/**
- * `PredictiveValidityResearcher` — concrete `Researcher` implementation
- * that drives selection from outcome-anchored predictive validity.
- *
- * Each method:
- *
- *   - `inspectFailures(runs)` — synthesizes failure modes from the
- *     bottom-quartile of `RunRecord`s on the configured proxy reward.
- *   - `proposeChange(failures)` — proposes steering changes that target
- *     the rubrics with the lowest predictive validity (decorative ones).
- *     Either reduce their weight in the composite, or recalibrate them.
- *   - `applyChange(changes, baseline)` — merges the proposed steering
- *     into the experiment plan.
- *   - `evaluateChange(plan)` — re-runs the predictive-validity check on
- *     the post-change runs and reports the delta.
- *
- * The result is a closed loop: the rubric weights drift toward the ones
- * that actually predict deployment outcomes, automatically. Pair with
- * `runRLCampaign` for the full auto-research story.
- */
-
-interface PredictiveValidityResearcherOptions {
-    outcomes: OutcomeStore;
-    outcomeMetrics: string[];
-    /** Score threshold below which a run counts as a "failure." Default 0.5. */
-    failureThreshold?: number;
-    /** Spearman bucket below which a rubric is "decorative." Default 0.4. */
-    decorativeThreshold?: number;
-    /** Optional steering-namespace prefix for proposed changes. Default `'rubric_weight'`. */
-    steeringNamespace?: string;
-    /** Override the rubric set the researcher inspects. Default: every numeric `outcome.raw` key seen. */
-    rubrics?: string[];
-    /**
-     * Snapshot stash hook — called with the most recent predictive-validity
-     * report. Useful when a downstream system wants to log rubric drift over
-     * time. Default no-op.
-     */
-    onReport?: (report: RubricPredictiveValidityReport) => void | Promise<void>;
-}
-/**
- * Concrete `Researcher` driven by `rubricPredictiveValidity`. The brain:
- * rubrics that don't predict deployment outcomes don't earn weight.
- */
-declare class PredictiveValidityResearcher implements Researcher {
-    private opts;
-    private lastReport;
-    constructor(opts: PredictiveValidityResearcherOptions);
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-    /**
-     * Run the predictive-validity check explicitly against a fresh RunRecord
-     * set. Updates the researcher's cached report so subsequent
-     * `proposeChange` calls have evidence to draw from.
-     */
-    runValidityCheck(runs: RunRecord[]): Promise<RubricPredictiveValidityReport>;
-    /**
-     * Force-feed a predictive-validity report into the researcher state —
-     * useful when the consumer ran the report out-of-band and wants the
-     * researcher's later proposals informed by it.
-     */
-    setReport(report: RubricPredictiveValidityReport): void;
-    getLastReport(): RubricPredictiveValidityReport | null;
-}
-
-/**
- * Validator-output verdict — substrate primitive for "did this output pass,
- * and how well?"
- *
- * Used by:
- *   - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
- *   - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
- *     Runtime keeps `Validator` because it's coupled to runtime-shaped
- *     `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
- *     itself is a substrate concept and lives here.
- *
- * Repo layering: agent-eval is the substrate (no upward deps). Both
- * agent-runtime and agent-knowledge consume this type FROM agent-eval —
- * never the other way around. See CLAUDE.md "Repo layering" for the rule.
- */
-/**
- * Minimal verdict shape — `valid` + `score` are required; `scores` +
- * `notes` are optional surface. Validators that need richer shapes
- * parameterise `Validator<Output, MyVerdict>` with their own type.
- *
- * Need structured extras? Extend DefaultVerdict with typed fields — never
- * serialize extras into `notes`.
- */
-interface DefaultVerdict {
-    /** Whether the output meets the validator's pass criteria. */
-    valid: boolean;
-    /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
-    score: number;
-    /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
-    scores?: Record<string, number>;
-    /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
-    notes?: string;
-}
-
-/**
- * Multi-layer verifier — ordered pipeline of verification layers.
- *
- * Different contract from {@link JudgeRunner} (which runs parallel
- * specs against a sandbox). MultiLayerVerifier is a DAG of layers
- * (install → typecheck → build → lint → serve → semantic → …) with
- * dependency-based skip, per-layer findings, soft-fail semantics, and
- * an aggregated `blendedScore` across all passed layers.
- *
- * Use when you want:
- *   - ordered stages where a failing upstream stage skips downstream ones
- *   - each stage produces rich `findings` (severity + message + evidence)
- *   - a single composite score across stages with per-stage weights
- *   - soft-fail stages whose failure doesn't abort the pipeline
- *
- * Use {@link JudgeRunner} when you want:
- *   - N independent judges running in parallel against the same artifact
- *   - no inter-judge dependencies
- *   - boolean `passed` per judge + overall
- *
- * Both primitives compose — JudgeRunner can be invoked as a single
- * layer inside a MultiLayerVerifier if that suits the caller.
- */
-
-type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
-type Severity = 'critical' | 'major' | 'minor' | 'info';
-interface Finding {
-    severity: Severity;
-    message: string;
-    evidence?: string;
-    /** Optional layer name the finding belongs to (set by the verifier if omitted). */
-    layer?: string;
-    /**
-     * Free-form structured payload — used by `multiToolchainLayer` to attach
-     * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc.
-     * Renderers MAY interrogate; agent-eval primitives never assume shape.
-     */
-    detail?: Record<string, unknown>;
-}
-interface LayerResult {
-    layer: string;
-    status: LayerStatus;
-    /** 0..1 score, optional — layers that don't produce a numeric score omit. */
-    score?: number;
-    durationMs: number;
-    findings: Finding[];
-    /** Short human-readable summary (one line). */
-    reason?: string;
-    /**
-     * Numeric layer-level diagnostics: error counts, warning counts,
-     * cyclomatic complexity, total adapter wall-time, etc. Keyed by
-     * diagnostic name; null = "diagnostic not applicable / not measured."
-     * Renderers that know the keys can display them; ones that don't,
-     * ignore. Free-form on purpose — consumers type the value shape in
-     * their own namespace.
-     */
-    diagnostics?: Record<string, number | null>;
-    /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
-    detail?: Record<string, unknown>;
-}
-/** Extends the substrate verdict spine: `valid` = `allPass` and `score` =
- *  `blendedScore` — derived where the report is aggregated, so spine
- *  consumers (drivers, gates) read this report without an adapter. */
-interface VerificationReport extends DefaultVerdict {
-    layers: LayerResult[];
-    passCount: number;
-    failCount: number;
-    skippedCount: number;
-    errorCount: number;
-    /** True iff at least one scored layer ran AND every scored layer passed. */
-    allPass: boolean;
-    /**
-     * Weighted mean of `score` across contributing layers. 0 when no layers
-     * contributed. See {@link Layer.failContributesToScore} for fail semantics.
-     */
-    blendedScore: number;
-    durationMs: number;
-    startedAt: string;
-    finishedAt: string;
-}
-
-/**
- * Verifiable reward channel.
- *
- * For RL on coding / math / theorem-proving / structured-output tasks, the
- * reward signal is *decidable* — a test passes or fails, a proof checks or
- * doesn't, an output validates against a schema or doesn't. These rewards
- * are dramatically more useful for RL training than LLM-judge scores
- * because they don't drift, can't be Goodhart-gamed by the policy in the
- * same way, and don't require a separate calibration loop.
- *
- * The `MultiLayerVerifier` already produces this signal — it just doesn't
- * surface it in a shape that's clean enough for RL training. This module
- * wraps the verifier output so consumers can:
- *
- *   1. Extract a clean `VerifiableReward` from a `VerificationReport`
- *   2. Distinguish *deterministic* rewards (compile, test, schema) from
- *      *probabilistic* rewards (judge) so they can be weighted differently
- *      in the RL training step
- *   3. Filter `RunRecord[]` to only those with a verifiable reward,
- *      producing the clean training set that DeepSeek-R1-style GRPO and
- *      AlphaProof-style search both depend on
- *
- * Why this matters: every credible 2025-2026 frontier RL result on coding
- * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,
- * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing
- * judge scores into the reward signal poisons the gradient. This module
- * is the seam.
- */
-
-type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
-interface VerifiableReward {
-    /** Scalar in [0, 1]. The RL training signal. */
-    value: number;
-    /** What produced the reward — different sources have different determinism. */
-    source: VerifiableRewardSource;
-    /**
-     * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte
-     * given the same inputs (compile, test, schema validation, sandbox exit code).
-     * `'probabilistic'` rewards depend on a stochastic component (LLM judge).
-     * Mixing these in the same training batch without separation is a known
-     * footgun in production RLHF pipelines.
-     */
-    determinism: 'deterministic' | 'probabilistic';
-    /**
-     * Confidence in the reward value. For deterministic sources this is 1.0
-     * (the bit either flipped or didn't). For judge sources this is the
-     * judge-reported confidence or — when missing — a calibrated prior.
-     */
-    confidence: number;
-    /** The layer / judge id that produced the signal, for provenance. */
-    origin: string;
-    /**
-     * Per-source contribution to `value`, keyed by layer/judge id. Single-source
-     * rewards carry one entry (`{ [origin]: value }`); composite rewards carry
-     * every contributing layer's score — the anti-scalar-collapse surface RL
-     * consumers weight per-source instead of trusting one blended number.
-     */
-    components: Record<string, number>;
-    /**
-     * @deprecated Read `components` for per-source reward values. Kept for
-     * published-API compatibility: single-source rewards carry the layer's
-     * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry
-     * the same per-layer scores `components` now holds.
-     */
-    breakdown?: Record<string, number>;
-}
-interface VerifiableRewardExtractionOptions {
-    /**
-     * Which layers count as deterministic-reward sources. The verifier doesn't
-     * tag layers as "this is verifiable"; the caller declares it via this list
-     * (or via the layer name → source mapping). Default treats common names
-     * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
-     * `sandbox`) as deterministic.
-     */
-    deterministicLayers?: string[];
-    /**
-     * Map layer name → reward source. Defaults to a sensible string-match.
-     */
-    sourceFor?: (layerName: string) => VerifiableRewardSource;
-    /**
-     * Whether to fall back to a probabilistic (judge) reward when no
-     * deterministic layer produced a numeric score. Default `true`. Set to
-     * `false` for "deterministic-only" training pipelines that should
-     * discard runs without a verifiable signal.
-     */
-    fallbackToJudge?: boolean;
-    /**
-     * Default confidence for probabilistic (judge) rewards when the judge
-     * doesn't report one. Default `0.7`.
-     */
-    judgeConfidenceFloor?: number;
-}
-/**
- * Extract a `VerifiableReward` from a `VerificationReport`.
- *
- * Strategy: prefer the deterministic layers (in order: test → compile →
- * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is
- * true, return `null` if no signal qualifies. When multiple deterministic
- * layers contribute, return a `'composite'` source with a weighted blend.
- */
-declare function extractVerifiableReward(report: VerificationReport, opts?: VerifiableRewardExtractionOptions): VerifiableReward | null;
-/**
- * Extract verifiable rewards from `RunRecord[]` produced via the
- * `verificationReportToRunRecord` adapter (which encodes per-layer scores
- * in `outcome.raw['layer.<name>']`). For records that don't carry layer
- * scores, returns `null` for that record.
- *
- * This is the canonical bridge from "campaign-shaped artifacts" to
- * "RL-training-ready reward signals": every record that has a clean
- * verifiable reward becomes a training datum, every record that doesn't
- * gets filtered out (or kept with `'probabilistic'` determinism for
- * separate downstream handling).
- */
-declare function extractVerifiableRewardsFromRecords(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
-    runId: string;
-    reward: VerifiableReward | null;
-}>;
-/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */
-declare function filterDeterministicallyRewarded(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
-    run: RunRecord;
-    reward: VerifiableReward;
-}>;
-
-/**
- * Reward hacking / Goodhart detection.
- *
- * Goodhart's Law says: when a measure becomes a target, it ceases to be
- * a good measure. In RLHF and agentic-RL settings this is the dominant
- * failure mode — the policy learns to produce outputs that score well on
- * the proxy reward (judge, rubric, test pass-rate) without producing
- * the underlying capability the proxy was meant to track.
- *
- * Krakovna et al. (2020, "Specification Gaming Examples in AI") and the
- * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.
- * 2023) converge on a few diagnostic signatures:
- *
- *   1. **Reward divergence:** the proxy reward grows while the held-out
- *      ground-truth signal stagnates or drops. Predictive validity over
- *      time captures this.
- *   2. **Distributional shift in outputs:** after RL, the policy produces
- *      outputs that no longer match the reference distribution — usually
- *      because it found a high-reward attractor that's degenerate (e.g.
- *      one-token responses, repetition, formatting tricks).
- *   3. **Disagreement between independent rewards:** if you train on
- *      reward A and a held-out independent reward B drops sharply, you're
- *      probably hacking A.
- *   4. **Calibration drift:** the verifiable / deterministic component of
- *      the reward is stable; the probabilistic / judge component drifts up
- *      while the deterministic component doesn't. The judge is being
- *      gamed.
- *
- * This module ships explicit detectors for all four signatures, plus a
- * combined verdict. The output is diagnostic — actionable signals,
- * not autoreject — because each signature has known false positives
- * (e.g., a policy that genuinely improves can show distributional shift).
- *
- * Differs from `rubricPredictiveValidity` (which is a *standing* check on
- * whether rubrics correlate with deployment outcomes) — this is a
- * *temporal* check on whether the reward-vs-truth gap is *widening over
- * time during a training run*.
- */
-
-type RewardHackingSignal = 'reward_divergence' | 'distribution_shift' | 'reward_disagreement' | 'judge_drift';
-interface RewardHackingFinding {
-    signal: RewardHackingSignal;
-    /** Severity in [0, 1]. >0.5 = strong signal. */
-    severity: number;
-    message: string;
-    /** Numeric evidence the consumer can render. */
-    detail: Record<string, number>;
-}
-interface RewardHackingReport {
-    findings: RewardHackingFinding[];
-    /**
-     * Composite verdict. `'clean'` if every signal severity < 0.3;
-     * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.
-     */
-    verdict: 'clean' | 'suspect' | 'gaming';
-    /** Rationale for the verdict, ready to paste into an audit log. */
-    rationale: string[];
-    /** Number of paired (proxy, truth) data points the report saw. */
-    n: number;
-}
-interface DetectRewardHackingInput {
-    /**
-     * Run records ordered by recency (oldest first). The detector segments
-     * them into prefix/suffix windows to compute "did the gap widen."
-     */
-    runs: RunRecord[];
-    /**
-     * The metric the policy was trained to optimize. Should be present on
-     * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
-     */
-    proxyOf?: (run: RunRecord) => number | null;
-    /**
-     * The held-out ground-truth metric. For RL on coding, this is typically
-     * test pass-rate. For RLHF, it's downstream task performance or human
-     * preference. For knowledge tasks, it's an independently-graded score.
-     */
-    truthOf?: (run: RunRecord) => number | null;
-    /**
-     * Independent secondary reward. Used for the `reward_disagreement`
-     * signal. Default uses the verifiable reward extractor (deterministic
-     * sources only).
-     */
-    secondaryRewardOf?: (run: RunRecord) => number | null;
-    /**
-     * Window size — how many of the most recent runs count as the "after"
-     * cohort. Default min(50, half the runs).
-     */
-    windowSize?: number;
-    /**
-     * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
-     * (gaming).
-     */
-    thresholds?: {
-        suspect?: number;
-        gaming?: number;
-    };
-    /**
-     * Verifiable-reward options used for the secondary-reward fallback.
-     */
-    verifiableRewardOptions?: VerifiableRewardExtractionOptions;
-}
-declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
-
-type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
-interface ChannelRollup {
-    channel: CostChannel;
-    calls: number;
-    inputTokens: number;
-    outputTokens: number;
-    reasoningTokens?: number;
-    cachedTokens: number;
-    cacheWriteTokens?: number;
-    costUsd: number;
-    unpricedCalls: number;
-    unknownUsageCalls: number;
-}
-interface CostLedgerSummary {
-    totalCalls: number;
-    pendingCalls: number;
-    unresolvedCalls: number;
-    reservedCostUsd: number;
-    inputTokens: number;
-    outputTokens: number;
-    reasoningTokens?: number;
-    cachedTokens: number;
-    cacheWriteTokens?: number;
-    totalCostUsd: number;
-    byChannel: ChannelRollup[];
-    unpricedModels: string[];
-    fullyPriced: boolean;
-    usageComplete: boolean;
-    accountingComplete: boolean;
-    incompleteReasons: string[];
-}
-
-/**
- * RawProviderSink — first-class persistence for the actual HTTP-level
- * request/response bodies of every LLM provider call.
- *
- * Why this is a separate sink from the structured `LlmSpan`:
- *
- *   - `LlmSpan` records the *intent* — model name, messages, output text,
- *     usage. It's what dashboards read; it's NOT enough for forensics.
- *   - When a downstream consumer reports "the verifier used the wrong route"
- *     or "tokens look right but reasoning was missing," the only way to
- *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
- *     a different `model` value than what actually answered); the raw
- *     response is ground truth.
- *
- * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
- * matrix runner / BuilderSession sets it up automatically) and every
- * request, response, and error is recorded — including retries, with the
- * attempt index attached so a flaky call's full event chain is recoverable.
- *
- * Redaction is enforced at sink time. The default redactor strips
- * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
- * payload field whose key matches `apiKey | api_key | bearer | password |
- * secret | token` (case-insensitive). Override via the sink constructor or
- * the per-call `redactor`. The `redactedFields` array on the persisted
- * event lets a reviewer see what was stripped without exposing the values.
- */
-type RawProviderDirection = 'request' | 'response' | 'error';
-interface RawProviderEvent {
-    /** Stable id. Generated by the sink if omitted. */
-    eventId: string;
-    /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
-    runId?: string;
-    spanId?: string;
-    /**
-     * Logical provider name. Free-form so callers can use whatever id matches
-     * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
-     * omitted, derived from `baseUrl` in `LlmClientOptions`.
-     */
-    provider: string;
-    model: string;
-    /** Endpoint path, e.g. `'/v1/chat/completions'`. */
-    endpoint: string;
-    /** Base URL used for the call (already-normalised — no trailing slash). */
-    baseUrl: string;
-    /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
-    attemptIndex: number;
-    direction: RawProviderDirection;
-    /** Unix ms. */
-    timestamp: number;
-    /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
-    durationMs?: number;
-    statusCode?: number;
-    requestHeaders?: Record<string, string>;
-    requestBody?: unknown;
-    responseHeaders?: Record<string, string>;
-    responseBody?: unknown;
-    /** Set on `direction: 'error'` events. */
-    errorMessage?: string;
-    /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
-    redactedFields: string[];
-}
-interface RawProviderSinkFilter {
-    runId?: string;
-    spanId?: string;
-    direction?: RawProviderDirection;
-    attemptIndex?: number;
-}
-interface RawProviderSink {
-    record(event: RawProviderEvent): Promise<void>;
-    /** Optional listing — implementations that durably persist (file, db) should support this. */
-    list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
-    /** Optional teardown for backed implementations. */
-    close?(): Promise<void>;
-}
-type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
-
-/**
- * LLM client with graceful degrade.
- *
- * OpenAI-compatible `/v1/chat/completions` client with:
- *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
- *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
- *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
- *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
- *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
- *     directly, cli-bridge subscriptions, and any router that speaks the spec.
- *
- * Usage:
- *   const { value, result } = await callLlmJson<MyType>(
- *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
- *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
- *   )
- *
- * This is THE llm-calling seam for agent-eval primitives that need structured
- * output (semantic concept judge, reviewer directives, critic scores). Primitives
- * that need free-form text use `callLlm` and parse output themselves.
- */
-
-interface LlmUsage {
-    promptTokens: number;
-    completionTokens: number;
-    totalTokens: number;
-    /** False when the provider omitted or malformed prompt/completion usage. */
-    captured?: boolean;
-    /** Proxies populate this when prompt caching is on. */
-    cachedPromptTokens?: number;
-}
-interface LlmCallResult {
-    /** The text content of the first choice. Empty string if none. */
-    content: string;
-    usage: LlmUsage;
-    /**
-     * Cost in USD. Pulled from proxy's `_response_cost` field when present;
-     * `null` when neither the proxy nor the caller can derive it.
-     */
-    costUsd: number | null;
-    /** Model name actually used (echoed from response). */
-    model: string;
-    /** Wall-clock duration of the HTTP call (last attempt, if retried). */
-    durationMs: number;
-    /**
-     * `finish_reason` echoed from the first choice (`stop`, `length`,
-     * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
-     * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
-     * (`length`) instead of treating a cut-off completion as complete. Note:
-     * `callLlm` does not itself reject on it — acting on this signal is the
-     * caller's responsibility (in-repo free-form drivers do not yet enforce it).
-     */
-    finishReason?: string | null;
-    /**
-     * True when `content.trim()` is empty. An empty completion is a silent zero
-     * for free-form `callLlm` callers; this flag is the signal a caller can
-     * inspect to fail loud rather than proceed on an empty string. `callLlm`
-     * surfaces it but does not throw on it.
-     */
-    contentEmpty?: boolean;
-    /** Raw response body. */
-    raw: Record<string, unknown>;
-}
-type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
-interface LlmClientOptions {
-    /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
-    baseUrl?: string;
-    /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
-    apiKey?: string;
-    bearer?: string;
-    /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
-    authHeader?: {
-        name: string;
-        value: string;
-    };
-    /** Stable provider idempotency key, reused across retries of this logical call. */
-    idempotencyKey?: string;
-    /** Default timeout in ms. Per-call can override. */
-    defaultTimeoutMs?: number;
-    /**
-     * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
-     * each attempt's per-attempt timeout controller, so aborting it cancels
-     * the in-flight fetch. A caller abort is FATAL: it is not retried even
-     * though an AbortError otherwise matches the transient patterns.
-     */
-    signal?: AbortSignal;
-    /**
-     * Cross-attempt wall-clock budget in ms, measured from the first attempt.
-     * Before launching each attempt the loop checks the remaining budget and
-     * stops retrying once it is exhausted, rather than waiting the full
-     * per-attempt timeout on every retry. Bounds total time independent of
-     * total attempts × `timeoutMs`.
-     */
-    deadlineMs?: number;
-    /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */
-    maxRetries?: number;
-    /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
-    fetch?: typeof fetch;
-    /**
-     * Optional raw HTTP capture sink. When provided, every request, response,
-     * and error (across all retry attempts) is recorded to the sink, with auth
-     * headers and credential-shaped body fields redacted by default. This is
-     * the layer-1 forensics primitive: structured `LlmSpan`s record intent,
-     * raw events record what actually crossed the wire.
-     */
-    rawSink?: RawProviderSink;
-    /**
-     * Logical provider id attached to raw events. When omitted, derived from
-     * `baseUrl` via `providerFromBaseUrl`.
-     */
-    provider?: string;
-    /** Trace context attached to raw events; populated by emitter-aware callers. */
-    traceContext?: {
-        runId?: string;
-        spanId?: string;
-    };
-    /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
-    redactor?: ProviderRedactor;
-}
-interface LlmRouteRequirements {
-    /**
-     * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
-     * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
-     * the public/free-tier router is a defect — the launch reviewer needs to
-     * know exactly which provider answered.
-     */
-    requireExplicitBaseUrl?: boolean;
-    /**
-     * Allowlist of acceptable base URLs. Strings match by prefix
-     * (case-insensitive); RegExps test against the full base URL.
-     */
-    allowedBaseUrls?: Array<string | RegExp>;
-    /** Blocklist that takes precedence over `allowedBaseUrls`. */
-    blockedBaseUrls?: Array<string | RegExp>;
-    /** Throw if no auth header / api key is configured. */
-    requireAuth?: boolean;
-    /**
-     * Logical provider id the configured `baseUrl` is expected to match (via
-     * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.
-     */
-    expectedProvider?: string;
-}
-
-/**
- * FailureClusterView — groups failed runs by (failureClass, triggerTool,
- * argHash-prefix) so weekly reviews can prioritize the top-N clusters.
- *
- * Each cluster includes: N runs, scenarios affected, representative
- * error message, a proposed mitigation hint (rule → action table).
- */
-
-interface FailureCluster {
-    failureClass: FailureClass;
-    /** Tool name when the trigger was a tool span, else undefined. */
-    toolName?: string;
-    /** First 16 chars of argHash — clusters similar args. */
-    argPrefix?: string;
-    /**
-     * Source dimension when the trigger was a judge span (e.g. `'format'`,
-     * `'safety'`, `'correctness'`). Lets cross-template aggregators
-     * group failures by the dimension that fired without overloading
-     * `argPrefix`. Optional — clusters without this field deserialize cleanly.
-     */
-    dimension?: string;
-    runCount: number;
-    scenarioIds: string[];
-    exampleError?: string;
-    exampleRunId: string;
-}
-interface FailureClusterReport {
-    clusters: FailureCluster[];
-    totalFailures: number;
-    totalRuns: number;
-}
-
-/**
- * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
- * than replacing it.
- *
- * Three artefacts:
- *
- *   - `summaryTable`           Markdown table of per-candidate means,
- *                            95% bootstrap CIs, BH-adjusted Wilcoxon
- *                            p-values, and Cohen's d versus a
- *                            comparator candidate.
- *   - `paretoChart`         Abstract spec for a cost vs quality
- *                            scatter, with gate decisions overlaid.
- *                            Returns numbers + labels — caller
- *                            chooses the plotting library.
- *   - `gainHistogram`
- *                            Per-item paired holdout deltas as a
- *                            histogram spec (bins + counts + median +
- *                            CI). Same "data, not images" contract.
- *
- * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
- * They aren't React components and they aren't PNGs; they are
- * what you'd hand to vega-lite, plotly, matplotlib, or your own
- * Canvas renderer to draw the actual figure.
- */
-
-interface SummaryTableRow {
-    candidateId: string;
-    n: number;
-    mean: number;
-    ciLow: number;
-    ciHigh: number;
-    /** BH-adjusted q-value vs comparator. NaN if no comparator. */
-    qValue: number;
-    /** Cohen's d vs comparator. NaN if no comparator. */
-    cohensD: number;
-}
-interface SummaryTable {
-    rows: SummaryTableRow[];
-    comparator: string | null;
-    split: 'search' | 'holdout';
-    /** Pre-rendered markdown — drop into a paper or PR. */
-    markdown: string;
-}
-interface ParetoPoint {
-    candidateId: string;
-    /** Mean USD cost per run on the chosen split. */
-    cost: number;
-    /** Mean score on the chosen split. */
-    quality: number;
-    /** Number of runs that informed this point. */
-    n: number;
-    /** Whether this candidate is on the Pareto frontier — high
-     *  quality, low cost, no dominator. */
-    onFrontier: boolean;
-    /** Optional gate verdict for this candidate, if a `GateDecision`
-     *  for it was passed in. */
-    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
-}
-interface ParetoFigureSpec {
-    kind: 'pareto-cost-quality';
-    split: 'search' | 'holdout';
-    points: ParetoPoint[];
-    axes: {
-        x: 'costUsd';
-        y: 'score';
-    };
-}
-interface GainDistributionBin {
-    /** Inclusive lower edge. */
-    lo: number;
-    /** Exclusive upper edge (or inclusive if it's the last bin). */
-    hi: number;
-    /** Number of pairs whose delta lands in this bin. */
-    count: number;
-}
-interface GainDistributionFigureSpec {
-    kind: 'gain-distribution';
-    candidateId: string;
-    comparator: string;
-    split: 'search' | 'holdout';
-    /** Number of pairs used. */
-    n: number;
-    bins: GainDistributionBin[];
-    median: number;
-    ci: {
-        low: number;
-        high: number;
-    };
-}
-type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
-interface ResearchReportOptions {
-    /** Human-readable report title. */
-    title?: string;
-    /** Comparator candidate id. Required for statistical decision guidance. */
-    comparator?: string;
-    /** Which split to use for the primary decision. Default 'holdout'. */
-    split?: 'search' | 'holdout';
-    /** Confidence level used by lower-level report helpers. Default 0.95. */
-    confidence?: number;
-    /** FDR threshold for q-values. Default 0.05. */
-    fdr?: number;
-    /**
-     * Soft floor on paired observations before issuing a directional
-     * promote / reject. Below this we report `needs_more_data` and surface the
-     * minimum detectable effect at the current N. Default 20 — chosen so the
-     * Wilcoxon signed-rank approximation is reasonable and so the paired
-     * bootstrap CI has non-degenerate coverage. Hard floor is enforced at
-     * `RESEARCH_REPORT_HARD_PAIR_FLOOR` (6) regardless of this value.
-     */
-    minPairs?: number;
-    /**
-     * Region of Practical Equivalence on the paired delta. When a candidate's
-     * paired-delta CI is fully contained in `[low, high]`, the decision is
-     * `equivalent` rather than `hold`. Sourced from the domain owner — there is
-     * no statistically-defensible default.
-     */
-    rope?: {
-        low: number;
-        high: number;
-    };
-    /**
-     * Power for the minimum detectable effect (MDE) reported on each candidate.
-     * Default 0.8.
-     */
-    mdePower?: number;
-    /**
-     * Two-sided alpha for the MDE. Default matches `fdr` so the reported MDE
-     * lines up with the test the report actually runs.
-     */
-    mdeAlpha?: number;
-    /** Optional held-out gate decisions keyed by candidate id. */
-    gateDecisions?: Record<string, GateDecision$1>;
-    /** Optional failure clusters from failureClusterView. */
-    failureClusters?: FailureClusterReport;
-    /** Build gain histograms for these candidates. Defaults to all non-comparator candidates. */
-    candidateIds?: string[];
-    /** Deterministic bootstrap seed passed to gainHistogram and the posterior helper. */
-    seed?: number;
-    /** Report timestamp. Defaults to current time. */
-    generatedAt?: string;
-    /**
-     * Hash of a preregistered protocol (e.g. `signManifest({...}).contentHash`).
-     * Embedded verbatim in the report so the analysis can be cited as the
-     * preregistered one rather than a post-hoc fishing expedition.
-     */
-    preregistrationHash?: string;
-}
-interface ResearchReportRecommendation {
-    decision: ResearchReportDecision;
-    candidateId: string | null;
-    rationale: string[];
-    risks: string[];
-    nextActions: string[];
-}
-interface ResearchReportCandidate {
-    candidateId: string;
-    n: number;
-    mean: number;
-    ciLow: number;
-    ciHigh: number;
-    qValue: number;
-    cohensD: number;
-    meanDeltaVsComparator: number | null;
-    pairedN: number;
-    medianGain: number | null;
-    meanGain: number | null;
-    gainCi: {
-        low: number;
-        high: number;
-    } | null;
-    /**
-     * Bayesian-bootstrap-style posterior summaries on the paired delta. Computed
-     * from the same resamples that produce the gain CI; interpretable as
-     * "fraction of resamples in which the candidate beats the comparator on
-     * matched pairs."
-     */
-    prGreaterThanZero: number | null;
-    prInRope: number | null;
-    /**
-     * Minimum detectable effect (in score units) at the candidate's paired N,
-     * the configured power, and the configured alpha. Standardised by the
-     * observed paired-delta SD and inverted via `requiredSampleSize`. Reported
-     * for every candidate so a `needs_more_data` verdict is actionable.
-     */
-    mde: number | null;
-    onParetoFrontier: boolean;
-    gate?: ParetoPoint['gate'];
-    decision: ResearchReportDecision;
-    decisionReason: string;
-}
-interface ResearchReportMethodology {
-    /**
-     * Plain-language assumptions the report depends on. Read these first when
-     * deciding whether the verdict is load-bearing for a launch decision.
-     */
-    assumptions: string[];
-    /** Tests and estimators the verdict was computed from. */
-    methods: string[];
-    /** Alternatives the author considered and why this report didn't take them. */
-    alternatives: string[];
-    /** Failure modes — when this report should NOT drive a decision. */
-    whenNotToApply: string[];
-    /** Citations for the methodological choices above. */
-    citations: string[];
-}
-interface ResearchReport {
-    kind: 'agent-eval-research-report';
-    title: string;
-    generatedAt: string;
-    split: 'search' | 'holdout';
-    comparator: string | null;
-    /**
-     * SHA-256 over the canonicalised set of `(runId, candidateId, split)` triples
-     * the report was computed from, plus the comparator and split. Stable across
-     * key insertion order; recomputable by the reader to verify provenance.
-     */
-    runFingerprint: string;
-    preregistrationHash: string | null;
-    rope: {
-        low: number;
-        high: number;
-    } | null;
-    executiveSummary: string[];
-    recommendation: ResearchReportRecommendation;
-    candidates: ResearchReportCandidate[];
-    summary: SummaryTable;
-    charts: {
-        pareto: ParetoFigureSpec;
-        gains: GainDistributionFigureSpec[];
-    };
-    methodology: ResearchReportMethodology;
-    failureClusters?: FailureClusterReport;
-    markdown: string;
-    html: string;
-}
-
-/**
- * TraceEmitter — hierarchical span builder that auto-parents using an
- * internal stack. One emitter per Run; emitters do NOT share state.
- *
- * Convenience methods (`llm`, `tool`, `retrieval`, `judge`, `sandbox`)
- * return a `SpanHandle` with `.end()` / `.fail()` so callers don't
- * have to thread spanIds manually. For async workflows that can't use
- * the stack (e.g. fan-out parallel calls), pass `parentSpanId`
- * explicitly.
- */
-
-interface SpanHandle<S extends Span = Span> {
-    span: S;
-    end(patch?: Partial<S>): Promise<void>;
-    fail(error: string | Error, patch?: Partial<S>): Promise<void>;
-}
-interface RunCompleteHookContext {
-    runId: string;
-    emitter: TraceEmitter;
-    store: TraceStore;
-    /** Outcome the caller passed to `endRun` (undefined for `abortRun`). */
-    outcome?: RunOutcome$1;
-    /** Final run status. */
-    status: 'completed' | 'failed' | 'aborted';
-}
-type RunCompleteHook = (ctx: RunCompleteHookContext) => Promise<void> | void;
-interface TraceEmitterOptions {
-    runId?: string;
-    /** Inject a clock for deterministic tests. */
-    now?: () => number;
-    /** Inject an id generator for deterministic tests. */
-    id?: () => string;
-    /**
-     * Hooks fired after `endRun` / `abortRun` writes the final run state.
-     * Designed for trace-analyst auto-execution, integrity assertions, and
-     * outbound notifications. Hooks run sequentially in the order supplied.
-     *
-     * By default a hook that throws is swallowed and logged as a `note` event
-     * on the run — auto-orchestration must not crash the underlying flow.
-     * Set `hookErrors: 'throw'` to propagate.
-     */
-    onRunComplete?: RunCompleteHook[];
-    /** `'swallow'` (default) | `'throw'`. */
-    hookErrors?: 'swallow' | 'throw';
-}
-declare class TraceEmitter {
-    private store;
-    private stack;
-    private _runId;
-    private now;
-    private id;
-    private hooks;
-    private hookErrors;
-    constructor(store: TraceStore, options?: TraceEmitterOptions);
-    get runId(): string;
-    get traceStore(): TraceStore;
-    /** Append a hook after construction (e.g. attach the trace analyst). */
-    addRunCompleteHook(hook: RunCompleteHook): void;
-    /**
-     * Begin a Run.
-     *
-     * `scenarioId` is required on the persisted Run shape — every Run downstream
-     * gets a non-empty scenarioId so filters and aggregations stay simple — but
-     * the INPUT here accepts it as optional. When omitted, startRun substitutes
-     * a sensible default (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) so
-     * runtime / operator / meta-eval runs that have no curated-scenario corpus
-     * to anchor to don't have to invent placeholder strings at the call site.
-     */
-    startRun(run: Omit<Run, 'runId' | 'scenarioId' | 'startedAt' | 'status'> & {
-        scenarioId?: string;
-    }): Promise<Run>;
-    endRun(outcome?: RunOutcome$1): Promise<void>;
-    abortRun(reason: string): Promise<void>;
-    private runHooks;
-    span<S extends Span = Span>(init: {
-        kind: SpanKind;
-        name: string;
-        parentSpanId?: string;
-        attributes?: Record<string, unknown>;
-    } & Partial<Omit<S, 'spanId' | 'runId' | 'startedAt' | 'kind' | 'name'>>): Promise<SpanHandle<S>>;
-    private handle;
-    private pop;
-    llm(init: Omit<LlmSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<LlmSpan>>;
-    tool(init: Omit<ToolSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<ToolSpan>>;
-    retrieval(init: Omit<RetrievalSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<RetrievalSpan>>;
-    recordJudge(verdict: Omit<JudgeSpan, 'spanId' | 'runId' | 'kind' | 'startedAt' | 'endedAt'>): Promise<JudgeSpan>;
-    sandbox(init: Omit<SandboxSpan, 'spanId' | 'runId' | 'kind' | 'startedAt'>): Promise<SpanHandle<SandboxSpan>>;
-    emit(event: {
-        kind: EventKind;
-        spanId?: string;
-        payload?: Record<string, unknown>;
-    }): Promise<TraceEvent>;
-    recordBudget(entry: Omit<BudgetLedgerEntry, 'runId' | 'timestamp'> & {
-        timestamp?: number;
-    }): Promise<BudgetLedgerEntry>;
-    recordArtifact(artifact: Omit<Artifact, 'artifactId' | 'runId'>): Promise<Artifact>;
-    /**
-     * Runs `fn` inside a span; auto-ends on success, auto-fails on throw.
-     * Returns the fn's return value. Use this for the 95% case.
-     */
-    within<T>(init: Parameters<TraceEmitter['span']>[0], fn: (handle: SpanHandle) => Promise<T>): Promise<T>;
-}
-
-/**
- * Run-completion integrity check — at end of run, verify the expected event
- * types were actually captured. The point is the launch-review failure mode:
- * a run *appears* successful but the raw provider events were never written,
- * so a downstream reviewer can't reconstruct what happened.
- *
- * Pattern:
- *
- *   const report = await assertRunCaptured(store, runId, {
- *     llmSpansMin: 1,
- *     judgeSpansMin: 1,
- *     rawSink: providerSink,                  // must have ≥ 1 event for this run
- *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events
- *   })
- *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue
- *
- * The function is read-only on the store and returns a structured report;
- * the caller chooses the failure mode (throw, mark run failed, log warning).
- * `throwIfRunIncomplete` is the convenient strict mode.
- */
-
-interface RunIntegrityExpectations {
-    /** Minimum LLM span count. Default 0 (no requirement). */
-    llmSpansMin?: number;
-    /** Minimum judge span count. Default 0. */
-    judgeSpansMin?: number;
-    /** Minimum tool span count. Default 0. */
-    toolSpansMin?: number;
-    /**
-     * Raw provider sink to consult for capture verification. When present,
-     * the check requires at least one raw event for the run.
-     */
-    rawSink?: RawProviderSink;
-    /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */
-    rawProviderEventsMin?: number;
-    /**
-     * Every LLM span must have at least one matching raw `request` event
-     * (matched by spanId). Catches the common bug where the structured span
-     * was emitted but the raw HTTP capture was wired to a different sink.
-     */
-    requireRawCoverageOfLlmSpans?: boolean;
-    /** Run outcome must be set (not null/undefined). Default false. */
-    requireOutcome?: boolean;
-}
-type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
-interface RunIntegrityIssue {
-    code: RunIntegrityIssueCode;
-    message: string;
-    detail?: Record<string, unknown>;
-}
-interface RunIntegrityReport {
-    ok: boolean;
-    runId: string;
-    llmSpanCount: number;
-    judgeSpanCount: number;
-    toolSpanCount: number;
-    rawProviderEventCount: number;
-    /**
-     * Coverage of LLM spans by raw provider events keyed on spanId.
-     * `total` is the number of LLM spans; `covered` is the count with at
-     * least one matching `request` raw event.
-     */
-    rawSpanCoverage: {
-        covered: number;
-        total: number;
-    };
-    issues: RunIntegrityIssue[];
-}
-
-/**
- * EvalCampaign — opinionated matrix runner that wires the four
- * capture-integrity directives by construction.
- *
- * The canonical benchmark shape — matrix runner → for each
- * (variant, scenario, seed) → start a TraceEmitter → call LLMs → end the
- * run → analyze — has a bug class at the integration boundary: raw
- * events not captured, route silently wrong, integrity not asserted,
- * analyst never run. The directives in `SKILL.md § Capture integrity`
- * are the mitigations.
- *
- * `EvalCampaign` is the structural fix — consumers don't wire the
- * integrity surface themselves; the campaign owns it. Specifically:
- *
- *   - calls `assertLlmRoute` once at preflight before any work runs
- *   - constructs a per-run `TraceStore` and `RawProviderSink` via factories
- *   - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]`
- *   - hands the runner an `LlmClientOptions` pre-wired with the sink and
- *     trace context — the runner can't accidentally call an LLM without
- *     capturing the raw HTTP envelope
- *   - calls `assertRunCaptured` after every `endRun` and routes failures
- *     through a configurable policy (`throw` / `mark_failed` / `log`)
- *   - assembles per-run `RunRecord`s and runs `researchReport` at the end
- *     so the campaign artifact is launch-decision-grade by default
- *   - embeds the campaign fingerprint (a SHA-256 over the canonicalised
- *     run set) and optional `preregistrationHash` in the report
- *
- * The runner contract is intentionally narrow: produce a `CampaignRunOutcome`
- * given a fully-wired `CampaignRunContext`. Everything orchestration-shaped
- * lives in the campaign. This is the inversion-of-control point — consumers
- * stop writing matrix runners and start writing scenario-runners.
- *
- * Out of scope for v1 (tracked in `docs/research-report-methodology.md`):
- *
- *   - Distributed/cluster execution (concurrency is local async)
- *   - Adaptive sampling / sequential interim looks
- *   - Resume from partial state across crashes
- *   - LLM-call retry beyond what `LlmClient` already does
- */
-
-interface CampaignVariant<V> {
-    id: string;
-    payload: V;
-}
-interface CampaignScenario {
-    scenarioId: string;
-    /** Free-form metadata propagated to runs and reports. */
-    tags?: Record<string, string>;
-}
-interface CampaignRunContext<V> {
-    /** Stable run id. The campaign generates this; the runner does not. */
-    runId: string;
-    /** Logical experiment id (campaignId by default; overridable per-run via opts). */
-    experimentId: string;
-    variant: V;
-    variantId: string;
-    scenarioId: string;
-    scenarioTags: Record<string, string>;
-    seed: number;
-    splitTag: RunSplitTag;
-    /**
-     * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired
-     * (analyst auto-execution if configured, plus integrity check). The
-     * runner MUST call `emitter.startRun` before doing any work and either
-     * `emitter.endRun` or `emitter.abortRun` before returning.
-     */
-    emitter: TraceEmitter;
-    store: TraceStore;
-    rawSink: RawProviderSink;
-    /**
-     * Pre-wired LLM client options — `rawSink` and `traceContext` are populated
-     * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The
-     * runner can spread additional fields if needed.
-     */
-    llmOpts: LlmClientOptions;
-}
-interface CampaignRunOutcome {
-    /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
-    pass: boolean;
-    /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
-    score: number;
-    /** Mandatory cost in USD. Use 0 + raw.cost_unknown=1 only if truly unknown. */
-    costUsd: number;
-    tokenUsage: RunTokenUsage;
-    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
-    model: string;
-    /** sha256 of the effective prompt sent to the model. */
-    promptHash: string;
-    /** sha256 of the effective config (model, temperature, tools, judges, splits). */
-    configHash: string;
-    /** Optional extra numeric metrics to land in `outcome.raw`. */
-    raw?: Record<string, number>;
-    /** Canonical cross-agent failure class from the shared `FAILURE_CLASSES`
-     *  taxonomy. Propagated to `RunRecord.failureClass` so campaign runs
-     *  aggregate failures in the same vocabulary as every other producer. */
-    failureClass?: FailureClass;
-    /** Optional free-form failure detail, scoped under `failureClass`. */
-    failureMode?: string;
-    /** Optional judge metadata when a judge was used. */
-    judgeMetadata?: RunJudgeMetadata;
-    /**
-     * Optional per-judge / per-dim breakdown for ensemble-judged runs.
-     * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.
-     * Single-judge or scalar-only runs leave this unset.
-     */
-    judgeScores?: JudgeScoresRecord;
-    /**
-     * Agent profile cell observed by the runner. When supplied, it overrides
-     * `EvalCampaignOptions.agentProfile` for this run and must match the
-     * outcome's `model` and `promptHash`.
-     */
-    agentProfile?: AgentProfileCell | AgentProfileCellInput;
-}
-type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
-type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
-interface EvalCampaignOptions<V> {
-    /**
-     * Stable id for the campaign. Used as the default `experimentId` on
-     * every run, and folded into the campaign fingerprint.
-     */
-    campaignId: string;
-    variants: CampaignVariant<V>[];
-    scenarios: CampaignScenario[];
-    /** Default `[0, 1, 2]`. */
-    seeds?: number[];
-    /** Default `'holdout'` — the split that anchors a launch decision. */
-    splitTag?: RunSplitTag;
-    /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */
-    commitSha: string;
-    /**
-     * LLM client config. Augmented per-run with `rawSink` and `traceContext`
-     * before being passed to the runner. The campaign asserts this config
-     * matches `routeRequirements` once at preflight.
-     */
-    llmOpts: LlmClientOptions;
-    /**
-     * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail
-     * loud if the campaign would silently fall back to the public router or
-     * run unauthenticated. Override with an empty object to disable.
-     */
-    routeRequirements?: LlmRouteRequirements;
-    /**
-     * Per-run TraceStore factory. Common shape: a fresh store per run keyed
-     * on `runId`. Implementations that share a store across the campaign
-     * are valid — the campaign only writes through `emitter`.
-     */
-    storeFactory: (params: CampaignFactoryParams) => TraceStore;
-    /**
-     * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`
-     * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;
-     * otherwise required. Forensic capture is non-negotiable in a campaign
-     * run — pass `NoopRawProviderSink` explicitly if you want to opt out.
-     */
-    rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink;
-    /**
-     * Filesystem root for default `rawSinkFactory`. Ignored if
-     * `rawSinkFactory` is supplied.
-     */
-    workDir?: string;
-    /**
-     * Extra `onRunComplete` hooks the campaign appends (after its own
-     * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.
-     */
-    onRunComplete?: RunCompleteHook[];
-    /**
-     * Per-run integrity expectations. Defaults to:
-     *   `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.
-     * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.
-     */
-    integrity?: RunIntegrityExpectations;
-    /** Behaviour when integrity fails. Default `'mark_failed'`. */
-    onIntegrityFailure?: CampaignIntegrityPolicy;
-    /**
-     * Per-run runner. Receives a fully-wired context; produces an outcome
-     * the campaign converts into a `RunRecord`.
-     */
-    runner: CampaignRunner<V>;
-    /**
-     * If set, the campaign computes `researchReport` at the end. `comparator`
-     * is a `variantId`. Other fields are forwarded verbatim.
-     */
-    report?: {
-        comparator?: string;
-    } & Omit<ResearchReportOptions, 'comparator' | 'preregistrationHash' | 'generatedAt'>;
-    /**
-     * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).
-     * Embedded in the campaign fingerprint and the research report.
-     */
-    preregistrationHash?: string;
-    /** Local concurrency. Default `1` (sequential). */
-    concurrency?: number;
-    /**
-     * Override the time source. Tests pass a mock to make wallMs deterministic.
-     */
-    now?: () => number;
-    /** Override the runId generator. Tests pin this. */
-    runId?: (params: CampaignFactoryParams) => string;
-    /**
-     * Agent profile cell for campaign runs. Static profiles can pass an object;
-     * routers or variant-specific harnesses can pass a factory. The campaign
-     * stamps the built cell onto every `RunRecord` and rejects profile/model or
-     * profile/prompt contradictions.
-     */
-    agentProfile?: AgentProfileCell | AgentProfileCellInput | ((params: CampaignFactoryParams & {
-        variant: V;
-        scenarioTags: Record<string, string>;
-    }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
-}
-interface CampaignFactoryParams {
-    campaignId: string;
-    runId: string;
-    variantId: string;
-    scenarioId: string;
-    seed: number;
-}
-interface FailedRun {
-    runId: string;
-    variantId: string;
-    scenarioId: string;
-    seed: number;
-    reason: string;
-    error?: string;
-}
-interface EvalCampaignResult {
-    campaignId: string;
-    /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
-    campaignFingerprint: string;
-    preregistrationHash: string | null;
-    /** Successful runs only. Failed runs land in `failedRuns`. */
-    runs: RunRecord[];
-    /** Integrity reports for every successful run. */
-    integrityReports: RunIntegrityReport[];
-    failedRuns: FailedRun[];
-    /** Computed when `report` is set on options. */
-    report?: ResearchReport;
-    startedAt: string;
-    endedAt: string;
-}
-declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
-
-/**
- * Always-valid sequential evaluation.
- *
- * `researchReport` assumes a single pre-specified analysis. Real
- * consumers run campaigns weekly / nightly / per-PR; each new run silently
- * inflates the false-discovery rate, because the BH-FDR guarantee is for
- * the *first* look, not the 47th. Without time-uniform inference,
- * launch-decision teams either (a) don't peek, which forfeits the cost
- * advantage of stop-when-decisive, or (b) peek and pretend they didn't,
- * which forfeits scientific validity.
- *
- * This module ships **e-value-based confidence sequences** for paired
- * bounded outcomes. The methodology is the predictable plug-in betting
- * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*
- * stopping time. Concretely:
- *
- *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,
- *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable
- *   plug-in), and the running e-value is
- *
- *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)
- *
- *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by
- *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null
- *   at any time without inflating the type-I error.
- *
- * Combined with `runEvalCampaign`, every consumer running rolling
- * campaigns gains the ability to ship the moment evidence is decisive,
- * stop-early on dead-on-arrival variants, and accumulate evidence across
- * partial runs without spending the FDR budget. No new sweep is wasted.
- *
- * References:
- *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
- *     Time-uniform, nonparametric, nonasymptotic confidence sequences.
- *     Annals of Statistics, 49(2), 1055–1080.
- *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded
- *     random variables by betting. JRSS B, 86(1), 1–27.
- */
-type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent';
-interface InterimReleaseConfidence {
-    candidates: Array<{
-        candidateId: string;
-        decision: SequentialDecision;
-        decisionFiredAt: number | null;
-        finalEvalue: number;
-        finalPValue: number;
-        pairs: number;
-        csLow: number;
-        csHigh: number;
-    }>;
-    /**
-     * Campaign-level recommendation: pick the strongest 'promote_now', else
-     * 'continue' if any candidate is still live, else 'reject_now' if every
-     * candidate is dead, else 'equivalent'.
-     */
-    recommendation: {
-        decision: SequentialDecision;
-        candidateId: string | null;
-    };
-}
-
-/**
- * `runRLCampaign` — top-level orchestrator that runs the matrix and
- * produces every RL-ready artifact in one call.
- *
- * Wires:
- *   1. `runEvalCampaign` for the matrix run (capture, integrity, hooks)
- *   2. `extractVerifiableReward` over each run, separating deterministic
- *      from probabilistic reward sources for the trainer
- *   3. `extractPreferences` to produce DPO/PPO/KTO triples
- *   4. `evaluateInterimReleaseConfidence` over paired deltas (anytime-valid)
- *   5. `rubricPredictiveValidity` against an outcome store, when provided
- *   6. `detectRewardHacking` as a standing hygiene check
- *   7. Trainer-format export rows ready for prime-rl / TRL / verl
- *
- * The output `RLCampaignResult` is a single, audit-ready artifact: every
- * stage's output is in there. The consumer's downstream fits in a single
- * line: pass `result.preferences` to their DPO trainer, `result.grpoRows`
- * to GRPO, `result.runs` plus `result.rewardSignals` to a custom RL loop.
- */
-
-interface RunRLCampaignOptions<V> extends EvalCampaignOptions<V> {
-    /** Preference-extraction options. Default uses paired-by-scenario-and-seed with min-margin 0.05. */
-    preferences?: ExtractPreferencesOptions;
-    /** Verifiable-reward extraction options. */
-    verifiableReward?: VerifiableRewardExtractionOptions;
-    /** Outcome store + metric names — when supplied, runs `rubricPredictiveValidity` post-campaign. */
-    outcomeStore?: OutcomeStore;
-    outcomeMetrics?: string[];
-    /** Anytime-valid sequential evaluation options. */
-    sequential?: {
-        alpha?: number;
-        bound?: number;
-        rope?: {
-            low: number;
-            high: number;
-        };
-    };
-    /** Trainer-format export lookups. When provided, the orchestrator builds the corresponding rows. */
-    trainerExport?: {
-        dpo?: DpoLookups;
-        grpo?: GrpoLookups;
-        sft?: SftLookups;
-    };
-}
-interface RLCampaignResult<V> {
-    campaign: EvalCampaignResult;
-    /** Per-run verifiable reward (deterministic when available, probabilistic fallback otherwise). */
-    rewardSignals: Array<{
-        runId: string;
-        reward: VerifiableReward | null;
-    }>;
-    /** Preference extraction report. */
-    preferences: PreferenceExtractionReport;
-    /** Anytime-valid interim verdict over the paired deltas (vs comparator). */
-    interimConfidence: InterimReleaseConfidence | null;
-    /** Standing reward-hacking hygiene check. */
-    rewardHacking: RewardHackingReport;
-    /** Predictive validity, when an outcome store was supplied. */
-    predictiveValidity: RubricPredictiveValidityReport | null;
-    /** Trainer-export rows, populated only for the formats the caller requested via `trainerExport`. */
-    trainerRows: {
-        dpo?: DpoExportRow[];
-        grpo?: GrpoExportRow[];
-        sft?: SftExportRow[];
-    };
-    /**
-     * One-line top-level summary the consumer can log.
-     */
-    summary: string;
-    /**
-     * Convenience type-tag — consumers can branch on `result.kind`.
-     */
-    kind: 'agent-eval-rl-campaign';
-    unusedVariant?: V;
-}
-declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult<V>>;
-
-/**
- * Analyst contract — the missing orchestration layer over agent-eval's
- * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
- * SemanticConceptJudge, JudgeFn, ...).
- *
- * Each existing primitive returns its own output shape. The Analyst
- * contract is the single envelope every primitive lifts into, so a
- * registry can run N analysts against a run and a single renderer can
- * compose findings without knowing which analyzer produced them.
- *
- * The contract is intentionally domain-agnostic: nothing here knows
- * about code, voice, RAG, or any particular agent stack. Analysts
- * declare what INPUT KIND they need (a trace store, an artifact dir,
- * a RunRecord, a JudgeInput, or `custom`), and the registry routes
- * the matching input from `AnalystRunInputs`.
- */
-
-interface EvidenceRef {
-    /**
-     * Where the evidence lives. `span` and `event` refer to OTLP trace
-     * elements; `artifact` to a file inside the run's artifact tree;
-     * `finding` to another AnalystFinding (cross-analyst chaining);
-     * `metric` to a named scalar reading the renderer knows how to read.
-     */
-    kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
-    uri: string;
-    excerpt?: string;
-}
-
-type PolicyEditSchemaVersion = 'policy-edit/v1';
-declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
-type PolicyEditAxis = (typeof POLICY_EDIT_AXES)[number];
-declare const POLICY_EDIT_TARGET_SURFACES: readonly ["prompt", "tool-contract", "runtime-config", "memory", "agent-profile", "code", "deployment"];
-type PolicyEditTargetSurface = (typeof POLICY_EDIT_TARGET_SURFACES)[number];
-type PolicyEditRisk = 'low' | 'medium' | 'high' | 'unknown';
-type PolicyEditGainDirection = 'increase' | 'decrease';
-type PolicyEditGainUnit = 'absolute' | 'relative' | 'percent' | 'score';
-interface PolicyEditTarget {
-    surface: PolicyEditTargetSurface;
-    /** Stable path inside the target surface, for example `system-prompt:tools`
-     * or `budget.maxTurns`. */
-    path?: string;
-    /** Optional canonical deployment identity. Store the existing cell, not a
-     * local profile shape. */
-    agentProfileCell?: AgentProfileCell;
-    /** Human label when the path is not enough for a readable audit trail. */
-    label?: string;
-}
-type PolicyEditChange = {
-    kind: 'text';
-    mode: 'append' | 'prepend' | 'replace';
-    value: string;
-    /** Required when `mode === 'replace'`; exact match only. */
-    find?: string;
-} | {
-    kind: 'json';
-    mode: 'set' | 'merge' | 'remove';
-    path: string;
-    value?: AgentProfileJson;
-};
-interface PolicyEditExpectedGain {
-    /** Metric this edit is expected to move, e.g. `holdout.composite`. */
-    metric: string;
-    direction: PolicyEditGainDirection;
-    /** Positive magnitude in the metric's native units. */
-    amount: number;
-    unit?: PolicyEditGainUnit;
-    rationale?: string;
-}
-interface PolicyEditSource {
-    findingIds: string[];
-    analystIds: string[];
-    evidenceRefs: EvidenceRef[];
-    /** Mirrors `AnalystFinding.derived_from_judge`; admission rejects it. */
-    derivedFromJudge?: boolean;
-}
-interface PolicyEdit {
-    schemaVersion: PolicyEditSchemaVersion;
-    editId: string;
-    axis: PolicyEditAxis;
-    target: PolicyEditTarget;
-    change: PolicyEditChange;
-    claim: string;
-    expectedGain: PolicyEditExpectedGain;
-    confidence: number;
-    risk: PolicyEditRisk;
-    source: PolicyEditSource;
-    rationale?: string;
-    validationPlan?: string;
-    metadata?: Record<string, unknown>;
-}
-declare const POLICY_EDIT_CANDIDATE_RECORD_SCHEMA: "tangle.policy-edit-candidate.v1";
-/** JSON-safe attribution carried with a measured candidate and its scores. */
-interface PolicyEditCandidateRecord {
-    schema: typeof POLICY_EDIT_CANDIDATE_RECORD_SCHEMA;
-    policyEdit: PolicyEdit;
-}
-
-/**
- * Pass A substrate types — `runCampaign` is the one primitive every
- * eval flow composes from. Three contracts in this file:
- *
- *   - `Scenario`            input set
- *   - `DispatchFn`          how to run one scenario → artifact
- *   - `CampaignResult`      defined output schema (the contract downstream tools depend on)
- *
- * Three more lifted from earlier substrate work (re-exported):
- *
- *   - `JudgeConfig`         pluggable dimensional scorer (0.38)
- *   - `Mutator`             optimization-loop surface mutator
- *   - `Gate`                promotion gate (`HeldOutGate` and friends adapt to this)
- *
- * No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
- * can build dashboards / CI gates / regression diffs against a stable schema.
- */
-
-/** Stable identifier + kind tag for any scenario. Consumers
- *  extend with their per-domain payload (persona, task, requirement, ...). */
-interface Scenario {
-    id: string;
-    kind: string;
-    tags?: string[];
-}
-/** Redacted identity of a complete scenario payload retained in campaign results. */
-interface CampaignScenarioIdentity extends Pick<Scenario, 'id' | 'kind'> {
-    scenarioDigest: `sha256:${string}`;
-}
-/** The canonical judge verdict shape — one declaration, shared by campaign
- *  judges and the multishot judge runner (which re-exports this type).
- *
- *  Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
- *  multishot runner emits 0-10. Cross-scale comparison must go through
- *  `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
- *  promotion-policy) — never renormalize a producer's values in place, as
- *  downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
- *  `>= 7` gates) key on the producer's native scale. */
-interface JudgeScore {
-    dimensions: Record<string, number>;
-    composite: number;
-    notes: string;
-    /** Provider metadata for display and diagnostics; accounting uses CostLedger receipts. */
-    llmCall?: LlmCallMetadata;
-    /** Set when the judge itself failed (call error, unparseable output).
-     *  `composite`/`dimensions` carry no signal — aggregators MUST exclude
-     *  failed scores from means instead of folding them into zeros. */
-    failed?: true;
-    /** Ensemble extras (populated by `ensembleJudge`): max per-dimension
-     *  spread across surviving judges — the inter-rater signal. */
-    maxDisagreement?: number;
-    /** Ensemble extras: judge identities whose verdict failed. */
-    failedJudges?: string[];
-    /** Ensemble extras: each surviving judge's per-dimension scores. */
-    perJudge?: Record<string, Record<string, number>>;
-}
-/** Five-valued verdict taxonomy (MOSS-paper alignment). */
-type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
-interface GateResult {
-    decision: GateDecision;
-    reasons: string[];
-    contributingGates: Array<{
-        name: string;
-        passed: boolean;
-        detail: unknown;
-    }>;
-    delta?: number;
-}
-/** Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage`
- *  (run-record.ts, same package) so a cell maps onto a `RunRecord` for the
- *  backend-integrity guard with ONE source of truth — a field added to
- *  `RunTokenUsage` is a compile error here, not a silent drift. */
-type CampaignTokenUsage = RunTokenUsage;
-interface CampaignCellResult<TArtifact> {
-    /** Manifest that produced this cell. Resumability refuses to reuse a cell
-     *  whose manifest differs from the current run. */
-    manifestHash?: string;
-    cellId: string;
-    scenarioId: string;
-    rep: number;
-    generation?: number;
-    artifact: TArtifact;
-    judgeScores: Record<string, JudgeScore>;
-    costUsd: number;
-    /** True when at least one priced receipt used the model table instead of a provider bill. */
-    costEstimated?: boolean;
-    /** Exact durable receipts required to reuse this cached result. */
-    costCallIds?: string[];
-    /** Agent-call token usage committed by `ctx.cost.runPaidCall`.
-     *  `{ input: 0, output: 0 }` when no paid agent call was recorded. */
-    tokenUsage: CampaignTokenUsage;
-    /** Concrete model from the latest committed agent receipt. Consumed by
-     *  `buildRunRecord` to pin the model when the declared profile uses a
-     *  runtime-resolved sentinel. */
-    resolvedModel?: string;
-    durationMs: number;
-    seed: number;
-    cached: boolean;
-    error?: string;
-}
-interface JudgeAggregate {
-    mean: number;
-    stdev: number;
-    ci95: [number, number];
-    n: number;
-}
-interface ScenarioAggregate {
-    meanComposite: number;
-    ci95: [number, number];
-    n: number;
-}
-interface GenerationRecord {
-    generationIndex: number;
-    candidates: GenerationCandidate[];
-    promoted: string[];
-}
-/** One scored candidate surface in a generation. `dimensions` + `scenarios`
- *  let a reflective proposer ground its next proposal on WHICH
- *  dimensions the candidate is weakest on and WHICH scenarios it best/worst
- *  handled — the evidence a blind `Mutator` cannot see. */
-interface GenerationCandidate {
-    surfaceHash: string;
-    composite: number;
-    ci95: [number, number];
-    /** Exact surface this candidate mutated. */
-    parentSurfaceHash?: string;
-    /** Measured search-split composite of the exact parent surface. */
-    parentComposite?: number;
-    /** Candidate composite minus its parent's composite. Present only when the
-     *  candidate completed the designed denominator. */
-    observedDeltaFromParent?: number;
-    /** Whether this candidate had a scorable result for every designed campaign
-     *  cell and was therefore eligible for ranking, promotion, and Pareto
-     *  selection. Older externally-authored records may omit this field; loop
-     *  records always populate it. */
-    eligibleForPromotion?: boolean;
-    /** Exact denominator receipt for selection eligibility. Scores stay
-     *  descriptive: an incomplete candidate is retained with its observed score
-     *  and errors instead of receiving an invented penalty. */
-    coverage?: {
-        expectedCells: number;
-        scorableCells: number;
-        unscorableCells: Array<{
-            cellId: string;
-            reason: string;
-        }>;
-    };
-    /** Mean score per judge dimension across all cells (scenarios × reps ×
-     *  judges that reported the dimension). */
-    dimensions: Record<string, number>;
-    /** Per-scenario composite (mean over reps + judges), plus the judge's
-     *  free-form `notes` for that scenario — the "why it scored low" evidence a
-     *  reflective proposer grounds its next edit on. Keep `notes` GENERALIZABLE
-     *  (which checks/lines/dimensions failed and how), NOT case-specific ground
-     *  truth: leaking expected answers into the prompt is memorization, and the
-     *  held-out gate would reject it anyway. */
-    scenarios: Array<{
-        scenarioId: string;
-        composite: number;
-        notes?: string;
-    }>;
-    /** Proposer-supplied short label for the change. Present when the proposer
-     *  returned a `ProposedCandidate`; absent for bare-surface mutators. */
-    label?: string;
-    /** Proposer-supplied rationale — WHY this candidate was proposed. The
-     *  "because rationale Z" the audit requires to survive to the result.
-     *  Present when the proposer returned a `ProposedCandidate`. */
-    rationale?: string;
-    /** Exact structured cause threaded from the proposer, when available. */
-    candidateRecord?: PolicyEditCandidateRecord;
-}
-interface CampaignAggregates {
-    byJudge: Record<string, JudgeAggregate>;
-    byScenario: Record<string, ScenarioAggregate>;
-    /** Canonical campaign accounting, including worker and judge calls. */
-    cost: CostLedgerSummary;
-    /** Compatibility alias of `cost.totalCostUsd`. */
-    totalCostUsd: number;
-    cellsExecuted: number;
-    cellsSkipped: number;
-    cellsCached: number;
-    cellsFailed: number;
-}
-interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
-    /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
-    manifestHash: string;
-    /** Canonical identity of the exact scenario payloads and replicate count. */
-    splitDigest: `sha256:${string}`;
-    seed: number;
-    /** Replicates designed for every scenario in this campaign. */
-    reps: number;
-    startedAt: string;
-    endedAt: string;
-    durationMs: number;
-    cells: Array<CampaignCellResult<TArtifact>>;
-    aggregates: CampaignAggregates;
-    optimization?: {
-        generations: GenerationRecord[];
-        winnerSurfaceHash?: string;
-    };
-    gate?: GateResult;
-    prUrl?: string;
-    runDir: string;
-    artifactsByPath: Record<string, string>;
-    /** Redacted identities that let consumers verify the exact scenario payloads
-     *  without retaining customer task content in the result. */
-    scenarios: Array<CampaignScenarioIdentity & Pick<TScenario, 'id' | 'kind'>>;
-}
-
-/**
- * Adapters: convert measurement outputs into the canonical `RunRecord[]`
- * artifact that `replayCache`, `pairedEvalueSequence`, and
- * `rubricPredictiveValidity` consume. Two sources:
- *   - `campaignToRunRecords` — the campaign substrate's per-cell results
- *     (the modern path: `runCampaign` / `runImprovementLoop` → records).
- *   - `verificationReportToRunRecord` — a `MultiLayerVerifier` report.
- *
- * Adapters are thin and explicit — every mandatory `RunRecord` field comes
- * from a caller-supplied context (`commitSha`, `model`, `promptHash`,
- * `configHash`) plus the cell's runtime data. The validator still rejects
- * bare-alias model strings — the caller snapshot-pins.
- */
-
-interface AdapterContext {
-    /** Logical experiment id — typically the campaign or sweep identifier. */
-    experimentId: string;
-    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
-    model: string;
-    /** Git SHA the harness was run from. */
-    commitSha: string;
-    /** Hash of the effective prompt sent to the model. */
-    promptHash: string;
-    /** Hash of the effective config (model, temperature, tools, judges, splits). */
-    configHash: string;
-    /** Default split tag. Default `'search'`. */
-    splitTag?: RunSplitTag;
-    /** Default cost in USD when the source doesn't record one. Default `0`. */
-    defaultCostUsd?: number;
-}
-/**
- * Convert a `CampaignResult` into canonical `RunRecord[]` — one record per
- * scored cell. The cell's mean judge composite becomes the split score; every
- * judge dimension is carried through to `outcome.raw`. A cell that errored
- * becomes a record with `failureMode: 'cell_error'` (kept, not dropped — an
- * unscored cell is signal). `candidateId` identifies the measured surface
- * (defaults to the campaign manifest hash).
- */
-declare function campaignToRunRecords(campaign: CampaignResult, ctx: AdapterContext & {
-    candidateId?: string;
-}): RunRecord[];
-/**
- * Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
- * `outcome.searchScore` (or `holdoutScore`) is `report.blendedScore`;
- * `outcome.raw` carries every layer's score + a pass indicator; `failureMode`
- * is the first failing layer's reason.
- */
-declare function verificationReportToRunRecord(report: VerificationReport, ctx: AdapterContext & {
-    candidateId: string;
-    scenarioId?: string;
-}, opts?: {
-    runId?: string;
-}): RunRecord;
-
-/**
- * Simulator fidelity — score a user SIMULATOR's realism against real-user
- * trace distributions.
- *
- * Synthetic-persona evals (`PersonaConfig`-driven canonical evals, fuzz
- * user-simulator objectives) stand in for real users in most of the numbers
- * we publish. The standing threat is the Sim2Real gap: a simulator that is
- * distributionally unlike production creates "easy mode" and silently
- * inflates every score built on it. This module measures that gap from the
- * SAME artifact both sides already produce — `RunRecord`s — so no new
- * capture pipeline is needed:
- *
- *   - `simFidelityReport` — per-feature Jensen-Shannon divergence between
- *     simulated and production record distributions, collapsed into a
- *     fidelity coefficient in [0,1].
- *   - `easyModeCheck` — the headline academic failure mode (sim inflates
- *     pass-rate over production) as its own named artifact.
- *
- * Every synthetic-persona eval result should publish its fidelity
- * coefficient alongside the score — a number from an unrepresentative
- * simulator is an unlabeled estimate. Wire-in points:
- *
- *   - canonical persona evals: pass the campaign's `RunRecord`s as
- *     `simulated` and intake-adapter output (`contract/intake`: OTel spans,
- *     feedback tables, coding-agent sessions) as `production`
- *   - the fuzz user-sim objective: use `1 - report.fidelity` as a realism
- *     penalty when searching over generated personas
- *   - the durable corpus (`./corpus`): both sides read straight from
- *     `readCorpus` — tag sim vs production by `experimentId`
- */
-
-/** Extracts a flat behavioral feature map from one record. `string` values
- *  are categorical, `number` values are quantile-bucketed over the union of
- *  both sides, `null` means the feature is absent on this record and is
- *  counted explicitly as its own category (never silently dropped). */
-type BehaviorFeatures = (record: RunRecord) => Record<string, string | number | null>;
-/** Reserved histogram category for `null` feature values. A capture-rate
- *  difference (one side instruments a signal, the other does not) registers
- *  as divergence by design: a simulator that produces no tool traces is not
- *  representative of production that does. */
-declare const ABSENT_CATEGORY = "(absent)";
-/** Minimum non-null observations PER SIDE for a feature to enter the
- *  fidelity mean. Below this the JSD estimate is sampling noise. */
-declare const DEFAULT_MIN_N_PER_FEATURE = 20;
-/** Quantile buckets used to discretize numeric features. Quartiles balance
- *  resolution against per-bucket sample size at the default minN. */
-declare const DEFAULT_QUANTILE_BUCKETS = 4;
-/** Fidelity at or above this → 'representative'; below → 'skewed'.
- *  1 − 0.8 = mean JSD 0.2 ≈ distributions that mostly overlap with one
- *  clearly shifted mode — the point where per-feature shifts start changing
- *  which failure classes an eval can even observe. */
-declare const REPRESENTATIVE_MIN_FIDELITY = 0.8;
-/**
- * Default feature set — ONLY fields verified present on both simulated and
- * production records:
- *
- *   - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
- *     validator (non-finite values read as absent rather than poisoning a
- *     bucket).
- *   - `failure_class` — optional taxonomy field; absent counted explicitly.
- *   - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
- *     `outcome.raw` counters the intake adapters and eval harnesses write
- *     (`turns_completed`, `assistant_messages`, `tool_errors`,
- *     `turns_aborted`); absent on records whose producer did not capture
- *     them, counted explicitly.
- *   - `completion_length` — from the optional `CorpusRecord` trajectory
- *     text; the message-length proxy when records come from the corpus.
- *
- * `RunRecord` carries event COUNTS, not event ordering, so
- * `tool_error_recovery` is a counts-only derivation: errors occurred and the
- * run still completed cleanly ('recovered') vs aborted or classified as a
- * failure ('unrecovered') — not a literal error→retry sequence check.
- */
-declare const defaultBehaviorFeatures: BehaviorFeatures;
-/**
- * Jensen-Shannon divergence between two categorical histograms (raw counts;
- * normalized internally). Log base 2 → bounded [0,1]: 0 = identical
- * distributions, 1 = disjoint support. Symmetric, defined even where the
- * supports differ — exactly the regime sim-vs-production comparison lives in.
- * Throws on zero-mass or negative/non-finite counts: an empty histogram has
- * no distribution and a silent 0 would read as "perfectly representative".
- */
-declare function jsDivergence(p: Record<string, number>, q: Record<string, number>): number;
-/**
- * Deterministic quantile edges over a value set (the UNION of both sides, so
- * sim and production land in the same buckets). Linear interpolation between
- * order statistics; duplicate edges from heavy ties collapse into fewer,
- * wider buckets. Returns `bucketCount - 1` edges before deduplication.
- */
-declare function quantileEdges(values: number[], bucketCount?: number): number[];
-/** Stable half-open bucket label for a value against quantile edges:
- *  `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
-declare function bucketLabel(value: number, edges: number[]): string;
-interface FeatureShift {
-    /** Category label (a string value, a numeric bucket, or `ABSENT_CATEGORY`). */
-    value: string;
-    /** Probability of this category among ALL simulated records (nulls included
-     *  via `ABSENT_CATEGORY`, so each side's shifts sum to 1). */
-    pSim: number;
-    /** Probability among ALL production records. */
-    pProd: number;
-}
-interface FeatureDivergence {
-    feature: string;
-    /** Jensen-Shannon divergence in [0,1] for this feature. */
-    divergence: number;
-    /** Largest |pSim − pProd| categories, descending — where the sim deviates. */
-    topShifts: FeatureShift[];
-    /** Non-null observations on the simulated side. */
-    nSim: number;
-    /** Non-null observations on the production side. */
-    nProd: number;
-}
-type FidelityVerdict = 'representative' | 'skewed' | 'insufficient-data';
-interface FidelityReport {
-    perDimension: FeatureDivergence[];
-    /** 1 − mean divergence over features with sufficient data. NaN when the
-     *  verdict is 'insufficient-data' — a 0 would read as "maximally skewed"
-     *  and silently poison downstream aggregation; check `verdict` first. */
-    fidelity: number;
-    /** Features excluded because either side had fewer than `minNPerFeature`
-     *  non-null observations. Named, never silently dropped. */
-    insufficientData: string[];
-    /** 'representative' when fidelity >= REPRESENTATIVE_MIN_FIDELITY (0.8),
-     *  'skewed' below, 'insufficient-data' when no feature met minN. */
-    verdict: FidelityVerdict;
-}
-interface SimFidelityOptions {
-    /** Feature extractor. Defaults to `defaultBehaviorFeatures`. */
-    features?: BehaviorFeatures;
-    /** Minimum non-null observations per side per feature. Default 20. */
-    minNPerFeature?: number;
-}
-/**
- * Compare a simulator's RunRecords against production RunRecords, feature by
- * feature. Numeric features are bucketed by deterministic quantiles of the
- * union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
- * inputs — "no records" is a wiring error, not a distribution.
- */
-declare function simFidelityReport(simulated: RunRecord[], production: RunRecord[], opts?: SimFidelityOptions): FidelityReport;
-interface EasyModeOptions {
-    /** A run passes when its score (holdout, else search) >= this. Default 0.5
-     *  — matches the pass-threshold convention across the rl/ primitives. */
-    passThreshold?: number;
-    /** Pass-rate gap above which the sim is flagged inflated. Default 0.1 —
-     *  a 10-point inflation is enough to flip most promotion gates. */
-    inflationTolerance?: number;
-}
-interface EasyModeReport {
-    simPassRate: number;
-    prodPassRate: number;
-    /** simPassRate − prodPassRate. Positive = the simulator is easier than reality. */
-    gap: number;
-    /** True when gap > inflationTolerance: numbers measured against this
-     *  simulator overstate production performance. */
-    inflated: boolean;
-}
-/**
- * The headline simulator failure mode as its own named artifact: a simulator
- * that creates "easy mode" inflates pass-rate relative to production, and
- * every score measured against it overstates reality. Throws on empty inputs
- * and on records carrying neither score — a silently-skipped record would
- * bias the very rate this check exists to keep honest.
- */
-declare function easyModeCheck(simulated: RunRecord[], production: RunRecord[], opts?: EasyModeOptions): EasyModeReport;
-
-/**
- * Bradley-Terry / Elo tournament evaluation.
- *
- * For multi-candidate sweeps, comparing every candidate's score against
- * a fixed comparator wastes information — the comparator becomes a high-
- * variance reference and rank flips between near-tied middle-rank
- * candidates are dominated by noise. Pairwise tournaments fix this:
- * every (i, j) pair contributes a comparison to a Bradley-Terry MLE that
- * estimates each candidate's strength on a unified scale.
- *
- * For online updating (rolling campaigns where new candidates arrive
- * over time), we also ship classical Elo with configurable K-factor.
- *
- * References:
- *   - Bradley, R. A., Terry, M. E. (1952). Rank analysis of incomplete
- *     block designs. Biometrika, 39(3/4), 324–345.
- *   - Hunter, D. R. (2004). MM algorithms for generalized Bradley-Terry
- *     models. Annals of Statistics, 32(1), 384–406. (The MLE algorithm
- *     used here.)
- *   - Elo, A. E. (1978). The Rating of Chess Players, Past and Present.
- *
- * This is a useful primitive because most LLM-eval communities (Chatbot
- * Arena, AlpacaEval, ELO-style ablation) have converged on pairwise
- * tournament eval as the most sample-efficient and most rank-stable
- * method when you have many candidates.
- */
-interface PairwiseOutcome {
-    /** Winner candidate id. */
-    winner: string;
-    /** Loser candidate id. */
-    loser: string;
-    /**
-     * Optional draw flag. When true, both candidates get half-credit
-     * (Bradley-Terry handles draws as half-wins for each side).
-     */
-    draw?: boolean;
-    /**
-     * Optional weight — useful if some pairwise comparisons are stronger
-     * signals than others (e.g. a paired test with a wider score gap is
-     * a more confident comparison). Default 1.
-     */
-    weight?: number;
-}
-interface BradleyTerryRating {
-    candidateId: string;
-    /** Latent strength θ ≥ 0 from the BT MLE. */
-    strength: number;
-    /** Log-strength = log(θ) — interpretable on a linear scale. */
-    logStrength: number;
-    /** Number of pairwise comparisons this candidate appears in. */
-    n: number;
-    /** Win count (+ 0.5 per draw). */
-    wins: number;
-}
-interface BradleyTerryFit {
-    ratings: BradleyTerryRating[];
-    /** Iterations of the MM algorithm before convergence. */
-    iterations: number;
-    /** Final maximum |θ_new - θ_old| / θ_old. */
-    finalDelta: number;
-    converged: boolean;
-}
-/**
- * Bradley-Terry MLE via Hunter's MM algorithm.
- *
- * Iteration: θ_i^new = W_i / Σ_{j ≠ i} N_ij / (θ_i + θ_j)
- *   where W_i = wins by i (+ 0.5 per draw), N_ij = total comparisons.
- *
- * Returns log-strengths normalized so the smallest is 0 (any constant
- * offset is unobservable in BT — only differences are identified).
- */
-declare function fitBradleyTerry(outcomes: PairwiseOutcome[], opts?: {
-    tolerance?: number;
-    maxIterations?: number;
-    smoothing?: number;
-}): BradleyTerryFit;
-/**
- * Online Elo updates. Use when comparisons arrive over time and you want
- * a running rating without re-fitting the full BT MLE on every update.
- *
- * Initialize ratings to `defaultRating` (1500 by default). Each call to
- * `applyEloUpdate` mutates the map in place and returns the deltas so
- * the caller can log per-comparison rating changes.
- */
-interface EloOptions {
-    /** Default rating for unseen candidates. Default 1500. */
-    defaultRating?: number;
-    /** K-factor controls the step size. Default 32 (FIDE-ish). */
-    kFactor?: number;
-}
-declare function applyEloUpdate(ratings: Map<string, number>, outcome: PairwiseOutcome, opts?: EloOptions): {
-    winnerDelta: number;
-    loserDelta: number;
-};
-/**
- * Build pairwise outcomes from the campaign artifact: for every scenario
- * shared by two candidates, the higher-scoring run wins. Useful when you
- * want a tournament view of an existing campaign without an additional
- * pairwise judge call.
- */
-interface BuildPairwiseFromCampaignInput {
-    runs: Array<{
-        candidateId: string;
-        /** Stable identifier for the matching unit (typically scenarioId). */
-        matchKey: string;
-        score: number;
-    }>;
-    /**
-     * Tied-score margin. Below this, the comparison is a draw. Default 0
-     * (no ties).
-     */
-    drawMargin?: number;
-}
-declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
-
-export { ABSENT_CATEGORY, type AdaptationCurve, type AdaptationPoint, type AdaptationRunner, type AdapterContext, type AdversarialMutation, type BehaviorFeatures, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type CellObservation, type CompareCurvesResult, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CorpusAppendResult, type CorpusRecord, type CurriculumAllocation, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, type DatasetFormat, type DeploymentOutcome, type DetectRewardHackingInput, type DpoExportRow, type DpoLookups, type EasyModeOptions, type EasyModeReport, type EloOptions, type ExtractPreferencesOptions, type ExtractStepRewardsOptions, type FeatureDivergence, type FeatureShift, type FidelityReport, type FidelityVerdict, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type GrpoExportRow, type GrpoLookups, type HarvestOptions, InMemoryOutcomeStore, type OffPolicyEstimate, type OffPolicyOptions, type OffPolicyTrajectory, type OutcomeStore, type PairwiseOutcome, type ParetoPointInput, PredictiveValidityResearcher, type PredictiveValidityResearcherOptions, type PreferenceExtractionReport, type PreferenceStrategy, type PreferenceTriple, type PrmExportRow, type PrmLookups, type PrmTrainingTriple, REPRESENTATIVE_MIN_FIDELITY, type RLCampaignResult, type RewardHackingFinding, type RewardHackingReport, type RewardHackingSignal, type RewardKind, type RewardStats, type RlDatasetBundle, type RlDatasetConfig, type RlDatasetManifest, type RlDatasetStats, type RunAdaptationCurveOptions, type RunComputeCurveOptions, type RunRLCampaignOptions, type RunwiseStepSummary, type ScenarioPerturbation, type ScenarioPerturbationKind, type SelfConsistencyOptions, type SelfConsistencyResult, type SftExportRow, type SftLookups, type SimFidelityOptions, type StepReward, type StepRewardJsonlRow, type StepScorer, type ThompsonCurriculumOptions, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, appendToCorpus, applyEloUpdate, bestOfN, bucketLabel, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, campaignToRunRecords, compareAdaptationCurves, datasheetToMarkdown, defaultBehaviorFeatures, detectRewardHacking, doublyRobust, easyModeCheck, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, inverseProbabilityWeighting, jsDivergence, observationsFromRunRecords, offPolicyEstimateAll, paretoFrontier, prmTrainingPairs, quantileEdges, readCorpus, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runEvalCampaign, runRLCampaign, runwiseStepRewardSummary, selfConsistency, selfNormalizedImportanceWeighting, shuffleOrder, simFidelityReport, stepRewardsToJsonl, thompsonCurriculum, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, varianceBasedCurriculum, verificationReportToRunRecord };