@tangle-network/agent-eval 0.86.0 → 0.90.0

package/dist/rl.d.ts

@@ -1,22 +1,26 @@
 export { O as OffPolicyEstimate, a as OffPolicyOptions, b as OffPolicyTrajectory, d as doublyRobust, i as inverseProbabilityWeighting, o as offPolicyEstimateAll, s as selfNormalizedImportanceWeighting } from './off-policy-DiwuKKg7.js';
-import { R as RunRecord, a as RunSplitTag } from './run-record-De9VarXR.js';
-import { g as CampaignResult } from './types-D7lLRYe9.js';
-import { a as VerificationReport } from './multi-layer-verifier-DlWCXuxL.js';
-import { S as Span } from './schema-m0gsnbt3.js';
-import { T as TraceStore } from './store-CKUAgsJz.js';
+import { P as PreferenceExtractionReport, D as DpoExportRow, G as GrpoExportRow, S as SftExportRow, E as ExtractPreferencesOptions, a as DpoLookups, b as GrpoLookups, c as SftLookups } from './corpus-BoR-041R.js';
+export { d as CorpusAppendResult, C as CorpusRecord, e as DatasetFormat, f as ExtractStepRewardsOptions, H as HarvestOptions, g as PreferenceStrategy, h as PreferenceTriple, i as PrmExportRow, j as PrmLookups, k as PrmTrainingTriple, R as RewardKind, l as RewardStats, m as RlDatasetBundle, n as RlDatasetConfig, o as RlDatasetManifest, p as RlDatasetStats, q as RunwiseStepSummary, r as StepReward, s as StepRewardJsonlRow, t as StepScorer, u as appendToCorpus, v as buildDatasetFromCorpus, w as buildRlDataset, x as datasheetToMarkdown, y as extractPreferences, z as extractStepRewards, A as prmTrainingPairs, B as readCorpus, F as runwiseStepRewardSummary, I as stepRewardsToJsonl, J as toAnthropicFormat, K as toDpoJsonl, L as toDpoRows, M as toGrpoJsonl, N as toGrpoRows, O as toPrmJsonl, Q as toPrmRows, T as toSftJsonl, U as toSftRows, V as toTRLFormat } from './corpus-BoR-041R.js';
+import { g as CampaignResult } from './types-BU-7W85F.js';
+import { a as RunSplitTag, R as RunRecord } from './run-record-e7vj1uZQ.js';
+import { a as VerificationReport } from './multi-layer-verifier-DUZXrPDA.js';
+export { A as AdversarialMutation, a as AdversarialScenario, b as AdversarialSearchOptions, c as AdversarialSearchReport, d as adversarialScenarioSearch } from './adversarial-DIVcDoI_.js';
 import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
 export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from './outcome-store-rnXLEqSn.js';
-import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-CnEl9Jc8.js';
-import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-BLPHBbNV.js';
-export { r as runEvalCampaign } from './researcher-BLPHBbNV.js';
+import { b as RubricPredictiveValidityReport } from './rubric-predictive-validity-Cy_W-hWZ.js';
+import { R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-DE6Gpnb4.js';
+export { r as runEvalCampaign } from './researcher-DE6Gpnb4.js';
 import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
-import './errors-Dwqw-T_m.js';
-import './llm-client-CuUg2Mn3.js';
+import './schema-m0gsnbt3.js';
+import './store-CKUAgsJz.js';
+import './errors-CzMUYo7b.js';
+import './verdict-C9MlYujm.js';
+import './llm-client-BeEcAokY.js';
 import './raw-provider-sink-C46HDghv.js';
-import './summary-report-Db0dDSWP.js';
+import './summary-report-DGmUucwQ.js';
 import './failure-cluster-CL7IVgkJ.js';
 import './emitter-DEZwY14K.js';
-import './integrity-CJzrpUua.js';
+import './integrity-VJ9A7aST.js';
 
 /**
  * Test-time compute scaling curves.
@@ -262,147 +266,6 @@ declare function injectIrrelevantClause<S extends {
     prompt: string;
 }>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
 
-/**
- * Preference dataset extraction — bridge from `RunRecord[]` to RL training.
- *
- * Production RLHF / DPO / KTO / SimPO pipelines need preference triples:
- * `(prompt, chosen, rejected)`. The campaign artifact already contains the
- * ingredients — every (variantId, scenarioId, seed) cell is a candidate
- * that ran the same prompt against the same scenario, scored by the same
- * judge — but turning that into a clean preference dataset requires
- * deciding *what counts as a preference*.
- *
- * This module ships three preference-extraction strategies with explicit
- * tradeoffs, plus a unified output type compatible with HuggingFace TRL,
- * Anthropic finetuning JSONL, and OpenAI fine-tuning APIs. The strategies
- * are deliberately not auto-magical — picking the wrong one corrupts the
- * gradient.
- *
- * Strategies:
- *
- *   1. **`paired-by-scenario-and-seed`** — exact-match comparisons. For
- *      each scenario × seed pair, compare every (variantA, variantB) on
- *      that exact (scenario, seed). Matches scenarios so the comparison
- *      isolates variant effects. Highest signal-to-noise; smallest
- *      dataset (only matched pairs count).
- *
- *   2. **`paired-by-scenario`** — looser matching. For each scenario,
- *      compare every (variantA, variantB) where both have ≥ 1 run on the
- *      same scenario. Aggregates across seeds to compute mean scores per
- *      (variant, scenario), then forms preferences from the means. More
- *      data, lower per-pair signal.
- *
- *   3. **`top-vs-bottom`** — coarsest. Within each scenario, the highest-
- *      scoring run is `chosen`, the lowest is `rejected`. Smallest dataset
- *      per scenario but biggest score gap per pair. Useful for early
- *      bootstrapping when you have few variants.
- *
- * The output `PreferenceTriple` is *agent-eval-canonical* but trivially
- * mappable to TRL's `DPODataset` shape (`prompt`, `chosen`, `rejected`)
- * via the `toTRLFormat` helper.
- */
-
-type PreferenceStrategy = 'paired-by-scenario-and-seed' | 'paired-by-scenario' | 'top-vs-bottom';
-interface PreferenceTriple {
-    /** The scenario (input) the variants were run against. */
-    scenarioId: string;
-    /** RunRecord ids on each side, for traceability. */
-    chosenRunId: string;
-    rejectedRunId: string;
-    /** Variant ids — load-bearing for the RL update. */
-    chosenVariantId: string;
-    rejectedVariantId: string;
-    /** The score gap between chosen and rejected. Larger = stronger signal. */
-    marginScore: number;
-    /**
-     * Optional `(chosen_score, rejected_score)` pair for soft-margin DPO
-     * variants. Omitted for `top-vs-bottom` runs that don't carry meaningful
-     * scalar gaps.
-     */
-    scores?: {
-        chosen: number;
-        rejected: number;
-    };
-    /** Tie-breaker — when multiple seeds match this scenario, the one used. */
-    seed?: number;
-    /**
-     * Free-form metadata propagated from the run records — e.g. original
-     * prompt-hash, model, etc. Lets the RL trainer reconstruct the prompt.
-     */
-    meta: {
-        chosenPromptHash: string;
-        rejectedPromptHash: string;
-        chosenConfigHash: string;
-        rejectedConfigHash: string;
-        chosenModel: string;
-        rejectedModel: string;
-    };
-}
-interface ExtractPreferencesOptions {
-    strategy?: PreferenceStrategy;
-    /**
-     * Minimum score gap required to admit a pair. Pairs below this are
-     * dropped — they're noise, not signal. Default 0.05 (5% of [0,1]).
-     */
-    minMargin?: number;
-    /**
-     * Optional split tag filter — restrict to runs from one split. Default
-     * `'holdout'` (the canonical "real" signal).
-     */
-    splitTag?: RunRecord['splitTag'];
-    /**
-     * Optional reward extractor that overrides `outcome.holdoutScore` /
-     * `outcome.searchScore`. Use to drive preferences off a verifiable
-     * reward instead of the headline score.
-     */
-    rewardOf?: (run: RunRecord) => number | null;
-}
-interface PreferenceExtractionReport {
-    pairs: PreferenceTriple[];
-    /** Number of (scenario, seed) cells inspected. */
-    cellsInspected: number;
-    /** Number of pairs filtered by `minMargin`. */
-    pairsBelowMargin: number;
-    /** Number of cells with only one variant (no comparison possible). */
-    cellsSingleton: number;
-    /** Strategy used. */
-    strategy: PreferenceStrategy;
-}
-/**
- * Convert `RunRecord[]` to preference triples for RL training.
- *
- * Returns a structured report so callers can see how much data was
- * dropped and why (low-margin pairs, singleton cells). For production
- * pipelines, you usually want to:
- *
- *   1. Run a campaign producing 5–10 variants × 50–200 scenarios × 3 seeds
- *   2. Call this with `strategy: 'paired-by-scenario-and-seed'` and a
- *      verifiable-reward extractor as `rewardOf`
- *   3. Pass `report.pairs` to `toTRLFormat` and pipe to your DPO trainer
- */
-declare function extractPreferences(runs: RunRecord[], opts?: ExtractPreferencesOptions): PreferenceExtractionReport;
-/**
- * TRL-compatible export. TRL's `DPODataset` is `{ prompt, chosen, rejected }`
- * but the prompt isn't stored on the RunRecord — only its hash. The caller
- * passes a `promptOf(promptHash)` lookup that the TRL trainer can use.
- */
-declare function toTRLFormat(triples: PreferenceTriple[], promptOf: (hash: string) => string): Array<{
-    prompt: string;
-    chosen: string;
-    rejected: string;
-}>;
-/**
- * Anthropic finetuning JSONL export — `{ system, user, assistant_chosen, assistant_rejected }`
- * shape. Same caveat as TRL: prompt + outputs are content the caller has
- * to map back from the run record / raw event log.
- */
-declare function toAnthropicFormat(triples: PreferenceTriple[]): Array<{
-    scenarioId: string;
-    chosenRunId: string;
-    rejectedRunId: string;
-    margin: number;
-}>;
-
 /**
  * Adapters: convert measurement outputs into the canonical `RunRecord[]`
  * artifact that `replayCache`, `pairedEvalueSequence`, and
@@ -624,7 +487,17 @@ interface VerifiableReward {
     /** The layer / judge id that produced the signal, for provenance. */
     origin: string;
     /**
-     * Any per-source breakdown the consumer might want — e.g. `{ tests_passed: 7, tests_total: 10 }`.
+     * Per-source contribution to `value`, keyed by layer/judge id. Single-source
+     * rewards carry one entry (`{ [origin]: value }`); composite rewards carry
+     * every contributing layer's score — the anti-scalar-collapse surface RL
+     * consumers weight per-source instead of trusting one blended number.
+     */
+    components: Record<string, number>;
+    /**
+     * @deprecated Read `components` for per-source reward values. Kept for
+     * published-API compatibility: single-source rewards carry the layer's
+     * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry
+     * the same per-layer scores `components` now holds.
      */
     breakdown?: Record<string, number>;
 }
@@ -904,507 +777,6 @@ declare function compareAdaptationCurves(a: AdaptationCurve, b: AdaptationCurve,
 /** First k at which the curve's per-scenario pass rate reliably hits the threshold. */
 declare function firstPassK(curve: AdaptationCurve, threshold?: number): number | null;
 
-/**
- * Adversarial scenario search.
- *
- * Capability evaluation on a fixed scenario set measures performance on
- * the distribution someone curated. Production failure modes live in the
- * tail — inputs the curator didn't think of, or actively avoided. The
- * adversarial-search primitive actively looks for them: starting from a
- * pool of scenarios where the policy already passes, it mutates them
- * (paraphrase, edge-case substitution, compositional combination) and
- * keeps the mutations that *break* the policy.
- *
- * This is not magic. It's the simplest version of the loop that AdA
- * (Open-Ended Adaptation, DeepMind 2023), POET, and Anthropic's
- * auto-jailbreak rigs all run: hill-climb against a failure indicator,
- * keep the survivors, repeat. We ship the harness; consumers supply the
- * mutation strategies and the failure detector.
- *
- * Why ship this in agent-eval and not as a separate red-team tool: every
- * piece of the standard adversarial loop is already in this package
- * (`runEvalCampaign` for the matrix run, `RawProviderSink` for capture,
- * `assertRunCaptured` for integrity, `pairedEvalueSequence` for stop
- * criteria). The adversarial primitive is just the *scenario-mutation
- * meta-loop* on top of that machinery.
- */
-interface AdversarialScenario<S> {
-    /** Stable id — used for deduplication and lineage tracking. */
-    id: string;
-    /** Generation index — 0 for seeds, 1 for first round of mutations, etc. */
-    generation: number;
-    /** Lineage — id of the parent scenario this was mutated from, if any. */
-    parentId: string | null;
-    scenario: S;
-    /** Score on the policy under test. Lower = adversarial signal. */
-    score: number | null;
-    /** Strategy that produced this mutation, for diagnostics. */
-    mutationStrategy: string | null;
-}
-interface AdversarialMutation<S> {
-    id: string;
-    /**
-     * Mutate one scenario. Return null to skip; return one or more new
-     * scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
-     */
-    mutate(parent: S, rng: () => number): Promise<S[]> | S[];
-}
-interface AdversarialSearchOptions<S> {
-    /** Initial scenarios — typically those the policy currently passes. */
-    seeds: S[];
-    /** Stable identifier extraction. */
-    mutateScenarioId: (s: S) => string;
-    /** Mutation strategies. */
-    mutations: AdversarialMutation<S>[];
-    /**
-     * Run the policy under test against one scenario, return a scalar score
-     * in [0, 1]. Lower = adversarial signal.
-     */
-    scoreFn: (s: S) => Promise<number>;
-    /**
-     * Threshold below which a scenario counts as a "failure" worth keeping.
-     * Default 0.5.
-     */
-    failureThreshold?: number;
-    /** Number of mutation rounds. Default 3. */
-    rounds?: number;
-    /** Children per parent per round. Default 4. */
-    childrenPerParent?: number;
-    /** Maximum total scenarios examined. Default Infinity. */
-    budget?: number;
-    /** Seed for the deterministic RNG. Default 1. */
-    seed?: number;
-}
-interface AdversarialSearchReport<S> {
-    scenarios: AdversarialScenario<S>[];
-    /** Discovered failures sorted by score ascending. */
-    failures: AdversarialScenario<S>[];
-    /** Round-by-round counts. */
-    byGeneration: Array<{
-        generation: number;
-        total: number;
-        failures: number;
-        meanScore: number;
-    }>;
-    /** Total scoreFn invocations consumed. */
-    scoreCalls: number;
-}
-declare function adversarialScenarioSearch<S>(opts: AdversarialSearchOptions<S>): Promise<AdversarialSearchReport<S>>;
-
-/**
- * Process reward extraction — step-level credit assignment from trace spans.
- *
- * RL on long-horizon agents needs *step-level* rewards, not run-level
- * ones. The classic credit-assignment problem (Sutton & Barto) requires
- * knowing which sub-decisions in a trajectory contributed to the
- * outcome. Modern systems (DeepSeek-R1, OpenAI o-series, Lightman et al.
- * "Let's Verify Step by Step" 2023) train *process reward models* (PRMs)
- * that score every step, then do RL with the PRM as the reward signal.
- *
- * This module extracts `StepReward[]` from trace spans — one per
- * meaningful step — and ships:
- *
- *   1. `extractStepRewards(store, runId, opts)` — span → step-reward
- *      conversion using configurable per-span scorers (LLM judge over the
- *      span output, deterministic checkers, or a learned PRM).
- *   2. `runwiseStepRewardSummary(stepRewards)` — aggregate the per-step
- *      signal into a credit-assignment-aware run-level score.
- *   3. `prmTrainingPairs(stepRewards, options)` — produce the
- *      `(prefix, suffix_chosen, suffix_rejected)` triples that PRM
- *      training pipelines consume.
- *
- * What we ship: the *extraction* and *aggregation* infrastructure plus
- * the data shape PRM training expects. We do NOT ship the actual PRM
- * training (gradient descent over a transformer is out of scope for a
- * TS package). The interface is the contract; downstream consumers wire
- * their preferred trainer.
- *
- * Caveat the panel will land: this is descriptive credit assignment
- * (which steps correlate with outcome), not causal credit assignment
- * (which steps caused outcome). For causal claims you need
- * counterfactual rollouts or a learned dynamics model. Future work; the
- * descriptive version is what production PRM training actually uses.
- */
-
-interface StepReward {
-    /** Trace span this reward attaches to. */
-    spanId: string;
-    runId: string;
-    /** Index in the trajectory (0-based, in started-at order). */
-    stepIndex: number;
-    /** Span kind (typically 'tool', 'llm', 'judge'). */
-    kind: Span['kind'];
-    /** Span name — for the consumer's downstream filtering. */
-    name: string;
-    /** Step-level reward in [0, 1]. */
-    reward: number;
-    /**
-     * Determinism class. Mirrors the verifiable-reward distinction:
-     * deterministic = test/compile/schema check; probabilistic = LLM judge.
-     */
-    determinism: 'deterministic' | 'probabilistic';
-    /** Optional rationale / evidence — the trainer typically discards. */
-    rationale?: string;
-    /** Optional weight — how much this step contributes to credit assignment. */
-    weight?: number;
-}
-interface StepScorer {
-    /** Span kinds this scorer applies to. */
-    appliesTo: Span['kind'][];
-    /** Returns null to skip the span; returns a `StepReward` shape (without index/runId/spanId, which are filled in). */
-    score(span: Span): Promise<Omit<StepReward, 'spanId' | 'runId' | 'stepIndex'>> | null | undefined;
-}
-interface ExtractStepRewardsOptions {
-    /**
-     * Ordered list of scorers. Each span runs through scorers in order;
-     * the first non-null result wins. If no scorer applies, the span is
-     * skipped (not all spans are training-worthy).
-     */
-    scorers: StepScorer[];
-    /** Optional filter — return null to drop the span entirely before scoring. */
-    preFilter?: (span: Span) => boolean;
-}
-declare function extractStepRewards(store: TraceStore, runId: string, opts: ExtractStepRewardsOptions): Promise<StepReward[]>;
-interface RunwiseStepSummary {
-    runId: string;
-    totalSteps: number;
-    meanReward: number;
-    /** Sum-of-rewards (weighted by `weight ?? 1`). Use as the run-level proxy. */
-    sumWeightedReward: number;
-    /** Fraction of steps where reward < 0.5 — proxy for "where the policy was wrong." */
-    failureFraction: number;
-    /** Maximum drop in reward between consecutive steps — diagnoses a step where things went sideways. */
-    worstStepDelta: number;
-    worstStepIndex: number | null;
-}
-declare function runwiseStepRewardSummary(stepRewards: StepReward[]): RunwiseStepSummary;
-interface PrmTrainingTriple {
-    /** Prefix run-id (or composite key) — the trajectory up to step k-1. */
-    prefixRunId: string;
-    prefixStepIndex: number;
-    /** The step that came next on a high-reward trajectory. */
-    chosenSpanId: string;
-    chosenReward: number;
-    /** A step from a divergent low-reward trajectory at the same prefix length. */
-    rejectedSpanId: string;
-    rejectedReward: number;
-    /** The prefix run came from this run; the rejected step came from `rejectedRunId`. */
-    rejectedRunId: string;
-    marginScore: number;
-}
-/**
- * Build PRM training triples. The shape: pair runs that share an early
- * prefix (same scenario, same first N steps) and diverge later — at the
- * point of divergence, the high-reward run's next step is `chosen`, the
- * low-reward run's next step is `rejected`. This is the canonical PRM
- * training data shape from Lightman et al. and DeepSeek-R1 process
- * supervision.
- *
- * Implementation note: we don't have a way to detect "same prefix" in
- * the general agent setting (token-level prefixes require hashing model
- * outputs). The current heuristic groups by `(scenarioId, prefixSpanName
- * sequence)` — runs are paired when their first K span names match. For
- * production use this should be replaced with a proper trajectory-prefix
- * hash; the heuristic is good enough for early-stage scaffolding.
- */
-declare function prmTrainingPairs(stepRewardsByRun: Map<string, StepReward[]>, opts?: {
-    minMargin?: number;
-    minPrefixLength?: number;
-}): PrmTrainingTriple[];
-
-/**
- * Trainer-format exporters.
- *
- * agent-eval produces canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`,
- * `StepReward[]`, `PrmTrainingTriple[]`). RL training pipelines consume
- * different shapes — Hugging Face TRL, Prime Intellect's prime-rl, OpenAI
- * fine-tuning, Anthropic finetuning, OpenRLHF, verl. Each has its own
- * JSONL conventions. Rather than ship N adapters, this module ships the
- * canonical formats most production pipelines accept and ergonomic helpers
- * for the rest.
- *
- * Shapes:
- *   - **DPO / IPO / KTO** — `{prompt, chosen, rejected}` JSONL. Consumed
- *     by HuggingFace TRL, prime-rl's offline DPO, OpenRLHF.
- *   - **GRPO offline** — `{prompt, completions[], rewards[]}` JSONL.
- *     Consumed by prime-rl GRPO, verl, OpenRLHF.
- *   - **SFT** — `{messages[]}` JSONL with chosen completion as the final
- *     assistant turn. Consumed by HF SFT trainers, OpenAI fine-tuning,
- *     Anthropic finetuning.
- *   - **PRM** — `{prompt, prefix_steps[], chosen_step, rejected_step}` JSONL.
- *     Consumed by Lightman-style PRM trainers and prime-rl's PRM mode.
- *
- * Why ship this in agent-eval rather than a separate adapter package: the
- * canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`, etc.) are
- * agent-eval's contract; without first-party exporters consumers reverse-
- * engineer the mapping every release. The exporters codify it.
- *
- * The exporters take callbacks for any field that isn't on the canonical
- * artifact (specifically: prompt + completion text, since the package
- * stores only their hashes by design — full text is the consumer's
- * trace store / raw event log).
- */
-
-interface DpoLookups {
-    /** Resolve the prompt text for a run (typically from a trace store / raw event sink). */
-    promptOf: (runId: string) => string | Promise<string>;
-    /** Resolve the assistant completion text for a run. */
-    completionOf: (runId: string) => string | Promise<string>;
-}
-interface DpoExportRow {
-    prompt: string;
-    chosen: string;
-    rejected: string;
-    /** Carried-through margin. Some KTO / IPO variants use this. */
-    margin?: number;
-    /** Free-form metadata for downstream filtering / sharding. */
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert preference triples to TRL-compatible DPO rows. The shape
- * `{prompt, chosen, rejected}` is the canonical HuggingFace DPODataset
- * entry; every major DPO trainer accepts it.
- */
-declare function toDpoRows(triples: PreferenceTriple[], lookups: DpoLookups): Promise<DpoExportRow[]>;
-/** Serialize DPO rows as JSONL. One line per row. */
-declare function toDpoJsonl(rows: DpoExportRow[]): string;
-interface GrpoLookups {
-    promptOf: (runId: string) => string | Promise<string>;
-    completionOf: (runId: string) => string | Promise<string>;
-    /** Optional: derive a custom reward from the run. Defaults to score. */
-    rewardOf?: (run: RunRecord) => number | null;
-}
-interface GrpoExportRow {
-    prompt: string;
-    completions: string[];
-    rewards: number[];
-    /** runIds in the same order as `completions[]` for traceability. */
-    runIds: string[];
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert RunRecord[] grouped by `(scenarioId)` into GRPO offline rows —
- * one row per scenario, with one completion per run on that scenario.
- *
- * GRPO (Shao et al. 2024 / DeepSeek-R1) trains on relative advantages
- * within a group of completions for the same prompt; this is the
- * canonical input format.
- */
-declare function toGrpoRows(runs: RunRecord[], lookups: GrpoLookups): Promise<GrpoExportRow[]>;
-declare function toGrpoJsonl(rows: GrpoExportRow[]): string;
-interface SftLookups {
-    promptOf: (runId: string) => string | Promise<string>;
-    completionOf: (runId: string) => string | Promise<string>;
-    /** Optional system message. Default omits. */
-    systemOf?: (run: RunRecord) => string | null | undefined;
-    /** Filter — return false to skip the run (e.g., low score, failed cases). */
-    include?: (run: RunRecord) => boolean;
-}
-interface SftExportRow {
-    messages: Array<{
-        role: 'system' | 'user' | 'assistant';
-        content: string;
-    }>;
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert RunRecord[] into Hugging Face / OpenAI / Anthropic-style
- * conversational SFT rows. By default every record becomes one row;
- * pass `include` to filter (e.g., keep only `score >= 0.8` for
- * rejection-sampling SFT).
- */
-declare function toSftRows(runs: RunRecord[], lookups: SftLookups): Promise<SftExportRow[]>;
-declare function toSftJsonl(rows: SftExportRow[]): string;
-interface PrmLookups {
-    /** Resolve the prompt text for a run. */
-    promptOf: (runId: string) => string | Promise<string>;
-    /** Resolve the trajectory step text for a (runId, spanId) pair. */
-    stepTextOf: (runId: string, spanId: string) => string | Promise<string>;
-    /** Optional: sequence of prefix span ids leading up to the divergence. */
-    prefixOf?: (runId: string, prefixStepIndex: number) => string[] | Promise<string[]>;
-}
-interface PrmExportRow {
-    prompt: string;
-    /** Span ids for the steps before divergence — caller resolves text via `stepTextOf`. */
-    prefixSpanIds: string[];
-    prefixStepText: string[];
-    chosenStep: string;
-    rejectedStep: string;
-    chosenReward: number;
-    rejectedReward: number;
-    marginScore: number;
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert PRM training triples to JSONL rows. Caller's `stepTextOf`
- * callback resolves span text from the consumer's trace store.
- */
-declare function toPrmRows(triples: PrmTrainingTriple[], lookups: PrmLookups): Promise<PrmExportRow[]>;
-declare function toPrmJsonl(rows: PrmExportRow[]): string;
-interface StepRewardJsonlRow {
-    runId: string;
-    spanId: string;
-    stepIndex: number;
-    reward: number;
-    determinism: 'deterministic' | 'probabilistic';
-    weight: number;
-}
-declare function stepRewardsToJsonl(stepRewards: StepReward[]): string;
-
-/**
- * RL dataset packaging + datasheet — the publishable, sellable bundle.
- *
- * The format exporters (`toGrpoRows` / `toSftRows` / `toDpoRows`) already
- * produce trainer-ready shapes (prime-rl GRPO, TRL DPO, conversational SFT).
- * What turns that into a dataset someone can PUBLISH or BUY is the provenance
- * + a datasheet: which models produced it, which prompt/agent versions, how the
- * reward was derived (deterministic verifiable vs probabilistic judge — the
- * credibility axis a buyer checks first), the split discipline, the reward
- * distribution, the quality gates, the license, and the intended/out-of-scope
- * uses. This module computes those facts from the `RunRecord[]` and renders a
- * "Datasheet for Datasets" (Gebru et al. 2018) card alongside the format files.
- *
- * It composes the existing `rl/exporters` — it does not reimplement any trainer
- * format. The renderers token-identity step (DeepSeek/Kimi/Qwen tokenization
- * with per-token loss masks) is a downstream Python stage that consumes the
- * `messages`/`completions` this bundle emits.
- */
-
-type RewardKind = 'deterministic' | 'probabilistic' | 'mixed';
-type DatasetFormat = 'grpo' | 'sft' | 'dpo';
-/** Caller-declared context — the qualitative half of the datasheet that can't
- *  be computed from records. */
-interface RlDatasetConfig {
-    name: string;
-    version: string;
-    /** Product/task domain, e.g. 'legal-m&a', 'tax-1040'. */
-    domain: string;
-    /** SPDX id or a named commercial license. Required — an unlicensed dataset
-     *  cannot be published or sold. */
-    license: string;
-    /** How the reward was produced. `kind: 'deterministic'` (a test/schema/XPath
-     *  decided it) is the credibility signal; 'probabilistic' = LLM-judge. */
-    reward: {
-        kind: RewardKind;
-        source: string;
-        description: string;
-    };
-    intendedUse: string;
-    outOfScope: string;
-    limitations: string;
-    /** ISO timestamp — passed in (the substrate forbids Date.now()). */
-    createdAtIso: string;
-    /** Default: ['grpo', 'sft']. */
-    formats?: DatasetFormat[];
-    /** Quality gates already run, recorded on the card for the buyer. */
-    qualityGates?: {
-        contaminationProbe?: 'passed' | 'failed' | 'not-run';
-        dedup?: boolean;
-        verifiableRewardFilter?: boolean;
-    };
-}
-interface RewardStats {
-    n: number;
-    mean: number;
-    median: number;
-    min: number;
-    max: number;
-    std: number;
-}
-interface RlDatasetStats {
-    records: number;
-    /** Record count per split — a publishable dataset must declare its holdout. */
-    splits: Record<RunSplitTag, number>;
-    reward: RewardStats;
-    /** Distinct snapshot-pinned models that produced the trajectories. */
-    models: string[];
-    /** Distinct effective-prompt hashes (the agent profile/prompt versions). */
-    promptHashes: string[];
-    commitShas: string[];
-    totalTokens: {
-        input: number;
-        output: number;
-    };
-    totalCostUsd: number;
-}
-interface RlDatasetManifest extends RlDatasetConfig {
-    formats: DatasetFormat[];
-    rowCounts: Partial<Record<DatasetFormat, number>>;
-    stats: RlDatasetStats;
-}
-interface RlDatasetBundle {
-    manifest: RlDatasetManifest;
-    /** Relative filename -> contents. Write these to a directory to publish. */
-    files: Record<string, string>;
-}
-/**
- * Package graded `RunRecord[]` into a publishable RL dataset bundle: the
- * trainer-format JSONL files + a manifest + a datasheet. DPO requires
- * pre-extracted preference triples (pass `preferences`); GRPO/SFT derive from
- * the records directly via the supplied lookups. Throws on an empty corpus —
- * an empty dataset must never be published.
- */
-declare function buildRlDataset(records: RunRecord[], lookups: GrpoLookups & SftLookups, config: RlDatasetConfig, preferences?: {
-    triples: PreferenceTriple[];
-    lookups: DpoLookups;
-}): Promise<RlDatasetBundle>;
-/** Render the "Datasheet for Datasets" card — the artifact a buyer reads. */
-declare function datasheetToMarkdown(m: RlDatasetManifest): string;
-
-/**
- * RL corpus — the durable, append-only accumulation of graded RunRecords that
- * every eval run deposits BY DEFAULT.
- *
- * The dataset is the free exhaust of the normal eval process: we run evals
- * constantly to get an agent production-ready, and those runs already produce
- * graded trajectories. Instead of writing them to an ephemeral run dir and
- * throwing them away, `appendToCorpus` accumulates them into a durable corpus;
- * `buildDatasetFromCorpus` later harvests the whole corpus into a publishable
- * bundle. No separate data-collection campaign — the data accrues from work we
- * do anyway. This is the "best things for free by our process" layer.
- *
- * Trajectory text rides on the record as top-level `prompt` / `completion`
- * (what the eval harnesses capture; the RunRecord validator ignores the extra
- * keys). The harvest reads them directly — no trace store round-trip needed.
- */
-
-/** A corpus record is a RunRecord carrying the trajectory text the harness
- *  captured. `prompt`/`completion` are top-level (the validator ignores extras). */
-type CorpusRecord = RunRecord & {
-    prompt?: string;
-    completion?: string;
-};
-interface CorpusAppendResult {
-    appended: number;
-    /** Skipped because a record with the same runId was already in the corpus
-     *  (idempotent appends — NOT re-run collapsing; re-runs get fresh runIds). */
-    skipped: number;
-    total: number;
-}
-/**
- * Append graded records to the corpus (append-only JSONL). Deduplicates by
- * `runId` against what's already on disk so re-running the same harness is
- * idempotent. Creates the file and parent dir. This is the call every eval
- * harness makes by default after producing its records.
- */
-declare function appendToCorpus(records: CorpusRecord[], corpusPath: string): CorpusAppendResult;
-/** Read the full corpus. Returns [] if the corpus does not exist yet. */
-declare function readCorpus(corpusPath: string): CorpusRecord[];
-interface HarvestOptions {
-    /** Keep only records scoring >= this (rejection-sampling for SFT). */
-    minScore?: number;
-    /** Keep only these splits (e.g. ['holdout'] for an eval-only dataset). */
-    splits?: RunRecord['splitTag'][];
-}
-/**
- * Harvest the accumulated corpus into a publishable RL dataset bundle. Reads
- * trajectory text from each record's top-level `prompt`/`completion`; records
- * missing either are excluded (a graded score with no trajectory can't train).
- * Optionally filters by score / split. Throws (via buildRlDataset) if nothing
- * survives — an empty dataset must never be published.
- */
-declare function buildDatasetFromCorpus(corpusPath: string, config: RlDatasetConfig, opts?: HarvestOptions): Promise<RlDatasetBundle>;
-
 /**
  * `PredictiveValidityResearcher` — concrete `Researcher` implementation
  * that drives selection from outcome-anchored predictive validity.
@@ -1651,4 +1023,170 @@ interface RLCampaignResult<V> {
 }
 declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult<V>>;
 
-export { type AdaptationCurve, type AdaptationPoint, type AdaptationRunner, type AdapterContext, type AdversarialMutation, type AdversarialScenario, type AdversarialSearchOptions, type AdversarialSearchReport, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type CellObservation, type CompareCurvesResult, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CorpusAppendResult, type CorpusRecord, type CurriculumAllocation, type DatasetFormat, type DetectRewardHackingInput, type DpoExportRow, type DpoLookups, type EloOptions, type ExtractPreferencesOptions, type ExtractStepRewardsOptions, type GrpoExportRow, type GrpoLookups, type HarvestOptions, OutcomeStore, type PairwiseOutcome, type ParetoPointInput, PredictiveValidityResearcher, type PredictiveValidityResearcherOptions, type PreferenceExtractionReport, type PreferenceStrategy, type PreferenceTriple, type PrmExportRow, type PrmLookups, type PrmTrainingTriple, type RLCampaignResult, type RewardHackingFinding, type RewardHackingReport, type RewardHackingSignal, type RewardKind, type RewardStats, type RlDatasetBundle, type RlDatasetConfig, type RlDatasetManifest, type RlDatasetStats, type RunAdaptationCurveOptions, type RunComputeCurveOptions, type RunRLCampaignOptions, type RunwiseStepSummary, type ScenarioPerturbation, type ScenarioPerturbationKind, type SelfConsistencyOptions, type SelfConsistencyResult, type SftExportRow, type SftLookups, type StepReward, type StepRewardJsonlRow, type StepScorer, type ThompsonCurriculumOptions, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, adversarialScenarioSearch, appendToCorpus, applyEloUpdate, bestOfN, buildDatasetFromCorpus, buildPairwiseFromCampaign, buildRlDataset, campaignToRunRecords, compareAdaptationCurves, datasheetToMarkdown, detectRewardHacking, extractPreferences, extractStepRewards, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, observationsFromRunRecords, paretoFrontier, prmTrainingPairs, readCorpus, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runRLCampaign, runwiseStepRewardSummary, selfConsistency, shuffleOrder, stepRewardsToJsonl, thompsonCurriculum, toAnthropicFormat, toDpoJsonl, toDpoRows, toGrpoJsonl, toGrpoRows, toPrmJsonl, toPrmRows, toSftJsonl, toSftRows, toTRLFormat, varianceBasedCurriculum, verificationReportToRunRecord };
+/**
+ * Simulator fidelity — score a user SIMULATOR's realism against real-user
+ * trace distributions.
+ *
+ * Synthetic-persona evals (`PersonaConfig`-driven canonical evals, fuzz
+ * user-simulator objectives) stand in for real users in most of the numbers
+ * we publish. The standing threat is the Sim2Real gap: a simulator that is
+ * distributionally unlike production creates "easy mode" and silently
+ * inflates every score built on it. This module measures that gap from the
+ * SAME artifact both sides already produce — `RunRecord`s — so no new
+ * capture pipeline is needed:
+ *
+ *   - `simFidelityReport` — per-feature Jensen-Shannon divergence between
+ *     simulated and production record distributions, collapsed into a
+ *     fidelity coefficient in [0,1].
+ *   - `easyModeCheck` — the headline academic failure mode (sim inflates
+ *     pass-rate over production) as its own named artifact.
+ *
+ * Every synthetic-persona eval result should publish its fidelity
+ * coefficient alongside the score — a number from an unrepresentative
+ * simulator is an unlabeled estimate. Wire-in points:
+ *
+ *   - canonical persona evals: pass the campaign's `RunRecord`s as
+ *     `simulated` and intake-adapter output (`contract/intake`: OTel spans,
+ *     feedback tables, coding-agent sessions) as `production`
+ *   - the fuzz user-sim objective: use `1 - report.fidelity` as a realism
+ *     penalty when searching over generated personas
+ *   - the durable corpus (`./corpus`): both sides read straight from
+ *     `readCorpus` — tag sim vs production by `experimentId`
+ */
+
+/** Extracts a flat behavioral feature map from one record. `string` values
+ *  are categorical, `number` values are quantile-bucketed over the union of
+ *  both sides, `null` means the feature is absent on this record and is
+ *  counted explicitly as its own category (never silently dropped). */
+type BehaviorFeatures = (record: RunRecord) => Record<string, string | number | null>;
+/** Reserved histogram category for `null` feature values. A capture-rate
+ *  difference (one side instruments a signal, the other does not) registers
+ *  as divergence by design: a simulator that produces no tool traces is not
+ *  representative of production that does. */
+declare const ABSENT_CATEGORY = "(absent)";
+/** Minimum non-null observations PER SIDE for a feature to enter the
+ *  fidelity mean. Below this the JSD estimate is sampling noise. */
+declare const DEFAULT_MIN_N_PER_FEATURE = 20;
+/** Quantile buckets used to discretize numeric features. Quartiles balance
+ *  resolution against per-bucket sample size at the default minN. */
+declare const DEFAULT_QUANTILE_BUCKETS = 4;
+/** Fidelity at or above this → 'representative'; below → 'skewed'.
+ *  1 − 0.8 = mean JSD 0.2 ≈ distributions that mostly overlap with one
+ *  clearly shifted mode — the point where per-feature shifts start changing
+ *  which failure classes an eval can even observe. */
+declare const REPRESENTATIVE_MIN_FIDELITY = 0.8;
+/**
+ * Default feature set — ONLY fields verified present on both simulated and
+ * production records:
+ *
+ *   - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
+ *     validator (non-finite values read as absent rather than poisoning a
+ *     bucket).
+ *   - `failure_class` — optional taxonomy field; absent counted explicitly.
+ *   - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
+ *     `outcome.raw` counters the intake adapters and eval harnesses write
+ *     (`turns_completed`, `assistant_messages`, `tool_errors`,
+ *     `turns_aborted`); absent on records whose producer did not capture
+ *     them, counted explicitly.
+ *   - `completion_length` — from the optional `CorpusRecord` trajectory
+ *     text; the message-length proxy when records come from the corpus.
+ *
+ * `RunRecord` carries event COUNTS, not event ordering, so
+ * `tool_error_recovery` is a counts-only derivation: errors occurred and the
+ * run still completed cleanly ('recovered') vs aborted or classified as a
+ * failure ('unrecovered') — not a literal error→retry sequence check.
+ */
+declare const defaultBehaviorFeatures: BehaviorFeatures;
+/**
+ * Jensen-Shannon divergence between two categorical histograms (raw counts;
+ * normalized internally). Log base 2 → bounded [0,1]: 0 = identical
+ * distributions, 1 = disjoint support. Symmetric, defined even where the
+ * supports differ — exactly the regime sim-vs-production comparison lives in.
+ * Throws on zero-mass or negative/non-finite counts: an empty histogram has
+ * no distribution and a silent 0 would read as "perfectly representative".
+ */
+declare function jsDivergence(p: Record<string, number>, q: Record<string, number>): number;
+/**
+ * Deterministic quantile edges over a value set (the UNION of both sides, so
+ * sim and production land in the same buckets). Linear interpolation between
+ * order statistics; duplicate edges from heavy ties collapse into fewer,
+ * wider buckets. Returns `bucketCount - 1` edges before deduplication.
+ */
+declare function quantileEdges(values: number[], bucketCount?: number): number[];
+/** Stable half-open bucket label for a value against quantile edges:
+ *  `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
+declare function bucketLabel(value: number, edges: number[]): string;
+interface FeatureShift {
+    /** Category label (a string value, a numeric bucket, or `ABSENT_CATEGORY`). */
+    value: string;
+    /** Probability of this category among ALL simulated records (nulls included
+     *  via `ABSENT_CATEGORY`, so each side's shifts sum to 1). */
+    pSim: number;
+    /** Probability among ALL production records. */
+    pProd: number;
+}
+interface FeatureDivergence {
+    feature: string;
+    /** Jensen-Shannon divergence in [0,1] for this feature. */
+    divergence: number;
+    /** Largest |pSim − pProd| categories, descending — where the sim deviates. */
+    topShifts: FeatureShift[];
+    /** Non-null observations on the simulated side. */
+    nSim: number;
+    /** Non-null observations on the production side. */
+    nProd: number;
+}
+type FidelityVerdict = 'representative' | 'skewed' | 'insufficient-data';
+interface FidelityReport {
+    perDimension: FeatureDivergence[];
+    /** 1 − mean divergence over features with sufficient data. NaN when the
+     *  verdict is 'insufficient-data' — a 0 would read as "maximally skewed"
+     *  and silently poison downstream aggregation; check `verdict` first. */
+    fidelity: number;
+    /** Features excluded because either side had fewer than `minNPerFeature`
+     *  non-null observations. Named, never silently dropped. */
+    insufficientData: string[];
+    /** 'representative' when fidelity >= REPRESENTATIVE_MIN_FIDELITY (0.8),
+     *  'skewed' below, 'insufficient-data' when no feature met minN. */
+    verdict: FidelityVerdict;
+}
+interface SimFidelityOptions {
+    /** Feature extractor. Defaults to `defaultBehaviorFeatures`. */
+    features?: BehaviorFeatures;
+    /** Minimum non-null observations per side per feature. Default 20. */
+    minNPerFeature?: number;
+}
+/**
+ * Compare a simulator's RunRecords against production RunRecords, feature by
+ * feature. Numeric features are bucketed by deterministic quantiles of the
+ * union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
+ * inputs — "no records" is a wiring error, not a distribution.
+ */
+declare function simFidelityReport(simulated: RunRecord[], production: RunRecord[], opts?: SimFidelityOptions): FidelityReport;
+interface EasyModeOptions {
+    /** A run passes when its score (holdout, else search) >= this. Default 0.5
+     *  — matches the pass-threshold convention across the rl/ primitives. */
+    passThreshold?: number;
+    /** Pass-rate gap above which the sim is flagged inflated. Default 0.1 —
+     *  a 10-point inflation is enough to flip most promotion gates. */
+    inflationTolerance?: number;
+}
+interface EasyModeReport {
+    simPassRate: number;
+    prodPassRate: number;
+    /** simPassRate − prodPassRate. Positive = the simulator is easier than reality. */
+    gap: number;
+    /** True when gap > inflationTolerance: numbers measured against this
+     *  simulator overstate production performance. */
+    inflated: boolean;
+}
+/**
+ * The headline simulator failure mode as its own named artifact: a simulator
+ * that creates "easy mode" inflates pass-rate relative to production, and
+ * every score measured against it overstates reality. Throws on empty inputs
+ * and on records carrying neither score — a silently-skipped record would
+ * bias the very rate this check exists to keep honest.
+ */
+declare function easyModeCheck(simulated: RunRecord[], production: RunRecord[], opts?: EasyModeOptions): EasyModeReport;
+
+export { ABSENT_CATEGORY, type AdaptationCurve, type AdaptationPoint, type AdaptationRunner, type AdapterContext, type BehaviorFeatures, type BradleyTerryFit, type BradleyTerryRating, type BuildPairwiseFromCampaignInput, type CellObservation, type CompareCurvesResult, type ComputeBestOfNOptions, type ComputeBestOfNResult, type ComputeCurve, type ComputeCurveBudget, type ComputeCurvePoint, type ContaminationProbeInput, type ContaminationProbeOptions, type ContaminationProbeReport, type CurriculumAllocation, DEFAULT_MIN_N_PER_FEATURE, DEFAULT_QUANTILE_BUCKETS, type DetectRewardHackingInput, DpoExportRow, DpoLookups, type EasyModeOptions, type EasyModeReport, type EloOptions, ExtractPreferencesOptions, type FeatureDivergence, type FeatureShift, type FidelityReport, type FidelityVerdict, GrpoExportRow, GrpoLookups, OutcomeStore, type PairwiseOutcome, type ParetoPointInput, PredictiveValidityResearcher, type PredictiveValidityResearcherOptions, PreferenceExtractionReport, REPRESENTATIVE_MIN_FIDELITY, type RLCampaignResult, type RewardHackingFinding, type RewardHackingReport, type RewardHackingSignal, type RunAdaptationCurveOptions, type RunComputeCurveOptions, type RunRLCampaignOptions, type ScenarioPerturbation, type ScenarioPerturbationKind, type SelfConsistencyOptions, type SelfConsistencyResult, SftExportRow, SftLookups, type SimFidelityOptions, type ThompsonCurriculumOptions, type VarianceCurriculumOptions, type VerifiableReward, type VerifiableRewardExtractionOptions, type VerifiableRewardSource, applyEloUpdate, bestOfN, bucketLabel, buildPairwiseFromCampaign, campaignToRunRecords, compareAdaptationCurves, defaultBehaviorFeatures, detectRewardHacking, easyModeCheck, extractVerifiableReward, extractVerifiableRewardsFromRecords, filterDeterministicallyRewarded, firstPassK, fitBradleyTerry, injectIrrelevantClause, jsDivergence, observationsFromRunRecords, paretoFrontier, quantileEdges, renameVariables, runAdaptationCurve, runComputeCurve, runContaminationProbe, runRLCampaign, selfConsistency, shuffleOrder, simFidelityReport, thompsonCurriculum, varianceBasedCurriculum, verificationReportToRunRecord };