@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/rl/compute-curves.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Test-time compute scaling curves.
+ *
+ * The test-time-compute frontier paper (Snell et al. 2024) and the
+ * subsequent o1-style scaling work both show that LLM-agent capability
+ * is a function of the compute budget at inference, not just of the
+ * training run. The right way to characterize a candidate is therefore
+ * a *curve* — score at compute budgets {1×, 4×, 16×, …} — not a single
+ * point.
+ *
+ * This module ships:
+ *
+ *   1. The compute-curve harness — `runComputeCurve(runner, budgets)` —
+ *      that evaluates one candidate at a sequence of compute budgets
+ *      and returns the (compute, score) curve.
+ *   2. A best-of-N evaluator — `bestOfN(runner, n, scoreFn)` — the
+ *      simplest test-time-compute scaling primitive: sample N
+ *      independent rollouts, return the best.
+ *   3. A self-consistency evaluator — `selfConsistency(runner, n)` —
+ *      the majority-vote variant of best-of-N for tasks with a small
+ *      categorical answer space.
+ *   4. Pareto-frontier extraction over multiple candidates — given
+ *      (candidate, compute, score) tuples, return the set of
+ *      candidate-compute combinations that aren't dominated.
+ *
+ * Caveat: "compute" here is the caller's notion of a compute unit. For
+ * agent eval that's typically wall-time × parallelism, or token budget,
+ * or LLM-call count. We accept whatever the caller provides; the curve
+ * is on whatever axis they pick.
+ */
+export interface ComputeCurveBudget {
+    /** Identifier — for the report. Common: '1x', '4x', '16x'. */
+    id: string;
+    /** Numeric value on the chosen axis (tokens, calls, USD, ms — caller picks). */
+    cost: number;
+    /** Free-form metadata (the caller can carry per-budget config). */
+    meta?: Record<string, unknown>;
+}
+export interface ComputeCurvePoint {
+    budgetId: string;
+    cost: number;
+    score: number;
+    /** Number of underlying samples used at this budget. */
+    samples: number;
+    /** Optional spread / variance information. */
+    std?: number;
+    /** Any extra metrics the runner returned. */
+    metrics?: Record<string, number>;
+}
+export interface ComputeCurve {
+    candidateId: string;
+    points: ComputeCurvePoint[];
+    /** Rough exponent fit: score ≈ a + b * log(cost). Useful for "how steep is the curve?" */
+    logSlope: number | null;
+    /** Best (highest-score) point on the curve. */
+    best: ComputeCurvePoint;
+}
+export interface RunComputeCurveOptions {
+    candidateId: string;
+    budgets: ComputeCurveBudget[];
+    /**
+     * Run the candidate at one budget. Returns the realized score plus
+     * optional spread + extra metrics.
+     */
+    runAtBudget: (budget: ComputeCurveBudget) => Promise<{
+        score: number;
+        samples: number;
+        std?: number;
+        metrics?: Record<string, number>;
+    }>;
+}
+export declare function runComputeCurve(opts: RunComputeCurveOptions): Promise<ComputeCurve>;
+export interface ComputeBestOfNOptions<O> {
+    /** Number of independent samples to draw. */
+    n: number;
+    /** Sampler — produces one rollout. */
+    sample: (sampleIdx: number) => Promise<O>;
+    /** Score one rollout. */
+    scoreFn: (rollout: O) => Promise<number> | number;
+}
+export interface ComputeBestOfNResult<O> {
+    best: O;
+    bestScore: number;
+    scores: number[];
+    meanScore: number;
+    /** Index of the best rollout, for diagnostics. */
+    bestIndex: number;
+}
+/** The simplest test-time scaling primitive. */
+export declare function bestOfN<O>(opts: ComputeBestOfNOptions<O>): Promise<ComputeBestOfNResult<O>>;
+export interface SelfConsistencyOptions<O> {
+    n: number;
+    sample: (sampleIdx: number) => Promise<O>;
+    /** Extract the canonical answer key (string) from a rollout. */
+    answerKey: (rollout: O) => string;
+}
+export interface SelfConsistencyResult<O> {
+    /** Modal answer (the majority vote). */
+    answer: string;
+    /** Fraction of samples voting for the modal answer in [0, 1]. */
+    agreement: number;
+    /** Histogram of all answers. */
+    histogram: Record<string, number>;
+    /** A representative rollout that voted for the modal answer. */
+    representative: O;
+    /** All rollouts. */
+    rollouts: O[];
+}
+/**
+ * Self-consistency / majority-vote test-time scaling. For tasks with a
+ * small categorical answer space (math problems, multiple choice).
+ */
+export declare function selfConsistency<O>(opts: SelfConsistencyOptions<O>): Promise<SelfConsistencyResult<O>>;
+/**
+ * Pareto frontier over (candidate, compute, score) tuples. A point is on
+ * the frontier iff no other point dominates it in both score (higher
+ * better) and cost (lower better). Returns the frontier sorted ascending
+ * by cost.
+ */
+export interface ParetoPointInput {
+    candidateId: string;
+    budgetId: string;
+    cost: number;
+    score: number;
+}
+export declare function paretoFrontier(points: ParetoPointInput[]): ParetoPointInput[];
+//# sourceMappingURL=compute-curves.d.ts.map

package/dist/rl/compute-curves.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compute-curves.d.ts","sourceRoot":"","sources":["../../src/rl/compute-curves.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAIH,MAAM,WAAW,kBAAkB;IACjC,8DAA8D;IAC9D,EAAE,EAAE,MAAM,CAAA;IACV,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAA;IACZ,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAA;IACf,8CAA8C;IAC9C,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,iBAAiB,EAAE,CAAA;IAC3B,0FAA0F;IAC1F,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,+CAA+C;IAC/C,IAAI,EAAE,iBAAiB,CAAA;CACxB;AAED,MAAM,WAAW,sBAAsB;IACrC,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,kBAAkB,EAAE,CAAA;IAC7B;;;OAGG;IACH,WAAW,EAAE,CAAC,MAAM,EAAE,kBAAkB,KAAK,OAAO,CAAC;QACnD,KAAK,EAAE,MAAM,CAAA;QACb,OAAO,EAAE,MAAM,CAAA;QACf,GAAG,CAAC,EAAE,MAAM,CAAA;QACZ,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KACjC,CAAC,CAAA;CACH;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAiBzF;AAED,MAAM,WAAW,qBAAqB,CAAC,CAAC;IACtC,6CAA6C;IAC7C,CAAC,EAAE,MAAM,CAAA;IACT,sCAAsC;IACtC,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAA;IACzC,yBAAyB;IACzB,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAA;CAClD;AAED,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,IAAI,EAAE,CAAC,CAAA;IACP,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,gDAAgD;AAChD,wBAAsB,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,qBAAqB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAmBjG;AAED,MAAM,WAAW,sBAAsB,CAAC,CAAC;IACvC,CAAC,EAAE,MAAM,CAAA;IACT,MAAM,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAA;IACzC,gEAAgE;IAChE,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,MAAM,CAAA;CAClC;AAED,MAAM,WAAW,qBAAqB,CAAC,CAAC;IACtC,wCAAwC;IACxC,MAAM,EAAE,MAAM,CAAA;IACd,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAA;IACjB,gCAAgC;IAChC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACjC,gEAAgE;IAChE,cAAc,EAAE,CAAC,CAAA;IACjB,oBAAoB;IACpB,QAAQ,EAAE,CAAC,EAAE,CAAA;CACd;AAED;;;GAGG;AACH,wBAAsB,eAAe,CAAC,CAAC,EACrC,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC,GAC9B,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CA0BnC;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACd;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,gBAAgB,EAAE,CAU7E"}

package/dist/rl/contamination.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Contamination probe — held-out perturbation tests.
+ *
+ * The bug class: once a benchmark scenario set is published, models train
+ * on it, and your scores become invalid. SWE-Bench-Verified, GPQA, and
+ * MMLU-Pro all exist because their predecessors got contaminated within
+ * months. The right defense is to keep a held-out *perturbed* version of
+ * every scenario — same task, slightly different surface — and check
+ * whether scores diverge significantly. Genuine capability transfers; rote
+ * memorization doesn't.
+ *
+ * This module ships the probe contract:
+ *
+ *   1. A `ScenarioPerturbation` strategy type — function that produces a
+ *      perturbed scenario from an original.
+ *   2. `runContaminationProbe({ originals, perturbed, scoreFn })` — runs
+ *      both halves and reports per-scenario score divergence + a global
+ *      contamination verdict via paired Wilcoxon.
+ *   3. Several stock perturbations: `renameVariables`, `shuffleOrder`,
+ *      `paraphrasePrompt`, `injectIrrelevantClause`. Each preserves the
+ *      task's structural difficulty while breaking surface memorization.
+ *
+ * The verdict is conservative: if the perturbed-vs-original score
+ * difference is statistically significant (BH-adjusted p < 0.05) AND
+ * the median drop is > 5 percentage points, we flag *contamination
+ * suspected*. False positives are possible (the perturbation might
+ * actually be harder); the default is to flag for review, not to
+ * autoreject.
+ */
+export type ScenarioPerturbationKind = 'rename_variables' | 'shuffle_order' | 'paraphrase' | 'inject_irrelevant_clause' | 'custom';
+export interface ScenarioPerturbation<S> {
+    kind: ScenarioPerturbationKind;
+    /** Apply to one scenario, return its perturbed sibling. */
+    apply: (scenario: S) => Promise<S> | S;
+    /** Optional id — for the report. */
+    id?: string;
+}
+export interface ContaminationProbeInput<S> {
+    /** Identity of every scenario. The probe's `runFingerprint` keys on these. */
+    scenarioId: (s: S) => string;
+    /** Original scenarios. */
+    originals: S[];
+    /**
+     * Either pre-computed perturbations (one per original, same order) OR a
+     * `perturbation` strategy that synthesizes them on the fly.
+     */
+    perturbed?: S[];
+    perturbation?: ScenarioPerturbation<S>;
+    /**
+     * Run the policy/agent against one scenario and return a scalar score
+     * in [0, 1]. The probe doesn't care what the policy is — that's the
+     * caller's contract.
+     */
+    scoreFn: (s: S) => Promise<number>;
+}
+export interface ContaminationProbeOptions {
+    /** Drop scores below this from the probe; treats partial failures separately. Default 0. */
+    scoreFloor?: number;
+    /**
+     * BH-FDR threshold for declaring contamination on each per-scenario
+     * delta. Default 0.05.
+     */
+    fdr?: number;
+    /**
+     * Minimum median per-scenario drop to flag global contamination. Default
+     * 0.05 (5 percentage points). Smaller drops may be noise.
+     */
+    minMedianDrop?: number;
+}
+export interface ContaminationProbeReport {
+    perScenario: Array<{
+        scenarioId: string;
+        originalScore: number;
+        perturbedScore: number;
+        delta: number;
+        /** Per-scenario q-value (single-test BH for a single scenario). Mainly for display. */
+        qValue: number;
+    }>;
+    /** Wilcoxon paired-test on the deltas. */
+    pairedTest: {
+        w: number;
+        p: number;
+    };
+    medianDelta: number;
+    meanDelta: number;
+    contaminationSuspected: boolean;
+    reason: string;
+    /** Number of scenarios processed. */
+    n: number;
+}
+export declare function runContaminationProbe<S>(input: ContaminationProbeInput<S>, opts?: ContaminationProbeOptions): Promise<ContaminationProbeReport>;
+/**
+ * Identifier-rename perturbation for code/text scenarios. Replaces every
+ * occurrence of the listed identifiers with synthesized aliases. Use when
+ * the scenario's structural difficulty is independent of variable names
+ * (e.g. SWE-Bench-style coding tasks).
+ */
+export declare function renameVariables<S extends {
+    prompt: string;
+}>(identifiers: string[], rename?: (name: string, idx: number) => string): ScenarioPerturbation<S>;
+/**
+ * Order-shuffle perturbation. Reshuffles a list-shaped section of the
+ * prompt (for QA scenarios that present options A/B/C/D — answer depends
+ * on the option labels, not order). Caller provides the section extractor.
+ */
+export declare function shuffleOrder<S extends {
+    prompt: string;
+}>(shuffleSection: (prompt: string, rng: () => number) => string, seed: number): ScenarioPerturbation<S>;
+/**
+ * Inject-irrelevant-clause perturbation. Adds a benign sentence that
+ * shouldn't change the answer. Tests for "did the model just memorize
+ * the input string."
+ */
+export declare function injectIrrelevantClause<S extends {
+    prompt: string;
+}>(clause: string, position?: 'prefix' | 'suffix'): ScenarioPerturbation<S>;
+//# sourceMappingURL=contamination.d.ts.map

package/dist/rl/contamination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contamination.d.ts","sourceRoot":"","sources":["../../src/rl/contamination.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAKH,MAAM,MAAM,wBAAwB,GAChC,kBAAkB,GAClB,eAAe,GACf,YAAY,GACZ,0BAA0B,GAC1B,QAAQ,CAAA;AAEZ,MAAM,WAAW,oBAAoB,CAAC,CAAC;IACrC,IAAI,EAAE,wBAAwB,CAAA;IAC9B,2DAA2D;IAC3D,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACtC,oCAAoC;IACpC,EAAE,CAAC,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,uBAAuB,CAAC,CAAC;IACxC,8EAA8E;IAC9E,UAAU,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,MAAM,CAAA;IAC5B,0BAA0B;IAC1B,SAAS,EAAE,CAAC,EAAE,CAAA;IACd;;;OAGG;IACH,SAAS,CAAC,EAAE,CAAC,EAAE,CAAA;IACf,YAAY,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAA;IACtC;;;;OAIG;IACH,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,yBAAyB;IACxC,4FAA4F;IAC5F,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,KAAK,CAAC;QACjB,UAAU,EAAE,MAAM,CAAA;QAClB,aAAa,EAAE,MAAM,CAAA;QACrB,cAAc,EAAE,MAAM,CAAA;QACtB,KAAK,EAAE,MAAM,CAAA;QACb,uFAAuF;QACvF,MAAM,EAAE,MAAM,CAAA;KACf,CAAC,CAAA;IACF,0CAA0C;IAC1C,UAAU,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IACpC,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,sBAAsB,EAAE,OAAO,CAAA;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,qCAAqC;IACrC,CAAC,EAAE,MAAM,CAAA;CACV;AAED,wBAAsB,qBAAqB,CAAC,CAAC,EAC3C,KAAK,EAAE,uBAAuB,CAAC,CAAC,CAAC,EACjC,IAAI,GAAE,yBAA8B,GACnC,OAAO,CAAC,wBAAwB,CAAC,CAgFnC;AAID;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,CAAC,SAAS;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,EAC1D,WAAW,EAAE,MAAM,EAAE,EACrB,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAyD,GAC/F,oBAAoB,CAAC,CAAC,CAAC,CAazB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,CAAC,SAAS;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,EACvD,cAAc,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,MAAM,KAAK,MAAM,EAC7D,IAAI,EAAE,MAAM,GACX,oBAAoB,CAAC,CAAC,CAAC,CAgBzB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,SAAS;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,EACjE,MAAM,EAAE,MAAM,EACd,QAAQ,GAAE,QAAQ,GAAG,QAAmB,GACvC,oBAAoB,CAAC,CAAC,CAAC,CASzB"}

package/dist/rl/corpus.d.ts

@@ -0,0 +1,55 @@
+/**
+ * RL corpus — the durable, append-only accumulation of graded RunRecords that
+ * every eval run deposits BY DEFAULT.
+ *
+ * The dataset is the free exhaust of the normal eval process: we run evals
+ * constantly to get an agent production-ready, and those runs already produce
+ * graded trajectories. Instead of writing them to an ephemeral run dir and
+ * throwing them away, `appendToCorpus` accumulates them into a durable corpus;
+ * `buildDatasetFromCorpus` later harvests the whole corpus into a publishable
+ * bundle. No separate data-collection campaign — the data accrues from work we
+ * do anyway. This is the "best things for free by our process" layer.
+ *
+ * Trajectory text rides on the record as top-level `prompt` / `completion`
+ * (what the eval harnesses capture; the RunRecord validator ignores the extra
+ * keys). The harvest reads them directly — no trace store round-trip needed.
+ */
+import type { RunRecord } from '../run-record';
+import { type RlDatasetBundle, type RlDatasetConfig } from './dataset';
+/** A corpus record is a RunRecord carrying the trajectory text the harness
+ *  captured. `prompt`/`completion` are top-level (the validator ignores extras). */
+export type CorpusRecord = RunRecord & {
+    prompt?: string;
+    completion?: string;
+};
+export interface CorpusAppendResult {
+    appended: number;
+    /** Skipped because a record with the same runId was already in the corpus
+     *  (idempotent appends — NOT re-run collapsing; re-runs get fresh runIds). */
+    skipped: number;
+    total: number;
+}
+/**
+ * Append graded records to the corpus (append-only JSONL). Deduplicates by
+ * `runId` against what's already on disk so re-running the same harness is
+ * idempotent. Creates the file and parent dir. This is the call every eval
+ * harness makes by default after producing its records.
+ */
+export declare function appendToCorpus(records: CorpusRecord[], corpusPath: string): CorpusAppendResult;
+/** Read the full corpus. Returns [] if the corpus does not exist yet. */
+export declare function readCorpus(corpusPath: string): CorpusRecord[];
+export interface HarvestOptions {
+    /** Keep only records scoring >= this (rejection-sampling for SFT). */
+    minScore?: number;
+    /** Keep only these splits (e.g. ['holdout'] for an eval-only dataset). */
+    splits?: RunRecord['splitTag'][];
+}
+/**
+ * Harvest the accumulated corpus into a publishable RL dataset bundle. Reads
+ * trajectory text from each record's top-level `prompt`/`completion`; records
+ * missing either are excluded (a graded score with no trajectory can't train).
+ * Optionally filters by score / split. Throws (via buildRlDataset) if nothing
+ * survives — an empty dataset must never be published.
+ */
+export declare function buildDatasetFromCorpus(corpusPath: string, config: RlDatasetConfig, opts?: HarvestOptions): Promise<RlDatasetBundle>;
+//# sourceMappingURL=corpus.d.ts.map

package/dist/rl/corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../../src/rl/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAkB,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAA;AAEtF;oFACoF;AACpF,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAE/E,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB;kFAC8E;IAC9E,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;CACd;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,kBAAkB,CAkB9F;AAED,yEAAyE;AACzE,wBAAgB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,YAAY,EAAE,CAO7D;AAOD,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,SAAS,CAAC,UAAU,CAAC,EAAE,CAAA;CACjC;AAED;;;;;;GAMG;AACH,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,eAAe,EACvB,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,eAAe,CAAC,CAe1B"}

package/dist/rl/corpus.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=corpus.test.d.ts.map

package/dist/rl/corpus.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.test.d.ts","sourceRoot":"","sources":["../../src/rl/corpus.test.ts"],"names":[],"mappings":""}

package/dist/rl/dataset.d.ts

@@ -0,0 +1,102 @@
+/**
+ * RL dataset packaging + datasheet — the publishable, sellable bundle.
+ *
+ * The format exporters (`toGrpoRows` / `toSftRows` / `toDpoRows`) already
+ * produce trainer-ready shapes (prime-rl GRPO, TRL DPO, conversational SFT).
+ * What turns that into a dataset someone can PUBLISH or BUY is the provenance
+ * + a datasheet: which models produced it, which prompt/agent versions, how the
+ * reward was derived (deterministic verifiable vs probabilistic judge — the
+ * credibility axis a buyer checks first), the split discipline, the reward
+ * distribution, the quality gates, the license, and the intended/out-of-scope
+ * uses. This module computes those facts from the `RunRecord[]` and renders a
+ * "Datasheet for Datasets" (Gebru et al. 2018) card alongside the format files.
+ *
+ * It composes the existing `rl/exporters` — it does not reimplement any trainer
+ * format. The renderers token-identity step (DeepSeek/Kimi/Qwen tokenization
+ * with per-token loss masks) is a downstream Python stage that consumes the
+ * `messages`/`completions` this bundle emits.
+ */
+import type { RunRecord, RunSplitTag } from '../run-record';
+import { type DpoLookups, type GrpoLookups, type SftLookups } from './exporters';
+import type { PreferenceTriple } from './preferences';
+export type RewardKind = 'deterministic' | 'probabilistic' | 'mixed';
+export type DatasetFormat = 'grpo' | 'sft' | 'dpo';
+/** Caller-declared context — the qualitative half of the datasheet that can't
+ *  be computed from records. */
+export interface RlDatasetConfig {
+    name: string;
+    version: string;
+    /** Product/task domain, e.g. 'legal-m&a', 'tax-1040'. */
+    domain: string;
+    /** SPDX id or a named commercial license. Required — an unlicensed dataset
+     *  cannot be published or sold. */
+    license: string;
+    /** How the reward was produced. `kind: 'deterministic'` (a test/schema/XPath
+     *  decided it) is the credibility signal; 'probabilistic' = LLM-judge. */
+    reward: {
+        kind: RewardKind;
+        source: string;
+        description: string;
+    };
+    intendedUse: string;
+    outOfScope: string;
+    limitations: string;
+    /** ISO timestamp — passed in (the substrate forbids Date.now()). */
+    createdAtIso: string;
+    /** Default: ['grpo', 'sft']. */
+    formats?: DatasetFormat[];
+    /** Quality gates already run, recorded on the card for the buyer. */
+    qualityGates?: {
+        contaminationProbe?: 'passed' | 'failed' | 'not-run';
+        dedup?: boolean;
+        verifiableRewardFilter?: boolean;
+    };
+}
+export interface RewardStats {
+    n: number;
+    mean: number;
+    median: number;
+    min: number;
+    max: number;
+    std: number;
+}
+export interface RlDatasetStats {
+    records: number;
+    /** Record count per split — a publishable dataset must declare its holdout. */
+    splits: Record<RunSplitTag, number>;
+    reward: RewardStats;
+    /** Distinct snapshot-pinned models that produced the trajectories. */
+    models: string[];
+    /** Distinct effective-prompt hashes (the agent profile/prompt versions). */
+    promptHashes: string[];
+    commitShas: string[];
+    totalTokens: {
+        input: number;
+        output: number;
+    };
+    totalCostUsd: number;
+}
+export interface RlDatasetManifest extends RlDatasetConfig {
+    formats: DatasetFormat[];
+    rowCounts: Partial<Record<DatasetFormat, number>>;
+    stats: RlDatasetStats;
+}
+export interface RlDatasetBundle {
+    manifest: RlDatasetManifest;
+    /** Relative filename -> contents. Write these to a directory to publish. */
+    files: Record<string, string>;
+}
+/**
+ * Package graded `RunRecord[]` into a publishable RL dataset bundle: the
+ * trainer-format JSONL files + a manifest + a datasheet. DPO requires
+ * pre-extracted preference triples (pass `preferences`); GRPO/SFT derive from
+ * the records directly via the supplied lookups. Throws on an empty corpus —
+ * an empty dataset must never be published.
+ */
+export declare function buildRlDataset(records: RunRecord[], lookups: GrpoLookups & SftLookups, config: RlDatasetConfig, preferences?: {
+    triples: PreferenceTriple[];
+    lookups: DpoLookups;
+}): Promise<RlDatasetBundle>;
+/** Render the "Datasheet for Datasets" card — the artifact a buyer reads. */
+export declare function datasheetToMarkdown(m: RlDatasetManifest): string;
+//# sourceMappingURL=dataset.d.ts.map

package/dist/rl/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/rl/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3D,OAAO,EACL,KAAK,UAAU,EACf,KAAK,WAAW,EAChB,KAAK,UAAU,EAOhB,MAAM,aAAa,CAAA;AACpB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAErD,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG,eAAe,GAAG,OAAO,CAAA;AACpE,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,CAAA;AAElD;gCACgC;AAChC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAA;IACd;uCACmC;IACnC,OAAO,EAAE,MAAM,CAAA;IACf;8EAC0E;IAC1E,MAAM,EAAE;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAA;IACjE,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,oEAAoE;IACpE,YAAY,EAAE,MAAM,CAAA;IACpB,gCAAgC;IAChC,OAAO,CAAC,EAAE,aAAa,EAAE,CAAA;IACzB,qEAAqE;IACrE,YAAY,CAAC,EAAE;QACb,kBAAkB,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAA;QACpD,KAAK,CAAC,EAAE,OAAO,CAAA;QACf,sBAAsB,CAAC,EAAE,OAAO,CAAA;KACjC,CAAA;CACF;AAED,MAAM,WAAW,WAAW;IAC1B,CAAC,EAAE,MAAM,CAAA;IACT,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;IACnC,MAAM,EAAE,WAAW,CAAA;IACnB,sEAAsE;IACtE,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,4EAA4E;IAC5E,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9C,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,OAAO,EAAE,aAAa,EAAE,CAAA;IACxB,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAA;IACjD,KAAK,EAAE,cAAc,CAAA;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,iBAAiB,CAAA;IAC3B,4EAA4E;IAC5E,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAgDD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,SAAS,EAAE,EACpB,OAAO,EAAE,WAAW,GAAG,UAAU,EACjC,MAAM,EAAE,eAAe,EACvB,WAAW,CAAC,EAAE;IAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,UAAU,CAAA;CAAE,GACjE,OAAO,CAAC,eAAe,CAAC,CAoC1B;AAMD,6EAA6E;AAC7E,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,iBAAiB,GAAG,MAAM,CAiDhE"}

package/dist/rl/dataset.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=dataset.test.d.ts.map

package/dist/rl/dataset.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.test.d.ts","sourceRoot":"","sources":["../../src/rl/dataset.test.ts"],"names":[],"mappings":""}

package/dist/rl/exporters.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Trainer-format exporters.
+ *
+ * agent-eval produces canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`,
+ * `StepReward[]`, `PrmTrainingTriple[]`). RL training pipelines consume
+ * different shapes — Hugging Face TRL, Prime Intellect's prime-rl, OpenAI
+ * fine-tuning, Anthropic finetuning, OpenRLHF, verl. Each has its own
+ * JSONL conventions. Rather than ship N adapters, this module ships the
+ * canonical formats most production pipelines accept and ergonomic helpers
+ * for the rest.
+ *
+ * Shapes:
+ *   - **DPO / IPO / KTO** — `{prompt, chosen, rejected}` JSONL. Consumed
+ *     by HuggingFace TRL, prime-rl's offline DPO, OpenRLHF.
+ *   - **GRPO offline** — `{prompt, completions[], rewards[]}` JSONL.
+ *     Consumed by prime-rl GRPO, verl, OpenRLHF.
+ *   - **SFT** — `{messages[]}` JSONL with chosen completion as the final
+ *     assistant turn. Consumed by HF SFT trainers, OpenAI fine-tuning,
+ *     Anthropic finetuning.
+ *   - **PRM** — `{prompt, prefix_steps[], chosen_step, rejected_step}` JSONL.
+ *     Consumed by Lightman-style PRM trainers and prime-rl's PRM mode.
+ *
+ * Why ship this in agent-eval rather than a separate adapter package: the
+ * canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`, etc.) are
+ * agent-eval's contract; without first-party exporters consumers reverse-
+ * engineer the mapping every release. The exporters codify it.
+ *
+ * The exporters take callbacks for any field that isn't on the canonical
+ * artifact (specifically: prompt + completion text, since the package
+ * stores only their hashes by design — full text is the consumer's
+ * trace store / raw event log).
+ */
+import type { RunRecord } from '../run-record';
+import type { PreferenceTriple } from './preferences';
+import type { PrmTrainingTriple, StepReward } from './process-reward';
+export interface DpoLookups {
+    /** Resolve the prompt text for a run (typically from a trace store / raw event sink). */
+    promptOf: (runId: string) => string | Promise<string>;
+    /** Resolve the assistant completion text for a run. */
+    completionOf: (runId: string) => string | Promise<string>;
+}
+export interface DpoExportRow {
+    prompt: string;
+    chosen: string;
+    rejected: string;
+    /** Carried-through margin. Some KTO / IPO variants use this. */
+    margin?: number;
+    /** Free-form metadata for downstream filtering / sharding. */
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert preference triples to TRL-compatible DPO rows. The shape
+ * `{prompt, chosen, rejected}` is the canonical HuggingFace DPODataset
+ * entry; every major DPO trainer accepts it.
+ */
+export declare function toDpoRows(triples: PreferenceTriple[], lookups: DpoLookups): Promise<DpoExportRow[]>;
+/** Serialize DPO rows as JSONL. One line per row. */
+export declare function toDpoJsonl(rows: DpoExportRow[]): string;
+export interface GrpoLookups {
+    promptOf: (runId: string) => string | Promise<string>;
+    completionOf: (runId: string) => string | Promise<string>;
+    /** Optional: derive a custom reward from the run. Defaults to score. */
+    rewardOf?: (run: RunRecord) => number | null;
+}
+export interface GrpoExportRow {
+    prompt: string;
+    completions: string[];
+    rewards: number[];
+    /** runIds in the same order as `completions[]` for traceability. */
+    runIds: string[];
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert RunRecord[] grouped by `(scenarioId)` into GRPO offline rows —
+ * one row per scenario, with one completion per run on that scenario.
+ *
+ * GRPO (Shao et al. 2024 / DeepSeek-R1) trains on relative advantages
+ * within a group of completions for the same prompt; this is the
+ * canonical input format.
+ */
+export declare function toGrpoRows(runs: RunRecord[], lookups: GrpoLookups): Promise<GrpoExportRow[]>;
+export declare function toGrpoJsonl(rows: GrpoExportRow[]): string;
+export interface SftLookups {
+    promptOf: (runId: string) => string | Promise<string>;
+    completionOf: (runId: string) => string | Promise<string>;
+    /** Optional system message. Default omits. */
+    systemOf?: (run: RunRecord) => string | null | undefined;
+    /** Filter — return false to skip the run (e.g., low score, failed cases). */
+    include?: (run: RunRecord) => boolean;
+}
+export interface SftExportRow {
+    messages: Array<{
+        role: 'system' | 'user' | 'assistant';
+        content: string;
+    }>;
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert RunRecord[] into Hugging Face / OpenAI / Anthropic-style
+ * conversational SFT rows. By default every record becomes one row;
+ * pass `include` to filter (e.g., keep only `score >= 0.8` for
+ * rejection-sampling SFT).
+ */
+export declare function toSftRows(runs: RunRecord[], lookups: SftLookups): Promise<SftExportRow[]>;
+export declare function toSftJsonl(rows: SftExportRow[]): string;
+export interface PrmLookups {
+    /** Resolve the prompt text for a run. */
+    promptOf: (runId: string) => string | Promise<string>;
+    /** Resolve the trajectory step text for a (runId, spanId) pair. */
+    stepTextOf: (runId: string, spanId: string) => string | Promise<string>;
+    /** Optional: sequence of prefix span ids leading up to the divergence. */
+    prefixOf?: (runId: string, prefixStepIndex: number) => string[] | Promise<string[]>;
+}
+export interface PrmExportRow {
+    prompt: string;
+    /** Span ids for the steps before divergence — caller resolves text via `stepTextOf`. */
+    prefixSpanIds: string[];
+    prefixStepText: string[];
+    chosenStep: string;
+    rejectedStep: string;
+    chosenReward: number;
+    rejectedReward: number;
+    marginScore: number;
+    meta?: Record<string, unknown>;
+}
+/**
+ * Convert PRM training triples to JSONL rows. Caller's `stepTextOf`
+ * callback resolves span text from the consumer's trace store.
+ */
+export declare function toPrmRows(triples: PrmTrainingTriple[], lookups: PrmLookups): Promise<PrmExportRow[]>;
+export declare function toPrmJsonl(rows: PrmExportRow[]): string;
+export interface StepRewardJsonlRow {
+    runId: string;
+    spanId: string;
+    stepIndex: number;
+    reward: number;
+    determinism: 'deterministic' | 'probabilistic';
+    weight: number;
+}
+export declare function stepRewardsToJsonl(stepRewards: StepReward[]): string;
+//# sourceMappingURL=exporters.d.ts.map

package/dist/rl/exporters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exporters.d.ts","sourceRoot":"","sources":["../../src/rl/exporters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAIrE,MAAM,WAAW,UAAU;IACzB,yFAAyF;IACzF,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACrD,uDAAuD;IACvD,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;CAC1D;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,gEAAgE;IAChE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,8DAA8D;IAC9D,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,gBAAgB,EAAE,EAC3B,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,YAAY,EAAE,CAAC,CAyBzB;AAED,qDAAqD;AACrD,wBAAgB,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,CAEvD;AAID,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACrD,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACzD,wEAAwE;IACxE,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,CAAA;CAC7C;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,oEAAoE;IACpE,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;;;;;;GAOG;AACH,wBAAsB,UAAU,CAC9B,IAAI,EAAE,SAAS,EAAE,EACjB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,aAAa,EAAE,CAAC,CAwC1B;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,MAAM,CAEzD;AAID,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACrD,YAAY,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACzD,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;IACxD,6EAA6E;IAC7E,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,OAAO,CAAA;CACtC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA0B/F;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,CAEvD;AAID,MAAM,WAAW,UAAU;IACzB,yCAAyC;IACzC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACrD,mEAAmE;IACnE,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACvE,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,KAAK,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CACpF;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,wFAAwF;IACxF,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,iBAAiB,EAAE,EAC5B,OAAO,EAAE,UAAU,GAClB,OAAO,CAAC,YAAY,EAAE,CAAC,CAgCzB;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,CAEvD;AAID,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,eAAe,GAAG,eAAe,CAAA;IAC9C,MAAM,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,MAAM,CAUpE"}

package/dist/rl/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * RL/data bridge.
+ *
+ * This subpath groups everything that turns eval output into training signal:
+ * stable adapters/rewards, dataset packaging, trainer-format exporters,
+ * process rewards, and closed-loop campaign research. Keeping it off the root
+ * import path makes the default API smaller without hiding useful capability.
+ */
+/** Advanced budget allocation across (variant, scenario) cells. */
+export * from './active-curriculum';
+/** Advanced adaptation eval — does the policy actually learn from feedback? */
+export * from './adaptation-eval';
+/** Advanced scenario search for inputs the policy fails on. */
+export * from './adversarial';
+/** @stable Compute curves: best-of-N, self-consistency, Pareto frontier across budgets. */
+export * from './compute-curves';
+/** @stable Held-out perturbation probes for benchmark contamination (paired Wilcoxon). */
+export * from './contamination';
+/** @stable Durable corpus: every eval run appends graded trajectories by default; harvest → buildRlDataset (datasets for free). */
+export * from './corpus';
+/** @stable Publishable/sellable dataset bundle: format exporters + provenance + a Datasheet-for-Datasets card. */
+export * from './dataset';
+/** Trainer-format exporters (HuggingFace datasets, JSONL, parquet). */
+export * from './exporters';
+/** @stable Off-policy value estimation: IPS, SNIPS, doubly-robust. */
+export * from './off-policy';
+/** Researcher that re-weights rubrics by deployment outcome correlation. */
+export * from './predictive-validity-researcher';
+/** @stable (chosen, rejected) preference triples for DPO / KTO / PPO. */
+export * from './preferences';
+/** Step-level rewards and process-reward training pairs. */
+export * from './process-reward';
+/** Reward-hacking signatures: reward divergence, distribution shift, judge drift. */
+export * from './reward-hacking';
+/** Closed-loop campaign runner: eval → preferences → mutate → re-eval. */
+export * from './rl-campaign';
+/** @stable Canonical `RunRecord` adapters: trials → records, verification reports → records. */
+export * from './run-record-adapters';
+/** Simulator fidelity between simulated and production RunRecords. */
+export * from './sim-fidelity';
+/** @stable Bradley-Terry MLE + online Elo for pairwise tournament ratings. */
+export * from './tournament';
+/** @stable Verifiable reward extraction (compile / test / schema) with judge-noise filtering. */
+export * from './verifiable-reward';
+/** @stable In-memory + filesystem stores for deployment outcomes;
+ *  consumed by `predictive-validity-researcher` to calibrate the gate
+ *  against observed downstream metrics, not just held-out judge scores. */
+export { type DeploymentOutcome, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, InMemoryOutcomeStore, type OutcomeStore, } from '../meta-eval/outcome-store';
+//# sourceMappingURL=index.d.ts.map

package/dist/rl/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rl/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,mEAAmE;AACnE,cAAc,qBAAqB,CAAA;AACnC,+EAA+E;AAC/E,cAAc,mBAAmB,CAAA;AACjC,+DAA+D;AAC/D,cAAc,eAAe,CAAA;AAC7B,2FAA2F;AAC3F,cAAc,kBAAkB,CAAA;AAChC,0FAA0F;AAC1F,cAAc,iBAAiB,CAAA;AAC/B,mIAAmI;AACnI,cAAc,UAAU,CAAA;AACxB,kHAAkH;AAClH,cAAc,WAAW,CAAA;AACzB,uEAAuE;AACvE,cAAc,aAAa,CAAA;AAC3B,sEAAsE;AACtE,cAAc,cAAc,CAAA;AAC5B,4EAA4E;AAC5E,cAAc,kCAAkC,CAAA;AAChD,yEAAyE;AACzE,cAAc,eAAe,CAAA;AAC7B,4DAA4D;AAC5D,cAAc,kBAAkB,CAAA;AAChC,qFAAqF;AACrF,cAAc,kBAAkB,CAAA;AAChC,0EAA0E;AAC1E,cAAc,eAAe,CAAA;AAC7B,gGAAgG;AAChG,cAAc,uBAAuB,CAAA;AACrC,sEAAsE;AACtE,cAAc,gBAAgB,CAAA;AAC9B,8EAA8E;AAC9E,cAAc,cAAc,CAAA;AAC5B,iGAAiG;AACjG,cAAc,qBAAqB,CAAA;AAQnC;;2EAE2E;AAC3E,OAAO,EACL,KAAK,iBAAiB,EACtB,sBAAsB,EACtB,KAAK,6BAA6B,EAClC,oBAAoB,EACpB,KAAK,YAAY,GAClB,MAAM,4BAA4B,CAAA"}