@tangle-network/agent-eval 0.109.1 → 0.110.0

package/dist/corpus-eBVwhCp1.d.ts

@@ -1,560 +0,0 @@
-import { R as RunRecord, b as RunSplitTag } from './run-record-I-Z3JNvO.js';
-import { S as Span } from './schema-m0gsnbt3.js';
-import { T as TraceStore } from './store-BcFXE6LG.js';
-
-/**
- * Preference dataset extraction — bridge from `RunRecord[]` to RL training.
- *
- * Production RLHF / DPO / KTO / SimPO pipelines need preference triples:
- * `(prompt, chosen, rejected)`. The campaign artifact already contains the
- * ingredients — every (variantId, scenarioId, seed) cell is a candidate
- * that ran the same prompt against the same scenario, scored by the same
- * judge — but turning that into a clean preference dataset requires
- * deciding *what counts as a preference*.
- *
- * This module ships three preference-extraction strategies with explicit
- * tradeoffs, plus a unified output type compatible with HuggingFace TRL,
- * Anthropic finetuning JSONL, and OpenAI fine-tuning APIs. The strategies
- * are deliberately not auto-magical — picking the wrong one corrupts the
- * gradient.
- *
- * Strategies:
- *
- *   1. **`paired-by-scenario-and-seed`** — exact-match comparisons. For
- *      each scenario × seed pair, compare every (variantA, variantB) on
- *      that exact (scenario, seed). Matches scenarios so the comparison
- *      isolates variant effects. Highest signal-to-noise; smallest
- *      dataset (only matched pairs count).
- *
- *   2. **`paired-by-scenario`** — looser matching. For each scenario,
- *      compare every (variantA, variantB) where both have ≥ 1 run on the
- *      same scenario. Aggregates across seeds to compute mean scores per
- *      (variant, scenario), then forms preferences from the means. More
- *      data, lower per-pair signal.
- *
- *   3. **`top-vs-bottom`** — coarsest. Within each scenario, the highest-
- *      scoring run is `chosen`, the lowest is `rejected`. Smallest dataset
- *      per scenario but biggest score gap per pair. Useful for early
- *      bootstrapping when you have few variants.
- *
- * The output `PreferenceTriple` is *agent-eval-canonical* but trivially
- * mappable to TRL's `DPODataset` shape (`prompt`, `chosen`, `rejected`)
- * via the `toTRLFormat` helper.
- */
-
-type PreferenceStrategy = 'paired-by-scenario-and-seed' | 'paired-by-scenario' | 'top-vs-bottom';
-interface PreferenceTriple {
-    /** The scenario (input) the variants were run against. */
-    scenarioId: string;
-    /** RunRecord ids on each side, for traceability. */
-    chosenRunId: string;
-    rejectedRunId: string;
-    /** Variant ids — load-bearing for the RL update. */
-    chosenVariantId: string;
-    rejectedVariantId: string;
-    /** The score gap between chosen and rejected. Larger = stronger signal. */
-    marginScore: number;
-    /**
-     * Optional `(chosen_score, rejected_score)` pair for soft-margin DPO
-     * variants. Omitted for `top-vs-bottom` runs that don't carry meaningful
-     * scalar gaps.
-     */
-    scores?: {
-        chosen: number;
-        rejected: number;
-    };
-    /** Tie-breaker — when multiple seeds match this scenario, the one used. */
-    seed?: number;
-    /**
-     * Free-form metadata propagated from the run records — e.g. original
-     * prompt-hash, model, etc. Lets the RL trainer reconstruct the prompt.
-     */
-    meta: {
-        chosenPromptHash: string;
-        rejectedPromptHash: string;
-        chosenConfigHash: string;
-        rejectedConfigHash: string;
-        chosenModel: string;
-        rejectedModel: string;
-    };
-}
-interface ExtractPreferencesOptions {
-    strategy?: PreferenceStrategy;
-    /**
-     * Minimum score gap required to admit a pair. Pairs below this are
-     * dropped — they're noise, not signal. Default 0.05 (5% of [0,1]).
-     */
-    minMargin?: number;
-    /**
-     * Optional split tag filter — restrict to runs from one split. Default
-     * `'holdout'` (the canonical "real" signal).
-     */
-    splitTag?: RunRecord['splitTag'];
-    /**
-     * Optional reward extractor that overrides `outcome.holdoutScore` /
-     * `outcome.searchScore`. Use to drive preferences off a verifiable
-     * reward instead of the headline score.
-     */
-    rewardOf?: (run: RunRecord) => number | null;
-}
-interface PreferenceExtractionReport {
-    pairs: PreferenceTriple[];
-    /** Number of (scenario, seed) cells inspected. */
-    cellsInspected: number;
-    /** Number of pairs filtered by `minMargin`. */
-    pairsBelowMargin: number;
-    /** Number of cells with only one variant (no comparison possible). */
-    cellsSingleton: number;
-    /** Strategy used. */
-    strategy: PreferenceStrategy;
-}
-/**
- * Convert `RunRecord[]` to preference triples for RL training.
- *
- * Returns a structured report so callers can see how much data was
- * dropped and why (low-margin pairs, singleton cells). For production
- * pipelines, you usually want to:
- *
- *   1. Run a campaign producing 5–10 variants × 50–200 scenarios × 3 seeds
- *   2. Call this with `strategy: 'paired-by-scenario-and-seed'` and a
- *      verifiable-reward extractor as `rewardOf`
- *   3. Pass `report.pairs` to `toTRLFormat` and pipe to your DPO trainer
- */
-declare function extractPreferences(runs: RunRecord[], opts?: ExtractPreferencesOptions): PreferenceExtractionReport;
-/**
- * TRL-compatible export. TRL's `DPODataset` is `{ prompt, chosen, rejected }`
- * but the prompt isn't stored on the RunRecord — only its hash. The caller
- * passes a `promptOf(promptHash)` lookup that the TRL trainer can use.
- */
-declare function toTRLFormat(triples: PreferenceTriple[], promptOf: (hash: string) => string): Array<{
-    prompt: string;
-    chosen: string;
-    rejected: string;
-}>;
-/**
- * Anthropic finetuning JSONL export — `{ system, user, assistant_chosen, assistant_rejected }`
- * shape. Same caveat as TRL: prompt + outputs are content the caller has
- * to map back from the run record / raw event log.
- */
-declare function toAnthropicFormat(triples: PreferenceTriple[]): Array<{
-    scenarioId: string;
-    chosenRunId: string;
-    rejectedRunId: string;
-    margin: number;
-}>;
-
-/**
- * Process reward extraction — step-level credit assignment from trace spans.
- *
- * RL on long-horizon agents needs *step-level* rewards, not run-level
- * ones. The classic credit-assignment problem (Sutton & Barto) requires
- * knowing which sub-decisions in a trajectory contributed to the
- * outcome. Modern systems (DeepSeek-R1, OpenAI o-series, Lightman et al.
- * "Let's Verify Step by Step" 2023) train *process reward models* (PRMs)
- * that score every step, then do RL with the PRM as the reward signal.
- *
- * This module extracts `StepReward[]` from trace spans — one per
- * meaningful step — and ships:
- *
- *   1. `extractStepRewards(store, runId, opts)` — span → step-reward
- *      conversion using configurable per-span scorers (LLM judge over the
- *      span output, deterministic checkers, or a learned PRM).
- *   2. `runwiseStepRewardSummary(stepRewards)` — aggregate the per-step
- *      signal into a credit-assignment-aware run-level score.
- *   3. `prmTrainingPairs(stepRewards, options)` — produce the
- *      `(prefix, suffix_chosen, suffix_rejected)` triples that PRM
- *      training pipelines consume.
- *
- * What we ship: the *extraction* and *aggregation* infrastructure plus
- * the data shape PRM training expects. We do NOT ship the actual PRM
- * training (gradient descent over a transformer is out of scope for a
- * TS package). The interface is the contract; downstream consumers wire
- * their preferred trainer.
- *
- * Caveat the panel will land: this is descriptive credit assignment
- * (which steps correlate with outcome), not causal credit assignment
- * (which steps caused outcome). For causal claims you need
- * counterfactual rollouts or a learned dynamics model. Future work; the
- * descriptive version is what production PRM training actually uses.
- */
-
-interface StepReward {
-    /** Trace span this reward attaches to. */
-    spanId: string;
-    runId: string;
-    /** Index in the trajectory (0-based, in started-at order). */
-    stepIndex: number;
-    /** Span kind (typically 'tool', 'llm', 'judge'). */
-    kind: Span['kind'];
-    /** Span name — for the consumer's downstream filtering. */
-    name: string;
-    /** Step-level reward in [0, 1]. */
-    reward: number;
-    /**
-     * Determinism class. Mirrors the verifiable-reward distinction:
-     * deterministic = test/compile/schema check; probabilistic = LLM judge.
-     */
-    determinism: 'deterministic' | 'probabilistic';
-    /** Optional rationale / evidence — the trainer typically discards. */
-    rationale?: string;
-    /** Optional weight — how much this step contributes to credit assignment. */
-    weight?: number;
-}
-interface StepScorer {
-    /** Span kinds this scorer applies to. */
-    appliesTo: Span['kind'][];
-    /** Returns null to skip the span; returns a `StepReward` shape (without index/runId/spanId, which are filled in). */
-    score(span: Span): Promise<Omit<StepReward, 'spanId' | 'runId' | 'stepIndex'>> | null | undefined;
-}
-interface ExtractStepRewardsOptions {
-    /**
-     * Ordered list of scorers. Each span runs through scorers in order;
-     * the first non-null result wins. If no scorer applies, the span is
-     * skipped (not all spans are training-worthy).
-     */
-    scorers: StepScorer[];
-    /** Optional filter — return null to drop the span entirely before scoring. */
-    preFilter?: (span: Span) => boolean;
-}
-declare function extractStepRewards(store: TraceStore, runId: string, opts: ExtractStepRewardsOptions): Promise<StepReward[]>;
-interface RunwiseStepSummary {
-    runId: string;
-    totalSteps: number;
-    meanReward: number;
-    /** Sum-of-rewards (weighted by `weight ?? 1`). Use as the run-level proxy. */
-    sumWeightedReward: number;
-    /** Fraction of steps where reward < 0.5 — proxy for "where the policy was wrong." */
-    failureFraction: number;
-    /** Maximum drop in reward between consecutive steps — diagnoses a step where things went sideways. */
-    worstStepDelta: number;
-    worstStepIndex: number | null;
-}
-declare function runwiseStepRewardSummary(stepRewards: StepReward[]): RunwiseStepSummary;
-interface PrmTrainingTriple {
-    /** Prefix run-id (or composite key) — the trajectory up to step k-1. */
-    prefixRunId: string;
-    prefixStepIndex: number;
-    /** The step that came next on a high-reward trajectory. */
-    chosenSpanId: string;
-    chosenReward: number;
-    /** A step from a divergent low-reward trajectory at the same prefix length. */
-    rejectedSpanId: string;
-    rejectedReward: number;
-    /** The prefix run came from this run; the rejected step came from `rejectedRunId`. */
-    rejectedRunId: string;
-    marginScore: number;
-}
-/**
- * Build PRM training triples. The shape: pair runs that share an early
- * prefix (same scenario, same first N steps) and diverge later — at the
- * point of divergence, the high-reward run's next step is `chosen`, the
- * low-reward run's next step is `rejected`. This is the canonical PRM
- * training data shape from Lightman et al. and DeepSeek-R1 process
- * supervision.
- *
- * Implementation note: we don't have a way to detect "same prefix" in
- * the general agent setting (token-level prefixes require hashing model
- * outputs). The current heuristic groups by `(scenarioId, prefixSpanName
- * sequence)` — runs are paired when their first K span names match. For
- * production use this should be replaced with a proper trajectory-prefix
- * hash; the heuristic is good enough for early-stage scaffolding.
- */
-declare function prmTrainingPairs(stepRewardsByRun: Map<string, StepReward[]>, opts?: {
-    minMargin?: number;
-    minPrefixLength?: number;
-}): PrmTrainingTriple[];
-
-/**
- * Trainer-format exporters.
- *
- * agent-eval produces canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`,
- * `StepReward[]`, `PrmTrainingTriple[]`). RL training pipelines consume
- * different shapes — Hugging Face TRL, Prime Intellect's prime-rl, OpenAI
- * fine-tuning, Anthropic finetuning, OpenRLHF, verl. Each has its own
- * JSONL conventions. Rather than ship N adapters, this module ships the
- * canonical formats most production pipelines accept and ergonomic helpers
- * for the rest.
- *
- * Shapes:
- *   - **DPO / IPO / KTO** — `{prompt, chosen, rejected}` JSONL. Consumed
- *     by HuggingFace TRL, prime-rl's offline DPO, OpenRLHF.
- *   - **GRPO offline** — `{prompt, completions[], rewards[]}` JSONL.
- *     Consumed by prime-rl GRPO, verl, OpenRLHF.
- *   - **SFT** — `{messages[]}` JSONL with chosen completion as the final
- *     assistant turn. Consumed by HF SFT trainers, OpenAI fine-tuning,
- *     Anthropic finetuning.
- *   - **PRM** — `{prompt, prefix_steps[], chosen_step, rejected_step}` JSONL.
- *     Consumed by Lightman-style PRM trainers and prime-rl's PRM mode.
- *
- * Why ship this in agent-eval rather than a separate adapter package: the
- * canonical artifacts (`RunRecord[]`, `PreferenceTriple[]`, etc.) are
- * agent-eval's contract; without first-party exporters consumers reverse-
- * engineer the mapping every release. The exporters codify it.
- *
- * The exporters take callbacks for any field that isn't on the canonical
- * artifact (specifically: prompt + completion text, since the package
- * stores only their hashes by design — full text is the consumer's
- * trace store / raw event log).
- */
-
-interface DpoLookups {
-    /** Resolve the prompt text for a run (typically from a trace store / raw event sink). */
-    promptOf: (runId: string) => string | Promise<string>;
-    /** Resolve the assistant completion text for a run. */
-    completionOf: (runId: string) => string | Promise<string>;
-}
-interface DpoExportRow {
-    prompt: string;
-    chosen: string;
-    rejected: string;
-    /** Carried-through margin. Some KTO / IPO variants use this. */
-    margin?: number;
-    /** Free-form metadata for downstream filtering / sharding. */
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert preference triples to TRL-compatible DPO rows. The shape
- * `{prompt, chosen, rejected}` is the canonical HuggingFace DPODataset
- * entry; every major DPO trainer accepts it.
- */
-declare function toDpoRows(triples: PreferenceTriple[], lookups: DpoLookups): Promise<DpoExportRow[]>;
-/** Serialize DPO rows as JSONL. One line per row. */
-declare function toDpoJsonl(rows: DpoExportRow[]): string;
-interface GrpoLookups {
-    promptOf: (runId: string) => string | Promise<string>;
-    completionOf: (runId: string) => string | Promise<string>;
-    /** Optional: derive a custom reward from the run. Defaults to score. */
-    rewardOf?: (run: RunRecord) => number | null;
-}
-interface GrpoExportRow {
-    prompt: string;
-    completions: string[];
-    rewards: number[];
-    /** runIds in the same order as `completions[]` for traceability. */
-    runIds: string[];
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert RunRecord[] grouped by `(scenarioId)` into GRPO offline rows —
- * one row per scenario, with one completion per run on that scenario.
- *
- * GRPO (Shao et al. 2024 / DeepSeek-R1) trains on relative advantages
- * within a group of completions for the same prompt; this is the
- * canonical input format.
- */
-declare function toGrpoRows(runs: RunRecord[], lookups: GrpoLookups): Promise<GrpoExportRow[]>;
-declare function toGrpoJsonl(rows: GrpoExportRow[]): string;
-interface SftLookups {
-    promptOf: (runId: string) => string | Promise<string>;
-    completionOf: (runId: string) => string | Promise<string>;
-    /** Optional system message. Default omits. */
-    systemOf?: (run: RunRecord) => string | null | undefined;
-    /** Filter — return false to skip the run (e.g., low score, failed cases). */
-    include?: (run: RunRecord) => boolean;
-}
-interface SftExportRow {
-    messages: Array<{
-        role: 'system' | 'user' | 'assistant';
-        content: string;
-    }>;
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert RunRecord[] into Hugging Face / OpenAI / Anthropic-style
- * conversational SFT rows. By default every record becomes one row;
- * pass `include` to filter (e.g., keep only `score >= 0.8` for
- * rejection-sampling SFT).
- */
-declare function toSftRows(runs: RunRecord[], lookups: SftLookups): Promise<SftExportRow[]>;
-declare function toSftJsonl(rows: SftExportRow[]): string;
-interface PrmLookups {
-    /** Resolve the prompt text for a run. */
-    promptOf: (runId: string) => string | Promise<string>;
-    /** Resolve the trajectory step text for a (runId, spanId) pair. */
-    stepTextOf: (runId: string, spanId: string) => string | Promise<string>;
-    /** Optional: sequence of prefix span ids leading up to the divergence. */
-    prefixOf?: (runId: string, prefixStepIndex: number) => string[] | Promise<string[]>;
-}
-interface PrmExportRow {
-    prompt: string;
-    /** Span ids for the steps before divergence — caller resolves text via `stepTextOf`. */
-    prefixSpanIds: string[];
-    prefixStepText: string[];
-    chosenStep: string;
-    rejectedStep: string;
-    chosenReward: number;
-    rejectedReward: number;
-    marginScore: number;
-    meta?: Record<string, unknown>;
-}
-/**
- * Convert PRM training triples to JSONL rows. Caller's `stepTextOf`
- * callback resolves span text from the consumer's trace store.
- */
-declare function toPrmRows(triples: PrmTrainingTriple[], lookups: PrmLookups): Promise<PrmExportRow[]>;
-declare function toPrmJsonl(rows: PrmExportRow[]): string;
-interface StepRewardJsonlRow {
-    runId: string;
-    spanId: string;
-    stepIndex: number;
-    reward: number;
-    determinism: 'deterministic' | 'probabilistic';
-    weight: number;
-}
-declare function stepRewardsToJsonl(stepRewards: StepReward[]): string;
-
-/**
- * RL dataset packaging + datasheet — the publishable, sellable bundle.
- *
- * The format exporters (`toGrpoRows` / `toSftRows` / `toDpoRows`) already
- * produce trainer-ready shapes (prime-rl GRPO, TRL DPO, conversational SFT).
- * What turns that into a dataset someone can PUBLISH or BUY is the provenance
- * + a datasheet: which models produced it, which prompt/agent versions, how the
- * reward was derived (deterministic verifiable vs probabilistic judge — the
- * credibility axis a buyer checks first), the split discipline, the reward
- * distribution, the quality gates, the license, and the intended/out-of-scope
- * uses. This module computes those facts from the `RunRecord[]` and renders a
- * "Datasheet for Datasets" (Gebru et al. 2018) card alongside the format files.
- *
- * It composes the existing `rl/exporters` — it does not reimplement any trainer
- * format. The renderers token-identity step (DeepSeek/Kimi/Qwen tokenization
- * with per-token loss masks) is a downstream Python stage that consumes the
- * `messages`/`completions` this bundle emits.
- */
-
-type RewardKind = 'deterministic' | 'probabilistic' | 'mixed';
-type DatasetFormat = 'grpo' | 'sft' | 'dpo';
-/** Caller-declared context — the qualitative half of the datasheet that can't
- *  be computed from records. */
-interface RlDatasetConfig {
-    name: string;
-    version: string;
-    /** Product/task domain, e.g. 'legal-m&a', 'tax-1040'. */
-    domain: string;
-    /** SPDX id or a named commercial license. Required — an unlicensed dataset
-     *  cannot be published or sold. */
-    license: string;
-    /** How the reward was produced. `kind: 'deterministic'` (a test/schema/XPath
-     *  decided it) is the credibility signal; 'probabilistic' = LLM-judge. */
-    reward: {
-        kind: RewardKind;
-        source: string;
-        description: string;
-    };
-    intendedUse: string;
-    outOfScope: string;
-    limitations: string;
-    /** ISO timestamp — passed in (the substrate forbids Date.now()). */
-    createdAtIso: string;
-    /** Default: ['grpo', 'sft']. */
-    formats?: DatasetFormat[];
-    /** Quality gates already run, recorded on the card for the buyer. */
-    qualityGates?: {
-        contaminationProbe?: 'passed' | 'failed' | 'not-run';
-        dedup?: boolean;
-        verifiableRewardFilter?: boolean;
-    };
-}
-interface RewardStats {
-    n: number;
-    mean: number;
-    median: number;
-    min: number;
-    max: number;
-    std: number;
-}
-interface RlDatasetStats {
-    records: number;
-    /** Record count per split — a publishable dataset must declare its holdout. */
-    splits: Record<RunSplitTag, number>;
-    reward: RewardStats;
-    /** Distinct snapshot-pinned models that produced the trajectories. */
-    models: string[];
-    /** Distinct effective-prompt hashes (the agent profile/prompt versions). */
-    promptHashes: string[];
-    commitShas: string[];
-    totalTokens: {
-        input: number;
-        output: number;
-    };
-    totalCostUsd: number;
-}
-interface RlDatasetManifest extends RlDatasetConfig {
-    formats: DatasetFormat[];
-    rowCounts: Partial<Record<DatasetFormat, number>>;
-    stats: RlDatasetStats;
-}
-interface RlDatasetBundle {
-    manifest: RlDatasetManifest;
-    /** Relative filename -> contents. Write these to a directory to publish. */
-    files: Record<string, string>;
-}
-/**
- * Package graded `RunRecord[]` into a publishable RL dataset bundle: the
- * trainer-format JSONL files + a manifest + a datasheet. DPO requires
- * pre-extracted preference triples (pass `preferences`); GRPO/SFT derive from
- * the records directly via the supplied lookups. Throws on an empty corpus —
- * an empty dataset must never be published.
- */
-declare function buildRlDataset(records: RunRecord[], lookups: GrpoLookups & SftLookups, config: RlDatasetConfig, preferences?: {
-    triples: PreferenceTriple[];
-    lookups: DpoLookups;
-}): Promise<RlDatasetBundle>;
-/** Render the "Datasheet for Datasets" card — the artifact a buyer reads. */
-declare function datasheetToMarkdown(m: RlDatasetManifest): string;
-
-/**
- * RL corpus — the durable, append-only accumulation of graded RunRecords that
- * every eval run deposits BY DEFAULT.
- *
- * The dataset is the free exhaust of the normal eval process: we run evals
- * constantly to get an agent production-ready, and those runs already produce
- * graded trajectories. Instead of writing them to an ephemeral run dir and
- * throwing them away, `appendToCorpus` accumulates them into a durable corpus;
- * `buildDatasetFromCorpus` later harvests the whole corpus into a publishable
- * bundle. No separate data-collection campaign — the data accrues from work we
- * do anyway. This is the "best things for free by our process" layer.
- *
- * Trajectory text rides on the record as top-level `prompt` / `completion`
- * (what the eval harnesses capture; the RunRecord validator ignores the extra
- * keys). The harvest reads them directly — no trace store round-trip needed.
- */
-
-/** A corpus record is a RunRecord carrying the trajectory text the harness
- *  captured. `prompt`/`completion` are top-level (the validator ignores extras). */
-type CorpusRecord = RunRecord & {
-    prompt?: string;
-    completion?: string;
-};
-interface CorpusAppendResult {
-    appended: number;
-    /** Skipped because a record with the same runId was already in the corpus
-     *  (idempotent appends — NOT re-run collapsing; re-runs get fresh runIds). */
-    skipped: number;
-    total: number;
-}
-/**
- * Append graded records to the corpus (append-only JSONL). Deduplicates by
- * `runId` against what's already on disk so re-running the same harness is
- * idempotent. Creates the file and parent dir. This is the call every eval
- * harness makes by default after producing its records.
- */
-declare function appendToCorpus(records: CorpusRecord[], corpusPath: string): CorpusAppendResult;
-/** Read the full corpus. Returns [] if the corpus does not exist yet. */
-declare function readCorpus(corpusPath: string): CorpusRecord[];
-interface HarvestOptions {
-    /** Keep only records scoring >= this (rejection-sampling for SFT). */
-    minScore?: number;
-    /** Keep only these splits (e.g. ['holdout'] for an eval-only dataset). */
-    splits?: RunRecord['splitTag'][];
-}
-/**
- * Harvest the accumulated corpus into a publishable RL dataset bundle. Reads
- * trajectory text from each record's top-level `prompt`/`completion`; records
- * missing either are excluded (a graded score with no trajectory can't train).
- * Optionally filters by score / split. Throws (via buildRlDataset) if nothing
- * survives — an empty dataset must never be published.
- */
-declare function buildDatasetFromCorpus(corpusPath: string, config: RlDatasetConfig, opts?: HarvestOptions): Promise<RlDatasetBundle>;
-
-export { prmTrainingPairs as A, readCorpus as B, type CorpusRecord as C, type DpoExportRow as D, type ExtractPreferencesOptions as E, runwiseStepRewardSummary as F, type GrpoExportRow as G, type HarvestOptions as H, stepRewardsToJsonl as I, toAnthropicFormat as J, toDpoJsonl as K, toDpoRows as L, toGrpoJsonl as M, toGrpoRows as N, toPrmJsonl as O, type PreferenceExtractionReport as P, toPrmRows as Q, type RewardKind as R, type SftExportRow as S, toSftJsonl as T, toSftRows as U, toTRLFormat as V, type DpoLookups as a, type GrpoLookups as b, type SftLookups as c, type CorpusAppendResult as d, type DatasetFormat as e, type ExtractStepRewardsOptions as f, type PreferenceStrategy as g, type PreferenceTriple as h, type PrmExportRow as i, type PrmLookups as j, type PrmTrainingTriple as k, type RewardStats as l, type RlDatasetBundle as m, type RlDatasetConfig as n, type RlDatasetManifest as o, type RlDatasetStats as p, type RunwiseStepSummary as q, type StepReward as r, type StepRewardJsonlRow as s, type StepScorer as t, appendToCorpus as u, buildDatasetFromCorpus as v, buildRlDataset as w, datasheetToMarkdown as x, extractPreferences as y, extractStepRewards as z };

package/dist/counterfactual-DlOz8PBx.d.ts

@@ -1,85 +0,0 @@
-import { T as TraceEmitter } from './emitter-C2rqGH_l.js';
-import { T as TraceStore } from './store-BcFXE6LG.js';
-import { T as Trajectory, a as TrajectoryStep } from './trajectory-2TkpSEVh.js';
-
-/**
- * Counterfactual replay — "what would have happened if we'd changed
- * exactly one thing at turn N?"
- *
- * The framework does NOT drive the agent — it sets up the replay
- * context (prior spans, prior state, mutation spec) and records the
- * resulting divergence. Consumers supply an `executeFrom(ctx)` callback
- * that runs their agent starting from turn N with the mutation applied.
- *
- * Counterfactual runs are recorded as a new Run with `layer='meta'` and
- * `parentRunId = originalRunId`, so downstream diff + correlation
- * pipelines see them natively.
- */
-
-type CounterfactualMutation = {
-    kind: 'swap-model';
-    at: number;
-    newModel: string;
-} | {
-    kind: 'swap-tool-result';
-    at: number;
-    newResult: unknown;
-} | {
-    kind: 'truncate-after';
-    at: number;
-} | {
-    kind: 'inject-system-message';
-    at: number;
-    content: string;
-} | {
-    kind: 'custom';
-    at: number;
-    describe: string;
-    apply: (step: TrajectoryStep) => TrajectoryStep;
-};
-interface CounterfactualContext {
-    originalRunId: string;
-    originalTrajectory: Trajectory;
-    /** Steps up to (but not including) the mutation point — the prefix the
-     *  replayed agent inherits as its prior conversation/tool history. */
-    prefix: TrajectoryStep[];
-    mutation: CounterfactualMutation;
-    /** Pre-applied mutation on the step at `mutation.at`. Consumers use this
-     *  as the FIRST step the replayed agent emits (they decide whether to
-     *  re-emit it or continue from there). */
-    mutatedStep: TrajectoryStep;
-}
-interface CounterfactualResult {
-    counterfactualRunId: string;
-    originalRunId: string;
-    mutation: CounterfactualMutation;
-    /** Structured delta summary — caller can extend via scoring. */
-    delta: {
-        originalOutcomeScore: number | null;
-        counterfactualOutcomeScore: number | null;
-        deltaScore: number | null;
-    };
-}
-interface CounterfactualRunner {
-    /**
-     * Execute the agent from `ctx.prefix` with the mutation applied.
-     * MUST emit spans into the provided emitter so they become part of
-     * the counterfactual run. MUST call emitter.endRun() with a verdict.
-     */
-    executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>;
-}
-declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
-/**
- * Aggregate a batch of counterfactuals into a simple attribution table:
- * which mutation kinds move outcomes most? (Useful when you run a grid
- * over the same trajectory — swap-model at every llm span, swap-tool
- * at every tool span — and want a ranked summary.)
- */
-declare function attributeCounterfactuals(results: CounterfactualResult[]): Array<{
-    mutationKind: CounterfactualMutation['kind'];
-    n: number;
-    meanAbsDelta: number;
-    meanSignedDelta: number;
-}>;
-
-export { type CounterfactualMutation as C, attributeCounterfactuals as a, type CounterfactualRunner as b, type CounterfactualContext as c, type CounterfactualResult as d, runCounterfactual as r };