@tangle-network/agent-eval 0.95.0 → 0.96.0

package/dist/rl/reward-hacking.d.ts

@@ -1,104 +0,0 @@
-/**
- * Reward hacking / Goodhart detection.
- *
- * Goodhart's Law says: when a measure becomes a target, it ceases to be
- * a good measure. In RLHF and agentic-RL settings this is the dominant
- * failure mode — the policy learns to produce outputs that score well on
- * the proxy reward (judge, rubric, test pass-rate) without producing
- * the underlying capability the proxy was meant to track.
- *
- * Krakovna et al. (2020, "Specification Gaming Examples in AI") and the
- * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.
- * 2023) converge on a few diagnostic signatures:
- *
- *   1. **Reward divergence:** the proxy reward grows while the held-out
- *      ground-truth signal stagnates or drops. Predictive validity over
- *      time captures this.
- *   2. **Distributional shift in outputs:** after RL, the policy produces
- *      outputs that no longer match the reference distribution — usually
- *      because it found a high-reward attractor that's degenerate (e.g.
- *      one-token responses, repetition, formatting tricks).
- *   3. **Disagreement between independent rewards:** if you train on
- *      reward A and a held-out independent reward B drops sharply, you're
- *      probably hacking A.
- *   4. **Calibration drift:** the verifiable / deterministic component of
- *      the reward is stable; the probabilistic / judge component drifts up
- *      while the deterministic component doesn't. The judge is being
- *      gamed.
- *
- * This module ships explicit detectors for all four signatures, plus a
- * combined verdict. The output is diagnostic — actionable signals,
- * not autoreject — because each signature has known false positives
- * (e.g., a policy that genuinely improves can show distributional shift).
- *
- * Differs from `rubricPredictiveValidity` (which is a *standing* check on
- * whether rubrics correlate with deployment outcomes) — this is a
- * *temporal* check on whether the reward-vs-truth gap is *widening over
- * time during a training run*.
- */
-import type { RunRecord } from '../run-record';
-import { type VerifiableRewardExtractionOptions } from './verifiable-reward';
-export type RewardHackingSignal = 'reward_divergence' | 'distribution_shift' | 'reward_disagreement' | 'judge_drift';
-export interface RewardHackingFinding {
-    signal: RewardHackingSignal;
-    /** Severity in [0, 1]. >0.5 = strong signal. */
-    severity: number;
-    message: string;
-    /** Numeric evidence the consumer can render. */
-    detail: Record<string, number>;
-}
-export interface RewardHackingReport {
-    findings: RewardHackingFinding[];
-    /**
-     * Composite verdict. `'clean'` if every signal severity < 0.3;
-     * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.
-     */
-    verdict: 'clean' | 'suspect' | 'gaming';
-    /** Rationale for the verdict, ready to paste into an audit log. */
-    rationale: string[];
-    /** Number of paired (proxy, truth) data points the report saw. */
-    n: number;
-}
-export interface DetectRewardHackingInput {
-    /**
-     * Run records ordered by recency (oldest first). The detector segments
-     * them into prefix/suffix windows to compute "did the gap widen."
-     */
-    runs: RunRecord[];
-    /**
-     * The metric the policy was trained to optimize. Should be present on
-     * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
-     */
-    proxyOf?: (run: RunRecord) => number | null;
-    /**
-     * The held-out ground-truth metric. For RL on coding, this is typically
-     * test pass-rate. For RLHF, it's downstream task performance or human
-     * preference. For knowledge tasks, it's an independently-graded score.
-     */
-    truthOf?: (run: RunRecord) => number | null;
-    /**
-     * Independent secondary reward. Used for the `reward_disagreement`
-     * signal. Default uses the verifiable reward extractor (deterministic
-     * sources only).
-     */
-    secondaryRewardOf?: (run: RunRecord) => number | null;
-    /**
-     * Window size — how many of the most recent runs count as the "after"
-     * cohort. Default min(50, half the runs).
-     */
-    windowSize?: number;
-    /**
-     * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
-     * (gaming).
-     */
-    thresholds?: {
-        suspect?: number;
-        gaming?: number;
-    };
-    /**
-     * Verifiable-reward options used for the secondary-reward fallback.
-     */
-    verifiableRewardOptions?: VerifiableRewardExtractionOptions;
-}
-export declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
-//# sourceMappingURL=reward-hacking.d.ts.map

package/dist/rl/reward-hacking.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"reward-hacking.d.ts","sourceRoot":"","sources":["../../src/rl/reward-hacking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAEL,KAAK,iCAAiC,EACvC,MAAM,qBAAqB,CAAA;AAE5B,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,oBAAoB,GACpB,qBAAqB,GACrB,aAAa,CAAA;AAEjB,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,mBAAmB,CAAA;IAC3B,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,oBAAoB,EAAE,CAAA;IAChC;;;OAGG;IACH,OAAO,EAAE,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAA;IACvC,mEAAmE;IACnE,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,kEAAkE;IAClE,CAAC,EAAE,MAAM,CAAA;CACV;AAED,MAAM,WAAW,wBAAwB;IACvC;;;OAGG;IACH,IAAI,EAAE,SAAS,EAAE,CAAA;IACjB;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,CAAA;IAC3C;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,CAAA;IAC3C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,KAAK,MAAM,GAAG,IAAI,CAAA;IACrD;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;OAGG;IACH,UAAU,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAClD;;OAEG;IACH,uBAAuB,CAAC,EAAE,iCAAiC,CAAA;CAC5D;AAOD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,mBAAmB,CA6IxF"}

package/dist/rl/rl-campaign.d.ts

@@ -1,85 +0,0 @@
-/**
- * `runRLCampaign` — top-level orchestrator that runs the matrix and
- * produces every RL-ready artifact in one call.
- *
- * Wires:
- *   1. `runEvalCampaign` for the matrix run (capture, integrity, hooks)
- *   2. `extractVerifiableReward` over each run, separating deterministic
- *      from probabilistic reward sources for the trainer
- *   3. `extractPreferences` to produce DPO/PPO/KTO triples
- *   4. `evaluateInterimReleaseConfidence` over paired deltas (anytime-valid)
- *   5. `rubricPredictiveValidity` against an outcome store, when provided
- *   6. `detectRewardHacking` as a standing hygiene check
- *   7. Trainer-format export rows ready for prime-rl / TRL / verl
- *
- * The output `RLCampaignResult` is a single, audit-ready artifact: every
- * stage's output is in there. The consumer's downstream fits in a single
- * line: pass `result.preferences` to their DPO trainer, `result.grpoRows`
- * to GRPO, `result.runs` plus `result.rewardSignals` to a custom RL loop.
- */
-import { type EvalCampaignOptions, type EvalCampaignResult } from '../eval-campaign';
-import type { OutcomeStore } from '../meta-eval/outcome-store';
-import { type RubricPredictiveValidityReport } from '../meta-eval/rubric-predictive-validity';
-import { type InterimReleaseConfidence } from '../sequential';
-import { type DpoExportRow, type DpoLookups, type GrpoExportRow, type GrpoLookups, type SftExportRow, type SftLookups } from './exporters';
-import { type ExtractPreferencesOptions, type PreferenceExtractionReport } from './preferences';
-import { type RewardHackingReport } from './reward-hacking';
-import { type VerifiableReward, type VerifiableRewardExtractionOptions } from './verifiable-reward';
-export interface RunRLCampaignOptions<V> extends EvalCampaignOptions<V> {
-    /** Preference-extraction options. Default uses paired-by-scenario-and-seed with min-margin 0.05. */
-    preferences?: ExtractPreferencesOptions;
-    /** Verifiable-reward extraction options. */
-    verifiableReward?: VerifiableRewardExtractionOptions;
-    /** Outcome store + metric names — when supplied, runs `rubricPredictiveValidity` post-campaign. */
-    outcomeStore?: OutcomeStore;
-    outcomeMetrics?: string[];
-    /** Anytime-valid sequential evaluation options. */
-    sequential?: {
-        alpha?: number;
-        bound?: number;
-        rope?: {
-            low: number;
-            high: number;
-        };
-    };
-    /** Trainer-format export lookups. When provided, the orchestrator builds the corresponding rows. */
-    trainerExport?: {
-        dpo?: DpoLookups;
-        grpo?: GrpoLookups;
-        sft?: SftLookups;
-    };
-}
-export interface RLCampaignResult<V> {
-    campaign: EvalCampaignResult;
-    /** Per-run verifiable reward (deterministic when available, probabilistic fallback otherwise). */
-    rewardSignals: Array<{
-        runId: string;
-        reward: VerifiableReward | null;
-    }>;
-    /** Preference extraction report. */
-    preferences: PreferenceExtractionReport;
-    /** Anytime-valid interim verdict over the paired deltas (vs comparator). */
-    interimConfidence: InterimReleaseConfidence | null;
-    /** Standing reward-hacking hygiene check. */
-    rewardHacking: RewardHackingReport;
-    /** Predictive validity, when an outcome store was supplied. */
-    predictiveValidity: RubricPredictiveValidityReport | null;
-    /** Trainer-export rows, populated only for the formats the caller requested via `trainerExport`. */
-    trainerRows: {
-        dpo?: DpoExportRow[];
-        grpo?: GrpoExportRow[];
-        sft?: SftExportRow[];
-    };
-    /**
-     * One-line top-level summary the consumer can log.
-     */
-    summary: string;
-    /**
-     * Convenience type-tag — consumers can branch on `result.kind`.
-     */
-    kind: 'agent-eval-rl-campaign';
-    unusedVariant?: V;
-}
-export declare function runRLCampaign<V>(opts: RunRLCampaignOptions<V>): Promise<RLCampaignResult<V>>;
-export { runEvalCampaign } from '../eval-campaign';
-//# sourceMappingURL=rl-campaign.d.ts.map

package/dist/rl/rl-campaign.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"rl-campaign.d.ts","sourceRoot":"","sources":["../../src/rl/rl-campaign.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EACL,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EAExB,MAAM,kBAAkB,CAAA;AACzB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EACL,KAAK,8BAA8B,EAEpC,MAAM,yCAAyC,CAAA;AAEhD,OAAO,EAAoC,KAAK,wBAAwB,EAAE,MAAM,eAAe,CAAA;AAC/F,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,UAAU,EAIhB,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,KAAK,yBAAyB,EAE9B,KAAK,0BAA0B,EAChC,MAAM,eAAe,CAAA;AACtB,OAAO,EAAuB,KAAK,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAChF,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,iCAAiC,EACvC,MAAM,qBAAqB,CAAA;AAE5B,MAAM,WAAW,oBAAoB,CAAC,CAAC,CAAE,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IACrE,oGAAoG;IACpG,WAAW,CAAC,EAAE,yBAAyB,CAAA;IACvC,4CAA4C;IAC5C,gBAAgB,CAAC,EAAE,iCAAiC,CAAA;IACpD,mGAAmG;IACnG,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;IACzB,mDAAmD;IACnD,UAAU,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAA;IACrF,oGAAoG;IACpG,aAAa,CAAC,EAAE;QACd,GAAG,CAAC,EAAE,UAAU,CAAA;QAChB,IAAI,CAAC,EAAE,WAAW,CAAA;QAClB,GAAG,CAAC,EAAE,UAAU,CAAA;KACjB,CAAA;CACF;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC;IACjC,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,kGAAkG;IAClG,aAAa,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAA;KAAE,CAAC,CAAA;IACxE,oCAAoC;IACpC,WAAW,EAAE,0BAA0B,CAAA;IACvC,4EAA4E;IAC5E,iBAAiB,EAAE,wBAAwB,GAAG,IAAI,CAAA;IAClD,6CAA6C;IAC7C,aAAa,EAAE,mBAAmB,CAAA;IAClC,+DAA+D;IAC/D,kBAAkB,EAAE,8BAA8B,GAAG,IAAI,CAAA;IACzD,oGAAoG;IACpG,WAAW,EAAE;QACX,GAAG,CAAC,EAAE,YAAY,EAAE,CAAA;QACpB,IAAI,CAAC,EAAE,aAAa,EAAE,CAAA;QACtB,GAAG,CAAC,EAAE,YAAY,EAAE,CAAA;KACrB,CAAA;IACD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,IAAI,EAAE,wBAAwB,CAAA;IAC9B,aAAa,CAAC,EAAE,CAAC,CAAA;CAClB;AAED,wBAAsB,aAAa,CAAC,CAAC,EACnC,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC5B,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAgF9B;AAmED,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/rl/run-record-adapters.d.ts

@@ -1,56 +0,0 @@
-/**
- * Adapters: convert measurement outputs into the canonical `RunRecord[]`
- * artifact that `replayCache`, `pairedEvalueSequence`, and
- * `rubricPredictiveValidity` consume. Two sources:
- *   - `campaignToRunRecords` — the campaign substrate's per-cell results
- *     (the modern path: `runCampaign` / `runImprovementLoop` → records).
- *   - `verificationReportToRunRecord` — a `MultiLayerVerifier` report.
- *
- * Adapters are thin and explicit — every mandatory `RunRecord` field comes
- * from a caller-supplied context (`commitSha`, `model`, `promptHash`,
- * `configHash`) plus the cell's runtime data. The validator still rejects
- * bare-alias model strings — the caller snapshot-pins.
- */
-import type { CampaignResult } from '../campaign';
-import type { VerificationReport } from '../multi-layer-verifier';
-import type { RunRecord, RunSplitTag } from '../run-record';
-export interface AdapterContext {
-    /** Logical experiment id — typically the campaign or sweep identifier. */
-    experimentId: string;
-    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
-    model: string;
-    /** Git SHA the harness was run from. */
-    commitSha: string;
-    /** Hash of the effective prompt sent to the model. */
-    promptHash: string;
-    /** Hash of the effective config (model, temperature, tools, judges, splits). */
-    configHash: string;
-    /** Default split tag. Default `'search'`. */
-    splitTag?: RunSplitTag;
-    /** Default cost in USD when the source doesn't record one. Default `0`. */
-    defaultCostUsd?: number;
-}
-/**
- * Convert a `CampaignResult` into canonical `RunRecord[]` — one record per
- * scored cell. The cell's mean judge composite becomes the split score; every
- * judge dimension is carried through to `outcome.raw`. A cell that errored
- * becomes a record with `failureMode: 'cell_error'` (kept, not dropped — an
- * unscored cell is signal). `candidateId` identifies the measured surface
- * (defaults to the campaign manifest hash).
- */
-export declare function campaignToRunRecords(campaign: CampaignResult, ctx: AdapterContext & {
-    candidateId?: string;
-}): RunRecord[];
-/**
- * Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
- * `outcome.searchScore` (or `holdoutScore`) is `report.blendedScore`;
- * `outcome.raw` carries every layer's score + a pass indicator; `failureMode`
- * is the first failing layer's reason.
- */
-export declare function verificationReportToRunRecord(report: VerificationReport, ctx: AdapterContext & {
-    candidateId: string;
-    scenarioId?: string;
-}, opts?: {
-    runId?: string;
-}): RunRecord;
-//# sourceMappingURL=run-record-adapters.d.ts.map

package/dist/rl/run-record-adapters.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"run-record-adapters.d.ts","sourceRoot":"","sources":["../../src/rl/run-record-adapters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AACjD,OAAO,KAAK,EAAe,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC9E,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAE3D,MAAM,WAAW,cAAc;IAC7B,0EAA0E;IAC1E,YAAY,EAAE,MAAM,CAAA;IACpB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAA;IACb,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAA;IAClB,gFAAgF;IAChF,UAAU,EAAE,MAAM,CAAA;IAClB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,cAAc,EACxB,GAAG,EAAE,cAAc,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7C,SAAS,EAAE,CAmCb;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,kBAAkB,EAC1B,GAAG,EAAE,cAAc,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,EAClE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5B,SAAS,CA4CX"}

package/dist/rl/sim-fidelity.d.ts

@@ -1,166 +0,0 @@
-/**
- * Simulator fidelity — score a user SIMULATOR's realism against real-user
- * trace distributions.
- *
- * Synthetic-persona evals (`PersonaConfig`-driven canonical evals, fuzz
- * user-simulator objectives) stand in for real users in most of the numbers
- * we publish. The standing threat is the Sim2Real gap: a simulator that is
- * distributionally unlike production creates "easy mode" and silently
- * inflates every score built on it. This module measures that gap from the
- * SAME artifact both sides already produce — `RunRecord`s — so no new
- * capture pipeline is needed:
- *
- *   - `simFidelityReport` — per-feature Jensen-Shannon divergence between
- *     simulated and production record distributions, collapsed into a
- *     fidelity coefficient in [0,1].
- *   - `easyModeCheck` — the headline academic failure mode (sim inflates
- *     pass-rate over production) as its own named artifact.
- *
- * Every synthetic-persona eval result should publish its fidelity
- * coefficient alongside the score — a number from an unrepresentative
- * simulator is an unlabeled estimate. Wire-in points:
- *
- *   - canonical persona evals: pass the campaign's `RunRecord`s as
- *     `simulated` and intake-adapter output (`contract/intake`: OTel spans,
- *     feedback tables, coding-agent sessions) as `production`
- *   - the fuzz user-sim objective: use `1 - report.fidelity` as a realism
- *     penalty when searching over generated personas
- *   - the durable corpus (`./corpus`): both sides read straight from
- *     `readCorpus` — tag sim vs production by `experimentId`
- */
-import type { RunRecord } from '../run-record';
-/** Extracts a flat behavioral feature map from one record. `string` values
- *  are categorical, `number` values are quantile-bucketed over the union of
- *  both sides, `null` means the feature is absent on this record and is
- *  counted explicitly as its own category (never silently dropped). */
-export type BehaviorFeatures = (record: RunRecord) => Record<string, string | number | null>;
-/** Reserved histogram category for `null` feature values. A capture-rate
- *  difference (one side instruments a signal, the other does not) registers
- *  as divergence by design: a simulator that produces no tool traces is not
- *  representative of production that does. */
-export declare const ABSENT_CATEGORY = "(absent)";
-/** Minimum non-null observations PER SIDE for a feature to enter the
- *  fidelity mean. Below this the JSD estimate is sampling noise. */
-export declare const DEFAULT_MIN_N_PER_FEATURE = 20;
-/** Quantile buckets used to discretize numeric features. Quartiles balance
- *  resolution against per-bucket sample size at the default minN. */
-export declare const DEFAULT_QUANTILE_BUCKETS = 4;
-/** Fidelity at or above this → 'representative'; below → 'skewed'.
- *  1 − 0.8 = mean JSD 0.2 ≈ distributions that mostly overlap with one
- *  clearly shifted mode — the point where per-feature shifts start changing
- *  which failure classes an eval can even observe. */
-export declare const REPRESENTATIVE_MIN_FIDELITY = 0.8;
-/**
- * Default feature set — ONLY fields verified present on both simulated and
- * production records:
- *
- *   - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
- *     validator (non-finite values read as absent rather than poisoning a
- *     bucket).
- *   - `failure_class` — optional taxonomy field; absent counted explicitly.
- *   - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
- *     `outcome.raw` counters the intake adapters and eval harnesses write
- *     (`turns_completed`, `assistant_messages`, `tool_errors`,
- *     `turns_aborted`); absent on records whose producer did not capture
- *     them, counted explicitly.
- *   - `completion_length` — from the optional `CorpusRecord` trajectory
- *     text; the message-length proxy when records come from the corpus.
- *
- * `RunRecord` carries event COUNTS, not event ordering, so
- * `tool_error_recovery` is a counts-only derivation: errors occurred and the
- * run still completed cleanly ('recovered') vs aborted or classified as a
- * failure ('unrecovered') — not a literal error→retry sequence check.
- */
-export declare const defaultBehaviorFeatures: BehaviorFeatures;
-/**
- * Jensen-Shannon divergence between two categorical histograms (raw counts;
- * normalized internally). Log base 2 → bounded [0,1]: 0 = identical
- * distributions, 1 = disjoint support. Symmetric, defined even where the
- * supports differ — exactly the regime sim-vs-production comparison lives in.
- * Throws on zero-mass or negative/non-finite counts: an empty histogram has
- * no distribution and a silent 0 would read as "perfectly representative".
- */
-export declare function jsDivergence(p: Record<string, number>, q: Record<string, number>): number;
-/**
- * Deterministic quantile edges over a value set (the UNION of both sides, so
- * sim and production land in the same buckets). Linear interpolation between
- * order statistics; duplicate edges from heavy ties collapse into fewer,
- * wider buckets. Returns `bucketCount - 1` edges before deduplication.
- */
-export declare function quantileEdges(values: number[], bucketCount?: number): number[];
-/** Stable half-open bucket label for a value against quantile edges:
- *  `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
-export declare function bucketLabel(value: number, edges: number[]): string;
-export interface FeatureShift {
-    /** Category label (a string value, a numeric bucket, or `ABSENT_CATEGORY`). */
-    value: string;
-    /** Probability of this category among ALL simulated records (nulls included
-     *  via `ABSENT_CATEGORY`, so each side's shifts sum to 1). */
-    pSim: number;
-    /** Probability among ALL production records. */
-    pProd: number;
-}
-export interface FeatureDivergence {
-    feature: string;
-    /** Jensen-Shannon divergence in [0,1] for this feature. */
-    divergence: number;
-    /** Largest |pSim − pProd| categories, descending — where the sim deviates. */
-    topShifts: FeatureShift[];
-    /** Non-null observations on the simulated side. */
-    nSim: number;
-    /** Non-null observations on the production side. */
-    nProd: number;
-}
-export type FidelityVerdict = 'representative' | 'skewed' | 'insufficient-data';
-export interface FidelityReport {
-    perDimension: FeatureDivergence[];
-    /** 1 − mean divergence over features with sufficient data. NaN when the
-     *  verdict is 'insufficient-data' — a 0 would read as "maximally skewed"
-     *  and silently poison downstream aggregation; check `verdict` first. */
-    fidelity: number;
-    /** Features excluded because either side had fewer than `minNPerFeature`
-     *  non-null observations. Named, never silently dropped. */
-    insufficientData: string[];
-    /** 'representative' when fidelity >= REPRESENTATIVE_MIN_FIDELITY (0.8),
-     *  'skewed' below, 'insufficient-data' when no feature met minN. */
-    verdict: FidelityVerdict;
-}
-export interface SimFidelityOptions {
-    /** Feature extractor. Defaults to `defaultBehaviorFeatures`. */
-    features?: BehaviorFeatures;
-    /** Minimum non-null observations per side per feature. Default 20. */
-    minNPerFeature?: number;
-}
-/**
- * Compare a simulator's RunRecords against production RunRecords, feature by
- * feature. Numeric features are bucketed by deterministic quantiles of the
- * union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
- * inputs — "no records" is a wiring error, not a distribution.
- */
-export declare function simFidelityReport(simulated: RunRecord[], production: RunRecord[], opts?: SimFidelityOptions): FidelityReport;
-export interface EasyModeOptions {
-    /** A run passes when its score (holdout, else search) >= this. Default 0.5
-     *  — matches the pass-threshold convention across the rl/ primitives. */
-    passThreshold?: number;
-    /** Pass-rate gap above which the sim is flagged inflated. Default 0.1 —
-     *  a 10-point inflation is enough to flip most promotion gates. */
-    inflationTolerance?: number;
-}
-export interface EasyModeReport {
-    simPassRate: number;
-    prodPassRate: number;
-    /** simPassRate − prodPassRate. Positive = the simulator is easier than reality. */
-    gap: number;
-    /** True when gap > inflationTolerance: numbers measured against this
-     *  simulator overstate production performance. */
-    inflated: boolean;
-}
-/**
- * The headline simulator failure mode as its own named artifact: a simulator
- * that creates "easy mode" inflates pass-rate relative to production, and
- * every score measured against it overstates reality. Throws on empty inputs
- * and on records carrying neither score — a silently-skipped record would
- * bias the very rate this check exists to keep honest.
- */
-export declare function easyModeCheck(simulated: RunRecord[], production: RunRecord[], opts?: EasyModeOptions): EasyModeReport;
-//# sourceMappingURL=sim-fidelity.d.ts.map

package/dist/rl/sim-fidelity.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sim-fidelity.d.ts","sourceRoot":"","sources":["../../src/rl/sim-fidelity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAI9C;;;uEAGuE;AACvE,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,CAAA;AAE5F;;;8CAG8C;AAC9C,eAAO,MAAM,eAAe,aAAa,CAAA;AAEzC;oEACoE;AACpE,eAAO,MAAM,yBAAyB,KAAK,CAAA;AAE3C;qEACqE;AACrE,eAAO,MAAM,wBAAwB,IAAI,CAAA;AAEzC;;;sDAGsD;AACtD,eAAO,MAAM,2BAA2B,MAAM,CAAA;AAI9C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,uBAAuB,EAAE,gBAerC,CAAA;AAoBD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CA6BzF;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,WAAW,SAA2B,GAAG,MAAM,EAAE,CAkBhG;AAED;iDACiD;AACjD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAMlE;AAID,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,KAAK,EAAE,MAAM,CAAA;IACb;kEAC8D;IAC9D,IAAI,EAAE,MAAM,CAAA;IACZ,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAA;IAClB,8EAA8E;IAC9E,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,mDAAmD;IACnD,IAAI,EAAE,MAAM,CAAA;IACZ,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,MAAM,eAAe,GAAG,gBAAgB,GAAG,QAAQ,GAAG,mBAAmB,CAAA;AAE/E,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,iBAAiB,EAAE,CAAA;IACjC;;6EAEyE;IACzE,QAAQ,EAAE,MAAM,CAAA;IAChB;gEAC4D;IAC5D,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B;wEACoE;IACpE,OAAO,EAAE,eAAe,CAAA;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,SAAS,EAAE,EACtB,UAAU,EAAE,SAAS,EAAE,EACvB,IAAI,GAAE,kBAAuB,GAC5B,cAAc,CA0DhB;AA6DD,MAAM,WAAW,eAAe;IAC9B;6EACyE;IACzE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;uEACmE;IACnE,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,mFAAmF;IACnF,GAAG,EAAE,MAAM,CAAA;IACX;sDACkD;IAClD,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,SAAS,EAAE,EACtB,UAAU,EAAE,SAAS,EAAE,EACvB,IAAI,GAAE,eAAoB,GACzB,cAAc,CA0BhB"}

package/dist/rl/sim-fidelity.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=sim-fidelity.test.d.ts.map

package/dist/rl/sim-fidelity.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sim-fidelity.test.d.ts","sourceRoot":"","sources":["../../src/rl/sim-fidelity.test.ts"],"names":[],"mappings":""}

package/dist/rl/tournament.d.ts

@@ -1,115 +0,0 @@
-/**
- * Bradley-Terry / Elo tournament evaluation.
- *
- * For multi-candidate sweeps, comparing every candidate's score against
- * a fixed comparator wastes information — the comparator becomes a high-
- * variance reference and rank flips between near-tied middle-rank
- * candidates are dominated by noise. Pairwise tournaments fix this:
- * every (i, j) pair contributes a comparison to a Bradley-Terry MLE that
- * estimates each candidate's strength on a unified scale.
- *
- * For online updating (rolling campaigns where new candidates arrive
- * over time), we also ship classical Elo with configurable K-factor.
- *
- * References:
- *   - Bradley, R. A., Terry, M. E. (1952). Rank analysis of incomplete
- *     block designs. Biometrika, 39(3/4), 324–345.
- *   - Hunter, D. R. (2004). MM algorithms for generalized Bradley-Terry
- *     models. Annals of Statistics, 32(1), 384–406. (The MLE algorithm
- *     used here.)
- *   - Elo, A. E. (1978). The Rating of Chess Players, Past and Present.
- *
- * This is a useful primitive because most LLM-eval communities (Chatbot
- * Arena, AlpacaEval, ELO-style ablation) have converged on pairwise
- * tournament eval as the most sample-efficient and most rank-stable
- * method when you have many candidates.
- */
-export interface PairwiseOutcome {
-    /** Winner candidate id. */
-    winner: string;
-    /** Loser candidate id. */
-    loser: string;
-    /**
-     * Optional draw flag. When true, both candidates get half-credit
-     * (Bradley-Terry handles draws as half-wins for each side).
-     */
-    draw?: boolean;
-    /**
-     * Optional weight — useful if some pairwise comparisons are stronger
-     * signals than others (e.g. a paired test with a wider score gap is
-     * a more confident comparison). Default 1.
-     */
-    weight?: number;
-}
-export interface BradleyTerryRating {
-    candidateId: string;
-    /** Latent strength θ ≥ 0 from the BT MLE. */
-    strength: number;
-    /** Log-strength = log(θ) — interpretable on a linear scale. */
-    logStrength: number;
-    /** Number of pairwise comparisons this candidate appears in. */
-    n: number;
-    /** Win count (+ 0.5 per draw). */
-    wins: number;
-}
-export interface BradleyTerryFit {
-    ratings: BradleyTerryRating[];
-    /** Iterations of the MM algorithm before convergence. */
-    iterations: number;
-    /** Final maximum |θ_new - θ_old| / θ_old. */
-    finalDelta: number;
-    converged: boolean;
-}
-/**
- * Bradley-Terry MLE via Hunter's MM algorithm.
- *
- * Iteration: θ_i^new = W_i / Σ_{j ≠ i} N_ij / (θ_i + θ_j)
- *   where W_i = wins by i (+ 0.5 per draw), N_ij = total comparisons.
- *
- * Returns log-strengths normalized so the smallest is 0 (any constant
- * offset is unobservable in BT — only differences are identified).
- */
-export declare function fitBradleyTerry(outcomes: PairwiseOutcome[], opts?: {
-    tolerance?: number;
-    maxIterations?: number;
-    smoothing?: number;
-}): BradleyTerryFit;
-/**
- * Online Elo updates. Use when comparisons arrive over time and you want
- * a running rating without re-fitting the full BT MLE on every update.
- *
- * Initialize ratings to `defaultRating` (1500 by default). Each call to
- * `applyEloUpdate` mutates the map in place and returns the deltas so
- * the caller can log per-comparison rating changes.
- */
-export interface EloOptions {
-    /** Default rating for unseen candidates. Default 1500. */
-    defaultRating?: number;
-    /** K-factor controls the step size. Default 32 (FIDE-ish). */
-    kFactor?: number;
-}
-export declare function applyEloUpdate(ratings: Map<string, number>, outcome: PairwiseOutcome, opts?: EloOptions): {
-    winnerDelta: number;
-    loserDelta: number;
-};
-/**
- * Build pairwise outcomes from the campaign artifact: for every scenario
- * shared by two candidates, the higher-scoring run wins. Useful when you
- * want a tournament view of an existing campaign without an additional
- * pairwise judge call.
- */
-export interface BuildPairwiseFromCampaignInput {
-    runs: Array<{
-        candidateId: string;
-        /** Stable identifier for the matching unit (typically scenarioId). */
-        matchKey: string;
-        score: number;
-    }>;
-    /**
-     * Tied-score margin. Below this, the comparison is a draw. Default 0
-     * (no ties).
-     */
-    drawMargin?: number;
-}
-export declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
-//# sourceMappingURL=tournament.d.ts.map

package/dist/rl/tournament.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tournament.d.ts","sourceRoot":"","sources":["../../src/rl/tournament.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,MAAM,WAAW,eAAe;IAC9B,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,CAAA;IACd;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAA;IACnB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAA;IAChB,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,CAAC,EAAE,MAAM,CAAA;IACT,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,kBAAkB,EAAE,CAAA;IAC7B,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAA;IAClB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,eAAe,EAAE,EAC3B,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5E,eAAe,CAqGjB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC5B,OAAO,EAAE,eAAe,EACxB,IAAI,GAAE,UAAe,GACpB;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAmB7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B;IAC7C,IAAI,EAAE,KAAK,CAAC;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,sEAAsE;QACtE,QAAQ,EAAE,MAAM,CAAA;QAChB,KAAK,EAAE,MAAM,CAAA;KACd,CAAC,CAAA;IACF;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,8BAA8B,GACpC,eAAe,EAAE,CA0BnB"}