@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/rl/run-record-adapters.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Adapters: convert measurement outputs into the canonical `RunRecord[]`
+ * artifact that `replayCache`, `pairedEvalueSequence`, and
+ * `rubricPredictiveValidity` consume. Two sources:
+ *   - `campaignToRunRecords` — the campaign substrate's per-cell results
+ *     (the modern path: `runCampaign` / `runImprovementLoop` → records).
+ *   - `verificationReportToRunRecord` — a `MultiLayerVerifier` report.
+ *
+ * Adapters are thin and explicit — every mandatory `RunRecord` field comes
+ * from a caller-supplied context (`commitSha`, `model`, `promptHash`,
+ * `configHash`) plus the cell's runtime data. The validator still rejects
+ * bare-alias model strings — the caller snapshot-pins.
+ */
+import type { CampaignResult } from '../campaign';
+import type { VerificationReport } from '../multi-layer-verifier';
+import type { RunRecord, RunSplitTag } from '../run-record';
+export interface AdapterContext {
+    /** Logical experiment id — typically the campaign or sweep identifier. */
+    experimentId: string;
+    /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
+    model: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** Hash of the effective prompt sent to the model. */
+    promptHash: string;
+    /** Hash of the effective config (model, temperature, tools, judges, splits). */
+    configHash: string;
+    /** Default split tag. Default `'search'`. */
+    splitTag?: RunSplitTag;
+    /** Default cost in USD when the source doesn't record one. Default `0`. */
+    defaultCostUsd?: number;
+}
+/**
+ * Convert a `CampaignResult` into canonical `RunRecord[]` — one record per
+ * scored cell. The cell's mean judge composite becomes the split score; every
+ * judge dimension is carried through to `outcome.raw`. A cell that errored
+ * becomes a record with `failureMode: 'cell_error'` (kept, not dropped — an
+ * unscored cell is signal). `candidateId` identifies the measured surface
+ * (defaults to the campaign manifest hash).
+ */
+export declare function campaignToRunRecords(campaign: CampaignResult, ctx: AdapterContext & {
+    candidateId?: string;
+}): RunRecord[];
+/**
+ * Convert a `MultiLayerVerifier` `VerificationReport` into a `RunRecord`.
+ * `outcome.searchScore` (or `holdoutScore`) is `report.blendedScore`;
+ * `outcome.raw` carries every layer's score + a pass indicator; `failureMode`
+ * is the first failing layer's reason.
+ */
+export declare function verificationReportToRunRecord(report: VerificationReport, ctx: AdapterContext & {
+    candidateId: string;
+    scenarioId?: string;
+}, opts?: {
+    runId?: string;
+}): RunRecord;
+//# sourceMappingURL=run-record-adapters.d.ts.map

package/dist/rl/run-record-adapters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-record-adapters.d.ts","sourceRoot":"","sources":["../../src/rl/run-record-adapters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AACjD,OAAO,KAAK,EAAe,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC9E,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAE3D,MAAM,WAAW,cAAc;IAC7B,0EAA0E;IAC1E,YAAY,EAAE,MAAM,CAAA;IACpB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAA;IACb,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAA;IAClB,gFAAgF;IAChF,UAAU,EAAE,MAAM,CAAA;IAClB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,2EAA2E;IAC3E,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,cAAc,EACxB,GAAG,EAAE,cAAc,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7C,SAAS,EAAE,CAmCb;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,kBAAkB,EAC1B,GAAG,EAAE,cAAc,GAAG;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,EAClE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5B,SAAS,CA4CX"}

package/dist/rl/sim-fidelity.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Simulator fidelity — score a user SIMULATOR's realism against real-user
+ * trace distributions.
+ *
+ * Synthetic-persona evals (`PersonaConfig`-driven canonical evals, fuzz
+ * user-simulator objectives) stand in for real users in most of the numbers
+ * we publish. The standing threat is the Sim2Real gap: a simulator that is
+ * distributionally unlike production creates "easy mode" and silently
+ * inflates every score built on it. This module measures that gap from the
+ * SAME artifact both sides already produce — `RunRecord`s — so no new
+ * capture pipeline is needed:
+ *
+ *   - `simFidelityReport` — per-feature Jensen-Shannon divergence between
+ *     simulated and production record distributions, collapsed into a
+ *     fidelity coefficient in [0,1].
+ *   - `easyModeCheck` — the headline academic failure mode (sim inflates
+ *     pass-rate over production) as its own named artifact.
+ *
+ * Every synthetic-persona eval result should publish its fidelity
+ * coefficient alongside the score — a number from an unrepresentative
+ * simulator is an unlabeled estimate. Wire-in points:
+ *
+ *   - canonical persona evals: pass the campaign's `RunRecord`s as
+ *     `simulated` and intake-adapter output (`contract/intake`: OTel spans,
+ *     feedback tables, coding-agent sessions) as `production`
+ *   - the fuzz user-sim objective: use `1 - report.fidelity` as a realism
+ *     penalty when searching over generated personas
+ *   - the durable corpus (`./corpus`): both sides read straight from
+ *     `readCorpus` — tag sim vs production by `experimentId`
+ */
+import type { RunRecord } from '../run-record';
+/** Extracts a flat behavioral feature map from one record. `string` values
+ *  are categorical, `number` values are quantile-bucketed over the union of
+ *  both sides, `null` means the feature is absent on this record and is
+ *  counted explicitly as its own category (never silently dropped). */
+export type BehaviorFeatures = (record: RunRecord) => Record<string, string | number | null>;
+/** Reserved histogram category for `null` feature values. A capture-rate
+ *  difference (one side instruments a signal, the other does not) registers
+ *  as divergence by design: a simulator that produces no tool traces is not
+ *  representative of production that does. */
+export declare const ABSENT_CATEGORY = "(absent)";
+/** Minimum non-null observations PER SIDE for a feature to enter the
+ *  fidelity mean. Below this the JSD estimate is sampling noise. */
+export declare const DEFAULT_MIN_N_PER_FEATURE = 20;
+/** Quantile buckets used to discretize numeric features. Quartiles balance
+ *  resolution against per-bucket sample size at the default minN. */
+export declare const DEFAULT_QUANTILE_BUCKETS = 4;
+/** Fidelity at or above this → 'representative'; below → 'skewed'.
+ *  1 − 0.8 = mean JSD 0.2 ≈ distributions that mostly overlap with one
+ *  clearly shifted mode — the point where per-feature shifts start changing
+ *  which failure classes an eval can even observe. */
+export declare const REPRESENTATIVE_MIN_FIDELITY = 0.8;
+/**
+ * Default feature set — ONLY fields verified present on both simulated and
+ * production records:
+ *
+ *   - `score`, `wall_ms`, `output_tokens` — mandatory per the `RunRecord`
+ *     validator (non-finite values read as absent rather than poisoning a
+ *     bucket).
+ *   - `failure_class` — optional taxonomy field; absent counted explicitly.
+ *   - `turn_count`, `tool_errors`, `tool_error_recovery` — derived from the
+ *     `outcome.raw` counters the intake adapters and eval harnesses write
+ *     (`turns_completed`, `assistant_messages`, `tool_errors`,
+ *     `turns_aborted`); absent on records whose producer did not capture
+ *     them, counted explicitly.
+ *   - `completion_length` — from the optional `CorpusRecord` trajectory
+ *     text; the message-length proxy when records come from the corpus.
+ *
+ * `RunRecord` carries event COUNTS, not event ordering, so
+ * `tool_error_recovery` is a counts-only derivation: errors occurred and the
+ * run still completed cleanly ('recovered') vs aborted or classified as a
+ * failure ('unrecovered') — not a literal error→retry sequence check.
+ */
+export declare const defaultBehaviorFeatures: BehaviorFeatures;
+/**
+ * Jensen-Shannon divergence between two categorical histograms (raw counts;
+ * normalized internally). Log base 2 → bounded [0,1]: 0 = identical
+ * distributions, 1 = disjoint support. Symmetric, defined even where the
+ * supports differ — exactly the regime sim-vs-production comparison lives in.
+ * Throws on zero-mass or negative/non-finite counts: an empty histogram has
+ * no distribution and a silent 0 would read as "perfectly representative".
+ */
+export declare function jsDivergence(p: Record<string, number>, q: Record<string, number>): number;
+/**
+ * Deterministic quantile edges over a value set (the UNION of both sides, so
+ * sim and production land in the same buckets). Linear interpolation between
+ * order statistics; duplicate edges from heavy ties collapse into fewer,
+ * wider buckets. Returns `bucketCount - 1` edges before deduplication.
+ */
+export declare function quantileEdges(values: number[], bucketCount?: number): number[];
+/** Stable half-open bucket label for a value against quantile edges:
+ *  `[-inf,e0)`, `[e0,e1)`, …, `[eLast,+inf)`. */
+export declare function bucketLabel(value: number, edges: number[]): string;
+export interface FeatureShift {
+    /** Category label (a string value, a numeric bucket, or `ABSENT_CATEGORY`). */
+    value: string;
+    /** Probability of this category among ALL simulated records (nulls included
+     *  via `ABSENT_CATEGORY`, so each side's shifts sum to 1). */
+    pSim: number;
+    /** Probability among ALL production records. */
+    pProd: number;
+}
+export interface FeatureDivergence {
+    feature: string;
+    /** Jensen-Shannon divergence in [0,1] for this feature. */
+    divergence: number;
+    /** Largest |pSim − pProd| categories, descending — where the sim deviates. */
+    topShifts: FeatureShift[];
+    /** Non-null observations on the simulated side. */
+    nSim: number;
+    /** Non-null observations on the production side. */
+    nProd: number;
+}
+export type FidelityVerdict = 'representative' | 'skewed' | 'insufficient-data';
+export interface FidelityReport {
+    perDimension: FeatureDivergence[];
+    /** 1 − mean divergence over features with sufficient data. NaN when the
+     *  verdict is 'insufficient-data' — a 0 would read as "maximally skewed"
+     *  and silently poison downstream aggregation; check `verdict` first. */
+    fidelity: number;
+    /** Features excluded because either side had fewer than `minNPerFeature`
+     *  non-null observations. Named, never silently dropped. */
+    insufficientData: string[];
+    /** 'representative' when fidelity >= REPRESENTATIVE_MIN_FIDELITY (0.8),
+     *  'skewed' below, 'insufficient-data' when no feature met minN. */
+    verdict: FidelityVerdict;
+}
+export interface SimFidelityOptions {
+    /** Feature extractor. Defaults to `defaultBehaviorFeatures`. */
+    features?: BehaviorFeatures;
+    /** Minimum non-null observations per side per feature. Default 20. */
+    minNPerFeature?: number;
+}
+/**
+ * Compare a simulator's RunRecords against production RunRecords, feature by
+ * feature. Numeric features are bucketed by deterministic quantiles of the
+ * union; nulls count as an explicit `ABSENT_CATEGORY`. Throws on empty
+ * inputs — "no records" is a wiring error, not a distribution.
+ */
+export declare function simFidelityReport(simulated: RunRecord[], production: RunRecord[], opts?: SimFidelityOptions): FidelityReport;
+export interface EasyModeOptions {
+    /** A run passes when its score (holdout, else search) >= this. Default 0.5
+     *  — matches the pass-threshold convention across the rl/ primitives. */
+    passThreshold?: number;
+    /** Pass-rate gap above which the sim is flagged inflated. Default 0.1 —
+     *  a 10-point inflation is enough to flip most promotion gates. */
+    inflationTolerance?: number;
+}
+export interface EasyModeReport {
+    simPassRate: number;
+    prodPassRate: number;
+    /** simPassRate − prodPassRate. Positive = the simulator is easier than reality. */
+    gap: number;
+    /** True when gap > inflationTolerance: numbers measured against this
+     *  simulator overstate production performance. */
+    inflated: boolean;
+}
+/**
+ * The headline simulator failure mode as its own named artifact: a simulator
+ * that creates "easy mode" inflates pass-rate relative to production, and
+ * every score measured against it overstates reality. Throws on empty inputs
+ * and on records carrying neither score — a silently-skipped record would
+ * bias the very rate this check exists to keep honest.
+ */
+export declare function easyModeCheck(simulated: RunRecord[], production: RunRecord[], opts?: EasyModeOptions): EasyModeReport;
+//# sourceMappingURL=sim-fidelity.d.ts.map

package/dist/rl/sim-fidelity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sim-fidelity.d.ts","sourceRoot":"","sources":["../../src/rl/sim-fidelity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAI9C;;;uEAGuE;AACvE,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,CAAA;AAE5F;;;8CAG8C;AAC9C,eAAO,MAAM,eAAe,aAAa,CAAA;AAEzC;oEACoE;AACpE,eAAO,MAAM,yBAAyB,KAAK,CAAA;AAE3C;qEACqE;AACrE,eAAO,MAAM,wBAAwB,IAAI,CAAA;AAEzC;;;sDAGsD;AACtD,eAAO,MAAM,2BAA2B,MAAM,CAAA;AAI9C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,uBAAuB,EAAE,gBAerC,CAAA;AAoBD;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CA6BzF;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,WAAW,SAA2B,GAAG,MAAM,EAAE,CAkBhG;AAED;iDACiD;AACjD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAMlE;AAID,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,KAAK,EAAE,MAAM,CAAA;IACb;kEAC8D;IAC9D,IAAI,EAAE,MAAM,CAAA;IACZ,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAA;IAClB,8EAA8E;IAC9E,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,mDAAmD;IACnD,IAAI,EAAE,MAAM,CAAA;IACZ,oDAAoD;IACpD,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,MAAM,eAAe,GAAG,gBAAgB,GAAG,QAAQ,GAAG,mBAAmB,CAAA;AAE/E,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,iBAAiB,EAAE,CAAA;IACjC;;6EAEyE;IACzE,QAAQ,EAAE,MAAM,CAAA;IAChB;gEAC4D;IAC5D,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B;wEACoE;IACpE,OAAO,EAAE,eAAe,CAAA;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,SAAS,EAAE,EACtB,UAAU,EAAE,SAAS,EAAE,EACvB,IAAI,GAAE,kBAAuB,GAC5B,cAAc,CA0DhB;AA6DD,MAAM,WAAW,eAAe;IAC9B;6EACyE;IACzE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;uEACmE;IACnE,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,mFAAmF;IACnF,GAAG,EAAE,MAAM,CAAA;IACX;sDACkD;IAClD,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,SAAS,EAAE,EACtB,UAAU,EAAE,SAAS,EAAE,EACvB,IAAI,GAAE,eAAoB,GACzB,cAAc,CA0BhB"}

package/dist/rl/sim-fidelity.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=sim-fidelity.test.d.ts.map

package/dist/rl/sim-fidelity.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sim-fidelity.test.d.ts","sourceRoot":"","sources":["../../src/rl/sim-fidelity.test.ts"],"names":[],"mappings":""}

package/dist/rl/tournament.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Bradley-Terry / Elo tournament evaluation.
+ *
+ * For multi-candidate sweeps, comparing every candidate's score against
+ * a fixed comparator wastes information — the comparator becomes a high-
+ * variance reference and rank flips between near-tied middle-rank
+ * candidates are dominated by noise. Pairwise tournaments fix this:
+ * every (i, j) pair contributes a comparison to a Bradley-Terry MLE that
+ * estimates each candidate's strength on a unified scale.
+ *
+ * For online updating (rolling campaigns where new candidates arrive
+ * over time), we also ship classical Elo with configurable K-factor.
+ *
+ * References:
+ *   - Bradley, R. A., Terry, M. E. (1952). Rank analysis of incomplete
+ *     block designs. Biometrika, 39(3/4), 324–345.
+ *   - Hunter, D. R. (2004). MM algorithms for generalized Bradley-Terry
+ *     models. Annals of Statistics, 32(1), 384–406. (The MLE algorithm
+ *     used here.)
+ *   - Elo, A. E. (1978). The Rating of Chess Players, Past and Present.
+ *
+ * This is a useful primitive because most LLM-eval communities (Chatbot
+ * Arena, AlpacaEval, ELO-style ablation) have converged on pairwise
+ * tournament eval as the most sample-efficient and most rank-stable
+ * method when you have many candidates.
+ */
+export interface PairwiseOutcome {
+    /** Winner candidate id. */
+    winner: string;
+    /** Loser candidate id. */
+    loser: string;
+    /**
+     * Optional draw flag. When true, both candidates get half-credit
+     * (Bradley-Terry handles draws as half-wins for each side).
+     */
+    draw?: boolean;
+    /**
+     * Optional weight — useful if some pairwise comparisons are stronger
+     * signals than others (e.g. a paired test with a wider score gap is
+     * a more confident comparison). Default 1.
+     */
+    weight?: number;
+}
+export interface BradleyTerryRating {
+    candidateId: string;
+    /** Latent strength θ ≥ 0 from the BT MLE. */
+    strength: number;
+    /** Log-strength = log(θ) — interpretable on a linear scale. */
+    logStrength: number;
+    /** Number of pairwise comparisons this candidate appears in. */
+    n: number;
+    /** Win count (+ 0.5 per draw). */
+    wins: number;
+}
+export interface BradleyTerryFit {
+    ratings: BradleyTerryRating[];
+    /** Iterations of the MM algorithm before convergence. */
+    iterations: number;
+    /** Final maximum |θ_new - θ_old| / θ_old. */
+    finalDelta: number;
+    converged: boolean;
+}
+/**
+ * Bradley-Terry MLE via Hunter's MM algorithm.
+ *
+ * Iteration: θ_i^new = W_i / Σ_{j ≠ i} N_ij / (θ_i + θ_j)
+ *   where W_i = wins by i (+ 0.5 per draw), N_ij = total comparisons.
+ *
+ * Returns log-strengths normalized so the smallest is 0 (any constant
+ * offset is unobservable in BT — only differences are identified).
+ */
+export declare function fitBradleyTerry(outcomes: PairwiseOutcome[], opts?: {
+    tolerance?: number;
+    maxIterations?: number;
+    smoothing?: number;
+}): BradleyTerryFit;
+/**
+ * Online Elo updates. Use when comparisons arrive over time and you want
+ * a running rating without re-fitting the full BT MLE on every update.
+ *
+ * Initialize ratings to `defaultRating` (1500 by default). Each call to
+ * `applyEloUpdate` mutates the map in place and returns the deltas so
+ * the caller can log per-comparison rating changes.
+ */
+export interface EloOptions {
+    /** Default rating for unseen candidates. Default 1500. */
+    defaultRating?: number;
+    /** K-factor controls the step size. Default 32 (FIDE-ish). */
+    kFactor?: number;
+}
+export declare function applyEloUpdate(ratings: Map<string, number>, outcome: PairwiseOutcome, opts?: EloOptions): {
+    winnerDelta: number;
+    loserDelta: number;
+};
+/**
+ * Build pairwise outcomes from the campaign artifact: for every scenario
+ * shared by two candidates, the higher-scoring run wins. Useful when you
+ * want a tournament view of an existing campaign without an additional
+ * pairwise judge call.
+ */
+export interface BuildPairwiseFromCampaignInput {
+    runs: Array<{
+        candidateId: string;
+        /** Stable identifier for the matching unit (typically scenarioId). */
+        matchKey: string;
+        score: number;
+    }>;
+    /**
+     * Tied-score margin. Below this, the comparison is a draw. Default 0
+     * (no ties).
+     */
+    drawMargin?: number;
+}
+export declare function buildPairwiseFromCampaign(input: BuildPairwiseFromCampaignInput): PairwiseOutcome[];
+//# sourceMappingURL=tournament.d.ts.map

package/dist/rl/tournament.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tournament.d.ts","sourceRoot":"","sources":["../../src/rl/tournament.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,MAAM,WAAW,eAAe;IAC9B,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAA;IACd,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,IAAI,CAAC,EAAE,OAAO,CAAA;IACd;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAA;IACnB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAA;IAChB,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,CAAC,EAAE,MAAM,CAAA;IACT,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,kBAAkB,EAAE,CAAA;IAC7B,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAA;IAClB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,OAAO,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,eAAe,EAAE,EAC3B,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5E,eAAe,CAqGjB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC5B,OAAO,EAAE,eAAe,EACxB,IAAI,GAAE,UAAe,GACpB;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAmB7C;AAED;;;;;GAKG;AACH,MAAM,WAAW,8BAA8B;IAC7C,IAAI,EAAE,KAAK,CAAC;QACV,WAAW,EAAE,MAAM,CAAA;QACnB,sEAAsE;QACtE,QAAQ,EAAE,MAAM,CAAA;QAChB,KAAK,EAAE,MAAM,CAAA;KACd,CAAC,CAAA;IACF;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,8BAA8B,GACpC,eAAe,EAAE,CA0BnB"}

package/dist/rl/verifiable-reward.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Verifiable reward channel.
+ *
+ * For RL on coding / math / theorem-proving / structured-output tasks, the
+ * reward signal is *decidable* — a test passes or fails, a proof checks or
+ * doesn't, an output validates against a schema or doesn't. These rewards
+ * are dramatically more useful for RL training than LLM-judge scores
+ * because they don't drift, can't be Goodhart-gamed by the policy in the
+ * same way, and don't require a separate calibration loop.
+ *
+ * The `MultiLayerVerifier` already produces this signal — it just doesn't
+ * surface it in a shape that's clean enough for RL training. This module
+ * wraps the verifier output so consumers can:
+ *
+ *   1. Extract a clean `VerifiableReward` from a `VerificationReport`
+ *   2. Distinguish *deterministic* rewards (compile, test, schema) from
+ *      *probabilistic* rewards (judge) so they can be weighted differently
+ *      in the RL training step
+ *   3. Filter `RunRecord[]` to only those with a verifiable reward,
+ *      producing the clean training set that DeepSeek-R1-style GRPO and
+ *      AlphaProof-style search both depend on
+ *
+ * Why this matters: every credible 2025-2026 frontier RL result on coding
+ * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,
+ * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing
+ * judge scores into the reward signal poisons the gradient. This module
+ * is the seam.
+ */
+import type { VerificationReport } from '../multi-layer-verifier';
+import type { RunRecord } from '../run-record';
+export type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
+export interface VerifiableReward {
+    /** Scalar in [0, 1]. The RL training signal. */
+    value: number;
+    /** What produced the reward — different sources have different determinism. */
+    source: VerifiableRewardSource;
+    /**
+     * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte
+     * given the same inputs (compile, test, schema validation, sandbox exit code).
+     * `'probabilistic'` rewards depend on a stochastic component (LLM judge).
+     * Mixing these in the same training batch without separation is a known
+     * footgun in production RLHF pipelines.
+     */
+    determinism: 'deterministic' | 'probabilistic';
+    /**
+     * Confidence in the reward value. For deterministic sources this is 1.0
+     * (the bit either flipped or didn't). For judge sources this is the
+     * judge-reported confidence or — when missing — a calibrated prior.
+     */
+    confidence: number;
+    /** The layer / judge id that produced the signal, for provenance. */
+    origin: string;
+    /**
+     * Per-source contribution to `value`, keyed by layer/judge id. Single-source
+     * rewards carry one entry (`{ [origin]: value }`); composite rewards carry
+     * every contributing layer's score — the anti-scalar-collapse surface RL
+     * consumers weight per-source instead of trusting one blended number.
+     */
+    components: Record<string, number>;
+    /**
+     * @deprecated Read `components` for per-source reward values. Kept for
+     * published-API compatibility: single-source rewards carry the layer's
+     * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry
+     * the same per-layer scores `components` now holds.
+     */
+    breakdown?: Record<string, number>;
+}
+export interface VerifiableRewardExtractionOptions {
+    /**
+     * Which layers count as deterministic-reward sources. The verifier doesn't
+     * tag layers as "this is verifiable"; the caller declares it via this list
+     * (or via the layer name → source mapping). Default treats common names
+     * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
+     * `sandbox`) as deterministic.
+     */
+    deterministicLayers?: string[];
+    /**
+     * Map layer name → reward source. Defaults to a sensible string-match.
+     */
+    sourceFor?: (layerName: string) => VerifiableRewardSource;
+    /**
+     * Whether to fall back to a probabilistic (judge) reward when no
+     * deterministic layer produced a numeric score. Default `true`. Set to
+     * `false` for "deterministic-only" training pipelines that should
+     * discard runs without a verifiable signal.
+     */
+    fallbackToJudge?: boolean;
+    /**
+     * Default confidence for probabilistic (judge) rewards when the judge
+     * doesn't report one. Default `0.7`.
+     */
+    judgeConfidenceFloor?: number;
+}
+/**
+ * Extract a `VerifiableReward` from a `VerificationReport`.
+ *
+ * Strategy: prefer the deterministic layers (in order: test → compile →
+ * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is
+ * true, return `null` if no signal qualifies. When multiple deterministic
+ * layers contribute, return a `'composite'` source with a weighted blend.
+ */
+export declare function extractVerifiableReward(report: VerificationReport, opts?: VerifiableRewardExtractionOptions): VerifiableReward | null;
+/**
+ * Extract verifiable rewards from `RunRecord[]` produced via the
+ * `verificationReportToRunRecord` adapter (which encodes per-layer scores
+ * in `outcome.raw['layer.<name>']`). For records that don't carry layer
+ * scores, returns `null` for that record.
+ *
+ * This is the canonical bridge from "campaign-shaped artifacts" to
+ * "RL-training-ready reward signals": every record that has a clean
+ * verifiable reward becomes a training datum, every record that doesn't
+ * gets filtered out (or kept with `'probabilistic'` determinism for
+ * separate downstream handling).
+ */
+export declare function extractVerifiableRewardsFromRecords(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+    runId: string;
+    reward: VerifiableReward | null;
+}>;
+/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */
+export declare function filterDeterministicallyRewarded(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+    run: RunRecord;
+    reward: VerifiableReward;
+}>;
+//# sourceMappingURL=verifiable-reward.d.ts.map

package/dist/rl/verifiable-reward.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verifiable-reward.d.ts","sourceRoot":"","sources":["../../src/rl/verifiable-reward.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAe,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC9E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC9B,SAAS,GACT,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,WAAW,CAAA;AAEf,MAAM,WAAW,gBAAgB;IAC/B,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAA;IACb,+EAA+E;IAC/E,MAAM,EAAE,sBAAsB,CAAA;IAC9B;;;;;;OAMG;IACH,WAAW,EAAE,eAAe,GAAG,eAAe,CAAA;IAC9C;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAA;IACd;;;;;OAKG;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iCAAiC;IAChD;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,sBAAsB,CAAA;IACzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAC9B;AA+BD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,kBAAkB,EAC1B,IAAI,GAAE,iCAAsC,GAC3C,gBAAgB,GAAG,IAAI,CAmEzB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,SAAS,EAAE,EACjB,IAAI,GAAE,iCAAsC,GAC3C,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAA;CAAE,CAAC,CA0E3D;AAED,2EAA2E;AAC3E,wBAAgB,+BAA+B,CAC7C,IAAI,EAAE,SAAS,EAAE,EACjB,IAAI,GAAE,iCAAsC,GAC3C,KAAK,CAAC;IAAE,GAAG,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,CAUrD"}