@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/campaign/gates/statistical-heldout.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Statistical held-out promotion machinery — the trustworthy core the
+ * point-estimate `heldout-delta` gate lacked.
+ *
+ * The shipped false positive it prevents: a winner re-scored against the
+ * baseline on the holdout read run-to-run model NOISE (e.g. 91 vs 95) as a
+ * "+4 lift" and shipped, because the gate compared point estimates with no
+ * confidence interval. Here we pair candidate vs baseline holdout observations
+ * and bootstrap a CI on the paired delta — a candidate ships only when the CI
+ * lower bound clears the effect-size threshold (the gain is real at the
+ * confidence level, not noise), and is blocked when a critical dimension
+ * (e.g. `hallucination_free` for a legal agent) significantly regresses even if
+ * the net composite rose (anti-Goodhart).
+ *
+ * Two traps this module is built around (both produce a NEW false positive if
+ * gotten wrong):
+ *   1. PAIRING GRANULARITY — pairs by FULL `cellId` (`scenario:rep`), never by
+ *      `scenarioId` (which averages reps away and destroys the within-pair
+ *      variance reduction that makes a paired bootstrap tighter than unpaired).
+ *      One paired observation per cell ⇒ reps multiply n.
+ *   2. SCALE — a judge may emit composites/dimensions on [0,1] or 0-100. The
+ *      threshold + tolerance are interpreted in the judge's NATIVE scale; the
+ *      per-dimension tolerance auto-scales off the observed baseline magnitudes
+ *      so `-0.10` on [0,1] doesn't silently become a no-op on a 0-100 dimension.
+ */
+import { type PairedBootstrapResult } from '../../statistics';
+import type { JudgeScore } from '../types';
+export interface PairedHoldout {
+    /** Baseline scalar per paired cell (same order as `after`/`cellIds`). */
+    before: number[];
+    /** Candidate scalar per paired cell. */
+    after: number[];
+    /** The full cellIds (`scenario:rep`) that paired, in order. */
+    cellIds: string[];
+}
+/**
+ * Pair candidate vs baseline holdout observations by FULL cellId. `select`
+ * pulls the scalar from a cell's judge reports (composite, or a named
+ * dimension); a cell contributes the mean of `select` across its judges. Cells
+ * whose scenario is not in `scenarioIds`, or where `select` is undefined for
+ * every judge on either side, are skipped on BOTH sides so the arrays stay
+ * paired. Throws when the two maps disagree on which holdout cells exist — a
+ * load-bearing invariant: the baseline + winner holdout campaigns run the same
+ * scenarios with the same seed base, so their cellIds MUST align; a mismatch
+ * means a silent pairing bug, not a soft fallback.
+ */
+export declare function pairHoldout(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, select: (s: JudgeScore) => number | undefined): PairedHoldout;
+export interface HeldoutSignificance {
+    paired: PairedHoldout;
+    bootstrap: PairedBootstrapResult;
+    /** n paired observations. */
+    n: number;
+    /** True iff n >= minProductiveRuns AND the CI lower bound clears the threshold. */
+    significant: boolean;
+    /** Set when n < minProductiveRuns — too little evidence to claim significance. */
+    fewRuns: boolean;
+}
+export interface HeldoutSignificanceOptions {
+    deltaThreshold?: number;
+    minProductiveRuns?: number;
+    confidence?: number;
+    resamples?: number;
+    /** Fixed by default for a deterministic, reproducible gate verdict. */
+    seed?: number;
+    statistic?: 'mean' | 'median';
+}
+/** Significance of the held-out composite lift: ship only when the paired
+ *  bootstrap CI lower bound on (candidate − baseline) exceeds `deltaThreshold`
+ *  (default 0 ⇒ "confidently positive"). Below `minProductiveRuns` paired
+ *  observations there is not enough evidence to claim significance → not
+ *  significant (`fewRuns`). Interpret `deltaThreshold` in the judge's native
+ *  composite scale. */
+export declare function heldoutSignificance(paired: PairedHoldout, opts?: HeldoutSignificanceOptions): HeldoutSignificance;
+export interface DimensionRegression {
+    dimension: string;
+    bootstrap: PairedBootstrapResult;
+    /** True iff the CI lower bound on (candidate − baseline) is below −tolerance:
+     *  the candidate may have regressed this dimension by more than tolerance. */
+    regressed: boolean;
+    tolerance: number;
+    n: number;
+}
+/** Detect the native scale of a set of scores: 0-100 when any magnitude clears
+ *  1.5, else [0,1]. Used to auto-scale the regression tolerance so a default
+ *  expressed for [0,1] is not silently a no-op on a 0-100 dimension. */
+export declare function detectScale(values: number[]): 1 | 100;
+/** Per-critical-dimension regression guard. For each dimension, pair the
+ *  candidate vs baseline values by full cellId and bootstrap the paired delta;
+ *  a dimension is "regressed" when the CI lower bound < −tolerance (conservative
+ *  — blocks if the credible worst case exceeds tolerance, which is the right
+ *  posture for safety dimensions like `hallucination_free`). When `tolerance`
+ *  is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100. */
+export declare function dimensionRegressions(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, criticalDimensions: string[], opts?: {
+    tolerance?: number;
+    confidence?: number;
+    resamples?: number;
+    seed?: number;
+}): DimensionRegression[];
+//# sourceMappingURL=statistical-heldout.d.ts.map

package/dist/campaign/gates/statistical-heldout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistical-heldout.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/statistical-heldout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,KAAK,qBAAqB,EAAmB,MAAM,kBAAkB,CAAA;AAC9E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAE1C,MAAM,WAAW,aAAa;IAC5B,yEAAyE;IACzE,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,wCAAwC;IACxC,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,EAClD,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,EACjD,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,MAAM,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,MAAM,GAAG,SAAS,GAC5C,aAAa,CA2Cf;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,aAAa,CAAA;IACrB,SAAS,EAAE,qBAAqB,CAAA;IAChC,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAA;IACT,mFAAmF;IACnF,WAAW,EAAE,OAAO,CAAA;IACpB,kFAAkF;IAClF,OAAO,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,0BAA0B;IACzC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,uEAAuE;IACvE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC9B;AAED;;;;;uBAKuB;AACvB,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,aAAa,EACrB,IAAI,GAAE,0BAA+B,GACpC,mBAAmB,CAarB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,qBAAqB,CAAA;IAChC;kFAC8E;IAC9E,SAAS,EAAE,OAAO,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,CAAC,EAAE,MAAM,CAAA;CACV;AAED;;wEAEwE;AACxE,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,CAErD;AAED;;;;;4DAK4D;AAC5D,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,EAClD,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,EACjD,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,EACxB,kBAAkB,EAAE,MAAM,EAAE,EAC5B,IAAI,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GACxF,mBAAmB,EAAE,CAqBvB"}

package/dist/campaign/gates/statistical-heldout.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=statistical-heldout.test.d.ts.map

package/dist/campaign/gates/statistical-heldout.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistical-heldout.test.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/statistical-heldout.test.ts"],"names":[],"mappings":""}