@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/statistics-D6Uebe_4.d.ts

@@ -1,968 +0,0 @@
-import { g as JudgeScore } from "./types-D216SgwM.js";
-//#region src/judge-calibration.d.ts
-/**
- * Judge calibration — measure judge quality against human gold + bias.
- *
- * Workflow:
- *   1. Build a golden set: {itemId, humanScore}[].
- *   2. Run candidate judges; each produces {itemId, score}.
- *   3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.
- *   4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted
- *      κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,
- *      and bootstrap CIs — use this for fine-grained judges where rounding
- *      to int discards information (e.g. 0.78 vs 0.81 both round to 1 and
- *      look "perfectly agreed" to integer κ).
- *   5. Run bias probes (positional, verbosity, self-preference) to
- *      detect systematic score inflation.
- *   6. For N≥2 judges on the same items, `continuousAgreement(scores)`
- *      reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.
- *
- * Returns actionable diagnostics, not a single number. Consumers then
- * decide whether to trust the judge, retrain it, or add a tie-breaker.
- */
-interface GoldenItem {
-  itemId: string;
-  humanScore: number;
-  /** Optional group used for per-group bias audits (e.g. model-of-output family). */
-  group?: string;
-}
-interface CandidateScore {
-  itemId: string;
-  score: number;
-  /** Optional — enables positional-bias analysis (did order matter?). */
-  positionOfAInput?: 'first' | 'second';
-}
-interface CalibrationResult {
-  n: number;
-  pearson: number;
-  /** Cohen's κ with quadratic weights over integer-rounded scores. */
-  kappa: number;
-  /** Mean absolute error vs human. */
-  mae: number;
-  /** Worst-5 miscalibrations (largest |judge - human|). */
-  worstItems: Array<{
-    itemId: string;
-    judge: number;
-    human: number;
-    delta: number;
-  }>;
-}
-/**
- * Measure judge quality against human gold labels: computes Cohen's κ, Pearson correlation, and MAE over matched item ids.
- */
-declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
-interface PositionalBiasResult {
-  /**
-   * Score delta (first-position - second-position) averaged across items
-   * presented in both positions. Non-zero = positional bias.
-   */
-  avgDelta: number;
-  n: number;
-}
-/**
- * Feed the same items to the judge twice with A/B swapped and pass all
- * results here. Items that don't appear in both positions are ignored.
- */
-declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
-interface VerbosityBiasResult {
-  /** Pearson correlation between output length and score. Strong positive = verbosity bias. */
-  pearson: number;
-  n: number;
-}
-declare function verbosityBias(samples: Array<{
-  outputLen: number;
-  score: number;
-}>): VerbosityBiasResult;
-interface SelfPreferenceResult {
-  /** Mean judge score when judge's family matches output's family. */
-  inFamilyMean: number;
-  outOfFamilyMean: number;
-  deltaMean: number;
-  n: number;
-}
-/**
- * Pass the same scenarios scored with judge-model X grading outputs from
- * model X (in-family) and model Y (out-of-family). Non-zero delta
- * indicates self-preference.
- */
-declare function selfPreference(samples: Array<{
-  score: number;
-  inFamily: boolean;
-}>): SelfPreferenceResult;
-interface ContinuousAgreement {
-  /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
-  weightedKappa: number;
-  /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
-  icc: number;
-  /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
-  pearson: number;
-  /** Spearman rank correlation (averaged over rater pairs if N>2). */
-  spearman: number;
-  /** 95% bootstrap percentile CIs over items. */
-  ci: {
-    icc: [number, number];
-    weightedKappa: [number, number];
-  };
-  /** Number of complete items (no NaN across raters). */
-  n: number;
-  /** Number of raters. */
-  raters: number;
-}
-interface ContinuousAgreementOptions {
-  /** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
-  bootstrap?: number;
-  /** κ weighting scheme. Default 'quadratic'. */
-  weights?: 'linear' | 'quadratic';
-  /** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */
-  seed?: number;
-  /** Confidence level for percentile CI. Default 0.95. */
-  ciLevel?: number;
-}
-/**
- * Inter-rater agreement on continuous (typically [0,1]) scores.
- *
- * `scores` has shape [n_items][n_raters]. Rows with any non-finite entry
- * are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
- * items remain.
- */
-declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
-interface ContinuousCalibrationResult extends CalibrationResult {
-  /** Cohen's κ_w computed on raw (un-rounded) scores. */
-  weightedKappaContinuous: number;
-  /** ICC(2,1) treating golden + candidate as two raters. */
-  icc: number;
-  spearman: number;
-  ci: {
-    icc: [number, number];
-    weightedKappa: [number, number];
-  };
-}
-/**
- * Extends `calibrateJudge` with continuous-value agreement metrics while
- * retaining its base calibration summary.
- */
-declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
-//#endregion
-//#region src/statistics.d.ts
-/** Identity: dimensions already follow "higher = better" by prompt convention
- *  (inverted dims like hallucination are scored 10 = best at the source). */
-declare const normalizeScores: (scores: JudgeScore[]) => JudgeScore[];
-/** Weighted mean — falls back to uniform weights when omitted */
-declare function weightedMean(scores: {
-  score: number;
-  weight?: number;
-}[]): number;
-/**
- * Percentile bootstrap confidence interval on the mean of `scores`.
- *
- * Descriptive spread. It is not a significance test, and at small n its bounds
- * are anti-conservative in the same way {@link pairedBootstrap}'s are — see
- * {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
- * the scores themselves, so the interval is reproducible either way.
- */
-declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
-  seed?: number;
-  resamples?: number;
-}): {
-  mean: number;
-  lower: number;
-  upper: number;
-};
-/**
- * Inter-rater reliability — Krippendorff's α under the squared-difference
- * metric, pooled across dimensions.
- *
- * Each inner array is one judge's scores. Items are matched by position
- * WITHIN a dimension: the k-th score a judge supplies carrying dimension
- * `d` is item k of `d`, and the ratings compared against each other are
- * the ones different judges gave to the same item. Every judge that scores
- * a dimension at all must supply the same number of scores for it —
- * ragged input cannot be aligned into items and throws rather than
- * comparing mismatched items.
- *
- * α = 1 − D_observed / D_expected: D_observed averages the squared
- * difference over within-item judge pairs, D_expected over every pair of
- * ratings irrespective of item. α = 1 is perfect agreement, 0 is chance,
- * negative is systematic disagreement.
- */
-declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
-/** How a rank test's p-value was actually computed. */
-type RankTestMethod = 'exact' | 'permutation' | 'asymptotic';
-/**
- * What the caller asks for. `'auto'` selects `'exact'` inside the enumeration
- * threshold and `'permutation'` above it, and never selects `'asymptotic'`.
- */
-type RankTestMethodRequest = 'auto' | 'exact' | 'asymptotic';
-interface RankTestOptions {
-  /** Default `'auto'`. `'asymptotic'` inside the exact-feasible range throws. */
-  method?: RankTestMethodRequest;
-  /** Resamples on the Monte Carlo permutation path. Default 100000. */
-  permutations?: number;
-  /** Seed for the permutation path. Omitted ⇒ derived from the data itself, so
-   *  the result is reproducible either way. */
-  seed?: number;
-}
-/** Maximum dynamic-programming cells used by an exact two-sample rank test. */
-declare const MANN_WHITNEY_EXACT_MAX_STATES = 8192;
-/** Maximum inner-loop transitions used by an exact two-sample rank test. */
-declare const MANN_WHITNEY_EXACT_MAX_WORK = 250000;
-/** Non-zero differences up to which the signed-rank null is enumerated exactly. */
-declare const WILCOXON_EXACT_MAX_N = 20;
-/** Resamples used when a rank test falls back to Monte Carlo permutation. */
-declare const DEFAULT_PERMUTATIONS = 100000;
-interface MannWhitneyResult {
-  /** `min(U_a, U_b)` — the conventional reported statistic. */
-  u: number;
-  /** U for sample `a`. Carries the direction of the effect, which `u` discards. */
-  uA: number;
-  /** Two-sided p-value. */
-  p: number;
-  /** How `p` was computed. */
-  method: RankTestMethod;
-  /** Smallest two-sided p this design can produce. `p` can never be below it. */
-  pFloor: number;
-}
-/**
- * Mann-Whitney U — two independent samples, no distributional assumption.
- *
- * Exact conditional (permutation) p by default when the dynamic program fits
- * {@link MANN_WHITNEY_EXACT_MAX_STATES} cells and
- * {@link MANN_WHITNEY_EXACT_MAX_WORK} transitions, seeded Monte Carlo
- * permutation above those limits. This keeps imbalanced designs such as 1+24
- * exact without admitting expensive balanced designs merely because they have
- * the same total size. Throws on non-finite input and on `method:
- * 'asymptotic'` where an exact answer is available. Empty input yields `p = 1,
- * pFloor = 1` — no design, no attainable evidence.
- */
-declare function mannWhitneyU(a: number[], b: number[], opts?: RankTestOptions): MannWhitneyResult;
-/** Partial credit: returns 0-1 ratio of current toward target */
-declare function partialCredit(current: number, target: number): number;
-interface PairedTTestResult {
-  /** Null when the statistic is undefined — see {@link pairedTTest}. */
-  t: number | null;
-  df: number;
-  /** Null exactly when `t` is null. */
-  p: number | null;
-}
-/**
- * Paired t-test — before/after measurements on the SAME items.
- * Pairing removes inter-item variance, giving tighter significance than
- * an unpaired test when comparing prompt v1 vs prompt v2 on identical
- * scenarios.
- *
- * Returns `t = p = null` where the statistic is undefined: fewer than two
- * pairs, or a non-zero constant delta whose observed variance is zero. A
- * constant shift carries no information about the variance it would have to
- * be compared against, so the honest answer is "undefined", not `p = 0` —
- * three observations cannot buy absolute certainty. This is the same contract
- * {@link pairedCohensDz} states for the same condition. An all-zero delta is
- * different: it is a measured null, and returns `t = 0, p = 1`.
- */
-declare function pairedTTest(before: number[], after: number[]): PairedTTestResult;
-interface WilcoxonSignedRankResult {
-  /** W⁺, the rank sum of the positive differences. (scipy reports
-   *  `min(W⁺, W⁻)`; compare statistics only after converting.) */
-  w: number;
-  /** Two-sided p-value. */
-  p: number;
-  /** How `p` was computed. */
-  method: RankTestMethod;
-  /** Smallest two-sided p this design can produce. */
-  pFloor: number;
-  /** Non-zero differences — zero differences are dropped and carry no rank. */
-  nNonZero: number;
-}
-/**
- * Wilcoxon signed-rank — paired, no distributional assumption on the deltas.
- *
- * Exact conditional (sign-flip) p by default at `n ≤
- * {@link WILCOXON_EXACT_MAX_N}` non-zero differences, seeded Monte Carlo
- * permutation above it. Throws on non-finite input and on `method:
- * 'asymptotic'` where an exact answer is available.
- *
- * `n` is the count of NON-ZERO differences: exact ties are dropped before
- * ranking, so a run of tied pairs shrinks the design and raises `pFloor`.
- * All-tied input yields `p = 1, pFloor = 1` — no attainable evidence, which
- * `pFloor` states rather than leaving `p = 1` to be read as a measured null.
- */
-declare function wilcoxonSignedRank(before: number[], after: number[], opts?: RankTestOptions): WilcoxonSignedRankResult;
-/**
- * Cohen's d — standardized effect size for two independent groups.
- * Positive d means group b has higher mean than group a.
- * Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
- *
- * Returns null where the standardized effect is undefined: fewer than two
- * observations in either group, or a zero pooled standard deviation with
- * unequal means. Null is NOT "no effect" — zero within-group spread across a
- * real mean gap is an unbounded effect, the opposite of negligible. Equal
- * means with zero spread is a genuine 0. Same contract as
- * {@link pairedCohensDz}.
- */
-declare function cohensD(a: number[], b: number[]): number | null;
-/**
- * Cohen's dz for paired observations: mean(after - before) divided by the
- * sample standard deviation of those within-pair deltas.
- *
- * Returns null when fewer than two pairs exist or a non-zero constant delta
- * has zero observed variance. In that case the standardized effect is
- * undefined, not an arbitrarily large finite number.
- */
-declare function pairedCohensDz(before: number[], after: number[]): number | null;
-type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
-/**
- * Cliff's delta — a non-parametric effect size for two independent samples.
- * `δ = (#(after > before) − #(after < before)) / (n_before · n_after)`,
- * ranging [-1, 1]. Positive ⇒ `after` tends to exceed `before` (improvement).
- *
- * Distribution-free counterpart to Cohen's d: no normality assumption, robust
- * to the bounded/skewed score distributions judges produce. Pairs with
- * `pairedBootstrap` / `wilcoxonSignedRank` for the non-parametric reporting
- * path. Returns 0 when either sample is empty.
- */
-declare function cliffsDelta(before: number[], after: number[]): number;
-/**
- * Map a Cliff's delta to a qualitative magnitude using the standard
- * Romano et al. thresholds (|δ|): <0.147 negligible, <0.33 small,
- * <0.474 medium, else large.
- */
-declare function interpretCliffs(delta: number): CliffsMagnitude;
-/**
- * Average-rank-with-ties transform (1-indexed). Tied values receive the mean
- * of the ranks they span, the standard correction for Spearman's ρ.
- */
-declare function ranks(xs: number[]): number[];
-/**
- * Pearson product-moment correlation coefficient r ∈ [-1, 1] between two
- * equal-length series. See the edge-case contract above: NaN for n < 2 or
- * unequal lengths, 1 when both series are constant, 0 when exactly one is.
- */
-declare function pearsonR(a: number[], b: number[]): number;
-/**
- * Spearman's rank correlation ρ — Pearson over the average-rank-with-ties
- * transform of each series. Same edge-case contract as {@link pearsonR}.
- */
-declare function spearmanR(a: number[], b: number[]): number;
-interface WeightedCompositeInput {
-  /** Per-dimension scores (typically 0..1). */
-  dims: Record<string, number>;
-  /** Weight per dimension. Every weighted dimension MUST be present in
-   *  `dims` — a weight for an absent dimension is a config error and throws,
-   *  because silently dropping it would renormalise the composite onto a
-   *  different denominator than intended. */
-  weights: Record<string, number>;
-  /** Optional pass threshold; when set, the result reports `pass`. */
-  threshold?: number;
-}
-interface WeightedCompositeResult {
-  composite: number;
-  pass?: boolean;
-}
-/**
- * Weighted composite over judge dimensions: `Σ(score_d · w_d) / Σ(w_d)` across
- * the weighted dimensions. The canonical replacement for the per-consumer
- * hand-rolled composite math (tax/legal/creative/gtm each ship a copy).
- *
- * Fail-loud: throws if a weighted dimension is missing from `dims`, if any
- * weight is negative, or if the weights sum to 0 — none of which can produce
- * a meaningful composite.
- */
-declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
-interface CorpusScoreRecord {
-  /** Stable identifier for the rated item (scenario, span, turn, …). */
-  itemId: string;
-  /** Identifier for the judge that produced this score. */
-  judgeName: string;
-  /** Dimension name (matches `JudgeScore.dimension`). */
-  dimension: string;
-  /** Numeric score; must be finite. */
-  score: number;
-}
-interface CorpusAgreementPerDimension extends ContinuousAgreement {
-  dimension: string;
-  /** Item IDs that contributed to this dimension's matrix (every judge scored them). */
-  itemIds: string[];
-  /** Judge IDs that contributed to this dimension's matrix. */
-  judgeIds: string[];
-}
-interface CorpusAgreementReport {
-  /** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
-  perDimension: CorpusAgreementPerDimension[];
-  /** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
-  overallIcc: number;
-  /** Mean weighted κ across dimensions (NaN if none finite). */
-  overallWeightedKappa: number;
-  /** Dimensions evaluated (sorted). */
-  dimensions: string[];
-  /** Judges seen across the corpus (sorted). */
-  judgeIds: string[];
-}
-interface CorpusAgreementOptions extends ContinuousAgreementOptions {
-  /**
-   * Restrict the audit to these dimensions. Default = every dimension
-   * that appears in the input. A dimension named here but absent from
-   * the input throws — silent omission would corrupt the overall metric.
-   */
-  dimensions?: string[];
-  /**
-   * Restrict the audit to these judges. Default = every judge that
-   * appears in the input. A judge named here but absent from a
-   * dimension throws (see "fail loud" below).
-   */
-  judges?: string[];
-}
-/**
- * Corpus-wide inter-rater agreement across N items × M judges × D dimensions.
- *
- * For each dimension, builds the [n_items][n_judges] matrix of scores
- * (keeping only items every judge rated on that dimension), then runs
- * `continuousAgreement` to get ICC(2,1), κ_w, Pearson, Spearman, and
- * bootstrap CIs. Reports a pooled mean across dimensions as a single
- * "is this judge panel reliable on this corpus?" number.
- *
- * Fail-loud contract:
- *   - Empty input throws.
- *   - Fewer than 2 judges or fewer than 2 items per dimension throws.
- *   - A judge present in some dimensions but with zero scored items on
- *     another dimension throws (would silently shrink the matrix).
- *   - Duplicate (itemId, judgeName, dimension) records throw.
- */
-declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
-/**
- * Convenience adapter for `JudgeScore[]` data keyed externally by item.
- *
- * Use when you have per-item arrays of `JudgeScore[]` (e.g. one
- * `ScenarioResult.judgeScores` per scenario) and want corpus-wide
- * agreement without manually flattening. `itemId` must be unique per
- * row of `itemsScores`.
- */
-declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
-  itemId: string;
-  scores: JudgeScore[];
-}>, opts?: CorpusAgreementOptions): CorpusAgreementReport;
-/**
- * Required N per arm for a two-sample comparison at target effect size,
- * alpha, and power. Normal-approximation formula:
- *   n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
- * where d is Cohen's d. Returns Infinity for effect ≤ 0.
- */
-declare function requiredSampleSize(opts: {
-  effect: number;
-  alpha?: number;
-  power?: number;
-  twoSided?: boolean;
-}): number;
-/**
- * Required number of paired observations for a target Cohen's dz.
- * Unlike the independent-groups formula, this has no two-arm factor of two.
- *
- * Normal quantiles with no t correction, so treat the result as a LOWER bound:
- * it returns 32 where the exact t-based answer is 34 at dz = 0.5, and 13 where
- * it is 15 at dz = 0.8 — a 6–13 % shortfall precisely in the range a caller
- * consults to decide whether 3–10 repetitions suffice.
- */
-declare function requiredPairedSampleSize(opts: {
-  effect: number;
-  alpha?: number;
-  power?: number;
-  twoSided?: boolean;
-}): number;
-/**
- * Minimum detectable paired effect (standardised units) for a target paired
- * sample size: d_min = (z_{1-α/2} + z_β) / sqrt(n_paired). Multiply by
- * sd(deltas) for score units; treat as a lower bound — Wilcoxon and bootstrap
- * have asymptotic relative efficiency below 1 vs the t-test on heavy tails.
- */
-declare function pairedMde(opts: {
-  nPaired: number;
-  alpha?: number;
-  power?: number;
-  twoSided?: boolean;
-}): number;
-/**
- * Number of paired observations needed for a McNemar test to reach a target
- * power — the pre-registration companion to {@link mcnemar}. Parametrised by the
- * expected discordant-cell probabilities `p10` (P[treatment wins on a pair]) and
- * `p01` (P[control wins]); concordant pairs carry no information, so the count
- * is driven entirely by the discordant rate. Lachin's (1992) asymptotic normal
- * approximation: with discordant rate `pDisc = p10 + p01` and marginal effect
- * `δ = p10 − p01`,
- *   n = ( z_{1-α/2}·√pDisc + z_{1-β}·√(pDisc − δ²) )² / δ².
- * Returns Infinity when there is no effect (p10 === p01). Asymptotic — at the
- * tiny discordant counts where the exact {@link mcnemar} differs from the normal
- * approximation, treat the result as a lower bound and prefer the discordant-pair
- * floor.
- */
-declare function mcnemarRequiredN(opts: {
-  p10: number;
-  p01: number;
-  alpha?: number;
-  power?: number;
-  twoSided?: boolean;
-}): number;
-/**
- * Power of a McNemar test at a given number of paired observations, the inverse
- * of {@link mcnemarRequiredN} (same Lachin asymptotic model, same parameters).
- * Returns a value in [0, 1]; equals `alpha` when there is no effect.
- */
-declare function mcnemarPower(opts: {
-  p10: number;
-  p01: number;
-  nPairs: number;
-  alpha?: number;
-  twoSided?: boolean;
-}): number;
-/**
- * Bonferroni adjustment: multiply every p-value by the test count, clamp at 1.
- *
- * Rejects at `p_adjusted ≤ alpha` — the boundary is inclusive, matching
- * {@link holm}, which uniformly dominates this correction and must therefore
- * never reject less. Validates its inputs on the same terms.
- */
-declare function bonferroni(pValues: readonly number[], alpha?: number): {
-  adjusted: number[];
-  significant: boolean[];
-};
-/**
- * Holm step-down family-wise error adjustment.
- *
- * P-values are sorted from smallest to largest, multiplied by their remaining
- * hypothesis count, and made monotonically non-decreasing before being mapped
- * back to input order. This uniformly dominates plain Bonferroni while keeping
- * strong family-wise error control under arbitrary dependence.
- */
-declare function holm(pValues: readonly number[], alpha?: number): {
-  adjusted: number[];
-  significant: boolean[];
-};
-/**
- * Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
- * significance at the target FDR; handles ties and preserves q monotonicity.
- *
- * Rejects at `q ≤ fdr` — the BH rule is inclusive at the boundary, so an
- * exactly-`fdr` q-value is a discovery.
- */
-declare function benjaminiHochberg(pValues: readonly number[], fdr?: number): {
-  qValues: number[];
-  significant: boolean[];
-};
-interface PairedBootstrapResult {
-  /** Number of paired observations. */
-  n: number;
-  /** Median of paired deltas (after − before). */
-  median: number;
-  /** Mean of paired deltas. */
-  mean: number;
-  /** Lower bound of the bootstrap CI on the chosen statistic. */
-  low: number;
-  /** Upper bound of the bootstrap CI on the chosen statistic. */
-  high: number;
-  /** Confidence level used (e.g. 0.95). */
-  confidence: number;
-  /** Number of bootstrap resamples used. */
-  resamples: number;
-  /** False below {@link BOOTSTRAP_GATE_MIN_N}. See {@link pairedBootstrap}. */
-  gateEligible: boolean;
-}
-/**
- * Pairs below which a percentile bootstrap interval is descriptive spread only.
- *
- * `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000
- * seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the
- * median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling
- * three points, not an implementation error — scipy's BCa gives 16.0 % on the
- * same n = 3 data — so no change to the estimator moves it. Below this floor
- * the decision belongs to the exact sign test or exact signed-rank test.
- */
-declare const BOOTSTRAP_GATE_MIN_N = 20;
-interface PairedBootstrapOptions {
-  /** Confidence level. Default 0.95. */
-  confidence?: number;
-  /** Bootstrap resample count. Default 2000. */
-  resamples?: number;
-  /** Statistic to bootstrap. Default 'median'. */
-  statistic?: 'median' | 'mean';
-  /** Deterministic seed. If omitted, derived from the deltas so the interval
-   *  is reproducible regardless. */
-  seed?: number;
-}
-/**
- * Paired bootstrap on (after − before) deltas. Returns a CI on the chosen
- * statistic (median by default); pairs are resampled with replacement. Throws
- * on unequal sample sizes.
- *
- * `low > threshold` carries the stated confidence ONLY at `n ≥
- * {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the
- * check fires under a true null several times more often than nominal, so the
- * interval is descriptive spread and a promotion must not turn on it.
- */
-declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
-/** Pre-registered direction for a one-sided paired sign test. */
-type SignTestAlternative = 'greater' | 'less';
-/** Exact one-sided sign-test result for paired numeric differences. */
-interface PairedSignTestResult {
-  /** Total supplied differences, including zero ties. */
-  n: number;
-  /** Strictly positive differences. */
-  positive: number;
-  /** Strictly negative differences. */
-  negative: number;
-  /** Zero differences excluded from the binomial test. */
-  ties: number;
-  /** Non-zero differences used by the binomial test. */
-  nNonTies: number;
-  /** Direction of the pre-registered alternative hypothesis. */
-  alternative: SignTestAlternative;
-  /** Exact one-sided p-value under P(positive) = P(negative) = 0.5. */
-  pValue: number;
-}
-/**
- * Exact one-sided sign test over paired differences.
- *
- * Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'`
- * tests whether positive signs are more likely than negative signs and returns
- * `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats
- * negative signs as successes instead. With a continuous difference
- * distribution this is the usual directional median test. Exact zero
- * differences are ties and do not enter the binomial denominator. All-tie and
- * empty inputs return p = 1. Every input difference must be finite, and the
- * direction must be chosen explicitly so a caller cannot select it after
- * seeing the signs.
- */
-declare function pairedSignTest(differences: readonly number[], alternative: SignTestAlternative): PairedSignTestResult;
-/** A binomial proportion estimate with a confidence interval. */
-interface ProportionInterval {
-  /** Point estimate successes / n (0 when n = 0). */
-  estimate: number;
-  /** Lower bound, clamped to [0, 1]. */
-  lower: number;
-  /** Upper bound, clamped to [0, 1]. */
-  upper: number;
-}
-/**
- * Wilson score interval for a binomial proportion. Correct at small n and near
- * 0/1, where the normal (Wald) approximation produces bounds outside [0, 1] and
- * understates coverage. Use this for any pass-rate / hit-rate / realness-rate
- * CI — the continuous `confidenceInterval` assumes the wrong distribution for a
- * proportion. `n = 0 ⇒ {0, 0, 0}`.
- */
-declare function wilson(successes: number, n: number, confidence?: number): ProportionInterval;
-/**
- * Are these per-item outcomes binary (every value exactly 0 or 1)?
- *
- * The discriminator a promotion gate needs before choosing a paired statistic.
- * On binary outcomes the paired delta vector lives in {-1, 0, +1} and is
- * normally dominated by zeros (both arms solve, or both arms miss, most items),
- * so its MEDIAN is pinned at exactly 0 no matter how large the real shift in
- * success rate is — and a bootstrap CI on that median collapses to [0, 0].
- * A gate keying on `ci.low > threshold` is then structurally unable to see
- * either a gain or a regression. Detect this shape and switch to the
- * paired-binary estimators ({@link mcnemar}, {@link pairedRiskDifference})
- * instead of silently answering "no" forever.
- *
- * Empty input is NOT binary: there is no evidence of the outcome's shape, and
- * defaulting an empty vector into the binary branch would pick a statistic on
- * no data at all.
- *
- * NOT the right discriminator for a gate. It recognises the literal {0, 1}
- * encoding and nothing else, so a pass/fail dimension emitted on 0-100 — which
- * judges in this codebase do routinely — reads as non-binary, and a single
- * partial-credit score in an otherwise pass/fail vector flips it to false while
- * leaving the median just as blind. Gates want {@link pairedBinaryScale} (any
- * two-point encoding). This predicate remains for callers that specifically
- * mean "literally 0/1".
- */
-declare function isBinaryOutcomeVector(values: ArrayLike<number>): boolean;
-/** Result of a McNemar paired-binary significance test. */
-interface McNemarResult {
-  /** Total paired observations. */
-  n: number;
-  /** Discordant pairs (b + c) — the only ones that carry signal. */
-  nDiscordant: number;
-  /** Pairs where treatment succeeded and control failed ("newly correct"). */
-  b: number;
-  /** Pairs where control succeeded and treatment failed ("newly wrong"). */
-  c: number;
-  /** Continuity-corrected chi-square statistic (reference; exact p drives the call). */
-  statistic: number;
-  /** Two-sided p-value. Exact (binomial sign test on discordant pairs). */
-  pValue: number;
-}
-/**
- * McNemar's test for paired binary outcomes — the correct significance test for
- * "does treatment change the success rate vs control on the SAME items". Only
- * discordant pairs (one arm right, the other wrong) carry information; concordant
- * pairs are uninformative, so a paired t-test / two-proportion z-test on the raw
- * rates is wrong here. The p-value is exact: under H0 the b "treatment-wins" are
- * Binomial(b + c, 0.5), so the two-sided p is the doubled binomial tail — correct
- * at the small discordant counts typical of eval runs (no continuity-corrected
- * chi-square approximation needed, though it is returned as `statistic` for
- * reference). Inputs are paired 0/1 (or boolean) arrays, control first to match
- * the module's (before, after) convention. Throws on unequal lengths.
- */
-declare function mcnemar(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>): McNemarResult;
-/** A paired binary effect size (treatment rate − control rate) with a CI. */
-interface RiskDifferenceResult {
-  /** Total paired observations. */
-  n: number;
-  /** Discordant pairs: treatment-win count. */
-  b: number;
-  /** Discordant pairs: control-win count. */
-  c: number;
-  /** Paired risk difference p(treatment) − p(control) = (b − c) / n. */
-  riskDifference: number;
-  /** Lower bound of the CI, clamped to [-1, 1]. */
-  lower: number;
-  /** Upper bound of the CI, clamped to [-1, 1]. */
-  upper: number;
-  /** Confidence level used. */
-  confidence: number;
-}
-/**
- * Paired risk difference (the effect-size companion to {@link mcnemar}): the
- * change in success rate p(treatment) − p(control) on matched items, which for
- * paired binary data equals (b − c) / n. The CI uses the paired variance from
- * the discordant counts, not the independent-samples formula (which overstates
- * the interval by ignoring the pairing). Inputs are paired 0/1 (or boolean)
- * arrays, control first. Throws on unequal lengths.
- *
- * REPORTING ONLY — do NOT decide a promotion on this interval. The CI is a Wald
- * normal approximation, which badly UNDERCOVERS when only a handful of pairs are
- * discordant: at n = 3 with b = 2, c = 0 it returns [0.133, 1.000], excluding 0,
- * while McNemar's exact test on the same data gives p = 0.50. A gate keying on
- * `lower > 0` would promote noise. Use {@link pairedRiskDifferenceExact}, whose
- * interval is dual to the exact test by construction, for any decision.
- */
-declare function pairedRiskDifference(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): RiskDifferenceResult;
-/** A paired binary effect size with an EXACT interval and the exact test that
- *  bounds it — one object so a caller cannot read the estimate without the
- *  significance it is entitled to. */
-interface ExactRiskDifferenceResult {
-  /** Total paired observations. */
-  n: number;
-  /** Discordant pairs: treatment-win count. */
-  b: number;
-  /** Discordant pairs: control-win count. */
-  c: number;
-  /** Discordant pairs (b + c) — the only ones carrying information. */
-  nDiscordant: number;
-  /** Paired risk difference p(treatment) − p(control) = (b − c) / n. */
-  riskDifference: number;
-  /** Exact conditional CI lower bound. 0 when there are no discordant pairs. */
-  lower: number;
-  /** Exact conditional CI upper bound. 0 when there are no discordant pairs. */
-  upper: number;
-  /** Confidence level used. */
-  confidence: number;
-  /** McNemar's exact two-sided p-value on the same discordant counts. */
-  pValue: number;
-}
-/**
- * Paired risk difference with the EXACT CONDITIONAL interval — the estimator a
- * promotion gate may decide on.
- *
- * Conditional on the number of discordant pairs m = b + c, the treatment-win
- * count b is Binomial(m, π) with π = P(treatment wins | discordant), and the
- * risk difference is an exact reparameterisation: RD = (2π − 1)·m/n. So a
- * Clopper-Pearson exact interval for π maps straight onto RD. This buys the
- * property the Wald interval in {@link pairedRiskDifference} does not have:
- *
- *   **`lower > 0` ⟺ McNemar's exact test rejects at α = 1 − confidence.**
- *
- * Clopper-Pearson excludes π = 0.5 exactly when the two-sided exact binomial
- * test of π = 0.5 rejects, and that test IS {@link mcnemar}'s p-value — so the
- * interval and the test can never disagree, and a gate keyed on `lower` cannot
- * promote what the exact test refuses. The exact p is returned in the same
- * object so the two are impossible to compute apart.
- *
- * The interval is conservative (exact intervals over-cover; conditioning on m
- * discards the concordant pairs' information about m itself). That is the
- * correct direction for a promotion gate: it refuses more often, never less.
- *
- * With m = 0 there are no discordant pairs and π is not identified: the result
- * is the degenerate [0, 0] with p = 1. That is NOT evidence of equivalence —
- * callers must treat a zero-width interval as "cannot decide", not as "no
- * difference". Inputs are paired 0/1 (or boolean) arrays, control first.
- * Throws on unequal lengths.
- */
-declare function pairedRiskDifferenceExact(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): ExactRiskDifferenceResult;
-/** A paired binary effect size with an interval that is valid at a NONZERO
- *  margin — the estimator a noninferiority decision may be made on. */
-interface ScoreRiskDifferenceResult {
-  /** Total paired observations. */
-  n: number;
-  /** Discordant pairs: treatment-win count. */
-  b: number;
-  /** Discordant pairs: control-win count. */
-  c: number;
-  /** Discordant pairs (b + c). */
-  nDiscordant: number;
-  /** Paired risk difference p(treatment) − p(control) = (b − c) / n. */
-  riskDifference: number;
-  /** Score-interval lower bound on the population risk difference. */
-  lower: number;
-  /** Score-interval upper bound on the population risk difference. */
-  upper: number;
-  /** Confidence level used. */
-  confidence: number;
-}
-/**
- * Paired risk difference with TANGO'S (1998) SCORE INTERVAL — the estimator a
- * promotion gate may decide on **at a nonzero margin**.
- *
- * {@link pairedRiskDifferenceExact} conditions on the observed discordant count
- * `m = b + c`, builds a Clopper-Pearson interval for the win share among those
- * `m` pairs, and multiplies by the observed `m/n`. That is exact for testing
- * RD = 0 — it is dual to McNemar — but it is NOT a confidence interval for the
- * population risk difference at a nonzero margin, because the sampling
- * variability of `m/n` itself is discarded. The gap is not academic: with the
- * production caller's `pairedDeltaThreshold: -0.05`, a process whose true risk
- * difference sits exactly on that margin clears a nominal-95 % `lower > margin`
- * check 24.75 % of the time at n = 40 and 43.95 % at n = 76 (2000 replicates
- * each) when the conditional interval decides.
- *
- * Tango's interval inverts the score test of RD = delta, which estimates the
- * nuisance loss rate under each hypothesised delta instead of fixing it at the
- * observed value, so `m` contributes its own uncertainty. It is the method
- * `ratesci::scorepairci` uses for paired risk-difference noninferiority, and it
- * is not conditional, so it stays valid as the margin moves away from zero.
- *
- * The bounds are found by bisecting `tangoScore(delta) = ±z` — the score is
- * monotone decreasing in delta, so each crossing is unique. Inputs are paired
- * 0/1 (or boolean) arrays, control first. Throws on unequal lengths.
- */
-declare function pairedRiskDifferenceScore(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): ScoreRiskDifferenceResult;
-/**
- * The common positive level `s` such that EVERY value across both paired arms is
- * exactly 0 or `s` — i.e. the outcome is pass/fail, whatever encoding it arrived
- * in. Returns null when the outcomes are not two-point, when the two arms use
- * different levels, or when no positive value was observed at all (all-zero
- * arms: the level is not identified, and there is nothing to decide anyway).
- *
- * This is the scale-aware successor to {@link isBinaryOutcomeVector}, which only
- * recognises literal {0, 1}. Judges in this codebase emit dimensions on 0-100 as
- * well as [0,1] (see `detectScale` in `campaign/gates/statistical-heldout.ts`),
- * so a pass/fail dimension routinely arrives as {0, 100} and a {0,1}-only test
- * silently sends it down the median path that cannot see it. Any positive level
- * is accepted, not just 1 and 100: for a two-point {0, s} outcome the mean paired
- * delta is exactly s·(b − c)/n, so the binary estimators apply after dividing by
- * s and rescaling the result back into the caller's native units.
- *
- * Non-finite values ⇒ null: an unusable outcome must not be classified as a
- * clean pass/fail shape.
- */
-declare function pairedBinaryScale(before: ArrayLike<number>, after: ArrayLike<number>): number | null;
-/** Fraction of paired observations whose delta is an exact tie (|after − before|
- *  < 1e-9). Throws on unequal sample sizes; 0 pairs ⇒ 0. */
-declare function pairedDeltaTieFraction(before: ArrayLike<number>, after: ArrayLike<number>): number;
-/**
- * The paired-delta statistic a DECISION is computed on, package-wide.
- *
- * The mean paired delta is the estimator that answers the question a promotion
- * gate asks — "by how much did the candidate move the score" — in the caller's
- * own units, and it equals the aggregate lift everyone quotes. The MEDIAN
- * answers a different question and loses the answer to this one in every regime
- * eval data actually lands in:
- *   - TWO-POINT (pass/fail) outcomes on any encoding: the delta vector lives in
- *     {−s, 0, +s} dominated by zeros, so the median and its whole bootstrap CI
- *     are pinned at exactly 0 however large the shift. (Decide these on
- *     {@link pairedRiskDifferenceExact} instead — same estimand, exact interval.)
- *   - TIE-DOMINATED outcomes: at half the pairs tied the sample median is 0 by
- *     construction, and `ci.low > threshold` then answers "no" forever at a
- *     non-negative threshold and "yes" forever at a negative one.
- *   - LOW-CARDINALITY outcomes, even well below half ties: judge dimensions on
- *     integer 0-100, and block scores like {⅔, 1} from averaging pass/fail
- *     leaves, put the median on a coarse lattice whose bootstrap percentiles
- *     land on atoms. Measured: 26 blocks of 3 pass/fail leaves carrying a real
- *     +12.8pp lift, only 23% of pairs tied, gives a median CI of [0, 0.333] —
- *     lower bound exactly 0, so a gate at threshold 0 refuses a real lift.
- * That last case is why there is no tie-fraction threshold here: any cutoff on
- * ties leaves the lattice case open on the other side of it.
- *
- * `heldoutSignificance` has defaulted to the mean since #316 for the same
- * reason. The median remains available per call site for callers who
- * specifically want outlier robustness and accept the blindness.
- */
-declare const DECISION_PAIRED_DELTA_STATISTIC: 'mean';
-/**
- * Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large
- * Language Models Trained on Code"). Given `n` independent samples for one
- * problem of which `c` pass, the probability that at least one of a random k of
- * them passes is 1 − C(n−c, k) / C(n, k). Estimating pass@k as "did any of the
- * first k pass" is biased high at small n; this is the variance-reduced estimator
- * averaged implicitly over all k-subsets. Average the per-problem values across
- * the suite for the corpus pass@k. Computed in the numerically stable product
- * form. Requires 1 ≤ k ≤ n and 0 ≤ c ≤ n.
- */
-declare function passAtK(n: number, c: number, k: number): number;
-interface EProcessOptions {
-  /** Type-I error budget. The process decides when wealth ≥ 1/alpha
-   *  (Ville's inequality). Default 0.05. */
-  alpha?: number;
-  /** Truncation bound on the predictable bet λ ∈ [0, maxBet]. Must satisfy
-   *  maxBet < 1/nullMean so every wealth factor stays strictly positive.
-   *  Default 0.5. */
-  maxBet?: number;
-  /** The null boundary m₀ for H0: E[x] ≤ m₀ on x ∈ [0,1]. Default 0.5
-   *  (the paired-delta encoding x = (d+1)/2 maps "no effect" to 1/2).
-   *  A pre-registered minEffect shifts this — see `sequentialPairedGate`. */
-  nullMean?: number;
-}
-interface EProcessStep {
-  /** Current wealth W_n — the e-value against H0 after n observations. */
-  wealth: number;
-  /** Observations consumed so far. */
-  n: number;
-  /** True from the first n where W_n ≥ 1/alpha onward (sticky). */
-  decided: boolean;
-}
-interface EProcessState extends EProcessStep {
-  alpha: number;
-  maxBet: number;
-  nullMean: number;
-  /** The decision boundary 1/alpha. */
-  threshold: number;
-  /** Observation count at the first threshold crossing; undefined until decided. */
-  decidedAtN?: number;
-}
-interface EProcess {
-  /** Consume one observation x ∈ [0,1]. Throws on non-finite / out-of-range
-   *  input — a silent clamp would corrupt the type-I guarantee. */
-  update(x: number): EProcessStep;
-  state(): EProcessState;
-}
-/**
- * Betting test-martingale for bounded observations — the e-process core of
- * anytime-valid sequential testing (Waudby-Smith & Ramdas, "Estimating means
- * of bounded random variables by betting", JRSS-B 2024).
- *
- * Observations x_i ∈ [0,1]; H0: E[x] ≤ m₀ (`nullMean`, default 1/2). Wealth
- *
- *   W_t = Π_{i≤t} (1 + λ_i (x_i − m₀)),  W_0 = 1
- *
- * with the truncated GROW-style plug-in bet computed from PRIOR observations:
- *
- *   λ_i = clamp((μ̂_{i−1} − m₀) / (σ̂²_{i−1} + (μ̂_{i−1} − m₀)²), 0, maxBet)
- *
- * where μ̂/σ̂² are the shrunk running estimates μ̂_t = (1/2 + Σx_i)/(t+1),
- * σ̂²_t = (1/4 + Σ(x_i − μ̂_i)²)/(t+1).
- *
- * PREDICTABILITY INVARIANT (load-bearing): λ_i is a function of x_1..x_{i−1}
- * ONLY — it may never see x_i. With λ_i ≥ 0 predictable, each factor has
- * E[1 + λ_i(x_i − m₀) | past] ≤ 1 under H0, so W is a nonnegative
- * supermartingale and Ville's inequality gives P(∃t: W_t ≥ 1/α) ≤ α — the
- * type-I guarantee holds at ANY data-dependent stopping time. λ_1 is always 0
- * (no prior evidence), so the first observation never moves wealth.
- *
- * `decided` latches at the first crossing W_t ≥ 1/α and never un-latches;
- * wealth keeps updating after the crossing (the e-process remains valid), but
- * the decision time is the first crossing.
- */
-declare function eProcess(opts?: EProcessOptions): EProcess;
-/** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
- *  cryptographic. Exported so e-process shuffles and bootstrap resampling
- *  share ONE PRNG implementation. Every distinct 32-bit seed gives a distinct
- *  stream, including 0. */
-declare function mulberry32(seed: number): () => number;
-//#endregion
-export { pairedCohensDz as $, WeightedCompositeInput as A, positionalBias as At, eProcess as B, RankTestMethod as C, GoldenItem as Ct, ScoreRiskDifferenceResult as D, calibrateJudge as Dt, RiskDifferenceResult as E, VerbosityBiasResult as Et, cliffsDelta as F, mannWhitneyU as G, interRaterReliability as H, cohensD as I, mcnemarRequiredN as J, mcnemar as K, confidenceInterval as L, WilcoxonSignedRankResult as M, verbosityBias as Mt, benjaminiHochberg as N, SignTestAlternative as O, calibrateJudgeContinuous as Ot, bonferroni as P, pairedBootstrap as Q, corpusInterRaterAgreement as R, ProportionInterval as S, ContinuousCalibrationResult as St, RankTestOptions as T, SelfPreferenceResult as Tt, interpretCliffs as U, holm as V, isBinaryOutcomeVector as W, normalizeScores as X, mulberry32 as Y, pairedBinaryScale as Z, McNemarResult as _, wilson as _t, CorpusAgreementReport as a, pairedSignTest as at, PairedSignTestResult as b, ContinuousAgreement as bt, DEFAULT_PERMUTATIONS as c, passAtK as ct, EProcessState as d, requiredPairedSampleSize as dt, pairedDeltaTieFraction as et, EProcessStep as f, requiredSampleSize as ft, MannWhitneyResult as g, wilcoxonSignedRank as gt, MANN_WHITNEY_EXACT_MAX_WORK as h, weightedMean as ht, CorpusAgreementPerDimension as i, pairedRiskDifferenceScore as it, WeightedCompositeResult as j, selfPreference as jt, WILCOXON_EXACT_MAX_N as k, continuousAgreement as kt, EProcess as l, pearsonR as lt, MANN_WHITNEY_EXACT_MAX_STATES as m, weightedComposite as mt, CliffsMagnitude as n, pairedRiskDifference as nt, CorpusScoreRecord as o, pairedTTest as ot, ExactRiskDifferenceResult as p, spearmanR as pt, mcnemarPower as q, CorpusAgreementOptions as r, pairedRiskDifferenceExact as rt, DECISION_PAIRED_DELTA_STATISTIC as s, partialCredit as st, BOOTSTRAP_GATE_MIN_N as t, pairedMde as tt, EProcessOptions as u, ranks as ut, PairedBootstrapOptions as v, CalibrationResult as vt, RankTestMethodRequest as w, PositionalBiasResult as wt, PairedTTestResult as x, ContinuousAgreementOptions as xt, PairedBootstrapResult as y, CandidateScore as yt, corpusInterRaterAgreementFromJudgeScores as z };
-//# sourceMappingURL=statistics-D6Uebe_4.d.ts.map