@tangle-network/agent-eval 0.133.1 → 0.133.3

package/dist/{statistics-DbvkkDPa.d.ts → statistics-D_4Snl-5.d.ts}

@@ -9,7 +9,14 @@ declare function weightedMean(scores: {
   score: number;
   weight?: number;
 }[]): number;
-/** Bootstrap confidence interval */
+/**
+ * Percentile bootstrap confidence interval on the mean of `scores`.
+ *
+ * Descriptive spread. It is not a significance test, and at small n its bounds
+ * are anti-conservative in the same way {@link pairedBootstrap}'s are — see
+ * {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
+ * the scores themselves, so the interval is reproducible either way.
+ */
 declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
   seed?: number;
   resamples?: number;
@@ -19,47 +26,136 @@ declare function confidenceInterval(scores: number[], confidence?: number, opts?
   upper: number;
 };
 /**
- * Inter-rater reliability — simplified Krippendorff's alpha.
+ * Inter-rater reliability — Krippendorff's α under the squared-difference
+ * metric, pooled across dimensions.
  *
- * Each inner array is one judge's scores for all items.
- * All arrays must have the same length (same items scored).
+ * Each inner array is one judge's scores. Items are matched by position
+ * WITHIN a dimension: the k-th score a judge supplies carrying dimension
+ * `d` is item k of `d`, and the ratings compared against each other are
+ * the ones different judges gave to the same item. Every judge that scores
+ * a dimension at all must supply the same number of scores for it —
+ * ragged input cannot be aligned into items and throws rather than
+ * comparing mismatched items.
+ *
+ * α = 1 − D_observed / D_expected: D_observed averages the squared
+ * difference over within-item judge pairs, D_expected over every pair of
+ * ratings irrespective of item. α = 1 is perfect agreement, 0 is chance,
+ * negative is systematic disagreement.
  */
 declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
+/** How a rank test's p-value was actually computed. */
+type RankTestMethod = 'exact' | 'permutation' | 'asymptotic';
 /**
- * Mann-Whitney U test for comparing two independent groups.
- * Returns U statistic and approximate p-value (normal approximation).
+ * What the caller asks for. `'auto'` selects `'exact'` inside the enumeration
+ * threshold and `'permutation'` above it, and never selects `'asymptotic'`.
  */
-declare function mannWhitneyU(a: number[], b: number[]): {
+type RankTestMethodRequest = 'auto' | 'exact' | 'asymptotic';
+interface RankTestOptions {
+  /** Default `'auto'`. `'asymptotic'` inside the exact-feasible range throws. */
+  method?: RankTestMethodRequest;
+  /** Resamples on the Monte Carlo permutation path. Default 100000. */
+  permutations?: number;
+  /** Seed for the permutation path. Omitted ⇒ derived from the data itself, so
+   *  the result is reproducible either way. */
+  seed?: number;
+}
+/** Maximum dynamic-programming cells used by an exact two-sample rank test. */
+declare const MANN_WHITNEY_EXACT_MAX_STATES = 8192;
+/** Maximum inner-loop transitions used by an exact two-sample rank test. */
+declare const MANN_WHITNEY_EXACT_MAX_WORK = 250000;
+/** Non-zero differences up to which the signed-rank null is enumerated exactly. */
+declare const WILCOXON_EXACT_MAX_N = 20;
+/** Resamples used when a rank test falls back to Monte Carlo permutation. */
+declare const DEFAULT_PERMUTATIONS = 100000;
+interface MannWhitneyResult {
+  /** `min(U_a, U_b)` — the conventional reported statistic. */
   u: number;
+  /** U for sample `a`. Carries the direction of the effect, which `u` discards. */
+  uA: number;
+  /** Two-sided p-value. */
   p: number;
-};
+  /** How `p` was computed. */
+  method: RankTestMethod;
+  /** Smallest two-sided p this design can produce. `p` can never be below it. */
+  pFloor: number;
+}
+/**
+ * Mann-Whitney U — two independent samples, no distributional assumption.
+ *
+ * Exact conditional (permutation) p by default when the dynamic program fits
+ * {@link MANN_WHITNEY_EXACT_MAX_STATES} cells and
+ * {@link MANN_WHITNEY_EXACT_MAX_WORK} transitions, seeded Monte Carlo
+ * permutation above those limits. This keeps imbalanced designs such as 1+24
+ * exact without admitting expensive balanced designs merely because they have
+ * the same total size. Throws on non-finite input and on `method:
+ * 'asymptotic'` where an exact answer is available. Empty input yields `p = 1,
+ * pFloor = 1` — no design, no attainable evidence.
+ */
+declare function mannWhitneyU(a: number[], b: number[], opts?: RankTestOptions): MannWhitneyResult;
 /** Partial credit: returns 0-1 ratio of current toward target */
 declare function partialCredit(current: number, target: number): number;
+interface PairedTTestResult {
+  /** Null when the statistic is undefined — see {@link pairedTTest}. */
+  t: number | null;
+  df: number;
+  /** Null exactly when `t` is null. */
+  p: number | null;
+}
 /**
  * Paired t-test — before/after measurements on the SAME items.
  * Pairing removes inter-item variance, giving tighter significance than
  * an unpaired test when comparing prompt v1 vs prompt v2 on identical
  * scenarios.
+ *
+ * Returns `t = p = null` where the statistic is undefined: fewer than two
+ * pairs, or a non-zero constant delta whose observed variance is zero. A
+ * constant shift carries no information about the variance it would have to
+ * be compared against, so the honest answer is "undefined", not `p = 0` —
+ * three observations cannot buy absolute certainty. This is the same contract
+ * {@link pairedCohensDz} states for the same condition. An all-zero delta is
+ * different: it is a measured null, and returns `t = 0, p = 1`.
  */
-declare function pairedTTest(before: number[], after: number[]): {
-  t: number;
-  df: number;
+declare function pairedTTest(before: number[], after: number[]): PairedTTestResult;
+interface WilcoxonSignedRankResult {
+  /** W⁺, the rank sum of the positive differences. (scipy reports
+   *  `min(W⁺, W⁻)`; compare statistics only after converting.) */
+  w: number;
+  /** Two-sided p-value. */
   p: number;
-};
+  /** How `p` was computed. */
+  method: RankTestMethod;
+  /** Smallest two-sided p this design can produce. */
+  pFloor: number;
+  /** Non-zero differences — zero differences are dropped and carry no rank. */
+  nNonZero: number;
+}
 /**
- * Wilcoxon signed-rank test — paired non-parametric alternative.
- * Use when the differences aren't normally distributed.
+ * Wilcoxon signed-rank — paired, no distributional assumption on the deltas.
+ *
+ * Exact conditional (sign-flip) p by default at `n ≤
+ * {@link WILCOXON_EXACT_MAX_N}` non-zero differences, seeded Monte Carlo
+ * permutation above it. Throws on non-finite input and on `method:
+ * 'asymptotic'` where an exact answer is available.
+ *
+ * `n` is the count of NON-ZERO differences: exact ties are dropped before
+ * ranking, so a run of tied pairs shrinks the design and raises `pFloor`.
+ * All-tied input yields `p = 1, pFloor = 1` — no attainable evidence, which
+ * `pFloor` states rather than leaving `p = 1` to be read as a measured null.
  */
-declare function wilcoxonSignedRank(before: number[], after: number[]): {
-  w: number;
-  p: number;
-};
+declare function wilcoxonSignedRank(before: number[], after: number[], opts?: RankTestOptions): WilcoxonSignedRankResult;
 /**
  * Cohen's d — standardized effect size for two independent groups.
  * Positive d means group b has higher mean than group a.
  * Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
+ *
+ * Returns null where the standardized effect is undefined: fewer than two
+ * observations in either group, or a zero pooled standard deviation with
+ * unequal means. Null is NOT "no effect" — zero within-group spread across a
+ * real mean gap is an unbounded effect, the opposite of negligible. Equal
+ * means with zero spread is a genuine 0. Same contract as
+ * {@link pairedCohensDz}.
  */
-declare function cohensD(a: number[], b: number[]): number;
+declare function cohensD(a: number[], b: number[]): number | null;
 /**
  * Cohen's dz for paired observations: mean(after - before) divided by the
  * sample standard deviation of those within-pair deltas.
@@ -215,6 +311,11 @@ declare function requiredSampleSize(opts: {
 /**
  * Required number of paired observations for a target Cohen's dz.
  * Unlike the independent-groups formula, this has no two-arm factor of two.
+ *
+ * Normal quantiles with no t correction, so treat the result as a LOWER bound:
+ * it returns 32 where the exact t-based answer is 34 at dz = 0.5, and 13 where
+ * it is 15 at dz = 0.8 — a 6–13 % shortfall precisely in the range a caller
+ * consults to decide whether 3–10 repetitions suffice.
  */
 declare function requiredPairedSampleSize(opts: {
   effect: number;
@@ -267,8 +368,14 @@ declare function mcnemarPower(opts: {
   alpha?: number;
   twoSided?: boolean;
 }): number;
-/** Bonferroni adjustment: multiply every p-value by the test count, clamp at 1. */
-declare function bonferroni(pValues: number[], alpha?: number): {
+/**
+ * Bonferroni adjustment: multiply every p-value by the test count, clamp at 1.
+ *
+ * Rejects at `p_adjusted ≤ alpha` — the boundary is inclusive, matching
+ * {@link holm}, which uniformly dominates this correction and must therefore
+ * never reject less. Validates its inputs on the same terms.
+ */
+declare function bonferroni(pValues: readonly number[], alpha?: number): {
   adjusted: number[];
   significant: boolean[];
 };
@@ -287,8 +394,11 @@ declare function holm(pValues: readonly number[], alpha?: number): {
 /**
  * Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
  * significance at the target FDR; handles ties and preserves q monotonicity.
+ *
+ * Rejects at `q ≤ fdr` — the BH rule is inclusive at the boundary, so an
+ * exactly-`fdr` q-value is a discovery.
  */
-declare function benjaminiHochberg(pValues: number[], fdr?: number): {
+declare function benjaminiHochberg(pValues: readonly number[], fdr?: number): {
   qValues: number[];
   significant: boolean[];
 };
@@ -307,7 +417,20 @@ interface PairedBootstrapResult {
   confidence: number;
   /** Number of bootstrap resamples used. */
   resamples: number;
+  /** False below {@link BOOTSTRAP_GATE_MIN_N}. See {@link pairedBootstrap}. */
+  gateEligible: boolean;
 }
+/**
+ * Pairs below which a percentile bootstrap interval is descriptive spread only.
+ *
+ * `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000
+ * seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the
+ * median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling
+ * three points, not an implementation error — scipy's BCa gives 16.0 % on the
+ * same n = 3 data — so no change to the estimator moves it. Below this floor
+ * the decision belongs to the exact sign test or exact signed-rank test.
+ */
+declare const BOOTSTRAP_GATE_MIN_N = 20;
 interface PairedBootstrapOptions {
   /** Confidence level. Default 0.95. */
   confidence?: number;
@@ -315,14 +438,19 @@ interface PairedBootstrapOptions {
   resamples?: number;
   /** Statistic to bootstrap. Default 'median'. */
   statistic?: 'median' | 'mean';
-  /** Deterministic seed. If omitted, uses Math.random(). */
+  /** Deterministic seed. If omitted, derived from the deltas so the interval
+   *  is reproducible regardless. */
   seed?: number;
 }
 /**
  * Paired bootstrap on (after − before) deltas. Returns a CI on the chosen
- * statistic (median by default); pairs are resampled with replacement. The
- * lower bound is what the promotion gate checks — `low > threshold` means the
- * gain is real at the confidence level. Throws on unequal sample sizes.
+ * statistic (median by default); pairs are resampled with replacement. Throws
+ * on unequal sample sizes.
+ *
+ * `low > threshold` carries the stated confidence ONLY at `n ≥
+ * {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the
+ * check fires under a true null several times more often than nominal, so the
+ * interval is descriptive spread and a promotion must not turn on it.
  */
 declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
 /** Pre-registered direction for a one-sided paired sign test. */
@@ -506,9 +634,9 @@ interface EProcess {
 declare function eProcess(opts?: EProcessOptions): EProcess;
 /** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
  *  cryptographic. Exported so e-process shuffles and bootstrap resampling
- *  share ONE PRNG implementation; a seed is REQUIRED (unseeded randomness in
- *  gate verdicts is non-reproducible by construction). */
+ *  share ONE PRNG implementation. Every distinct 32-bit seed gives a distinct
+ *  stream, including 0. */
 declare function mulberry32(seed: number): () => number;
 //#endregion
-export { mannWhitneyU as A, pairedSignTest as B, confidenceInterval as C, holm as D, eProcess as E, normalizeScores as F, ranks as G, partialCredit as H, pairedBootstrap as I, spearmanR as J, requiredPairedSampleSize as K, pairedCohensDz as L, mcnemarPower as M, mcnemarRequiredN as N, interRaterReliability as O, mulberry32 as P, wilson as Q, pairedMde as R, cohensD as S, corpusInterRaterAgreementFromJudgeScores as T, passAtK as U, pairedTTest as V, pearsonR as W, weightedMean as X, weightedComposite as Y, wilcoxonSignedRank as Z, WeightedCompositeInput as _, CorpusScoreRecord as a, bonferroni as b, EProcessState as c, PairedBootstrapOptions as d, PairedBootstrapResult as f, SignTestAlternative as g, RiskDifferenceResult as h, CorpusAgreementReport as i, mcnemar as j, interpretCliffs as k, EProcessStep as l, ProportionInterval as m, CorpusAgreementOptions as n, EProcess as o, PairedSignTestResult as p, requiredSampleSize as q, CorpusAgreementPerDimension as r, EProcessOptions as s, CliffsMagnitude as t, McNemarResult as u, WeightedCompositeResult as v, corpusInterRaterAgreement as w, cliffsDelta as x, benjaminiHochberg as y, pairedRiskDifference as z };
-//# sourceMappingURL=statistics-DbvkkDPa.d.ts.map
+export { partialCredit as $, benjaminiHochberg as A, interpretCliffs as B, RankTestOptions as C, WeightedCompositeInput as D, WILCOXON_EXACT_MAX_N as E, corpusInterRaterAgreement as F, mulberry32 as G, mcnemar as H, corpusInterRaterAgreementFromJudgeScores as I, pairedCohensDz as J, normalizeScores as K, eProcess as L, cliffsDelta as M, cohensD as N, WeightedCompositeResult as O, confidenceInterval as P, pairedTTest as Q, holm as R, RankTestMethodRequest as S, SignTestAlternative as T, mcnemarPower as U, mannWhitneyU as V, mcnemarRequiredN as W, pairedRiskDifference as X, pairedMde as Y, pairedSignTest as Z, PairedBootstrapResult as _, CorpusAgreementReport as a, spearmanR as at, ProportionInterval as b, EProcess as c, wilcoxonSignedRank as ct, EProcessStep as d, passAtK as et, MANN_WHITNEY_EXACT_MAX_STATES as f, PairedBootstrapOptions as g, McNemarResult as h, CorpusAgreementPerDimension as i, requiredSampleSize as it, bonferroni as j, WilcoxonSignedRankResult as k, EProcessOptions as l, wilson as lt, MannWhitneyResult as m, CliffsMagnitude as n, ranks as nt, CorpusScoreRecord as o, weightedComposite as ot, MANN_WHITNEY_EXACT_MAX_WORK as p, pairedBootstrap as q, CorpusAgreementOptions as r, requiredPairedSampleSize as rt, DEFAULT_PERMUTATIONS as s, weightedMean as st, BOOTSTRAP_GATE_MIN_N as t, pearsonR as tt, EProcessState as u, PairedSignTestResult as v, RiskDifferenceResult as w, RankTestMethod as x, PairedTTestResult as y, interRaterReliability as z };
+//# sourceMappingURL=statistics-D_4Snl-5.d.ts.map

package/dist/statistics-D_4Snl-5.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics-D_4Snl-5.d.ts","names":[],"sources":["../src/statistics.ts"],"mappings":";;;;;cAaa,kBAAe,QAAY,iBAAe;;iBAGvC,aAAa;EAAU;EAAe;;;;;;;;;;iBAoBtC,mBACd,kBACA,qBACA;EAAQ;EAAe;;EACpB;EAAc;EAAe;;;;;;;;;;;;;;;;;;;iBAiDlB,sBAAsB,aAAa;;KAkFvC;;;;;KAMA;UAEK;;EAEf,SAAS;;EAET;;;EAGA;;;cAIW;;cAEA;;cAEA;;cAEA;UAEI;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER;;;;;;;;;;;;;;iBAec,aACd,aACA,aACA,OAAM,kBACL;;iBAwFa,cAAc,iBAAiB;UAK9B;;EAEf;EACA;;EAEA;;;;;;;;;;;;;;;;iBAiBc,YAAY,kBAAkB,kBAAkB;UAyB/C;;;EAGf;;EAEA;;EAEA,QAAQ;;EAER;;EAEA;;;;;;;;;;;;;;;iBAgBc,mBACd,kBACA,iBACA,OAAM,kBACL;;;;;;;;;;;;;iBAmFa,QAAQ,aAAa;;;;;;;;;iBAqBrB,eAAe,kBAAkB;KAoBrC;;;;;;;;;;;iBAYI,YAAY,kBAAkB;;;;;;iBAkB9B,gBAAgB,gBAAgB;;;;;iBAuBhC,MAAM;;;;;;iBAmBN,SAAS,aAAa;;;;;iBAuBtB,UAAU,aAAa;UAKtB;;EAEf,MAAM;;;;;EAKN,SAAS;;EAET;;UAGe;EACf;EACA;;;;;;;;;;;iBAYc,kBAAkB,OAAO,yBAAyB;UA4CjD;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe,oCAAoC;EACnD;;EAEA;;EAEA;;UAGe;;EAEf,cAAc;;EAEd;;EAEA;;EAEA;;EAEA;;UAGe,+BAA+B;;;;;;EAM9C;;;;;;EAMA;;;;;;;;;;;;;;;;;;iBAmBc,0BACd,SAAS,qBACT,OAAM,yBACL;;;;;;;;;iBA0Ha,yCACd,aAAa;EAAQ;EAAgB,QAAQ;IAC7C,OAAM,yBACL;;;;;;;iBA8Ba,mBAAmB;EACjC;EACA;EACA;EACA;;;;;;;;;;;iBAsBc,yBAAyB;EACvC;EACA;EACA;EACA;;;;;;;;iBAkBc,UAAU;EACxB;EACA;EACA;EACA;;;;;;;;;;;;;;;;iBAyBc,iBAAiB;EAC/B;EACA;EACA;EACA;EACA;;;;;;;iBAyBc,aAAa;EAC3B;EACA;EACA;EACA;EACA;;;;;;;;;iBAwBc,WACd,4BACA;EACG;EAAoB;;;;;;;;;;iBAgBT,KACd,4BACA;EACG;EAAoB;;;;;;;;;iBA4BT,kBACd,4BACA;EACG;EAAmB;;UAuCP;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;cAaW;UAEI;;EAEf;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;iBAac,gBACd,kBACA,iBACA,OAAM,yBACL;;KA0DS;;UAGK;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;;;;;;;;;iBAgBc,eACd,gCACA,aAAa,sBACZ;;UAgDc;;EAEf;;EAEA;;EAEA;;;;;;;;;iBAUc,OAAO,mBAAmB,WAAW,sBAAoB;;UAmBxD;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;iBAec,QACd,SAAS,6BACT,WAAW,8BACV;;UAmBc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;iBAWc,qBACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;;;;;;;;;iBAyCa,QAAQ,WAAW,WAAW;UAgE7B;;;EAGf;;;;EAIA;;;;EAIA;;UAGe;;EAEf;;EAEA;;EAEA;;UAGe,sBAAsB;EACrC;EACA;EACA;;EAEA;;EAEA;;UAGe;;;EAGf,OAAO,YAAY;EACnB,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8BK,SAAS,OAAM,kBAAuB;;;;;iBAsatC,WAAW"}