@tangle-network/agent-eval 0.204.0 → 0.205.1

package/docs/concepts.md

@@ -414,3 +414,24 @@ release decision.
 - **Reading a verdict someone else produced?** Read [verdicts.md](./verdicts.md) for what `certification` carries and what an absent one means.
 - **Grading a finding by executing its repair?** Read [trace-repair-grader.md](./trace-repair-grader.md), and [trajectory-replay.md](./trajectory-replay.md) for re-executing a recorded failure.
 - **Checking package ownership?** Read [charter.md](./charter.md) for the implemented foundations and host responsibilities.
+
+## Independent sample statistics
+
+The root export owns `summarizeSample`, `sampleMedian`, and `welchsTTest`.
+`summarizeSample` reports finite sample facts, interpolated quartiles and a seeded median-bootstrap spread.
+An empty sample returns `null`; a zero mean leaves relative dispersion (`cv`) unavailable.
+Fewer than three observations retain a null `medianInterval`, not a confidence interval fabricated from the observed range.
+The bootstrap interval describes the observed sample, not a significance test or generalization guarantee.
+Confidence and resample count are explicit, and nonfinite input is refused.
+Stability labels and decisions remain consumer policy.
+
+`welchsTTest(a,b)` compares independent samples with positive direction meaning B minus A.
+Its existing `status: 'ok'` result includes means, difference, standard error, t statistic, fractional degrees of freedom, two-sided p-value, effect size and 95% interval.
+Too few samples or zero within-group variance retains the existing `insufficient-sample` or `zero-variance` status, observed means and difference, NaN inferential fields and a null interval.
+An unavailable test is not evidence that the groups are equivalent.
+Use the existing `cohensD` alongside it; an undefined standardized effect stays null.
+
+The root also exports contingency-table inference, normal and Student-t tails, and centered linear-regression slope inference.
+`twoByTwoPValue` chooses Fisher exact inference for sparse expected counts and the Pearson chi-squared tail otherwise.
+A regression with insufficient residual degrees of freedom or constant x retains a null slope p-value.
+These primitives do not choose product acceptance thresholds or authorize release.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.204.0",
+  "version": "0.205.1",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

package/dist/baseline-CVKsfWTg.js

@@ -1,150 +0,0 @@
-import { n as studentTQuantile, t as studentTCdf } from "./student-t-BBZYka0E.js";
-import { n as cohensD } from "./effect-sizes-DiH8MGOH.js";
-//#region src/baseline.ts
-/**
-* Baseline regression detection.
-*
-* Lifted from ADC baseline.ts. Every promotion-blocking signal boils down
-* to: "is this run measurably worse than baseline?" — with enough
-* statistical rigor to distinguish noise from drift.
-*
-* Uses:
-*   - Welch's t-test (unequal variance) for per-metric mean comparison
-*   - Cohen's d for effect size magnitude
-*   - IQR for stability flag (unstable samples can't be trusted for comparisons)
-*
-* Returns a structured verdict: improved | regressed | stable | unstable.
-*/
-/**
-* Compare candidate samples against baseline per metric. Verdict logic:
-*   - unstable: IQR/|mean| > threshold on either set — not enough signal
-*   - improved: meaningful effect in the "better" direction AND p < alpha
-*   - regressed: meaningful effect in the "worse" direction AND p < alpha
-*   - stable: otherwise (no significant change)
-*/
-function compareToBaseline(samples, options = {}) {
-	const effectThreshold = options.effectThreshold ?? .5;
-	const alpha = options.alpha ?? .05;
-	const cvThreshold = options.unstableCvThreshold ?? .3;
-	const metrics = samples.map((s) => {
-		const comparison = welchsTTest(s.baseline, s.candidate);
-		if (comparison.status === "insufficient-sample") throw new Error(`compareToBaseline: need ≥2 samples per side for "${s.metric}"`);
-		const { meanA: bMean, meanB: cMean, delta, cohensD: d, t, df, p } = comparison;
-		const baselineIqr = iqr(s.baseline);
-		const candidateIqr = iqr(s.candidate);
-		const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold;
-		const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold;
-		const stable = baselineStable && candidateStable;
-		const reportedIqr = Math.max(baselineIqr, candidateIqr);
-		const effectClears = d === null || Math.abs(d) >= effectThreshold;
-		let verdict;
-		if (!stable) verdict = "unstable";
-		else if (comparison.status === "ok" && p < alpha && effectClears) verdict = (s.higherIsBetter ? delta > 0 : delta < 0) ? "improved" : "regressed";
-		else verdict = "stable";
-		return {
-			metric: s.metric,
-			baselineMean: bMean,
-			candidateMean: cMean,
-			delta,
-			cohensD: d,
-			welchT: t,
-			welchDf: df,
-			welchP: p,
-			stable,
-			iqr: reportedIqr,
-			verdict
-		};
-	});
-	return {
-		metrics,
-		hasRegression: metrics.some((m) => m.verdict === "regressed"),
-		hasUnstable: metrics.some((m) => m.verdict === "unstable")
-	};
-}
-function mean(xs) {
-	return xs.reduce((a, b) => a + b, 0) / xs.length;
-}
-/** Inter-quartile range; 0 when the sample has no spread. */
-function iqr(xs) {
-	if (xs.length === 0) return 0;
-	const sorted = [...xs].sort((a, b) => a - b);
-	const q = (p) => {
-		const idx = p * (sorted.length - 1);
-		const lo = Math.floor(idx);
-		const hi = Math.ceil(idx);
-		return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo);
-	};
-	return q(.75) - q(.25);
-}
-/**
-* Welch's t-test and 95% interval for two independent samples.
-*
-* Uses the Student-t distribution at every finite degree of freedom. Fewer
-* than two observations per side and zero observed standard error cannot
-* define a t reference distribution; those cases return an explicit status,
-* `NaN` inferential fields, and no interval rather than fabricated certainty.
-*/
-function welchsTTest(a, b) {
-	assertFiniteSample("a", a);
-	assertFiniteSample("b", b);
-	const mA = mean(a);
-	const mB = mean(b);
-	const delta = mB - mA;
-	if (a.length < 2 || b.length < 2) return {
-		status: "insufficient-sample",
-		meanA: mA,
-		meanB: mB,
-		delta,
-		standardError: NaN,
-		t: NaN,
-		df: NaN,
-		p: NaN,
-		ci95: null,
-		cohensD: null
-	};
-	const vA = variance(a, mA);
-	const vB = variance(b, mB);
-	const seSquared = vA / a.length + vB / b.length;
-	const d = cohensD(a, b);
-	if (seSquared === 0) return {
-		status: "zero-variance",
-		meanA: mA,
-		meanB: mB,
-		delta,
-		standardError: 0,
-		t: delta === 0 ? 0 : Math.sign(delta) * Number.POSITIVE_INFINITY,
-		df: NaN,
-		p: NaN,
-		ci95: null,
-		cohensD: null
-	};
-	const standardError = Math.sqrt(seSquared);
-	const t = delta / standardError;
-	const df = seSquared * seSquared / ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1));
-	const p = 2 * (1 - studentTCdf(Math.abs(t), df));
-	if (d === null) throw new Error("welchsTTest: non-zero standard error produced no pooled effect size");
-	const halfWidth = studentTQuantile(.975, df) * standardError;
-	return {
-		status: "ok",
-		meanA: mA,
-		meanB: mB,
-		delta,
-		standardError,
-		t,
-		df,
-		p,
-		ci95: [delta - halfWidth, delta + halfWidth],
-		cohensD: d
-	};
-}
-function variance(xs, m) {
-	return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1);
-}
-function assertFiniteSample(name, sample) {
-	const invalid = sample.find((value) => !Number.isFinite(value));
-	if (invalid !== void 0) throw new RangeError(`welchsTTest: sample ${name} must contain only finite values, got ${invalid}`);
-}
-//#endregion
-export { iqr as n, welchsTTest as r, compareToBaseline as t };
-
-//# sourceMappingURL=baseline-CVKsfWTg.js.map

package/dist/baseline-CVKsfWTg.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"baseline-CVKsfWTg.js","names":[],"sources":["../src/baseline.ts"],"sourcesContent":["/**\n * Baseline regression detection.\n *\n * Lifted from ADC baseline.ts. Every promotion-blocking signal boils down\n * to: \"is this run measurably worse than baseline?\" — with enough\n * statistical rigor to distinguish noise from drift.\n *\n * Uses:\n *   - Welch's t-test (unequal variance) for per-metric mean comparison\n *   - Cohen's d for effect size magnitude\n *   - IQR for stability flag (unstable samples can't be trusted for comparisons)\n *\n * Returns a structured verdict: improved | regressed | stable | unstable.\n */\n\nimport { studentTCdf, studentTQuantile } from './math/student-t'\nimport { cohensD } from './statistics'\n\nexport interface MetricSamples {\n  /** Stable metric key (e.g. \"overallScore\", \"firstTokenMs\"). */\n  metric: string\n  /** Whether higher values are better. */\n  higherIsBetter: boolean\n  baseline: number[]\n  candidate: number[]\n}\n\nexport interface MetricVerdict {\n  metric: string\n  baselineMean: number\n  candidateMean: number\n  delta: number\n  /** Null when both samples are constant with unequal means — the\n   *  standardized effect is unbounded there, not zero. */\n  cohensD: number | null\n  welchT: number\n  welchDf: number\n  welchP: number\n  stable: boolean\n  /** IQR of the combined samples — used as a rough stability indicator. */\n  iqr: number\n  verdict: 'improved' | 'regressed' | 'stable' | 'unstable'\n}\n\nexport interface BaselineReport {\n  metrics: MetricVerdict[]\n  /** True if any critical metric regressed. */\n  hasRegression: boolean\n  /** True if any metric is unstable (too noisy to judge). */\n  hasUnstable: boolean\n}\n\nexport interface BaselineOptions {\n  /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */\n  effectThreshold?: number\n  /** p-value threshold for statistical significance (default 0.05). */\n  alpha?: number\n  /** IQR/mean ratio above which samples are flagged unstable (default 0.30). */\n  unstableCvThreshold?: number\n}\n\nexport type WelchTestStatus = 'ok' | 'insufficient-sample' | 'zero-variance'\n\n/**\n * Full unequal-variance comparison for two independent samples.\n *\n * `meanB`, `delta`, `t`, and `cohensD` are oriented as `b - a`. Inferential\n * fields are `NaN` and `ci95` is null when the sample count or observed\n * variance cannot define a Student-t result. `status` makes that state\n * explicit so callers cannot mistake it for evidence of no difference.\n */\ninterface WelchTestResultBase {\n  meanA: number\n  meanB: number\n  delta: number\n}\n\nexport type WelchTestResult =\n  | (WelchTestResultBase & {\n      status: 'ok'\n      standardError: number\n      t: number\n      df: number\n      p: number\n      ci95: [number, number]\n      cohensD: number\n    })\n  | (WelchTestResultBase & {\n      status: 'insufficient-sample' | 'zero-variance'\n      standardError: number\n      t: number\n      df: number\n      p: number\n      ci95: null\n      cohensD: null\n    })\n\n/**\n * Compare candidate samples against baseline per metric. Verdict logic:\n *   - unstable: IQR/|mean| > threshold on either set — not enough signal\n *   - improved: meaningful effect in the \"better\" direction AND p < alpha\n *   - regressed: meaningful effect in the \"worse\" direction AND p < alpha\n *   - stable: otherwise (no significant change)\n */\nexport function compareToBaseline(\n  samples: MetricSamples[],\n  options: BaselineOptions = {},\n): BaselineReport {\n  const effectThreshold = options.effectThreshold ?? 0.5\n  const alpha = options.alpha ?? 0.05\n  const cvThreshold = options.unstableCvThreshold ?? 0.3\n\n  const metrics: MetricVerdict[] = samples.map((s) => {\n    const comparison = welchsTTest(s.baseline, s.candidate)\n    if (comparison.status === 'insufficient-sample') {\n      throw new Error(`compareToBaseline: need ≥2 samples per side for \"${s.metric}\"`)\n    }\n    const { meanA: bMean, meanB: cMean, delta, cohensD: d, t, df, p } = comparison\n    // Stability is per-side: a comparison is trustworthy only when BOTH\n    // samples are internally consistent. Combining the sides would flag\n    // large-but-real deltas as \"unstable\" which is exactly what we want\n    // to detect.\n    const baselineIqr = iqr(s.baseline)\n    const candidateIqr = iqr(s.candidate)\n    const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold\n    const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold\n    const stable = baselineStable && candidateStable\n    const reportedIqr = Math.max(baselineIqr, candidateIqr)\n\n    // Both sides are guaranteed ≥ 2 samples above, so a null d means zero\n    // pooled spread across a real mean gap: an unbounded standardized effect,\n    // which clears any finite threshold.\n    const effectClears = d === null || Math.abs(d) >= effectThreshold\n\n    let verdict: MetricVerdict['verdict']\n    if (!stable) {\n      verdict = 'unstable'\n    } else if (comparison.status === 'ok' && p < alpha && effectClears) {\n      const candidateIsBetter = s.higherIsBetter ? delta > 0 : delta < 0\n      verdict = candidateIsBetter ? 'improved' : 'regressed'\n    } else {\n      verdict = 'stable'\n    }\n\n    return {\n      metric: s.metric,\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      cohensD: d,\n      welchT: t,\n      welchDf: df,\n      welchP: p,\n      stable,\n      iqr: reportedIqr,\n      verdict,\n    }\n  })\n\n  return {\n    metrics,\n    hasRegression: metrics.some((m) => m.verdict === 'regressed'),\n    hasUnstable: metrics.some((m) => m.verdict === 'unstable'),\n  }\n}\n\nfunction mean(xs: number[]): number {\n  return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\n/** Inter-quartile range; 0 when the sample has no spread. */\nexport function iqr(xs: number[]): number {\n  if (xs.length === 0) return 0\n  const sorted = [...xs].sort((a, b) => a - b)\n  const q = (p: number) => {\n    const idx = p * (sorted.length - 1)\n    const lo = Math.floor(idx)\n    const hi = Math.ceil(idx)\n    return sorted[lo]! + (sorted[hi]! - sorted[lo]!) * (idx - lo)\n  }\n  return q(0.75) - q(0.25)\n}\n\n/**\n * Welch's t-test and 95% interval for two independent samples.\n *\n * Uses the Student-t distribution at every finite degree of freedom. Fewer\n * than two observations per side and zero observed standard error cannot\n * define a t reference distribution; those cases return an explicit status,\n * `NaN` inferential fields, and no interval rather than fabricated certainty.\n */\nexport function welchsTTest(a: number[], b: number[]): WelchTestResult {\n  assertFiniteSample('a', a)\n  assertFiniteSample('b', b)\n  const mA = mean(a)\n  const mB = mean(b)\n  const delta = mB - mA\n  if (a.length < 2 || b.length < 2) {\n    return {\n      status: 'insufficient-sample',\n      meanA: mA,\n      meanB: mB,\n      delta,\n      standardError: Number.NaN,\n      t: Number.NaN,\n      df: Number.NaN,\n      p: Number.NaN,\n      ci95: null,\n      cohensD: null,\n    }\n  }\n\n  const vA = variance(a, mA)\n  const vB = variance(b, mB)\n  const seSquared = vA / a.length + vB / b.length\n  const d = cohensD(a, b)\n  if (seSquared === 0) {\n    return {\n      status: 'zero-variance',\n      meanA: mA,\n      meanB: mB,\n      delta,\n      standardError: 0,\n      t: delta === 0 ? 0 : Math.sign(delta) * Number.POSITIVE_INFINITY,\n      df: Number.NaN,\n      p: Number.NaN,\n      ci95: null,\n      cohensD: null,\n    }\n  }\n\n  const standardError = Math.sqrt(seSquared)\n  const t = delta / standardError\n  const df =\n    (seSquared * seSquared) /\n    ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1))\n  const p = 2 * (1 - studentTCdf(Math.abs(t), df))\n  if (d === null) {\n    throw new Error('welchsTTest: non-zero standard error produced no pooled effect size')\n  }\n  const halfWidth = studentTQuantile(0.975, df) * standardError\n  return {\n    status: 'ok',\n    meanA: mA,\n    meanB: mB,\n    delta,\n    standardError,\n    t,\n    df,\n    p,\n    ci95: [delta - halfWidth, delta + halfWidth],\n    cohensD: d,\n  }\n}\n\nfunction variance(xs: number[], m: number): number {\n  return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1)\n}\n\nfunction assertFiniteSample(name: string, sample: readonly number[]): void {\n  const invalid = sample.find((value) => !Number.isFinite(value))\n  if (invalid !== undefined) {\n    throw new RangeError(\n      `welchsTTest: sample ${name} must contain only finite values, got ${invalid}`,\n    )\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAwGA,SAAgB,kBACd,SACA,UAA2B,CAAC,GACZ;CAChB,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,MAAM,QAAQ,QAAQ,SAAS;CAC/B,MAAM,cAAc,QAAQ,uBAAuB;CAEnD,MAAM,UAA2B,QAAQ,KAAK,MAAM;EAClD,MAAM,aAAa,YAAY,EAAE,UAAU,EAAE,SAAS;EACtD,IAAI,WAAW,WAAW,uBACxB,MAAM,IAAI,MAAM,oDAAoD,EAAE,OAAO,EAAE;EAEjF,MAAM,EAAE,OAAO,OAAO,OAAO,OAAO,OAAO,SAAS,GAAG,GAAG,IAAI,MAAM;EAKpE,MAAM,cAAc,IAAI,EAAE,QAAQ;EAClC,MAAM,eAAe,IAAI,EAAE,SAAS;EACpC,MAAM,iBAAiB,cAAc,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;EACxE,MAAM,kBAAkB,eAAe,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;EAC1E,MAAM,SAAS,kBAAkB;EACjC,MAAM,cAAc,KAAK,IAAI,aAAa,YAAY;EAKtD,MAAM,eAAe,MAAM,QAAQ,KAAK,IAAI,CAAC,KAAK;EAElD,IAAI;EACJ,IAAI,CAAC,QACH,UAAU;OACL,IAAI,WAAW,WAAW,QAAQ,IAAI,SAAS,cAEpD,WAD0B,EAAE,iBAAiB,QAAQ,IAAI,QAAQ,KACnC,aAAa;OAE3C,UAAU;EAGZ,OAAO;GACL,QAAQ,EAAE;GACV,cAAc;GACd,eAAe;GACf;GACA,SAAS;GACT,QAAQ;GACR,SAAS;GACT,QAAQ;GACR;GACA,KAAK;GACL;EACF;CACF,CAAC;CAED,OAAO;EACL;EACA,eAAe,QAAQ,MAAM,MAAM,EAAE,YAAY,WAAW;EAC5D,aAAa,QAAQ,MAAM,MAAM,EAAE,YAAY,UAAU;CAC3D;AACF;AAEA,SAAS,KAAK,IAAsB;CAClC,OAAO,GAAG,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;;AAGA,SAAgB,IAAI,IAAsB;CACxC,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC3C,MAAM,KAAK,MAAc;EACvB,MAAM,MAAM,KAAK,OAAO,SAAS;EACjC,MAAM,KAAK,KAAK,MAAM,GAAG;EACzB,MAAM,KAAK,KAAK,KAAK,GAAG;EACxB,OAAO,OAAO,OAAQ,OAAO,MAAO,OAAO,QAAS,MAAM;CAC5D;CACA,OAAO,EAAE,GAAI,IAAI,EAAE,GAAI;AACzB;;;;;;;;;AAUA,SAAgB,YAAY,GAAa,GAA8B;CACrE,mBAAmB,KAAK,CAAC;CACzB,mBAAmB,KAAK,CAAC;CACzB,MAAM,KAAK,KAAK,CAAC;CACjB,MAAM,KAAK,KAAK,CAAC;CACjB,MAAM,QAAQ,KAAK;CACnB,IAAI,EAAE,SAAS,KAAK,EAAE,SAAS,GAC7B,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA,eAAe;EACf,GAAG;EACH,IAAI;EACJ,GAAG;EACH,MAAM;EACN,SAAS;CACX;CAGF,MAAM,KAAK,SAAS,GAAG,EAAE;CACzB,MAAM,KAAK,SAAS,GAAG,EAAE;CACzB,MAAM,YAAY,KAAK,EAAE,SAAS,KAAK,EAAE;CACzC,MAAM,IAAI,QAAQ,GAAG,CAAC;CACtB,IAAI,cAAc,GAChB,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA,eAAe;EACf,GAAG,UAAU,IAAI,IAAI,KAAK,KAAK,KAAK,IAAI,OAAO;EAC/C,IAAI;EACJ,GAAG;EACH,MAAM;EACN,SAAS;CACX;CAGF,MAAM,gBAAgB,KAAK,KAAK,SAAS;CACzC,MAAM,IAAI,QAAQ;CAClB,MAAM,KACH,YAAY,cACX,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS,MAAM,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS;CAC9E,MAAM,IAAI,KAAK,IAAI,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE;CAC9C,IAAI,MAAM,MACR,MAAM,IAAI,MAAM,qEAAqE;CAEvF,MAAM,YAAY,iBAAiB,MAAO,EAAE,IAAI;CAChD,OAAO;EACL,QAAQ;EACR,OAAO;EACP,OAAO;EACP;EACA;EACA;EACA;EACA;EACA,MAAM,CAAC,QAAQ,WAAW,QAAQ,SAAS;EAC3C,SAAS;CACX;AACF;AAEA,SAAS,SAAS,IAAc,GAAmB;CACjD,OAAO,GAAG,QAAQ,KAAK,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,KAAK,GAAG,SAAS;AACrE;AAEA,SAAS,mBAAmB,MAAc,QAAiC;CACzE,MAAM,UAAU,OAAO,MAAM,UAAU,CAAC,OAAO,SAAS,KAAK,CAAC;CAC9D,IAAI,YAAY,KAAA,GACd,MAAM,IAAI,WACR,uBAAuB,KAAK,wCAAwC,SACtE;AAEJ"}