@graphorin/evals 0.5.0 → 0.6.1

package/dist/stats.d.ts

@@ -0,0 +1,115 @@
+//#region src/stats.d.ts
+/**
+ * Shared eval statistics (audit 2026-07-04, E8; closes the evals-05 /
+ * evals-08 gap): confidence intervals, pass^k, and paired significance
+ * for benchmark deltas. Everything here is pure and dependency-free so
+ * the runner, the regression gate, and the benchmarks can share one
+ * implementation instead of each inlining its own mean/stddev.
+ *
+ * Statistical choices, briefly:
+ * - **Wilson score interval** for pass-rate CIs. Eval suites are small
+ *   (n in the tens) and pass rates hug 0/1, exactly where the normal
+ *   approximation interval collapses; Wilson stays sane there.
+ * - **pass^k** ("pass-hat-k", the fraction of base cases that pass in
+ *   ALL k repeat iterations) as the stability metric for `iterations>1`
+ *   runs - a mean pass rate hides a case that flips every other run.
+ * - **McNemar's test** (continuity-corrected normal approximation) for
+ *   "did the pass rate really change vs the baseline": the same cases
+ *   ran in both reports, so a PAIRED test on the discordant cases has
+ *   far more power than comparing two independent proportions - and it
+ *   is sample-size aware, which the fixed-tolerance regression gate is
+ *   not.
+ *
+ * @packageDocumentation
+ */
+/**
+ * Arithmetic mean; `0` for an empty list.
+ *
+ * @stable
+ */
+declare function mean(values: ReadonlyArray<number>): number;
+/**
+ * Sample standard deviation (n-1 denominator); `0` for n < 2.
+ *
+ * @stable
+ */
+declare function sampleStddev(values: ReadonlyArray<number>): number;
+/**
+ * Wilson score interval for a binomial proportion.
+ *
+ * @param passed number of successes
+ * @param total  number of trials
+ * @param z      normal quantile (default 1.96 = 95% two-sided)
+ * @returns `{ lo, hi }` clamped to [0, 1]; `{ lo: 0, hi: 1 }` when `total` is 0
+ *
+ * @stable
+ */
+declare function wilsonInterval(passed: number, total: number, z?: number): {
+  readonly lo: number;
+  readonly hi: number;
+};
+/**
+ * Strip the `-iter-N` disambiguation suffix the runner appends under
+ * `iterations > 1` (EB-6), recovering the base case id.
+ *
+ * @stable
+ */
+declare function stripIterationSuffix(caseId: string): string;
+/**
+ * pass^k over per-iteration case outcomes: group by base case id and
+ * report the fraction of base cases whose EVERY iteration passed.
+ *
+ * @param outcomes one entry per executed case iteration
+ * @returns `k` = the largest group size observed, `baseCases` = number of
+ *   distinct base cases, `value` = the pass^k fraction (0 when no cases)
+ *
+ * @stable
+ */
+declare function passHatK(outcomes: ReadonlyArray<{
+  readonly caseId: string;
+  readonly pass: boolean;
+}>): {
+  readonly k: number;
+  readonly baseCases: number;
+  readonly value: number;
+};
+/**
+ * Result of {@link pairedPassSignificance}.
+ *
+ * @stable
+ */
+interface PairedSignificance {
+  /** Cases present in BOTH runs (the paired sample). */
+  readonly pairs: number;
+  /** baseline-pass -> current-fail count (the regressions). */
+  readonly regressed: number;
+  /** baseline-fail -> current-pass count (the improvements). */
+  readonly improved: number;
+  /**
+   * Two-sided p-value from McNemar's test (continuity-corrected normal
+   * approximation over the discordant pairs). `1` when there are no
+   * discordant pairs - identical outcomes are never "significant".
+   */
+  readonly pValue: number;
+}
+/**
+ * McNemar's paired test on per-case pass outcomes of two eval runs.
+ * Only cases present in both maps are compared (by base case id).
+ *
+ * @stable
+ */
+declare function pairedPassSignificance(current: ReadonlyMap<string, boolean>, baseline: ReadonlyMap<string, boolean>): PairedSignificance;
+/**
+ * Collapse per-iteration outcomes to per-base-case pass (a base case
+ * passes when EVERY iteration passed) - the paired unit for
+ * {@link pairedPassSignificance}.
+ *
+ * @stable
+ */
+declare function passByBaseCase(outcomes: ReadonlyArray<{
+  readonly caseId: string;
+  readonly pass: boolean;
+}>): ReadonlyMap<string, boolean>;
+//#endregion
+export { PairedSignificance, mean, pairedPassSignificance, passByBaseCase, passHatK, sampleStddev, stripIterationSuffix, wilsonInterval };
+//# sourceMappingURL=stats.d.ts.map

package/dist/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","names":[],"sources":["../src/stats.ts"],"sourcesContent":[],"mappings":";;AA6BA;AAYA;AAkBA;AAyBA;AAcA;AA0BA;AAqBA;;;;;AA6BA;;;;;;;;;;;;;;;;iBAjJgB,IAAA,SAAa;;;;;;iBAYb,YAAA,SAAqB;;;;;;;;;;;iBAkBrB,cAAA;;;;;;;;;;iBAyBA,oBAAA;;;;;;;;;;;iBAcA,QAAA,WACJ;;;;;;;;;;;;;UAyBK,kBAAA;;;;;;;;;;;;;;;;;;;;iBAqBD,sBAAA,UACL,wCACC,+BACT;;;;;;;;iBA0Ba,cAAA,WACJ;;;KACT"}

package/dist/stats.js

@@ -0,0 +1,180 @@
+//#region src/stats.ts
+/**
+* Shared eval statistics (audit 2026-07-04, E8; closes the evals-05 /
+* evals-08 gap): confidence intervals, pass^k, and paired significance
+* for benchmark deltas. Everything here is pure and dependency-free so
+* the runner, the regression gate, and the benchmarks can share one
+* implementation instead of each inlining its own mean/stddev.
+*
+* Statistical choices, briefly:
+* - **Wilson score interval** for pass-rate CIs. Eval suites are small
+*   (n in the tens) and pass rates hug 0/1, exactly where the normal
+*   approximation interval collapses; Wilson stays sane there.
+* - **pass^k** ("pass-hat-k", the fraction of base cases that pass in
+*   ALL k repeat iterations) as the stability metric for `iterations>1`
+*   runs - a mean pass rate hides a case that flips every other run.
+* - **McNemar's test** (continuity-corrected normal approximation) for
+*   "did the pass rate really change vs the baseline": the same cases
+*   ran in both reports, so a PAIRED test on the discordant cases has
+*   far more power than comparing two independent proportions - and it
+*   is sample-size aware, which the fixed-tolerance regression gate is
+*   not.
+*
+* @packageDocumentation
+*/
+/**
+* Arithmetic mean; `0` for an empty list.
+*
+* @stable
+*/
+function mean(values) {
+	if (values.length === 0) return 0;
+	let sum = 0;
+	for (const v of values) sum += v;
+	return sum / values.length;
+}
+/**
+* Sample standard deviation (n-1 denominator); `0` for n < 2.
+*
+* @stable
+*/
+function sampleStddev(values) {
+	if (values.length < 2) return 0;
+	const m = mean(values);
+	let acc = 0;
+	for (const v of values) acc += (v - m) * (v - m);
+	return Math.sqrt(acc / (values.length - 1));
+}
+/**
+* Wilson score interval for a binomial proportion.
+*
+* @param passed number of successes
+* @param total  number of trials
+* @param z      normal quantile (default 1.96 = 95% two-sided)
+* @returns `{ lo, hi }` clamped to [0, 1]; `{ lo: 0, hi: 1 }` when `total` is 0
+*
+* @stable
+*/
+function wilsonInterval(passed, total, z = 1.96) {
+	if (total <= 0) return {
+		lo: 0,
+		hi: 1
+	};
+	const p = passed / total;
+	const z2 = z * z;
+	const denom = 1 + z2 / total;
+	const centre = p + z2 / (2 * total);
+	const margin = z * Math.sqrt(p * (1 - p) / total + z2 / (4 * total * total));
+	return {
+		lo: Math.max(0, (centre - margin) / denom),
+		hi: Math.min(1, (centre + margin) / denom)
+	};
+}
+const ITERATION_SUFFIX_RE = /-iter-\d+$/;
+/**
+* Strip the `-iter-N` disambiguation suffix the runner appends under
+* `iterations > 1` (EB-6), recovering the base case id.
+*
+* @stable
+*/
+function stripIterationSuffix(caseId) {
+	return caseId.replace(ITERATION_SUFFIX_RE, "");
+}
+/**
+* pass^k over per-iteration case outcomes: group by base case id and
+* report the fraction of base cases whose EVERY iteration passed.
+*
+* @param outcomes one entry per executed case iteration
+* @returns `k` = the largest group size observed, `baseCases` = number of
+*   distinct base cases, `value` = the pass^k fraction (0 when no cases)
+*
+* @stable
+*/
+function passHatK(outcomes) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const o of outcomes) {
+		const base = stripIterationSuffix(o.caseId);
+		const g = groups.get(base) ?? {
+			total: 0,
+			passed: 0
+		};
+		g.total += 1;
+		if (o.pass) g.passed += 1;
+		groups.set(base, g);
+	}
+	if (groups.size === 0) return {
+		k: 0,
+		baseCases: 0,
+		value: 0
+	};
+	let k = 0;
+	let allPass = 0;
+	for (const g of groups.values()) {
+		if (g.total > k) k = g.total;
+		if (g.passed === g.total) allPass += 1;
+	}
+	return {
+		k,
+		baseCases: groups.size,
+		value: allPass / groups.size
+	};
+}
+/**
+* McNemar's paired test on per-case pass outcomes of two eval runs.
+* Only cases present in both maps are compared (by base case id).
+*
+* @stable
+*/
+function pairedPassSignificance(current, baseline) {
+	let pairs = 0;
+	let regressed = 0;
+	let improved = 0;
+	for (const [caseId, basePass] of baseline) {
+		const curPass = current.get(caseId);
+		if (curPass === void 0) continue;
+		pairs += 1;
+		if (basePass && !curPass) regressed += 1;
+		else if (!basePass && curPass) improved += 1;
+	}
+	const discordant = regressed + improved;
+	if (discordant === 0) return {
+		pairs,
+		regressed,
+		improved,
+		pValue: 1
+	};
+	const z = (Math.abs(regressed - improved) - 1) / Math.sqrt(discordant);
+	const pValue = z <= 0 ? 1 : Math.min(1, 2 * (1 - normalCdf(z)));
+	return {
+		pairs,
+		regressed,
+		improved,
+		pValue
+	};
+}
+/**
+* Collapse per-iteration outcomes to per-base-case pass (a base case
+* passes when EVERY iteration passed) - the paired unit for
+* {@link pairedPassSignificance}.
+*
+* @stable
+*/
+function passByBaseCase(outcomes) {
+	const out = /* @__PURE__ */ new Map();
+	for (const o of outcomes) {
+		const base = stripIterationSuffix(o.caseId);
+		const prev = out.get(base);
+		out.set(base, prev === void 0 ? o.pass : prev && o.pass);
+	}
+	return out;
+}
+/** Standard normal CDF via the Abramowitz-Stegun 7.1.26 erf approximation. */
+function normalCdf(x) {
+	const t = 1 / (1 + .3275911 * (Math.abs(x) / Math.SQRT2));
+	const erf = 1 - ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - .284496736) * t + .254829592) * t * Math.exp(-(x * x) / 2);
+	return x >= 0 ? .5 * (1 + erf) : .5 * (1 - erf);
+}
+
+//#endregion
+export { mean, pairedPassSignificance, passByBaseCase, passHatK, sampleStddev, stripIterationSuffix, wilsonInterval };
+//# sourceMappingURL=stats.js.map

package/dist/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","names":[],"sources":["../src/stats.ts"],"sourcesContent":["/**\n * Shared eval statistics (audit 2026-07-04, E8; closes the evals-05 /\n * evals-08 gap): confidence intervals, pass^k, and paired significance\n * for benchmark deltas. Everything here is pure and dependency-free so\n * the runner, the regression gate, and the benchmarks can share one\n * implementation instead of each inlining its own mean/stddev.\n *\n * Statistical choices, briefly:\n * - **Wilson score interval** for pass-rate CIs. Eval suites are small\n *   (n in the tens) and pass rates hug 0/1, exactly where the normal\n *   approximation interval collapses; Wilson stays sane there.\n * - **pass^k** (\"pass-hat-k\", the fraction of base cases that pass in\n *   ALL k repeat iterations) as the stability metric for `iterations>1`\n *   runs - a mean pass rate hides a case that flips every other run.\n * - **McNemar's test** (continuity-corrected normal approximation) for\n *   \"did the pass rate really change vs the baseline\": the same cases\n *   ran in both reports, so a PAIRED test on the discordant cases has\n *   far more power than comparing two independent proportions - and it\n *   is sample-size aware, which the fixed-tolerance regression gate is\n *   not.\n *\n * @packageDocumentation\n */\n\n/**\n * Arithmetic mean; `0` for an empty list.\n *\n * @stable\n */\nexport function mean(values: ReadonlyArray<number>): number {\n  if (values.length === 0) return 0;\n  let sum = 0;\n  for (const v of values) sum += v;\n  return sum / values.length;\n}\n\n/**\n * Sample standard deviation (n-1 denominator); `0` for n < 2.\n *\n * @stable\n */\nexport function sampleStddev(values: ReadonlyArray<number>): number {\n  if (values.length < 2) return 0;\n  const m = mean(values);\n  let acc = 0;\n  for (const v of values) acc += (v - m) * (v - m);\n  return Math.sqrt(acc / (values.length - 1));\n}\n\n/**\n * Wilson score interval for a binomial proportion.\n *\n * @param passed number of successes\n * @param total  number of trials\n * @param z      normal quantile (default 1.96 = 95% two-sided)\n * @returns `{ lo, hi }` clamped to [0, 1]; `{ lo: 0, hi: 1 }` when `total` is 0\n *\n * @stable\n */\nexport function wilsonInterval(\n  passed: number,\n  total: number,\n  z = 1.96,\n): { readonly lo: number; readonly hi: number } {\n  if (total <= 0) return { lo: 0, hi: 1 };\n  const p = passed / total;\n  const z2 = z * z;\n  const denom = 1 + z2 / total;\n  const centre = p + z2 / (2 * total);\n  const margin = z * Math.sqrt((p * (1 - p)) / total + z2 / (4 * total * total));\n  return {\n    lo: Math.max(0, (centre - margin) / denom),\n    hi: Math.min(1, (centre + margin) / denom),\n  };\n}\n\nconst ITERATION_SUFFIX_RE = /-iter-\\d+$/;\n\n/**\n * Strip the `-iter-N` disambiguation suffix the runner appends under\n * `iterations > 1` (EB-6), recovering the base case id.\n *\n * @stable\n */\nexport function stripIterationSuffix(caseId: string): string {\n  return caseId.replace(ITERATION_SUFFIX_RE, '');\n}\n\n/**\n * pass^k over per-iteration case outcomes: group by base case id and\n * report the fraction of base cases whose EVERY iteration passed.\n *\n * @param outcomes one entry per executed case iteration\n * @returns `k` = the largest group size observed, `baseCases` = number of\n *   distinct base cases, `value` = the pass^k fraction (0 when no cases)\n *\n * @stable\n */\nexport function passHatK(\n  outcomes: ReadonlyArray<{ readonly caseId: string; readonly pass: boolean }>,\n): { readonly k: number; readonly baseCases: number; readonly value: number } {\n  const groups = new Map<string, { total: number; passed: number }>();\n  for (const o of outcomes) {\n    const base = stripIterationSuffix(o.caseId);\n    const g = groups.get(base) ?? { total: 0, passed: 0 };\n    g.total += 1;\n    if (o.pass) g.passed += 1;\n    groups.set(base, g);\n  }\n  if (groups.size === 0) return { k: 0, baseCases: 0, value: 0 };\n  let k = 0;\n  let allPass = 0;\n  for (const g of groups.values()) {\n    if (g.total > k) k = g.total;\n    if (g.passed === g.total) allPass += 1;\n  }\n  return { k, baseCases: groups.size, value: allPass / groups.size };\n}\n\n/**\n * Result of {@link pairedPassSignificance}.\n *\n * @stable\n */\nexport interface PairedSignificance {\n  /** Cases present in BOTH runs (the paired sample). */\n  readonly pairs: number;\n  /** baseline-pass -> current-fail count (the regressions). */\n  readonly regressed: number;\n  /** baseline-fail -> current-pass count (the improvements). */\n  readonly improved: number;\n  /**\n   * Two-sided p-value from McNemar's test (continuity-corrected normal\n   * approximation over the discordant pairs). `1` when there are no\n   * discordant pairs - identical outcomes are never \"significant\".\n   */\n  readonly pValue: number;\n}\n\n/**\n * McNemar's paired test on per-case pass outcomes of two eval runs.\n * Only cases present in both maps are compared (by base case id).\n *\n * @stable\n */\nexport function pairedPassSignificance(\n  current: ReadonlyMap<string, boolean>,\n  baseline: ReadonlyMap<string, boolean>,\n): PairedSignificance {\n  let pairs = 0;\n  let regressed = 0;\n  let improved = 0;\n  for (const [caseId, basePass] of baseline) {\n    const curPass = current.get(caseId);\n    if (curPass === undefined) continue;\n    pairs += 1;\n    if (basePass && !curPass) regressed += 1;\n    else if (!basePass && curPass) improved += 1;\n  }\n  const discordant = regressed + improved;\n  if (discordant === 0) return { pairs, regressed, improved, pValue: 1 };\n  // Continuity-corrected normal approximation to the binomial(discordant, 0.5).\n  const z = (Math.abs(regressed - improved) - 1) / Math.sqrt(discordant);\n  const pValue = z <= 0 ? 1 : Math.min(1, 2 * (1 - normalCdf(z)));\n  return { pairs, regressed, improved, pValue };\n}\n\n/**\n * Collapse per-iteration outcomes to per-base-case pass (a base case\n * passes when EVERY iteration passed) - the paired unit for\n * {@link pairedPassSignificance}.\n *\n * @stable\n */\nexport function passByBaseCase(\n  outcomes: ReadonlyArray<{ readonly caseId: string; readonly pass: boolean }>,\n): ReadonlyMap<string, boolean> {\n  const out = new Map<string, boolean>();\n  for (const o of outcomes) {\n    const base = stripIterationSuffix(o.caseId);\n    const prev = out.get(base);\n    out.set(base, prev === undefined ? o.pass : prev && o.pass);\n  }\n  return out;\n}\n\n/** Standard normal CDF via the Abramowitz-Stegun 7.1.26 erf approximation. */\nfunction normalCdf(x: number): number {\n  const t = 1 / (1 + 0.3275911 * (Math.abs(x) / Math.SQRT2));\n  const erf =\n    1 -\n    ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) *\n      t *\n      Math.exp(-(x * x) / 2);\n  return x >= 0 ? 0.5 * (1 + erf) : 0.5 * (1 - erf);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAgB,KAAK,QAAuC;AAC1D,KAAI,OAAO,WAAW,EAAG,QAAO;CAChC,IAAI,MAAM;AACV,MAAK,MAAM,KAAK,OAAQ,QAAO;AAC/B,QAAO,MAAM,OAAO;;;;;;;AAQtB,SAAgB,aAAa,QAAuC;AAClE,KAAI,OAAO,SAAS,EAAG,QAAO;CAC9B,MAAM,IAAI,KAAK,OAAO;CACtB,IAAI,MAAM;AACV,MAAK,MAAM,KAAK,OAAQ,SAAQ,IAAI,MAAM,IAAI;AAC9C,QAAO,KAAK,KAAK,OAAO,OAAO,SAAS,GAAG;;;;;;;;;;;;AAa7C,SAAgB,eACd,QACA,OACA,IAAI,MAC0C;AAC9C,KAAI,SAAS,EAAG,QAAO;EAAE,IAAI;EAAG,IAAI;EAAG;CACvC,MAAM,IAAI,SAAS;CACnB,MAAM,KAAK,IAAI;CACf,MAAM,QAAQ,IAAI,KAAK;CACvB,MAAM,SAAS,IAAI,MAAM,IAAI;CAC7B,MAAM,SAAS,IAAI,KAAK,KAAM,KAAK,IAAI,KAAM,QAAQ,MAAM,IAAI,QAAQ,OAAO;AAC9E,QAAO;EACL,IAAI,KAAK,IAAI,IAAI,SAAS,UAAU,MAAM;EAC1C,IAAI,KAAK,IAAI,IAAI,SAAS,UAAU,MAAM;EAC3C;;AAGH,MAAM,sBAAsB;;;;;;;AAQ5B,SAAgB,qBAAqB,QAAwB;AAC3D,QAAO,OAAO,QAAQ,qBAAqB,GAAG;;;;;;;;;;;;AAahD,SAAgB,SACd,UAC4E;CAC5E,MAAM,yBAAS,IAAI,KAAgD;AACnE,MAAK,MAAM,KAAK,UAAU;EACxB,MAAM,OAAO,qBAAqB,EAAE,OAAO;EAC3C,MAAM,IAAI,OAAO,IAAI,KAAK,IAAI;GAAE,OAAO;GAAG,QAAQ;GAAG;AACrD,IAAE,SAAS;AACX,MAAI,EAAE,KAAM,GAAE,UAAU;AACxB,SAAO,IAAI,MAAM,EAAE;;AAErB,KAAI,OAAO,SAAS,EAAG,QAAO;EAAE,GAAG;EAAG,WAAW;EAAG,OAAO;EAAG;CAC9D,IAAI,IAAI;CACR,IAAI,UAAU;AACd,MAAK,MAAM,KAAK,OAAO,QAAQ,EAAE;AAC/B,MAAI,EAAE,QAAQ,EAAG,KAAI,EAAE;AACvB,MAAI,EAAE,WAAW,EAAE,MAAO,YAAW;;AAEvC,QAAO;EAAE;EAAG,WAAW,OAAO;EAAM,OAAO,UAAU,OAAO;EAAM;;;;;;;;AA6BpE,SAAgB,uBACd,SACA,UACoB;CACpB,IAAI,QAAQ;CACZ,IAAI,YAAY;CAChB,IAAI,WAAW;AACf,MAAK,MAAM,CAAC,QAAQ,aAAa,UAAU;EACzC,MAAM,UAAU,QAAQ,IAAI,OAAO;AACnC,MAAI,YAAY,OAAW;AAC3B,WAAS;AACT,MAAI,YAAY,CAAC,QAAS,cAAa;WAC9B,CAAC,YAAY,QAAS,aAAY;;CAE7C,MAAM,aAAa,YAAY;AAC/B,KAAI,eAAe,EAAG,QAAO;EAAE;EAAO;EAAW;EAAU,QAAQ;EAAG;CAEtE,MAAM,KAAK,KAAK,IAAI,YAAY,SAAS,GAAG,KAAK,KAAK,KAAK,WAAW;CACtE,MAAM,SAAS,KAAK,IAAI,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,UAAU,EAAE,EAAE;AAC/D,QAAO;EAAE;EAAO;EAAW;EAAU;EAAQ;;;;;;;;;AAU/C,SAAgB,eACd,UAC8B;CAC9B,MAAM,sBAAM,IAAI,KAAsB;AACtC,MAAK,MAAM,KAAK,UAAU;EACxB,MAAM,OAAO,qBAAqB,EAAE,OAAO;EAC3C,MAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,MAAI,IAAI,MAAM,SAAS,SAAY,EAAE,OAAO,QAAQ,EAAE,KAAK;;AAE7D,QAAO;;;AAIT,SAAS,UAAU,GAAmB;CACpC,MAAM,IAAI,KAAK,IAAI,YAAa,KAAK,IAAI,EAAE,GAAG,KAAK;CACnD,MAAM,MACJ,QACI,cAAc,IAAI,eAAe,IAAI,eAAe,IAAI,cAAe,IAAI,cAC7E,IACA,KAAK,IAAI,EAAE,IAAI,KAAK,EAAE;AAC1B,QAAO,KAAK,IAAI,MAAO,IAAI,OAAO,MAAO,IAAI"}

package/dist/types.d.ts

@@ -4,7 +4,7 @@ import { Case as Case$1, Dataset as Dataset$1, EvalCaseResult, EvalReport, EvalR
 
 /**
  * Agent shape consumed by the runner. Anything with a `run(input)`
- * method satisfies the contract — the framework's own `Agent` type
+ * method satisfies the contract - the framework's own `Agent` type
  * matches by structural typing.
  *
  * @stable
@@ -38,7 +38,7 @@ interface RunOptions<I, O> {
   /**
    * How to handle `signal` abort. Default (`false`): stop dispatching and
    * resolve with a **partial** {@link EvalReport} (`aborted: true`) covering
-   * the cases that finished — a long judged run shouldn't lose all completed
+   * the cases that finished - a long judged run shouldn't lose all completed
    * work to a Ctrl+C. Set `true` to throw the abort reason instead.
    */
   readonly throwOnAbort?: boolean;
@@ -74,6 +74,18 @@ interface RegressionOptions {
    * absolute duration gate; leave unset to ignore duration entirely.
    */
   readonly maxAvgDurationIncreaseMs?: number;
+  /**
+   * E8 (evals-05/08): when `true`, a `pass-rate-drop` finding is only kept if
+   * McNemar's paired test over the shared cases rejects "no real change" at
+   * {@link significanceAlpha} - a fixed percentage tolerance is blind to
+   * sample size (a 5pp drop is one case in a 20-case suite). Off by default
+   * so existing gates keep their exact behavior; the computed `pValue` is
+   * attached to the finding either way whenever both reports carry per-case
+   * results.
+   */
+  readonly requireSignificance?: boolean;
+  /** Significance level for {@link requireSignificance}. Default `0.05`. */
+  readonly significanceAlpha?: number;
 }
 /**
  * Result of {@link detectRegressions}.
@@ -92,6 +104,12 @@ interface RegressionFinding {
   readonly scorer?: string;
   readonly message: string;
   readonly delta: number;
+  /**
+   * Two-sided McNemar p-value over the cases shared by both reports
+   * (E8; only on `pass-rate-drop` findings, and only when both reports
+   * carry per-case results to pair on).
+   */
+  readonly pValue?: number;
 }
 //#endregion
 export { AgentLike, type Case$1 as Case, type Dataset$1 as Dataset, type EvalCaseResult, type EvalReport$1 as EvalReport, ProgressEvent, RegressionFinding, RegressionOptions, RegressionReport, type RunEvalOptions, RunOptions, type ScoreResult$1 as ScoreResult, type Scorer$1 as Scorer };

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;AAiD4C,UAnB3B,SAmB2B,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EAAV,GAAA,CAAA,KAAA,EAlBrB,CAkBqB,EAAA,GAKd,CALc,EAAA;IAAd,MAAA,CAAA,EAlBa,WAkBb;EAKA,CAAA,CAAA,EAvB6B,OAuB7B,CAvBqC,CAuBrC,CAAA;;;AAgBpB;AAaA;AAoBA;;AAEqB,UAlEJ,UAkEI,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EACU,SAAA,KAAA,EAlEb,SAkEa,CAlEH,CAkEG,EAlEA,CAkEA,CAAA;EAAG,SAAA,OAAA,EAAA;IAAd,SAAA,KAAA,EAhEA,aAgEA,CAAA;MACY,SAAA,EAAA,CAAA,EAAA,MAAA;MAAG,SAAA,KAAA,EA/Db,CA+Da;MAAd,SAAA,QAAA,CAAA,EA9DK,CA8DL;MAAU,SAAA,QAAA,CAAA,EA7DL,QA6DK,CA7DI,MA6DJ,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA;IAId,CAAA,CAAA;;oBA9DG,cAAc,OAAO,GAAG;;;;;oBAKxB;;;;;;;;;;;;gCAYY;;;UAIf,aAAA;;;;;;;;;;;;UAaA,iBAAA;;;;;;;;;;;;;;;;;;;UAoBA;;qBAEI,cAAc;oBACf,WAAW,GAAG;qBACb,WAAW,GAAG;;;UAIlB,iBAAA"}
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"sourcesContent":[],"mappings":";;;;;;;;;;;AAiD4C,UAnB3B,SAmB2B,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EAAV,GAAA,CAAA,KAAA,EAlBrB,CAkBqB,EAAA,GAKd,CALc,EAAA;IAAd,MAAA,CAAA,EAlBa,WAkBb;EAKA,CAAA,CAAA,EAvB6B,OAuB7B,CAvBqC,CAuBrC,CAAA;;;AAgBpB;AAaA;AAgCA;;AAEqB,UA9EJ,UA8EI,CAAA,CAAA,EAAA,CAAA,CAAA,CAAA;EACU,SAAA,KAAA,EA9Eb,SA8Ea,CA9EH,CA8EG,EA9EA,CA8EA,CAAA;EAAG,SAAA,OAAA,EAAA;IAAd,SAAA,KAAA,EA5EA,aA4EA,CAAA;MACY,SAAA,EAAA,CAAA,EAAA,MAAA;MAAG,SAAA,KAAA,EA3Eb,CA2Ea;MAAd,SAAA,QAAA,CAAA,EA1EK,CA0EL;MAAU,SAAA,QAAA,CAAA,EAzEL,QAyEK,CAzEI,MAyEJ,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA;IAId,CAAA,CAAA;;oBA1EG,cAAc,OAAO,GAAG;;;;;oBAKxB;;;;;;;;;;;;gCAYY;;;UAIf,aAAA;;;;;;;;;;;;UAaA,iBAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAgCA;;qBAEI,cAAc;oBACf,WAAW,GAAG;qBACb,WAAW,GAAG;;;UAIlB,iBAAA"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@graphorin/evals",
-  "version": "0.5.0",
-  "description": "Eval framework for the Graphorin framework. Ships scorer libraries (`code/` exact-match, regex, JSON path, predicates; `llm/` LLM-judge; `prebuilt/` toxicity / factuality / helpfulness), dataset loaders (JSONL, CSV, from-traces, generic iterable), reporters (terminal, markdown, JSON, JUnit XML), a parallel runner with `concurrency` config, and regression detection that compares the current run against a stored baseline. Builds on the eval primitives shipped from `@graphorin/observability` (RB-17 / DEC-152 — full orchestrator lives here, post-MVP, decoupled from the core observability bundle so consumers do not pay the dataset / reporter cost when only the inline runner is needed). Created and maintained by Oleksiy Stepurenko.",
+  "version": "0.6.1",
+  "description": "Eval framework for the Graphorin framework. Ships scorer libraries (`code/` exact-match, regex, JSON path, predicates; `llm/` LLM-judge; `prebuilt/` toxicity / factuality / helpfulness), dataset loaders (JSONL, CSV, from-traces, generic iterable), reporters (terminal, markdown, JSON, JUnit XML), a parallel runner with `concurrency` config, and regression detection that compares the current run against a stored baseline. Builds on the eval primitives shipped from `@graphorin/observability` (RB-17 / DEC-152 - full orchestrator lives here, post-MVP, decoupled from the core observability bundle so consumers do not pay the dataset / reporter cost when only the inline runner is needed). Created and maintained by Oleksiy Stepurenko.",
   "license": "MIT",
   "author": "Oleksiy Stepurenko",
   "homepage": "https://github.com/o-stepper/graphorin/tree/main/packages/evals",
@@ -64,8 +64,8 @@
     "LICENSE"
   ],
   "dependencies": {
-    "@graphorin/core": "0.5.0",
-    "@graphorin/observability": "0.5.0"
+    "@graphorin/core": "0.6.1",
+    "@graphorin/observability": "0.6.1"
   },
   "publishConfig": {
     "access": "public",