autor3search-typescript 0.1.0

package/dist/state/runnaming.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Shared naming constants and the `-tag`-inference helper, previously
+ * copied (with an identical comment explaining why) across `cmd-baseline.ts`,
+ * `cmd-eval.ts`, `cmd-stop.ts`, `cmd-status.ts` and `pipeline/eval.ts`.
+ *
+ * The duplication was a deliberate, documented choice during earlier tasks
+ * (a scheduling constraint: `cmd-baseline.ts` was already-shipped code a
+ * later task was not to modify) rather than an oversight. That constraint
+ * no longer applies -- per Ruling 44 ("eliminate duplication when you can,
+ * guard it only when you cannot"), these five copies are consolidated here.
+ * A future change to the branch-naming scheme or the worktree/frozen layout
+ * now cannot silently diverge between call sites by being made in one copy
+ * and forgotten in the others.
+ */
+/** Prefix `baseline` uses for the run branch it creates: `<prefix><tag>`. */
+export declare const BRANCH_PREFIX = "autor3search-typescript/";
+/** Directory name, under the per-run state dir, of the pinned baseline worktree. */
+export declare const WORKTREE_DIRNAME = "baseline-worktree";
+/** Directory name, under the per-run state dir, of the frozen test/bench snapshot. */
+export declare const FROZEN_DIRNAME = "frozen";
+/**
+ * `-tag`'s default: the tag encoded in the current run branch's name.
+ * Every command that accepts `-tag` falls back to this when it is omitted,
+ * so a human or agent already checked out on a run branch never has to
+ * repeat the tag on every invocation.
+ */
+export declare function inferTagFromBranch(repoRoot: string): Promise<string>;

package/dist/state/runnaming.js

@@ -0,0 +1,47 @@
+import { currentBranch } from '../gitx/git.js';
+/**
+ * Shared naming constants and the `-tag`-inference helper, previously
+ * copied (with an identical comment explaining why) across `cmd-baseline.ts`,
+ * `cmd-eval.ts`, `cmd-stop.ts`, `cmd-status.ts` and `pipeline/eval.ts`.
+ *
+ * The duplication was a deliberate, documented choice during earlier tasks
+ * (a scheduling constraint: `cmd-baseline.ts` was already-shipped code a
+ * later task was not to modify) rather than an oversight. That constraint
+ * no longer applies -- per Ruling 44 ("eliminate duplication when you can,
+ * guard it only when you cannot"), these five copies are consolidated here.
+ * A future change to the branch-naming scheme or the worktree/frozen layout
+ * now cannot silently diverge between call sites by being made in one copy
+ * and forgotten in the others.
+ */
+/** Prefix `baseline` uses for the run branch it creates: `<prefix><tag>`. */
+export const BRANCH_PREFIX = 'autor3search-typescript/';
+/** Directory name, under the per-run state dir, of the pinned baseline worktree. */
+export const WORKTREE_DIRNAME = 'baseline-worktree';
+/** Directory name, under the per-run state dir, of the frozen test/bench snapshot. */
+export const FROZEN_DIRNAME = 'frozen';
+/**
+ * `-tag`'s default: the tag encoded in the current run branch's name.
+ * Every command that accepts `-tag` falls back to this when it is omitted,
+ * so a human or agent already checked out on a run branch never has to
+ * repeat the tag on every invocation.
+ */
+export async function inferTagFromBranch(repoRoot) {
+    let branch;
+    try {
+        branch = await currentBranch(repoRoot);
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        throw new Error(`-tag was not given, and the current branch could not be determined: ${message}`);
+    }
+    if (!branch.startsWith(BRANCH_PREFIX)) {
+        throw new Error(`-tag was not given, and the current branch (${JSON.stringify(branch)}) is not a run branch ` +
+            `(expected "${BRANCH_PREFIX}<tag>"). Pass -tag <tag> explicitly.`);
+    }
+    const tag = branch.slice(BRANCH_PREFIX.length);
+    if (tag === '') {
+        throw new Error(`-tag was not given, and the current branch (${JSON.stringify(branch)}) names no tag`);
+    }
+    return tag;
+}
+//# sourceMappingURL=runnaming.js.map

package/dist/state/runnaming.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runnaming.js","sourceRoot":"","sources":["../../src/state/runnaming.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAE9C;;;;;;;;;;;;;GAaG;AAEH,6EAA6E;AAC7E,MAAM,CAAC,MAAM,aAAa,GAAG,0BAA0B,CAAA;AAEvD,oFAAoF;AACpF,MAAM,CAAC,MAAM,gBAAgB,GAAG,mBAAmB,CAAA;AAEnD,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAA;AAEtC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACvD,IAAI,MAAc,CAAA;IAClB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAA;IACxC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CAAC,uEAAuE,OAAO,EAAE,CAAC,CAAA;IACnG,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,+CAA+C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,wBAAwB;YAC3F,cAAc,aAAa,sCAAsC,CACpE,CAAA;IACH,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;IAC9C,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;IACxG,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC"}

package/dist/state/stop.d.ts

@@ -0,0 +1,28 @@
+export interface StopRequest {
+    requestedAt: string;
+    /** True when the user also wants the in-flight eval abandoned. */
+    force: boolean;
+}
+export declare function stopPath(dir: string): string;
+/**
+ * A human runs `stop` in another terminal while an eval may be mid-read of
+ * this exact file. Write via a temp file plus rename (atomic within a
+ * directory) so a reader never observes a half-written stop.json.
+ */
+export declare function requestStop(dir: string, force: boolean): Promise<void>;
+/**
+ * Returns null when no stop has been requested (ENOENT) or when stop.json
+ * can't be parsed. The parse case is genuinely expected here -- a human's
+ * `stop` process could be killed mid-write despite the rename above, or a
+ * reader could race a writer that has not finished flushing yet -- so it must
+ * never crash the eval.
+ *
+ * Any other read failure (EACCES, a different uid, a transient I/O error,
+ * ...) is a different fact entirely: the request may well exist and simply
+ * be unreadable. Silently reporting that as "no request" would mean a user
+ * runs `stop`, it succeeds, and the eval side never observes it -- no error,
+ * no log, nothing. So only ENOENT (plus a genuine parse failure) collapses
+ * to null; everything else propagates.
+ */
+export declare function readStop(dir: string): Promise<StopRequest | null>;
+export declare function clearStop(dir: string): Promise<void>;

package/dist/state/stop.js

@@ -0,0 +1,59 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+export function stopPath(dir) {
+    return path.join(dir, 'stop.json');
+}
+/**
+ * A human runs `stop` in another terminal while an eval may be mid-read of
+ * this exact file. Write via a temp file plus rename (atomic within a
+ * directory) so a reader never observes a half-written stop.json.
+ */
+export async function requestStop(dir, force) {
+    await mkdir(dir, { recursive: true });
+    const rec = { requestedAt: new Date().toISOString(), force };
+    const tmpPath = path.join(dir, `.stop.json.${process.pid}.${randomUUID()}.tmp`);
+    await writeFile(tmpPath, JSON.stringify(rec));
+    try {
+        await rename(tmpPath, stopPath(dir));
+    }
+    catch (err) {
+        await rm(tmpPath, { force: true });
+        throw err;
+    }
+}
+/**
+ * Returns null when no stop has been requested (ENOENT) or when stop.json
+ * can't be parsed. The parse case is genuinely expected here -- a human's
+ * `stop` process could be killed mid-write despite the rename above, or a
+ * reader could race a writer that has not finished flushing yet -- so it must
+ * never crash the eval.
+ *
+ * Any other read failure (EACCES, a different uid, a transient I/O error,
+ * ...) is a different fact entirely: the request may well exist and simply
+ * be unreadable. Silently reporting that as "no request" would mean a user
+ * runs `stop`, it succeeds, and the eval side never observes it -- no error,
+ * no log, nothing. So only ENOENT (plus a genuine parse failure) collapses
+ * to null; everything else propagates.
+ */
+export async function readStop(dir) {
+    let text;
+    try {
+        text = await readFile(stopPath(dir), 'utf8');
+    }
+    catch (err) {
+        if (err.code === 'ENOENT')
+            return null;
+        throw err;
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+}
+export async function clearStop(dir) {
+    await rm(stopPath(dir), { force: true });
+}
+//# sourceMappingURL=stop.js.map

package/dist/state/stop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stop.js","sourceRoot":"","sources":["../../src/state/stop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzE,OAAO,IAAI,MAAM,WAAW,CAAA;AAQ5B,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;AACpC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAW,EAAE,KAAc;IAC3D,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrC,MAAM,GAAG,GAAgB,EAAE,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,CAAA;IACzE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,MAAM,CAAC,CAAA;IAC/E,MAAM,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAA;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAClC,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAW;IACxC,IAAI,IAAY,CAAA;IAChB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAA;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAA;QACjE,MAAM,GAAG,CAAA;IACX,CAAC;IACD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAgB,CAAA;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW;IACzC,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;AAC1C,CAAC"}

package/dist/stats/delta.d.ts

@@ -0,0 +1,29 @@
+/**
+ * The significance threshold. Deliberately a constant and not a config key:
+ * it is the one number the optimizing agent must not be able to argue with.
+ */
+export declare const ALPHA = 0.05;
+/** One benchmark compared across the two sides. */
+export interface Delta {
+    name: string;
+    /** Median nanoseconds per operation on the baseline. */
+    baseNs: number;
+    /** Median nanoseconds per operation on the candidate. */
+    candNs: number;
+    /** Percent change; negative is faster. */
+    pctChange: number;
+    /** Two-sided p-value. */
+    p: number;
+    /** p < ALPHA, uncorrected. The honest statistic, not the KEEP threshold. */
+    significant: boolean;
+    /** Whether the exact test was used. */
+    exact: boolean;
+}
+export declare function compare(name: string, base: readonly number[], cand: readonly number[]): Delta;
+/**
+ * Compares every benchmark, requiring both sides to declare exactly the same
+ * set. A mismatch is an error rather than an intersection: silently comparing
+ * only the overlap would let a candidate drop a benchmark it regressed and
+ * still be scored.
+ */
+export declare function compareAll(base: ReadonlyMap<string, number[]>, cand: ReadonlyMap<string, number[]>): Delta[];

package/dist/stats/delta.js

@@ -0,0 +1,41 @@
+import { median } from './geomean.js';
+import { mannWhitneyU } from './mannwhitney.js';
+/**
+ * The significance threshold. Deliberately a constant and not a config key:
+ * it is the one number the optimizing agent must not be able to argue with.
+ */
+export const ALPHA = 0.05;
+export function compare(name, base, cand) {
+    const baseNs = median(base);
+    const candNs = median(cand);
+    const { p, exact } = mannWhitneyU(base, cand);
+    return {
+        name,
+        baseNs,
+        candNs,
+        pctChange: ((candNs - baseNs) / baseNs) * 100,
+        p,
+        significant: p < ALPHA,
+        exact,
+    };
+}
+/**
+ * Compares every benchmark, requiring both sides to declare exactly the same
+ * set. A mismatch is an error rather than an intersection: silently comparing
+ * only the overlap would let a candidate drop a benchmark it regressed and
+ * still be scored.
+ */
+export function compareAll(base, cand) {
+    const missing = [...base.keys()].filter((k) => !cand.has(k)).sort();
+    if (missing.length > 0) {
+        throw new Error(`candidate is missing benchmarks present in the baseline: ${missing.join(', ')}`);
+    }
+    const extra = [...cand.keys()].filter((k) => !base.has(k)).sort();
+    if (extra.length > 0) {
+        throw new Error(`candidate declares unexpected benchmarks absent from the baseline: ${extra.join(', ')}`);
+    }
+    return [...base.keys()]
+        .sort()
+        .map((name) => compare(name, base.get(name), cand.get(name)));
+}
+//# sourceMappingURL=delta.js.map

package/dist/stats/delta.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"delta.js","sourceRoot":"","sources":["../../src/stats/delta.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAE/C;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,CAAA;AAmBzB,MAAM,UAAU,OAAO,CACrB,IAAY,EACZ,IAAuB,EACvB,IAAuB;IAEvB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;IAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAC7C,OAAO;QACL,IAAI;QACJ,MAAM;QACN,MAAM;QACN,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG;QAC7C,CAAC;QACD,WAAW,EAAE,CAAC,GAAG,KAAK;QACtB,KAAK;KACN,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CACxB,IAAmC,EACnC,IAAmC;IAEnC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACnE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,4DAA4D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjF,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACjE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,sEAAsE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACzF,CAAA;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;SACpB,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAC,CAAA;AACnE,CAAC"}

package/dist/stats/geomean.d.ts

@@ -0,0 +1,4 @@
+/** Geometric mean. Used to combine per-benchmark ratios into one score. */
+export declare function geoMean(values: readonly number[]): number;
+/** Median. Does not modify the caller's array. */
+export declare function median(values: readonly number[]): number;

package/dist/stats/geomean.js

@@ -0,0 +1,21 @@
+/** Geometric mean. Used to combine per-benchmark ratios into one score. */
+export function geoMean(values) {
+    if (values.length === 0)
+        throw new Error('geoMean: empty value set');
+    let sumLog = 0;
+    for (const v of values) {
+        if (!(v > 0))
+            throw new Error(`geoMean: values must be positive, got ${v}`);
+        sumLog += Math.log(v);
+    }
+    return Math.exp(sumLog / values.length);
+}
+/** Median. Does not modify the caller's array. */
+export function median(values) {
+    if (values.length === 0)
+        throw new Error('median: empty value set');
+    const xs = values.slice().sort((a, b) => a - b);
+    const mid = xs.length >> 1;
+    return xs.length % 2 === 1 ? xs[mid] : (xs[mid - 1] + xs[mid]) / 2;
+}
+//# sourceMappingURL=geomean.js.map

package/dist/stats/geomean.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geomean.js","sourceRoot":"","sources":["../../src/stats/geomean.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,MAAM,UAAU,OAAO,CAAC,MAAyB;IAC/C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;IACpE,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,EAAE,CAAC,CAAA;QAC3E,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;AACzC,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,MAAM,CAAC,MAAyB;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;IACnE,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/C,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,CAAA;IAC1B,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,EAAE,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAA;AACvE,CAAC"}

package/dist/stats/mannwhitney.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Exact and approximate Mann-Whitney U test.
+ *
+ * Benchmark rounds give us small samples (default 10 per side), where the
+ * normal approximation is poor. For small n with no ties we enumerate the
+ * exact null distribution instead; that is also what makes the p-value floor
+ * in `minAchievableP` a real, reportable property rather than an estimate.
+ */
+export interface MannWhitneyResult {
+    /** The U statistic for the smaller of the two rank sums. */
+    u: number;
+    /** Two-sided p-value, in [0, 1]. */
+    p: number;
+    /** True when the exact null distribution was used. */
+    exact: boolean;
+}
+/**
+ * The smallest two-sided p-value the exact test can ever produce at these
+ * sample sizes: 2 / C(n1+n2, n1). No degree of separation between the samples
+ * can go below it. For equal sizes this is the 2/C(2n,n) floor that makes a
+ * Bonferroni-corrected KEEP unreachable once there are enough benchmarks.
+ */
+export declare function minAchievableP(n1: number, n2: number): number;
+/**
+ * Two-sided Mann-Whitney U test. Neither input is modified.
+ */
+export declare function mannWhitneyU(a: readonly number[], b: readonly number[]): MannWhitneyResult;

package/dist/stats/mannwhitney.js

@@ -0,0 +1,158 @@
+/**
+ * Exact and approximate Mann-Whitney U test.
+ *
+ * Benchmark rounds give us small samples (default 10 per side), where the
+ * normal approximation is poor. For small n with no ties we enumerate the
+ * exact null distribution instead; that is also what makes the p-value floor
+ * in `minAchievableP` a real, reportable property rather than an estimate.
+ */
+/**
+ * Largest n1*n2 for which we build the exact distribution.
+ *
+ * The DP table's total footprint grows much faster than n1*n2 itself: at
+ * n1=n2=50 (2_500) it holds 1,628,226 Float64 entries, about 12 MB; at
+ * n1=n2=100 (10_000) — a value `count` can reach with no upper bound in
+ * config, and one the README will tell users to raise on a noisy machine —
+ * it holds 25,512,701 entries, about 195 MB, once per benchmark per
+ * experiment. That is a plausible way to OOM a memory-constrained CI runner,
+ * not a theoretical edge case. Above this bound we fall back to the normal
+ * approximation, which is essentially exact once each sample exceeds ~50
+ * observations.
+ */
+const MAX_EXACT_CELLS = 2_500;
+/**
+ * Counts of the U statistic under the null hypothesis.
+ *
+ * table[u] is the number of arrangements of n1 and n2 observations giving
+ * statistic u, via c(i, j, u) = c(i-1, j, u-j) + c(i, j-1, u).
+ */
+function uCounts(n1, n2) {
+    const table = [];
+    for (let i = 0; i <= n1; i++) {
+        const row = [];
+        for (let j = 0; j <= n2; j++)
+            row.push(new Float64Array(i * j + 1));
+        table.push(row);
+    }
+    table[0][0][0] = 1;
+    for (let i = 0; i <= n1; i++) {
+        for (let j = 0; j <= n2; j++) {
+            if (i === 0 && j === 0)
+                continue;
+            const cur = table[i][j];
+            for (let u = 0; u <= i * j; u++) {
+                let v = 0;
+                if (i > 0 && u - j >= 0 && u - j <= (i - 1) * j)
+                    v += table[i - 1][j][u - j];
+                if (j > 0 && u <= i * (j - 1))
+                    v += table[i][j - 1][u];
+                cur[u] = v;
+            }
+        }
+    }
+    return table[n1][n2];
+}
+/** Binomial coefficient, exact for the sizes we use. */
+function binomial(n, k) {
+    let r = 1;
+    for (let i = 1; i <= k; i++)
+        r = (r * (n - k + i)) / i;
+    return Math.round(r);
+}
+/**
+ * The smallest two-sided p-value the exact test can ever produce at these
+ * sample sizes: 2 / C(n1+n2, n1). No degree of separation between the samples
+ * can go below it. For equal sizes this is the 2/C(2n,n) floor that makes a
+ * Bonferroni-corrected KEEP unreachable once there are enough benchmarks.
+ */
+export function minAchievableP(n1, n2) {
+    return Math.min(1, 2 / binomial(n1 + n2, n1));
+}
+/** Tie-averaged ranks of `values`, in the original element order. */
+function rank(values) {
+    const idx = values.map((v, i) => ({ v, i }));
+    idx.sort((a, b) => a.v - b.v);
+    const ranks = new Array(values.length);
+    let i = 0;
+    while (i < idx.length) {
+        let j = i;
+        while (j + 1 < idx.length && idx[j + 1].v === idx[i].v)
+            j++;
+        // Ranks are 1-based; tied entries share their average rank.
+        const avg = (i + j + 2) / 2;
+        for (let k = i; k <= j; k++)
+            ranks[idx[k].i] = avg;
+        i = j + 1;
+    }
+    return ranks;
+}
+function hasTies(values) {
+    return new Set(values).size !== values.length;
+}
+/** Normal CDF via the complementary error function. */
+function normalCdf(z) {
+    // Abramowitz & Stegun 7.1.26 applied to erf.
+    const sign = z < 0 ? -1 : 1;
+    const x = Math.abs(z) / Math.SQRT2;
+    const t = 1 / (1 + 0.3275911 * x);
+    const y = 1 -
+        ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t +
+            0.254829592) *
+            t *
+            Math.exp(-x * x);
+    return 0.5 * (1 + sign * y);
+}
+/**
+ * Two-sided Mann-Whitney U test. Neither input is modified.
+ */
+export function mannWhitneyU(a, b) {
+    if (a.length === 0 || b.length === 0) {
+        throw new Error('mannWhitneyU: each sample needs at least one observation');
+    }
+    // These slices are not what stops caller mutation today: rank() sorts a
+    // derived array of {v, i} pairs rather than `combined`, and `combined`
+    // (built with concat() below) is already a fresh array regardless of
+    // whether xa/xb are copies. The "does not mutate its inputs" guarantee
+    // currently comes entirely from rank()'s design. We keep these copies
+    // anyway — they are cheap, and they keep that guarantee true even if
+    // rank() is later changed to sort its argument in place.
+    const xa = a.slice();
+    const xb = b.slice();
+    const n1 = xa.length;
+    const n2 = xb.length;
+    const combined = xa.concat(xb);
+    const ranks = rank(combined);
+    let rankSumA = 0;
+    for (let i = 0; i < n1; i++)
+        rankSumA += ranks[i];
+    const uA = rankSumA - (n1 * (n1 + 1)) / 2;
+    const uB = n1 * n2 - uA;
+    const u = Math.min(uA, uB);
+    const ties = hasTies(combined);
+    if (!ties && n1 * n2 <= MAX_EXACT_CELLS) {
+        const counts = uCounts(n1, n2);
+        let atOrBelow = 0;
+        for (let k = 0; k <= u; k++)
+            atOrBelow += counts[k];
+        const total = binomial(n1 + n2, n1);
+        const p = Math.min(1, (2 * atOrBelow) / total);
+        return { u, p, exact: true };
+    }
+    // Normal approximation with a tie correction on the variance.
+    const mean = (n1 * n2) / 2;
+    const n = n1 + n2;
+    const counts = new Map();
+    for (const v of combined)
+        counts.set(v, (counts.get(v) ?? 0) + 1);
+    let tieTerm = 0;
+    for (const c of counts.values())
+        tieTerm += c * c * c - c;
+    const variance = ((n1 * n2) / 12) * (n + 1 - tieTerm / (n * (n - 1)));
+    if (variance <= 0)
+        return { u, p: 1, exact: false };
+    // Continuity correction.
+    const z = (Math.abs(u - mean) - 0.5) / Math.sqrt(variance);
+    const p = Math.min(1, 2 * (1 - normalCdf(z)));
+    return { u, p, exact: false };
+}
+//# sourceMappingURL=mannwhitney.js.map

package/dist/stats/mannwhitney.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mannwhitney.js","sourceRoot":"","sources":["../../src/stats/mannwhitney.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAWH;;;;;;;;;;;;GAYG;AACH,MAAM,eAAe,GAAG,KAAK,CAAA;AAE7B;;;;;GAKG;AACH,SAAS,OAAO,CAAC,EAAU,EAAE,EAAU;IACrC,MAAM,KAAK,GAAqB,EAAE,CAAA;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAmB,EAAE,CAAA;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACnE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACjB,CAAC;IACD,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAQ;YAChC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAE,CAAA;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACT,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC;oBAAE,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,CAAE,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;gBAC/E,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;oBAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,CAAE,CAAA;gBACzD,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YACZ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,EAAE,CAAE,CAAC,EAAE,CAAE,CAAA;AACxB,CAAC;AAED,wDAAwD;AACxD,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACtD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,EAAU,EAAE,EAAU;IACnD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;AAC/C,CAAC;AAED,qEAAqE;AACrE,SAAS,IAAI,CAAC,MAAyB;IACrC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAC5C,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7B,MAAM,KAAK,GAAG,IAAI,KAAK,CAAS,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9C,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG,CAAC,CAAA;QACT,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;YAAE,CAAC,EAAE,CAAA;QAC7D,4DAA4D;QAC5D,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAA;QACnD,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACX,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,OAAO,CAAC,MAAyB;IACxC,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,CAAA;AAC/C,CAAC;AAED,uDAAuD;AACvD,SAAS,SAAS,CAAC,CAAS;IAC1B,6CAA6C;IAC7C,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAA;IAClC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAA;IACjC,MAAM,CAAC,GACL,CAAC;QACD,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC;YAC1E,WAAW,CAAC;YACZ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACpB,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAA;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,CAAoB,EACpB,CAAoB;IAEpB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;IAC7E,CAAC;IACD,wEAAwE;IACxE,uEAAuE;IACvE,qEAAqE;IACrE,uEAAuE;IACvE,sEAAsE;IACtE,qEAAqE;IACrE,yDAAyD;IACzD,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAA;IACpB,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAA;IAEpB,MAAM,QAAQ,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5B,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE;QAAE,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAE,CAAA;IAElD,MAAM,EAAE,GAAG,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACzC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;IACvB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;IAE1B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC9B,IAAI,CAAC,IAAI,IAAI,EAAE,GAAG,EAAE,IAAI,eAAe,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QAC9B,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;YAAE,SAAS,IAAI,MAAM,CAAC,CAAC,CAAE,CAAA;QACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAA;QACnC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,CAAA;QAC9C,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IAC9B,CAAC;IAED,8DAA8D;IAC9D,MAAM,IAAI,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAA;IAC1B,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,CAAA;IACjB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACjE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACzD,MAAM,QAAQ,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,IAAI,QAAQ,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;IACnD,yBAAyB;IACzB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC1D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7C,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;AAC/B,CAAC"}

package/dist/verdict/verdict.d.ts

@@ -0,0 +1,46 @@
+import { type Delta } from '../stats/delta.js';
+export type Status = 'keep' | 'discard' | 'fail' | 'crash';
+export declare const EXIT_CODES: Record<Status, number>;
+export type DiscardReason = 
+/** Nothing measurably moved. */
+'no_significant_improvement'
+/** It really did speed things up, by less than min_effect_pct. */
+ | 'improvement_below_min_effect'
+/** At least one benchmark got significantly and materially worse. */
+ | 'significant_regression';
+export interface Verdict {
+    status: Status;
+    /** geomean of candidate/baseline time ratios. Below 1 is faster. */
+    score: number;
+    reason?: DiscardReason;
+    /** Names of benchmarks that triggered the regression guard. */
+    regressions: string[];
+    correctedAlpha: number;
+    /** Never changes the decision; explains when it cannot be read at face value. */
+    warnings: string[];
+}
+export interface VerdictInput {
+    deltas: Delta[];
+    minEffectPct: number;
+    maxRegressPct: number;
+    /** Measured rounds per side, for the sample-size warnings. */
+    rounds: number;
+}
+export declare function decide(input: VerdictInput): Verdict;
+/**
+ * The smallest per-side round count whose exact p-floor clears ALPHA/k, or
+ * `undefined` if no count up to `MAX_REQUIRED_COUNT` does.
+ *
+ * The brief's version returned the cap itself in that case, which reads as
+ * "raising count to 64 fixes it" when 64 in fact still cannot clear the
+ * bar — misleading advice for a caller that acts on it and then watches
+ * every experiment keep discarding anyway. Reporting "unreachable"
+ * explicitly instead lets the caller be told to reduce benchmark count
+ * instead of chasing a round count that was never going to help.
+ *
+ * Exported for testing: the case this guards against needs k on the order
+ * of 1e36 (floor(64, 64) is about 8.35e-38) to reach the cap for real,
+ * which is unreachable through `decide`'s own Delta[] input without
+ * constructing an array of that size.
+ */
+export declare function requiredCount(k: number): number | undefined;

package/dist/verdict/verdict.js

@@ -0,0 +1,102 @@
+import { geoMean } from '../stats/geomean.js';
+import { ALPHA } from '../stats/delta.js';
+import { minAchievableP } from '../stats/mannwhitney.js';
+export const EXIT_CODES = {
+    keep: 0,
+    discard: 1,
+    fail: 2,
+    crash: 3,
+};
+/** At 95% confidence the median's interval needs at least this many samples. */
+const MIN_ROUNDS_FOR_CI = 6;
+export function decide(input) {
+    const { deltas, minEffectPct, maxRegressPct, rounds } = input;
+    if (deltas.length === 0) {
+        throw new Error('cannot decide with no benchmarks compared');
+    }
+    // decide() must be total: its Delta[] input comes from the statistics
+    // layer, which this module does not control. Left unchecked, a zero or
+    // negative baseNs/candNs (division, or a 0/0 ratio) or an out-of-range p
+    // would either throw geoMean's generic "must be positive" error with no
+    // benchmark attached, or -- for a positive/zero ratio, which geoMean's
+    // check happily lets through -- silently produce an infinite score. Reject
+    // it explicitly here instead, naming the benchmark, so a corrupt upstream
+    // Delta is diagnosable at the boundary rather than three frames deep or
+    // not diagnosed at all.
+    for (const delta of deltas) {
+        if (!(delta.baseNs > 0) || !Number.isFinite(delta.baseNs)) {
+            throw new Error(`invalid delta for "${delta.name}": baseNs must be a positive finite number, got ${delta.baseNs}`);
+        }
+        if (!(delta.candNs > 0) || !Number.isFinite(delta.candNs)) {
+            throw new Error(`invalid delta for "${delta.name}": candNs must be a positive finite number, got ${delta.candNs}`);
+        }
+        if (!(delta.p >= 0 && delta.p <= 1)) {
+            throw new Error(`invalid delta for "${delta.name}": p must be a number in [0, 1], got ${delta.p}`);
+        }
+    }
+    const score = geoMean(deltas.map((d) => d.candNs / d.baseNs));
+    const k = deltas.length;
+    const correctedAlpha = ALPHA / k;
+    const warnings = [];
+    if (rounds < MIN_ROUNDS_FOR_CI) {
+        warnings.push(`only ${rounds} rounds per side: a 95% confidence interval for the median needs at least ` +
+            `${MIN_ROUNDS_FOR_CI} observations, so below that it is unbounded`);
+    }
+    const floor = minAchievableP(rounds, rounds);
+    if (floor > correctedAlpha) {
+        const suggestion = requiredCount(k);
+        const advice = suggestion === undefined
+            ? `No per-side round count up to ${MAX_REQUIRED_COUNT} would clear it either — reduce the ` +
+                `number of benchmarks compared instead.`
+            : `Raise count to at least ${suggestion}.`;
+        warnings.push(`no KEEP was reachable: with ${rounds} rounds per side the smallest attainable two-sided ` +
+            `p-value is ${floor.toExponential(2)}, above the Bonferroni-corrected threshold of ` +
+            `${correctedAlpha.toExponential(2)} for ${k} benchmarks — every experiment would discard ` +
+            `regardless of what changed. ${advice}`);
+    }
+    // Harm first, and deliberately at the UNCORRECTED alpha. The Bonferroni
+    // correction only ever makes significance harder to declare; applying it
+    // here would make real regressions easier to miss. Conservative about
+    // accepting a win, liberal about catching damage.
+    const regressions = deltas
+        .filter((d) => d.significant && d.pctChange > maxRegressPct)
+        .map((d) => d.name);
+    if (regressions.length > 0) {
+        return { status: 'discard', score, reason: 'significant_regression', regressions, correctedAlpha, warnings };
+    }
+    const anyCorrectedSignificantWin = deltas.some((d) => d.p < correctedAlpha && d.pctChange < 0);
+    if (!anyCorrectedSignificantWin) {
+        return { status: 'discard', score, reason: 'no_significant_improvement', regressions, correctedAlpha, warnings };
+    }
+    const threshold = 1 - minEffectPct / 100;
+    if (!(score < threshold)) {
+        return { status: 'discard', score, reason: 'improvement_below_min_effect', regressions, correctedAlpha, warnings };
+    }
+    return { status: 'keep', score, regressions, correctedAlpha, warnings };
+}
+/** Largest per-side round count `requiredCount` will search up to. */
+const MAX_REQUIRED_COUNT = 64;
+/**
+ * The smallest per-side round count whose exact p-floor clears ALPHA/k, or
+ * `undefined` if no count up to `MAX_REQUIRED_COUNT` does.
+ *
+ * The brief's version returned the cap itself in that case, which reads as
+ * "raising count to 64 fixes it" when 64 in fact still cannot clear the
+ * bar — misleading advice for a caller that acts on it and then watches
+ * every experiment keep discarding anyway. Reporting "unreachable"
+ * explicitly instead lets the caller be told to reduce benchmark count
+ * instead of chasing a round count that was never going to help.
+ *
+ * Exported for testing: the case this guards against needs k on the order
+ * of 1e36 (floor(64, 64) is about 8.35e-38) to reach the cap for real,
+ * which is unreachable through `decide`'s own Delta[] input without
+ * constructing an array of that size.
+ */
+export function requiredCount(k) {
+    for (let n = 4; n <= MAX_REQUIRED_COUNT; n++) {
+        if (minAchievableP(n, n) < ALPHA / k)
+            return n;
+    }
+    return undefined;
+}
+//# sourceMappingURL=verdict.js.map

package/dist/verdict/verdict.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict.js","sourceRoot":"","sources":["../../src/verdict/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC7C,OAAO,EAAE,KAAK,EAAc,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAIxD,MAAM,CAAC,MAAM,UAAU,GAA2B;IAChD,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC;IACV,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAA;AA8BD,gFAAgF;AAChF,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAE3B,MAAM,UAAU,MAAM,CAAC,KAAmB;IACxC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,KAAK,CAAA;IAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;IAC9D,CAAC;IAED,sEAAsE;IACtE,uEAAuE;IACvE,yEAAyE;IACzE,wEAAwE;IACxE,uEAAuE;IACvE,2EAA2E;IAC3E,0EAA0E;IAC1E,wEAAwE;IACxE,wBAAwB;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,CAAC,IAAI,mDAAmD,KAAK,CAAC,MAAM,EAAE,CAClG,CAAA;QACH,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CACb,sBAAsB,KAAK,CAAC,IAAI,mDAAmD,KAAK,CAAC,MAAM,EAAE,CAClG,CAAA;QACH,CAAC;QACD,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,IAAI,wCAAwC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAA;QACpG,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;IAC7D,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,MAAM,cAAc,GAAG,KAAK,GAAG,CAAC,CAAA;IAEhC,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,IAAI,MAAM,GAAG,iBAAiB,EAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,CACX,QAAQ,MAAM,4EAA4E;YACxF,GAAG,iBAAiB,8CAA8C,CACrE,CAAA;IACH,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5C,IAAI,KAAK,GAAG,cAAc,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,MAAM,GACV,UAAU,KAAK,SAAS;YACtB,CAAC,CAAC,iCAAiC,kBAAkB,sCAAsC;gBACzF,wCAAwC;YAC1C,CAAC,CAAC,2BAA2B,UAAU,GAAG,CAAA;QAC9C,QAAQ,CAAC,IAAI,CACX,+BAA+B,MAAM,qDAAqD;YACxF,cAAc,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,gDAAgD;YACpF,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,+CAA+C;YAC1F,+BAA+B,MAAM,EAAE,CAC1C,CAAA;IACH,CAAC;IAED,wEAAwE;IACxE,yEAAyE;IACzE,sEAAsE;IACtE,kDAAkD;IAClD,MAAM,WAAW,GAAG,MAAM;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,GAAG,aAAa,CAAC;SAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACrB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAA;IAC9G,CAAC;IAED,MAAM,0BAA0B,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;IAC9F,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAA;IAClH,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,GAAG,YAAY,GAAG,GAAG,CAAA;IACxC,IAAI,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,8BAA8B,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAA;IACpH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAA;AACzE,CAAC;AAED,sEAAsE;AACtE,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,aAAa,CAAC,CAAS;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,kBAAkB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,IAAI,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC;YAAE,OAAO,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC"}

package/dist/version.d.ts

@@ -0,0 +1,2 @@
+/** The published package version. Kept in step with package.json by hand. */
+export declare const VERSION = "0.1.0";

package/dist/version.js

@@ -0,0 +1,3 @@
+/** The published package version. Kept in step with package.json by hand. */
+export const VERSION = '0.1.0';
+//# sourceMappingURL=version.js.map