@namzu/sdk 62.12.0 → 62.14.0

package/dist/eval/bootstrap.d.ts

@@ -0,0 +1,122 @@
+/**
+ * Intervals for an eval, by resampling cases.
+ *
+ * A pass rate printed alone invites conclusions the data cannot carry: on 80
+ * tasks a difference of -1.25 points came with a 95% interval from -10.0 to
+ * +7.5, which says almost nothing. The interval here is a percentile
+ * bootstrap over CASES. A case is the unit that was sampled from the world, so
+ * a case is what gets resampled, and all of a case's repeats travel with it:
+ * repeats of one case are not independent evidence about the suite, and
+ * treating them as such would narrow the interval for free.
+ *
+ * Nothing in here reads the clock or an unseeded random source. The generator
+ * is a small deterministic one fed by a seed that the report records, so the
+ * same cases and the same seed give the same interval.
+ */
+/** Fewest cases an interval is computed for. Below it the report says why. */
+export declare const MIN_CASES_FOR_INTERVAL = 5;
+/** Resamples drawn per interval. */
+export declare const BOOTSTRAP_RESAMPLES = 2000;
+/** The seed used when none is given. A fixed number, so a report is reproducible. */
+export declare const DEFAULT_EVAL_SEED = 1;
+/** Coverage of every interval in this module. */
+export declare const INTERVAL_LEVEL = 0.95;
+/**
+ * What one case contributes: the sum of its judged runs' scores, how many of
+ * those runs passed, and how many were judged at all. `judged` is at least 1;
+ * a case with no judged run has nothing to say and is left out by the caller.
+ */
+export interface CaseTally {
+    readonly scoreSum: number;
+    readonly passes: number;
+    readonly judged: number;
+}
+/**
+ * An interval, or the reason there is none.
+ *
+ * `estimate` is the number the interval is around, and is present even when
+ * no interval is computed, so a reader always has the point value and the
+ * explanation for why it carries no bounds.
+ */
+export type Interval = {
+    readonly computed: true;
+    readonly estimate: number;
+    readonly low: number;
+    readonly high: number;
+} | {
+    readonly computed: false;
+    readonly estimate: number | null;
+    readonly reason: string;
+};
+/** The method and its settings, written next to the numbers so they can be reproduced. */
+export interface BootstrapMethod {
+    readonly kind: 'percentile bootstrap over cases, repeats kept together';
+    readonly level: typeof INTERVAL_LEVEL;
+    readonly resamples: number;
+    readonly seed: number;
+    /** Cases the interval was drawn over. */
+    readonly cases: number;
+    readonly minimumCases: number;
+}
+export interface CaseIntervals {
+    readonly method: BootstrapMethod;
+    /** Mean score over judged runs. */
+    readonly mean: Interval;
+    /** Passed runs over judged runs. */
+    readonly passRate: Interval;
+}
+/** The same for a paired difference, second side minus first. */
+export interface DifferenceIntervals {
+    readonly method: BootstrapMethod;
+    readonly mean: Interval;
+    readonly passRate: Interval;
+}
+/**
+ * A bad `--repeat` or `--seed`. A RangeError, so existing handlers still see
+ * one, and marked with a code so a CLI can tell it from a RangeError a suite's
+ * own code threw (and from one thrown by another copy of this class).
+ */
+export declare class EvalUsageError extends RangeError {
+    readonly code = "EVAL_USAGE";
+}
+/** True for an {@link EvalUsageError}, even one built by another copy of the SDK. */
+export declare function isEvalUsageError(err: unknown): err is EvalUsageError;
+/**
+ * Validate a seed: an integer that fits in 32 bits, unsigned.
+ * Thrown as an EvalUsageError so a CLI can turn it into a usage error.
+ */
+export declare function validateSeed(value: number | undefined): number;
+/**
+ * mulberry32: a 32-bit generator with one word of state. Not for secrets;
+ * its job is to be the same on every machine for a given seed.
+ */
+export declare function createSeededRandom(seed: number): () => number;
+export declare function pooledMeanAndRate(tallies: readonly CaseTally[], draw?: Int32Array): [number, number];
+/**
+ * 95% intervals for the mean score and the pass rate over a set of cases.
+ *
+ * Each resample draws as many cases as there are, with replacement, and pools
+ * their runs. Cases without a judged run must not be passed in.
+ */
+export declare function bootstrapCases(tallies: readonly CaseTally[], seed: number): CaseIntervals;
+/** One case seen from both sides of a comparison. */
+export interface PairedTally {
+    readonly a: CaseTally;
+    readonly b: CaseTally;
+}
+/**
+ * 95% intervals for the difference of two runs over the same cases, `b` minus
+ * `a`. A resample draws PAIRS, so the two sides of a case always travel
+ * together; that is what makes the interval a paired one, and it is much
+ * narrower than two separate intervals when the cases are the thing that
+ * varies.
+ */
+export declare function bootstrapPairedDifference(pairs: readonly PairedTally[], seed: number): DifferenceIntervals;
+/**
+ * Exact two-sided sign test on discordant pairs: under no difference each flip
+ * is equally likely to go either way, so the count going one way is
+ * Binomial(n, 1/2). Returns twice the smaller tail, capped at 1; with no flips
+ * at all the answer is 1.
+ */
+export declare function exactSignTestP(oneWay: number, otherWay: number): number;
+//# sourceMappingURL=bootstrap.d.ts.map

package/dist/eval/bootstrap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.d.ts","sourceRoot":"","sources":["../../src/eval/bootstrap.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8EAA8E;AAC9E,eAAO,MAAM,sBAAsB,IAAI,CAAA;AAEvC,oCAAoC;AACpC,eAAO,MAAM,mBAAmB,OAAO,CAAA;AAEvC,qFAAqF;AACrF,eAAO,MAAM,iBAAiB,IAAI,CAAA;AAElC,iDAAiD;AACjD,eAAO,MAAM,cAAc,OAAO,CAAA;AAElC;;;;GAIG;AACH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,GACjB;IACA,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACpB,GACD;IACA,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACtB,CAAA;AAEJ,0FAA0F;AAC1F,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,IAAI,EAAE,wDAAwD,CAAA;IACvE,QAAQ,CAAC,KAAK,EAAE,OAAO,cAAc,CAAA;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAC7B;AAED,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAChC,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,oCAAoC;IACpC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;CAC3B;AAED,iEAAiE;AACjE,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAA;IAChC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;CAC3B;AAED;;;;GAIG;AACH,qBAAa,cAAe,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,gBAAe;CAC5B;AAED,qFAAqF;AACrF,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,cAAc,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAQ9D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAS7D;AAyDD,wBAAgB,iBAAiB,CAChC,OAAO,EAAE,SAAS,SAAS,EAAE,EAC7B,IAAI,CAAC,EAAE,UAAU,GACf,CAAC,MAAM,EAAE,MAAM,CAAC,CAYlB;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CA6BzF;AAED,qDAAqD;AACrD,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAA;CACrB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACxC,KAAK,EAAE,SAAS,WAAW,EAAE,EAC7B,IAAI,EAAE,MAAM,GACV,mBAAmB,CAoCrB;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAgBvE"}

package/dist/eval/bootstrap.js

@@ -0,0 +1,227 @@
+/**
+ * Intervals for an eval, by resampling cases.
+ *
+ * A pass rate printed alone invites conclusions the data cannot carry: on 80
+ * tasks a difference of -1.25 points came with a 95% interval from -10.0 to
+ * +7.5, which says almost nothing. The interval here is a percentile
+ * bootstrap over CASES. A case is the unit that was sampled from the world, so
+ * a case is what gets resampled, and all of a case's repeats travel with it:
+ * repeats of one case are not independent evidence about the suite, and
+ * treating them as such would narrow the interval for free.
+ *
+ * Nothing in here reads the clock or an unseeded random source. The generator
+ * is a small deterministic one fed by a seed that the report records, so the
+ * same cases and the same seed give the same interval.
+ */
+/** Fewest cases an interval is computed for. Below it the report says why. */
+export const MIN_CASES_FOR_INTERVAL = 5;
+/** Resamples drawn per interval. */
+export const BOOTSTRAP_RESAMPLES = 2000;
+/** The seed used when none is given. A fixed number, so a report is reproducible. */
+export const DEFAULT_EVAL_SEED = 1;
+/** Coverage of every interval in this module. */
+export const INTERVAL_LEVEL = 0.95;
+/**
+ * A bad `--repeat` or `--seed`. A RangeError, so existing handlers still see
+ * one, and marked with a code so a CLI can tell it from a RangeError a suite's
+ * own code threw (and from one thrown by another copy of this class).
+ */
+export class EvalUsageError extends RangeError {
+    code = 'EVAL_USAGE';
+}
+/** True for an {@link EvalUsageError}, even one built by another copy of the SDK. */
+export function isEvalUsageError(err) {
+    return err instanceof RangeError && err.code === 'EVAL_USAGE';
+}
+/**
+ * Validate a seed: an integer that fits in 32 bits, unsigned.
+ * Thrown as an EvalUsageError so a CLI can turn it into a usage error.
+ */
+export function validateSeed(value) {
+    if (value === undefined)
+        return DEFAULT_EVAL_SEED;
+    if (!Number.isInteger(value) || value < 0 || value > 0xffffffff) {
+        throw new EvalUsageError(`seed must be an integer from 0 to 4294967295, or omitted; received ${String(value)}`);
+    }
+    return value;
+}
+/**
+ * mulberry32: a 32-bit generator with one word of state. Not for secrets;
+ * its job is to be the same on every machine for a given seed.
+ */
+export function createSeededRandom(seed) {
+    let state = validateSeed(seed) >>> 0;
+    return () => {
+        state = (state + 0x6d2b79f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+function quantile(sorted, p) {
+    const position = p * (sorted.length - 1);
+    const lower = Math.floor(position);
+    const upper = Math.ceil(position);
+    const low = sorted[lower];
+    const high = sorted[upper];
+    return low + (high - low) * (position - lower);
+}
+/**
+ * Draw `resamples` samples of `n` indices with replacement, hand each to
+ * `statistics`, and return the 2.5th and 97.5th percentiles of the two numbers
+ * it returns. Both statistics see the same draw, so the two intervals come
+ * from one set of resampled suites.
+ */
+function bootstrap(n, seed, statistics) {
+    const random = createSeededRandom(seed);
+    const draw = new Int32Array(n);
+    const means = new Float64Array(BOOTSTRAP_RESAMPLES);
+    const rates = new Float64Array(BOOTSTRAP_RESAMPLES);
+    for (let b = 0; b < BOOTSTRAP_RESAMPLES; b++) {
+        for (let i = 0; i < n; i++)
+            draw[i] = Math.floor(random() * n);
+        const [mean, rate] = statistics(draw);
+        means[b] = mean;
+        rates[b] = rate;
+    }
+    means.sort();
+    rates.sort();
+    const lowP = (1 - INTERVAL_LEVEL) / 2;
+    const highP = 1 - lowP;
+    return {
+        mean: [quantile(means, lowP), quantile(means, highP)],
+        passRate: [quantile(rates, lowP), quantile(rates, highP)],
+    };
+}
+function methodOf(seed, cases) {
+    return {
+        kind: 'percentile bootstrap over cases, repeats kept together',
+        level: INTERVAL_LEVEL,
+        resamples: BOOTSTRAP_RESAMPLES,
+        seed,
+        cases,
+        minimumCases: MIN_CASES_FOR_INTERVAL,
+    };
+}
+function tooFew(one, many, n) {
+    return `only ${n} ${n === 1 ? one : many}; an interval needs at least ${MIN_CASES_FOR_INTERVAL}`;
+}
+export function pooledMeanAndRate(tallies, draw) {
+    let score = 0;
+    let passes = 0;
+    let judged = 0;
+    const count = draw ? draw.length : tallies.length;
+    for (let i = 0; i < count; i++) {
+        const t = tallies[draw ? draw[i] : i];
+        score += t.scoreSum;
+        passes += t.passes;
+        judged += t.judged;
+    }
+    return judged === 0 ? [Number.NaN, Number.NaN] : [score / judged, passes / judged];
+}
+/**
+ * 95% intervals for the mean score and the pass rate over a set of cases.
+ *
+ * Each resample draws as many cases as there are, with replacement, and pools
+ * their runs. Cases without a judged run must not be passed in.
+ */
+export function bootstrapCases(tallies, seed) {
+    const checked = validateSeed(seed);
+    const n = tallies.length;
+    const method = methodOf(checked, n);
+    const [meanEstimate, rateEstimate] = n === 0 ? [null, null] : pooledMeanAndRate(tallies);
+    if (n < MIN_CASES_FOR_INTERVAL) {
+        const reason = tooFew('case with a judged run', 'cases with a judged run', n);
+        return {
+            method,
+            mean: { computed: false, estimate: meanEstimate, reason },
+            passRate: { computed: false, estimate: rateEstimate, reason },
+        };
+    }
+    const result = bootstrap(n, checked, (draw) => pooledMeanAndRate(tallies, draw));
+    return {
+        method,
+        mean: {
+            computed: true,
+            estimate: meanEstimate,
+            low: result.mean[0],
+            high: result.mean[1],
+        },
+        passRate: {
+            computed: true,
+            estimate: rateEstimate,
+            low: result.passRate[0],
+            high: result.passRate[1],
+        },
+    };
+}
+/**
+ * 95% intervals for the difference of two runs over the same cases, `b` minus
+ * `a`. A resample draws PAIRS, so the two sides of a case always travel
+ * together; that is what makes the interval a paired one, and it is much
+ * narrower than two separate intervals when the cases are the thing that
+ * varies.
+ */
+export function bootstrapPairedDifference(pairs, seed) {
+    const checked = validateSeed(seed);
+    const n = pairs.length;
+    const method = methodOf(checked, n);
+    const aSide = pairs.map((p) => p.a);
+    const bSide = pairs.map((p) => p.b);
+    const difference = (draw) => {
+        const [bMean, bRate] = pooledMeanAndRate(bSide, draw);
+        const [aMean, aRate] = pooledMeanAndRate(aSide, draw);
+        return [bMean - aMean, bRate - aRate];
+    };
+    const [meanEstimate, rateEstimate] = n === 0 ? [null, null] : difference();
+    if (n < MIN_CASES_FOR_INTERVAL) {
+        const reason = tooFew('paired case', 'paired cases', n);
+        return {
+            method,
+            mean: { computed: false, estimate: meanEstimate, reason },
+            passRate: { computed: false, estimate: rateEstimate, reason },
+        };
+    }
+    const result = bootstrap(n, checked, (draw) => difference(draw));
+    return {
+        method,
+        mean: {
+            computed: true,
+            estimate: meanEstimate,
+            low: result.mean[0],
+            high: result.mean[1],
+        },
+        passRate: {
+            computed: true,
+            estimate: rateEstimate,
+            low: result.passRate[0],
+            high: result.passRate[1],
+        },
+    };
+}
+/**
+ * Exact two-sided sign test on discordant pairs: under no difference each flip
+ * is equally likely to go either way, so the count going one way is
+ * Binomial(n, 1/2). Returns twice the smaller tail, capped at 1; with no flips
+ * at all the answer is 1.
+ */
+export function exactSignTestP(oneWay, otherWay) {
+    const n = oneWay + otherWay;
+    if (n === 0)
+        return 1;
+    const k = Math.min(oneWay, otherWay);
+    const logFactorial = new Float64Array(n + 1);
+    for (let i = 2; i <= n; i++)
+        logFactorial[i] = logFactorial[i - 1] + Math.log(i);
+    let tail = 0;
+    for (let i = 0; i <= k; i++) {
+        const logTerm = logFactorial[n] -
+            logFactorial[i] -
+            logFactorial[n - i] -
+            n * Math.LN2;
+        tail += Math.exp(logTerm);
+    }
+    return Math.min(1, 2 * tail);
+}
+//# sourceMappingURL=bootstrap.js.map

package/dist/eval/bootstrap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.js","sourceRoot":"","sources":["../../src/eval/bootstrap.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8EAA8E;AAC9E,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAEvC,oCAAoC;AACpC,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAA;AAEvC,qFAAqF;AACrF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAElC,iDAAiD;AACjD,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA;AA2DlC;;;;GAIG;AACH,MAAM,OAAO,cAAe,SAAQ,UAAU;IACpC,IAAI,GAAG,YAAY,CAAA;CAC5B;AAED,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC5C,OAAO,GAAG,YAAY,UAAU,IAAK,GAA0B,CAAC,IAAI,KAAK,YAAY,CAAA;AACtF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,KAAyB;IACrD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,iBAAiB,CAAA;IACjD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACjE,MAAM,IAAI,cAAc,CACvB,sEAAsE,MAAM,CAAC,KAAK,CAAC,EAAE,CACrF,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,IAAI,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACpC,OAAO,GAAG,EAAE;QACX,KAAK,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAClC,IAAI,CAAC,GAAG,KAAK,CAAA;QACb,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QACpC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAA;QACzC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAA;IAC7C,CAAC,CAAA;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,MAAoB,EAAE,CAAS;IAChD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAW,CAAA;IACnC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAW,CAAA;IACpC,OAAO,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAA;AAC/C,CAAC;AAED;;;;;GAKG;AACH,SAAS,SAAS,CACjB,CAAS,EACT,IAAY,EACZ,UAA2D;IAE3D,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC,CAAA;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,mBAAmB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;QAC9D,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;QACf,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;IAChB,CAAC;IACD,KAAK,CAAC,IAAI,EAAE,CAAA;IACZ,KAAK,CAAC,IAAI,EAAE,CAAA;IACZ,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAA;IACrC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAA;IACtB,OAAO;QACN,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,QAAQ,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;KACzD,CAAA;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC5C,OAAO;QACN,IAAI,EAAE,wDAAwD;QAC9D,KAAK,EAAE,cAAc;QACrB,SAAS,EAAE,mBAAmB;QAC9B,IAAI;QACJ,KAAK;QACL,YAAY,EAAE,sBAAsB;KACpC,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY,EAAE,CAAS;IACnD,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,gCAAgC,sBAAsB,EAAE,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,OAA6B,EAC7B,IAAiB;IAEjB,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAA;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,CAAY,CAAC,CAAC,CAAC,CAAC,CAAc,CAAA;QAC9D,KAAK,IAAI,CAAC,CAAC,QAAQ,CAAA;QACnB,MAAM,IAAI,CAAC,CAAC,MAAM,CAAA;QAClB,MAAM,IAAI,CAAC,CAAC,MAAM,CAAA;IACnB,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAA;AACnF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,OAA6B,EAAE,IAAY;IACzE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAA;IACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnC,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;IACxF,IAAI,CAAC,GAAG,sBAAsB,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,CAAC,CAAC,CAAA;QAC7E,OAAO;YACN,MAAM;YACN,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;YACzD,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;SAC7D,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;IAChF,OAAO;QACN,MAAM;QACN,IAAI,EAAE;YACL,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;SACpB;QACD,QAAQ,EAAE;YACT,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;SACxB;KACD,CAAA;AACF,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACxC,KAA6B,EAC7B,IAAY;IAEZ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAA;IACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACnC,MAAM,UAAU,GAAG,CAAC,IAAiB,EAAoB,EAAE;QAC1D,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACrD,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACrD,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAA;IACtC,CAAC,CAAA;IACD,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAA;IAC1E,IAAI,CAAC,GAAG,sBAAsB,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,EAAE,cAAc,EAAE,CAAC,CAAC,CAAA;QACvD,OAAO;YACN,MAAM;YACN,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;YACzD,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;SAC7D,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,MAAM;QACN,IAAI,EAAE;YACL,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;SACpB;QACD,QAAQ,EAAE;YACT,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;SACxB;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB;IAC9D,MAAM,CAAC,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC3B,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IACrB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,YAAY,CAAC,CAAC,CAAC,GAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAC5F,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GACX,YAAY,CAAC,CAAC,CAAY;YAC1B,YAAY,CAAC,CAAC,CAAY;YAC1B,YAAY,CAAC,CAAC,GAAG,CAAC,CAAY;YAC/B,CAAC,GAAG,IAAI,CAAC,GAAG,CAAA;QACb,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC1B,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;AAC7B,CAAC"}

package/dist/eval/compare.d.ts

@@ -0,0 +1,109 @@
+import { type DifferenceIntervals } from './bootstrap.js';
+import { type CaseRuns } from './report-stats.js';
+/**
+ * Two eval reports, side by side, paired by suite and case id.
+ *
+ * Two pass rates a few points apart read as a difference, and at the size of
+ * a hand-built suite they are usually not one. The comparison pairs the cases
+ * both sides ran, takes the difference over those, and puts an interval on it
+ * by resampling the pairs. It says "no difference can be claimed at this
+ * size" whenever that interval includes zero, and it says what else differs
+ * between the two runs, because a difference that comes with a new model and
+ * a new harness version and a new effort cannot be credited to any of them.
+ */
+/** One suite of a loaded report file. */
+export interface LoadedSuite {
+    readonly id: string;
+    readonly cases: readonly CaseRuns[];
+    readonly repeats: number;
+    /** Whatever the file said about how the suite ran; absent in an older file. */
+    readonly facts: Readonly<Record<string, unknown>> | undefined;
+}
+export interface LoadedReportFile {
+    readonly label: string;
+    readonly suites: readonly LoadedSuite[];
+}
+/**
+ * Read the JSON `namzu eval --out` writes. Tolerant of a file from an older
+ * version: only `suites[].suite` and each case's name, mean and status are
+ * required; repeats, splits and disclosure are read when they are there.
+ * Throws an Error whose message names what is wrong, for a person to read.
+ */
+export declare function loadEvalReportFile(json: unknown, label: string): LoadedReportFile;
+export type DisclosureDimension = 'harness' | 'suites' | 'model' | 'route' | 'effort' | 'repeats';
+export interface DisclosureDifference {
+    readonly dimension: DisclosureDimension;
+    readonly key: string;
+    readonly a: string;
+    readonly b: string;
+}
+export interface DisclosureUnknown {
+    readonly dimension: DisclosureDimension;
+    readonly key: string;
+    /** Which side does not say; the other side's value is given. */
+    readonly missingIn: 'a' | 'b';
+    readonly value: string;
+}
+export interface DisclosureComparison {
+    readonly differences: readonly DisclosureDifference[];
+    /** Facts one side states and the other does not. */
+    readonly unknown: readonly DisclosureUnknown[];
+    /** Dimensions in which at least one stated fact differs, in a fixed order. */
+    readonly differingDimensions: readonly DisclosureDimension[];
+    /** Set when more than one dimension differs. */
+    readonly warning?: string;
+}
+export interface CaseRef {
+    readonly suite: string;
+    readonly case: string;
+}
+export interface SideNumbers {
+    /** Over the paired cases only. */
+    readonly mean: number | null;
+    readonly passRate: number | null;
+}
+export interface Flips {
+    /** Cases that passed in every run on side B and not on side A. */
+    readonly onlyBPasses: number;
+    /** The reverse: passed in every run on side A and not on side B. */
+    readonly onlyAPasses: number;
+    readonly bothPass: number;
+    readonly bothFail: number;
+    /** Exact two-sided sign test on the two flip counts; 1 with no flips. */
+    readonly pValue: number;
+    readonly test: 'exact two-sided sign test on the cases that flipped';
+}
+export interface EvalComparison {
+    readonly seed: number;
+    readonly a: {
+        readonly label: string;
+        readonly cases: number;
+    };
+    readonly b: {
+        readonly label: string;
+        readonly cases: number;
+    };
+    readonly onlyInA: readonly CaseRef[];
+    readonly onlyInB: readonly CaseRef[];
+    /** Present on both sides but with no judged run on one of them. */
+    readonly notJudged: readonly CaseRef[];
+    readonly paired: number;
+    readonly sideA: SideNumbers;
+    readonly sideB: SideNumbers;
+    /** B minus A. */
+    readonly difference: DifferenceIntervals;
+    readonly flips: Flips;
+    readonly verdict: {
+        readonly claim: 'better' | 'worse' | 'none';
+        /** The one sentence. */
+        readonly sentence: string;
+        /** Why, in plain words. */
+        readonly detail: string;
+    };
+    readonly disclosure: DisclosureComparison;
+}
+export declare function compareEvalReports(a: LoadedReportFile, b: LoadedReportFile, options?: {
+    seed?: number;
+}): EvalComparison;
+export declare function formatEvalComparison(c: EvalComparison): string;
+//# sourceMappingURL=compare.d.ts.map

package/dist/eval/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,KAAK,mBAAmB,EAOxB,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,KAAK,QAAQ,EAAoD,MAAM,mBAAmB,CAAA;AAGnG;;;;;;;;;;GAUG;AAEH,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CACvC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAoCjF;AAID,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAgFjG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,GAAG,GAAG,GAAG,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CACzB;AA2CD,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC;AAED,MAAM,WAAW,KAAK;IACrB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,qDAAqD,CAAA;CACpE;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,SAAS,OAAO,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,iBAAiB;IACjB,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAA;IACxC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;QAC3C,wBAAwB;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;QACzB,2BAA2B;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACvB,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAA;CACzC;AAaD,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,gBAAgB,EACnB,CAAC,EAAE,gBAAgB,EACnB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,cAAc,CAgFhB;AAyDD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAqC9D"}