@namzu/sdk 62.12.0 → 62.13.0

package/dist/eval/bootstrap.js

@@ -0,0 +1,227 @@
+/**
+ * Intervals for an eval, by resampling cases.
+ *
+ * A pass rate printed alone invites conclusions the data cannot carry: on 80
+ * tasks a difference of -1.25 points came with a 95% interval from -10.0 to
+ * +7.5, which says almost nothing. The interval here is a percentile
+ * bootstrap over CASES. A case is the unit that was sampled from the world, so
+ * a case is what gets resampled, and all of a case's repeats travel with it:
+ * repeats of one case are not independent evidence about the suite, and
+ * treating them as such would narrow the interval for free.
+ *
+ * Nothing in here reads the clock or an unseeded random source. The generator
+ * is a small deterministic one fed by a seed that the report records, so the
+ * same cases and the same seed give the same interval.
+ */
+/** Fewest cases an interval is computed for. Below it the report says why. */
+export const MIN_CASES_FOR_INTERVAL = 5;
+/** Resamples drawn per interval. */
+export const BOOTSTRAP_RESAMPLES = 2000;
+/** The seed used when none is given. A fixed number, so a report is reproducible. */
+export const DEFAULT_EVAL_SEED = 1;
+/** Coverage of every interval in this module. */
+export const INTERVAL_LEVEL = 0.95;
+/**
+ * A bad `--repeat` or `--seed`. A RangeError, so existing handlers still see
+ * one, and marked with a code so a CLI can tell it from a RangeError a suite's
+ * own code threw (and from one thrown by another copy of this class).
+ */
+export class EvalUsageError extends RangeError {
+    code = 'EVAL_USAGE';
+}
+/** True for an {@link EvalUsageError}, even one built by another copy of the SDK. */
+export function isEvalUsageError(err) {
+    return err instanceof RangeError && err.code === 'EVAL_USAGE';
+}
+/**
+ * Validate a seed: an integer that fits in 32 bits, unsigned.
+ * Thrown as an EvalUsageError so a CLI can turn it into a usage error.
+ */
+export function validateSeed(value) {
+    if (value === undefined)
+        return DEFAULT_EVAL_SEED;
+    if (!Number.isInteger(value) || value < 0 || value > 0xffffffff) {
+        throw new EvalUsageError(`seed must be an integer from 0 to 4294967295, or omitted; received ${String(value)}`);
+    }
+    return value;
+}
+/**
+ * mulberry32: a 32-bit generator with one word of state. Not for secrets;
+ * its job is to be the same on every machine for a given seed.
+ */
+export function createSeededRandom(seed) {
+    let state = validateSeed(seed) >>> 0;
+    return () => {
+        state = (state + 0x6d2b79f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+function quantile(sorted, p) {
+    const position = p * (sorted.length - 1);
+    const lower = Math.floor(position);
+    const upper = Math.ceil(position);
+    const low = sorted[lower];
+    const high = sorted[upper];
+    return low + (high - low) * (position - lower);
+}
+/**
+ * Draw `resamples` samples of `n` indices with replacement, hand each to
+ * `statistics`, and return the 2.5th and 97.5th percentiles of the two numbers
+ * it returns. Both statistics see the same draw, so the two intervals come
+ * from one set of resampled suites.
+ */
+function bootstrap(n, seed, statistics) {
+    const random = createSeededRandom(seed);
+    const draw = new Int32Array(n);
+    const means = new Float64Array(BOOTSTRAP_RESAMPLES);
+    const rates = new Float64Array(BOOTSTRAP_RESAMPLES);
+    for (let b = 0; b < BOOTSTRAP_RESAMPLES; b++) {
+        for (let i = 0; i < n; i++)
+            draw[i] = Math.floor(random() * n);
+        const [mean, rate] = statistics(draw);
+        means[b] = mean;
+        rates[b] = rate;
+    }
+    means.sort();
+    rates.sort();
+    const lowP = (1 - INTERVAL_LEVEL) / 2;
+    const highP = 1 - lowP;
+    return {
+        mean: [quantile(means, lowP), quantile(means, highP)],
+        passRate: [quantile(rates, lowP), quantile(rates, highP)],
+    };
+}
+function methodOf(seed, cases) {
+    return {
+        kind: 'percentile bootstrap over cases, repeats kept together',
+        level: INTERVAL_LEVEL,
+        resamples: BOOTSTRAP_RESAMPLES,
+        seed,
+        cases,
+        minimumCases: MIN_CASES_FOR_INTERVAL,
+    };
+}
+function tooFew(one, many, n) {
+    return `only ${n} ${n === 1 ? one : many}; an interval needs at least ${MIN_CASES_FOR_INTERVAL}`;
+}
+export function pooledMeanAndRate(tallies, draw) {
+    let score = 0;
+    let passes = 0;
+    let judged = 0;
+    const count = draw ? draw.length : tallies.length;
+    for (let i = 0; i < count; i++) {
+        const t = tallies[draw ? draw[i] : i];
+        score += t.scoreSum;
+        passes += t.passes;
+        judged += t.judged;
+    }
+    return judged === 0 ? [Number.NaN, Number.NaN] : [score / judged, passes / judged];
+}
+/**
+ * 95% intervals for the mean score and the pass rate over a set of cases.
+ *
+ * Each resample draws as many cases as there are, with replacement, and pools
+ * their runs. Cases without a judged run must not be passed in.
+ */
+export function bootstrapCases(tallies, seed) {
+    const checked = validateSeed(seed);
+    const n = tallies.length;
+    const method = methodOf(checked, n);
+    const [meanEstimate, rateEstimate] = n === 0 ? [null, null] : pooledMeanAndRate(tallies);
+    if (n < MIN_CASES_FOR_INTERVAL) {
+        const reason = tooFew('case with a judged run', 'cases with a judged run', n);
+        return {
+            method,
+            mean: { computed: false, estimate: meanEstimate, reason },
+            passRate: { computed: false, estimate: rateEstimate, reason },
+        };
+    }
+    const result = bootstrap(n, checked, (draw) => pooledMeanAndRate(tallies, draw));
+    return {
+        method,
+        mean: {
+            computed: true,
+            estimate: meanEstimate,
+            low: result.mean[0],
+            high: result.mean[1],
+        },
+        passRate: {
+            computed: true,
+            estimate: rateEstimate,
+            low: result.passRate[0],
+            high: result.passRate[1],
+        },
+    };
+}
+/**
+ * 95% intervals for the difference of two runs over the same cases, `b` minus
+ * `a`. A resample draws PAIRS, so the two sides of a case always travel
+ * together; that is what makes the interval a paired one, and it is much
+ * narrower than two separate intervals when the cases are the thing that
+ * varies.
+ */
+export function bootstrapPairedDifference(pairs, seed) {
+    const checked = validateSeed(seed);
+    const n = pairs.length;
+    const method = methodOf(checked, n);
+    const aSide = pairs.map((p) => p.a);
+    const bSide = pairs.map((p) => p.b);
+    const difference = (draw) => {
+        const [bMean, bRate] = pooledMeanAndRate(bSide, draw);
+        const [aMean, aRate] = pooledMeanAndRate(aSide, draw);
+        return [bMean - aMean, bRate - aRate];
+    };
+    const [meanEstimate, rateEstimate] = n === 0 ? [null, null] : difference();
+    if (n < MIN_CASES_FOR_INTERVAL) {
+        const reason = tooFew('paired case', 'paired cases', n);
+        return {
+            method,
+            mean: { computed: false, estimate: meanEstimate, reason },
+            passRate: { computed: false, estimate: rateEstimate, reason },
+        };
+    }
+    const result = bootstrap(n, checked, (draw) => difference(draw));
+    return {
+        method,
+        mean: {
+            computed: true,
+            estimate: meanEstimate,
+            low: result.mean[0],
+            high: result.mean[1],
+        },
+        passRate: {
+            computed: true,
+            estimate: rateEstimate,
+            low: result.passRate[0],
+            high: result.passRate[1],
+        },
+    };
+}
+/**
+ * Exact two-sided sign test on discordant pairs: under no difference each flip
+ * is equally likely to go either way, so the count going one way is
+ * Binomial(n, 1/2). Returns twice the smaller tail, capped at 1; with no flips
+ * at all the answer is 1.
+ */
+export function exactSignTestP(oneWay, otherWay) {
+    const n = oneWay + otherWay;
+    if (n === 0)
+        return 1;
+    const k = Math.min(oneWay, otherWay);
+    const logFactorial = new Float64Array(n + 1);
+    for (let i = 2; i <= n; i++)
+        logFactorial[i] = logFactorial[i - 1] + Math.log(i);
+    let tail = 0;
+    for (let i = 0; i <= k; i++) {
+        const logTerm = logFactorial[n] -
+            logFactorial[i] -
+            logFactorial[n - i] -
+            n * Math.LN2;
+        tail += Math.exp(logTerm);
+    }
+    return Math.min(1, 2 * tail);
+}
+//# sourceMappingURL=bootstrap.js.map

package/dist/eval/bootstrap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.js","sourceRoot":"","sources":["../../src/eval/bootstrap.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8EAA8E;AAC9E,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAEvC,oCAAoC;AACpC,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAA;AAEvC,qFAAqF;AACrF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAElC,iDAAiD;AACjD,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA;AA2DlC;;;;GAIG;AACH,MAAM,OAAO,cAAe,SAAQ,UAAU;IACpC,IAAI,GAAG,YAAY,CAAA;CAC5B;AAED,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC5C,OAAO,GAAG,YAAY,UAAU,IAAK,GAA0B,CAAC,IAAI,KAAK,YAAY,CAAA;AACtF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,KAAyB;IACrD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,iBAAiB,CAAA;IACjD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACjE,MAAM,IAAI,cAAc,CACvB,sEAAsE,MAAM,CAAC,KAAK,CAAC,EAAE,CACrF,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,IAAI,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACpC,OAAO,GAAG,EAAE;QACX,KAAK,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAClC,IAAI,CAAC,GAAG,KAAK,CAAA;QACb,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QACpC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAA;QACzC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAA;IAC7C,CAAC,CAAA;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,MAAoB,EAAE,CAAS;IAChD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAW,CAAA;IACnC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAW,CAAA;IACpC,OAAO,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAA;AAC/C,CAAC;AAED;;;;;GAKG;AACH,SAAS,SAAS,CACjB,CAAS,EACT,IAAY,EACZ,UAA2D;IAE3D,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,mBAAmB,CAAC,CAAA;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,mBAAmB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;QAC9D,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;QACf,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAA;IAChB,CAAC;IACD,KAAK,CAAC,IAAI,EAAE,CAAA;IACZ,KAAK,CAAC,IAAI,EAAE,CAAA;IACZ,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAA;IACrC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAA;IACtB,OAAO;QACN,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrD,QAAQ,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;KACzD,CAAA;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC5C,OAAO;QACN,IAAI,EAAE,wDAAwD;QAC9D,KAAK,EAAE,cAAc;QACrB,SAAS,EAAE,mBAAmB;QAC9B,IAAI;QACJ,KAAK;QACL,YAAY,EAAE,sBAAsB;KACpC,CAAA;AACF,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY,EAAE,CAAS;IACnD,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,gCAAgC,sBAAsB,EAAE,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,OAA6B,EAC7B,IAAiB;IAEjB,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAA;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,CAAC,CAAY,CAAC,CAAC,CAAC,CAAC,CAAc,CAAA;QAC9D,KAAK,IAAI,CAAC,CAAC,QAAQ,CAAA;QACnB,MAAM,IAAI,CAAC,CAAC,MAAM,CAAA;QAClB,MAAM,IAAI,CAAC,CAAC,MAAM,CAAA;IACnB,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAA;AACnF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,OAA6B,EAAE,IAAY;IACzE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAA;IACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnC,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;IACxF,IAAI,CAAC,GAAG,sBAAsB,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,wBAAwB,EAAE,yBAAyB,EAAE,CAAC,CAAC,CAAA;QAC7E,OAAO;YACN,MAAM;YACN,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;YACzD,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;SAC7D,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;IAChF,OAAO;QACN,MAAM;QACN,IAAI,EAAE;YACL,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;SACpB;QACD,QAAQ,EAAE;YACT,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;SACxB;KACD,CAAA;AACF,CAAC;AAQD;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACxC,KAA6B,EAC7B,IAAY;IAEZ,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAA;IACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACnC,MAAM,UAAU,GAAG,CAAC,IAAiB,EAAoB,EAAE;QAC1D,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACrD,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACrD,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAA;IACtC,CAAC,CAAA;IACD,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAA;IAC1E,IAAI,CAAC,GAAG,sBAAsB,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,EAAE,cAAc,EAAE,CAAC,CAAC,CAAA;QACvD,OAAO;YACN,MAAM;YACN,IAAI,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;YACzD,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE;SAC7D,CAAA;IACF,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,MAAM;QACN,IAAI,EAAE;YACL,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;SACpB;QACD,QAAQ,EAAE;YACT,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,YAAsB;YAChC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACvB,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;SACxB;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB;IAC9D,MAAM,CAAC,GAAG,MAAM,GAAG,QAAQ,CAAA;IAC3B,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IACrB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,YAAY,CAAC,CAAC,CAAC,GAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAC5F,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GACX,YAAY,CAAC,CAAC,CAAY;YAC1B,YAAY,CAAC,CAAC,CAAY;YAC1B,YAAY,CAAC,CAAC,GAAG,CAAC,CAAY;YAC/B,CAAC,GAAG,IAAI,CAAC,GAAG,CAAA;QACb,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAC1B,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;AAC7B,CAAC"}

package/dist/eval/compare.d.ts

@@ -0,0 +1,109 @@
+import { type DifferenceIntervals } from './bootstrap.js';
+import { type CaseRuns } from './report-stats.js';
+/**
+ * Two eval reports, side by side, paired by suite and case id.
+ *
+ * Two pass rates a few points apart read as a difference, and at the size of
+ * a hand-built suite they are usually not one. The comparison pairs the cases
+ * both sides ran, takes the difference over those, and puts an interval on it
+ * by resampling the pairs. It says "no difference can be claimed at this
+ * size" whenever that interval includes zero, and it says what else differs
+ * between the two runs, because a difference that comes with a new model and
+ * a new harness version and a new effort cannot be credited to any of them.
+ */
+/** One suite of a loaded report file. */
+export interface LoadedSuite {
+    readonly id: string;
+    readonly cases: readonly CaseRuns[];
+    readonly repeats: number;
+    /** Whatever the file said about how the suite ran; absent in an older file. */
+    readonly facts: Readonly<Record<string, unknown>> | undefined;
+}
+export interface LoadedReportFile {
+    readonly label: string;
+    readonly suites: readonly LoadedSuite[];
+}
+/**
+ * Read the JSON `namzu eval --out` writes. Tolerant of a file from an older
+ * version: only `suites[].suite` and each case's name, mean and status are
+ * required; repeats, splits and disclosure are read when they are there.
+ * Throws an Error whose message names what is wrong, for a person to read.
+ */
+export declare function loadEvalReportFile(json: unknown, label: string): LoadedReportFile;
+export type DisclosureDimension = 'harness' | 'suites' | 'model' | 'route' | 'effort' | 'repeats';
+export interface DisclosureDifference {
+    readonly dimension: DisclosureDimension;
+    readonly key: string;
+    readonly a: string;
+    readonly b: string;
+}
+export interface DisclosureUnknown {
+    readonly dimension: DisclosureDimension;
+    readonly key: string;
+    /** Which side does not say; the other side's value is given. */
+    readonly missingIn: 'a' | 'b';
+    readonly value: string;
+}
+export interface DisclosureComparison {
+    readonly differences: readonly DisclosureDifference[];
+    /** Facts one side states and the other does not. */
+    readonly unknown: readonly DisclosureUnknown[];
+    /** Dimensions in which at least one stated fact differs, in a fixed order. */
+    readonly differingDimensions: readonly DisclosureDimension[];
+    /** Set when more than one dimension differs. */
+    readonly warning?: string;
+}
+export interface CaseRef {
+    readonly suite: string;
+    readonly case: string;
+}
+export interface SideNumbers {
+    /** Over the paired cases only. */
+    readonly mean: number | null;
+    readonly passRate: number | null;
+}
+export interface Flips {
+    /** Cases that passed in every run on side B and not on side A. */
+    readonly onlyBPasses: number;
+    /** The reverse: passed in every run on side A and not on side B. */
+    readonly onlyAPasses: number;
+    readonly bothPass: number;
+    readonly bothFail: number;
+    /** Exact two-sided sign test on the two flip counts; 1 with no flips. */
+    readonly pValue: number;
+    readonly test: 'exact two-sided sign test on the cases that flipped';
+}
+export interface EvalComparison {
+    readonly seed: number;
+    readonly a: {
+        readonly label: string;
+        readonly cases: number;
+    };
+    readonly b: {
+        readonly label: string;
+        readonly cases: number;
+    };
+    readonly onlyInA: readonly CaseRef[];
+    readonly onlyInB: readonly CaseRef[];
+    /** Present on both sides but with no judged run on one of them. */
+    readonly notJudged: readonly CaseRef[];
+    readonly paired: number;
+    readonly sideA: SideNumbers;
+    readonly sideB: SideNumbers;
+    /** B minus A. */
+    readonly difference: DifferenceIntervals;
+    readonly flips: Flips;
+    readonly verdict: {
+        readonly claim: 'better' | 'worse' | 'none';
+        /** The one sentence. */
+        readonly sentence: string;
+        /** Why, in plain words. */
+        readonly detail: string;
+    };
+    readonly disclosure: DisclosureComparison;
+}
+export declare function compareEvalReports(a: LoadedReportFile, b: LoadedReportFile, options?: {
+    seed?: number;
+}): EvalComparison;
+export declare function formatEvalComparison(c: EvalComparison): string;
+//# sourceMappingURL=compare.d.ts.map

package/dist/eval/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/eval/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,KAAK,mBAAmB,EAOxB,MAAM,gBAAgB,CAAA;AAEvB,OAAO,EAAE,KAAK,QAAQ,EAAoD,MAAM,mBAAmB,CAAA;AAGnG;;;;;;;;;;GAUG;AAEH,yCAAyC;AACzC,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAA;IACnC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAA;CAC7D;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAA;CACvC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAoCjF;AAID,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAA;AAgFjG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAA;IACvC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,GAAG,GAAG,GAAG,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACrD,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC9C,8EAA8E;IAC9E,QAAQ,CAAC,mBAAmB,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAC5D,gDAAgD;IAChD,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CACzB;AA2CD,MAAM,WAAW,OAAO;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,kCAAkC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC;AAED,MAAM,WAAW,KAAK;IACrB,kEAAkE;IAClE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,qDAAqD,CAAA;CACpE;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,CAAC,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC9D,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,QAAQ,CAAC,OAAO,EAAE,SAAS,OAAO,EAAE,CAAA;IACpC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,SAAS,OAAO,EAAE,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,iBAAiB;IACjB,QAAQ,CAAC,UAAU,EAAE,mBAAmB,CAAA;IACxC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;IACrB,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;QAC3C,wBAAwB;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;QACzB,2BAA2B;QAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACvB,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAA;CACzC;AAaD,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,gBAAgB,EACnB,CAAC,EAAE,gBAAgB,EACnB,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,cAAc,CAgFhB;AAyDD,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAqC9D"}

package/dist/eval/compare.js

@@ -0,0 +1,324 @@
+import { bootstrapPairedDifference, exactSignTestP, pooledMeanAndRate, validateSeed, } from './bootstrap.js';
+import { UNKNOWN } from './disclosure.js';
+import { duplicateRun, groupRuns, passedEveryRun, tallyOf } from './report-stats.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * Read the JSON `namzu eval --out` writes. Tolerant of a file from an older
+ * version: only `suites[].suite` and each case's name, mean and status are
+ * required; repeats, splits and disclosure are read when they are there.
+ * Throws an Error whose message names what is wrong, for a person to read.
+ */
+export function loadEvalReportFile(json, label) {
+    if (!isRecord(json) || !Array.isArray(json.suites)) {
+        throw new Error(`${label} is not an eval report: it has no "suites" list.`);
+    }
+    const suites = [];
+    for (const [index, entry] of json.suites.entries()) {
+        if (!isRecord(entry) || typeof entry.suite !== 'string' || !isRecord(entry.report)) {
+            throw new Error(`${label}: suites[${index}] needs a "suite" id and a "report".`);
+        }
+        const report = entry.report;
+        if (!Array.isArray(report.cases)) {
+            throw new Error(`${label}: suite "${entry.suite}" has no "cases" list.`);
+        }
+        const runs = [];
+        for (const [caseIndex, run] of report.cases.entries()) {
+            if (!isRecord(run) || typeof run.case !== 'string' || !Number.isFinite(run.mean)) {
+                throw new Error(`${label}: suite "${entry.suite}", case ${caseIndex} needs a name in "case" and a finite number in "mean".`);
+            }
+            runs.push(run);
+        }
+        const twice = duplicateRun(runs);
+        if (twice !== undefined) {
+            throw new Error(`${label}: suite "${entry.suite}" has the case "${twice}" more than once, so its runs cannot be told from one case's repeats. A case is named by its id; the report needs each case once per repeat.`);
+        }
+        suites.push({
+            id: entry.suite,
+            cases: groupRuns(runs),
+            repeats: typeof report.repeats === 'number' ? report.repeats : 1,
+            facts: isRecord(report.disclosure) ? report.disclosure : undefined,
+        });
+    }
+    return { label, suites };
+}
+const DIMENSION_ORDER = [
+    'harness',
+    'suites',
+    'model',
+    'route',
+    'effort',
+    'repeats',
+];
+function known(value) {
+    return typeof value === 'string' && value !== UNKNOWN && value.length > 0 ? value : null;
+}
+function canonical(value) {
+    if (value === undefined || value === null || value === UNKNOWN)
+        return null;
+    if (typeof value === 'string')
+        return value;
+    return JSON.stringify(value, (_key, v) => isRecord(v)
+        ? Object.fromEntries(Object.entries(v).sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0)))
+        : v);
+}
+function rolesFact(value) {
+    if (!isRecord(value))
+        return null;
+    const parts = [];
+    for (const role of Object.keys(value).sort()) {
+        const who = value[role];
+        const provider = isRecord(who) ? known(who.provider) : null;
+        const model = isRecord(who) ? known(who.model) : null;
+        if (provider === null || model === null)
+            return null;
+        parts.push(`${role}=${provider}/${model}`);
+    }
+    return parts.length === 0 ? 'no model' : parts.join('; ');
+}
+function firstKnown(suites, field) {
+    for (const suite of suites) {
+        const value = known(suite.facts?.[field]);
+        if (value !== null)
+            return value;
+    }
+    return null;
+}
+function factsOf(file, suiteIds) {
+    const facts = [
+        {
+            dimension: 'harness',
+            key: 'harness version',
+            value: firstKnown(file.suites, 'harnessVersion'),
+        },
+        { dimension: 'harness', key: 'sdk version', value: firstKnown(file.suites, 'sdkVersion') },
+        { dimension: 'harness', key: 'commit', value: firstKnown(file.suites, 'commit') },
+    ];
+    for (const id of suiteIds) {
+        const suite = file.suites.find((s) => s.id === id);
+        if (!suite)
+            continue;
+        const f = suite.facts;
+        facts.push({ dimension: 'suites', key: `${id} version`, value: known(f?.suiteVersion) }, { dimension: 'model', key: `${id} mode`, value: known(f?.mode) }, { dimension: 'model', key: `${id} models`, value: rolesFact(f?.roles) }, { dimension: 'route', key: `${id} routing`, value: canonical(f?.routing) }, { dimension: 'effort', key: `${id} reasoning effort`, value: known(f?.reasoningEffort) }, 
+        // A file from before repeats existed ran every case once.
+        { dimension: 'repeats', key: `${id} repeats`, value: String(suite.repeats) });
+    }
+    return facts;
+}
+function compareDisclosure(a, b, sharedSuites) {
+    const left = new Map(factsOf(a, sharedSuites).map((f) => [f.key, f]));
+    const right = new Map(factsOf(b, sharedSuites).map((f) => [f.key, f]));
+    const differences = [];
+    const unknown = [];
+    for (const [key, fa] of left) {
+        const fb = right.get(key);
+        if (fa.value !== null && fb.value !== null) {
+            if (fa.value !== fb.value) {
+                differences.push({ dimension: fa.dimension, key, a: fa.value, b: fb.value });
+            }
+        }
+        else if (fa.value !== null || fb.value !== null) {
+            unknown.push({
+                dimension: fa.dimension,
+                key,
+                missingIn: fa.value === null ? 'a' : 'b',
+                value: (fa.value ?? fb.value),
+            });
+        }
+    }
+    const differingDimensions = DIMENSION_ORDER.filter((d) => differences.some((x) => x.dimension === d));
+    return {
+        differences,
+        unknown,
+        differingDimensions,
+        ...(differingDimensions.length > 1
+            ? {
+                warning: `More than one thing differs between these two runs (${differingDimensions.join(', ')}), so a difference in the scores cannot be attributed to any one of them.`,
+            }
+            : {}),
+    };
+}
+function sorted(refs) {
+    return refs.sort((x, y) => x.suite === y.suite ? (x.case < y.case ? -1 : 1) : x.suite < y.suite ? -1 : 1);
+}
+function signed(value, digits = 3) {
+    const text = value.toFixed(digits);
+    return value > 0 ? `+${text}` : text;
+}
+export function compareEvalReports(a, b, options = {}) {
+    const seed = validateSeed(options.seed);
+    const index = (file) => {
+        const map = new Map();
+        for (const suite of file.suites) {
+            for (const c of suite.cases) {
+                map.set(`${suite.id}\u0000${c.name}`, { ref: { suite: suite.id, case: c.name }, runs: c });
+            }
+        }
+        return map;
+    };
+    const left = index(a);
+    const right = index(b);
+    const onlyInA = [];
+    const onlyInB = [];
+    const notJudged = [];
+    const pairs = [];
+    let onlyBPasses = 0;
+    let onlyAPasses = 0;
+    let bothPass = 0;
+    let bothFail = 0;
+    for (const [key, l] of left) {
+        const r = right.get(key);
+        if (!r) {
+            onlyInA.push(l.ref);
+            continue;
+        }
+        const ta = tallyOf(l.runs.runs);
+        const tb = tallyOf(r.runs.runs);
+        if (!ta || !tb) {
+            notJudged.push(l.ref);
+            continue;
+        }
+        pairs.push({ a: ta, b: tb });
+        const passA = passedEveryRun(l.runs.runs);
+        const passB = passedEveryRun(r.runs.runs);
+        if (passA && passB)
+            bothPass++;
+        else if (!passA && !passB)
+            bothFail++;
+        else if (passB)
+            onlyBPasses++;
+        else
+            onlyAPasses++;
+    }
+    for (const [key, r] of right)
+        if (!left.has(key))
+            onlyInB.push(r.ref);
+    const aSide = pairs.map((p) => p.a);
+    const bSide = pairs.map((p) => p.b);
+    const numbers = (tallies) => {
+        if (tallies.length === 0)
+            return { mean: null, passRate: null };
+        const [mean, passRate] = pooledMeanAndRate(tallies);
+        return { mean, passRate };
+    };
+    const difference = bootstrapPairedDifference(pairs, seed);
+    const flips = {
+        onlyBPasses,
+        onlyAPasses,
+        bothPass,
+        bothFail,
+        pValue: exactSignTestP(onlyBPasses, onlyAPasses),
+        test: 'exact two-sided sign test on the cases that flipped',
+    };
+    const verdict = decide(difference.mean, pairs.length, flips);
+    const sharedSuites = a.suites.map((s) => s.id).filter((id) => b.suites.some((s) => s.id === id));
+    return {
+        seed,
+        a: { label: a.label, cases: left.size },
+        b: { label: b.label, cases: right.size },
+        onlyInA: sorted(onlyInA),
+        onlyInB: sorted(onlyInB),
+        notJudged: sorted(notJudged),
+        paired: pairs.length,
+        sideA: numbers(aSide),
+        sideB: numbers(bSide),
+        difference,
+        flips,
+        verdict,
+        disclosure: compareDisclosure(a, b, sharedSuites),
+    };
+}
+/** The sign test must be at least this convincing before the sentence names a winner. */
+const CLAIM_P = 0.05;
+/** Bounds are judged at the precision they are printed at. */
+function atPrintedPrecision(value) {
+    return Math.round(value * 1000) / 1000 || 0;
+}
+function decide(mean, paired, flips) {
+    const none = 'no difference can be claimed at this size';
+    if (!mean.computed) {
+        return { claim: 'none', sentence: none, detail: `no interval: ${mean.reason}.` };
+    }
+    const range = `[${mean.low.toFixed(3)}, ${mean.high.toFixed(3)}]`;
+    // A bound is zero when it prints as zero: sums that differ by rounding only
+    // (the same scores added in another order) must not read as a difference.
+    const low = atPrintedPrecision(mean.low);
+    const high = atPrintedPrecision(mean.high);
+    const excludesZero = (low > 0 || high < 0) && atPrintedPrecision(mean.estimate) !== 0;
+    if (!excludesZero) {
+        return {
+            claim: 'none',
+            sentence: none,
+            detail: `the 95% interval ${range} for the difference in mean score over ${paired} paired cases includes zero.`,
+        };
+    }
+    const flipped = flips.onlyAPasses + flips.onlyBPasses;
+    if (flipped > 0 && flips.pValue >= CLAIM_P) {
+        return {
+            claim: 'none',
+            sentence: none,
+            detail: `the 95% interval ${range} over ${paired} paired cases excludes zero, but the exact test on the ${flipped} case${flipped === 1 ? '' : 's'} that flipped gives p = ${flips.pValue.toFixed(4)}, not below ${CLAIM_P}; two checks that disagree are not a finding.`,
+        };
+    }
+    const better = low > 0;
+    const size = Math.abs(mean.estimate).toFixed(3);
+    return {
+        claim: better ? 'better' : 'worse',
+        sentence: `B is ${better ? 'better' : 'worse'} than A by ${size} in mean score, 95% interval ${range}`,
+        detail: `the interval over ${paired} paired cases excludes zero${flipped > 0 ? ` and the exact test on the flips agrees (p = ${flips.pValue.toFixed(4)})` : ''}.`,
+    };
+}
+function listCases(title, refs) {
+    if (refs.length === 0)
+        return [`${title}: none`];
+    const shown = refs.slice(0, 20).map((r) => `  ${r.suite} / ${r.case}`);
+    if (refs.length > shown.length)
+        shown.push(`  and ${refs.length - shown.length} more`);
+    return [`${title} (${refs.length}):`, ...shown];
+}
+function intervalText(interval, show) {
+    if (!interval.computed)
+        return `no interval: ${interval.reason}`;
+    return `95% paired interval [${show(interval.low)}, ${show(interval.high)}]`;
+}
+export function formatEvalComparison(c) {
+    const score = (n) => (n === null ? 'none' : n.toFixed(3));
+    const rate = (n) => (n === null ? 'none' : `${(n * 100).toFixed(1)}%`);
+    const points = (n) => `${signed(n * 100, 1)} points`;
+    const lines = [
+        `A: ${c.a.label} (${c.a.cases} cases)`,
+        `B: ${c.b.label} (${c.b.cases} cases)`,
+        `Cases are paired by suite and id; ${c.paired} paired${c.notJudged.length > 0 ? `, ${c.notJudged.length} more present on both but not judged on one side` : ''}. Seed ${c.seed}.`,
+        '',
+        ...listCases('Only in A', c.onlyInA),
+        ...listCases('Only in B', c.onlyInB),
+        '',
+        `Mean score: A ${score(c.sideA.mean)}, B ${score(c.sideB.mean)}${c.difference.mean.estimate !== null ? `, difference ${signed(c.difference.mean.estimate)}` : ''}; ${intervalText(c.difference.mean, (n) => signed(n))}`,
+        `Pass rate: A ${rate(c.sideA.passRate)}, B ${rate(c.sideB.passRate)}${c.difference.passRate.estimate !== null ? `, difference ${points(c.difference.passRate.estimate)}` : ''}; ${intervalText(c.difference.passRate, (n) => points(n))}`,
+        `Flips: ${c.flips.onlyBPasses} case${c.flips.onlyBPasses === 1 ? '' : 's'} passed on B and not on A, ${c.flips.onlyAPasses} passed on A and not on B (${c.flips.bothPass} passed on both, ${c.flips.bothFail} failed on both). A case passes when every one of its runs passed. ${c.flips.test}: p = ${c.flips.pValue.toFixed(4)}`,
+        '',
+        `${c.verdict.sentence}.`,
+        `  (${c.verdict.detail})`,
+        '',
+    ];
+    const d = c.disclosure;
+    if (d.differences.length === 0) {
+        lines.push('What the two reports say about how they were produced: no stated difference.');
+    }
+    else {
+        lines.push('What differs in how the two reports were produced:');
+        for (const x of d.differences)
+            lines.push(`  ${x.key}: A ${x.a}, B ${x.b}`);
+    }
+    if (d.unknown.length > 0) {
+        lines.push('Stated on one side only, so it cannot be compared:');
+        for (const x of d.unknown) {
+            lines.push(`  ${x.key}: ${x.missingIn === 'a' ? 'A does not say' : 'B does not say'}, the other says ${x.value}`);
+        }
+    }
+    if (d.warning)
+        lines.push('', `Warning: ${d.warning}`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=compare.js.map