@cogitator-ai/evals 0.11.0 → 0.12.0

package/dist/pairwise.d.ts

@@ -0,0 +1,53 @@
+import type { EvalSuiteResult } from './eval-suite.js';
+import type { JudgeContext } from './metrics/llm-judge.js';
+export interface PairwiseOptions {
+    /** What makes one answer better than the other. Default: correct, relevant, complete, clear */
+    criteria?: string;
+    /** The name the result reports. Default `pairwise` */
+    name?: string;
+}
+export type PairwiseSide = 'baseline' | 'challenger';
+export type PairwiseOutcome = PairwiseSide | 'tie';
+/** One of the two verdicts on a case */
+export interface PairwiseVerdict {
+    /** The side shown to the judge first, as answer A */
+    first: PairwiseSide;
+    /** The side the judge preferred, or `tie` */
+    preferred: PairwiseOutcome;
+    reasoning?: string;
+}
+export interface PairwiseCase {
+    /** Position of the case in the dataset */
+    index: number;
+    input: string;
+    winner: PairwiseOutcome;
+    /** The verdict with the baseline first, then the one with the challenger first */
+    verdicts: PairwiseVerdict[];
+    /** The side that errored and so lost the case without a judge call */
+    errored?: PairwiseSide;
+    /** Why the judge gave no verdict in one of the orders, which makes the case a tie */
+    error?: string;
+}
+export interface PairwiseResult {
+    name: string;
+    wins: {
+        baseline: number;
+        challenger: number;
+    };
+    ties: number;
+    /** Cases compared: those at least one side answered */
+    cases: number;
+    /** Challenger wins over the decided cases (wins of either side), NaN when none was decided */
+    winRate: number;
+    /** Cases the judge failed on in at least one order, counted as ties */
+    judgeErrors: number;
+    /** USD the judge calls cost, as far as the judge reported usage */
+    cost: number;
+    perCase: PairwiseCase[];
+}
+/**
+ * Asks the judge which side answered each case better, see `EvalComparisonOptions.pairwise`.
+ * Cases both sides errored on are left out
+ */
+export declare function judgePairwise(baseline: EvalSuiteResult, challenger: EvalSuiteResult, judge: JudgeContext, options: PairwiseOptions, concurrency: number, signal?: AbortSignal): Promise<PairwiseResult>;
+//# sourceMappingURL=pairwise.d.ts.map

package/dist/pairwise.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.d.ts","sourceRoot":"","sources":["../src/pairwise.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAUxD,MAAM,WAAW,eAAe;IAC9B,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,YAAY,CAAC;AAErD,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,KAAK,CAAC;AAEnD,wCAAwC;AACxC,MAAM,WAAW,eAAe;IAC9B,qDAAqD;IACrD,KAAK,EAAE,YAAY,CAAC;IACpB,6CAA6C;IAC7C,SAAS,EAAE,eAAe,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,kFAAkF;IAClF,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,sEAAsE;IACtE,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,8FAA8F;IAC9F,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AA4GD;;;GAGG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,eAAe,EACzB,UAAU,EAAE,eAAe,EAC3B,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,eAAe,EACxB,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,cAAc,CAAC,CAmCzB"}

package/dist/pairwise.js

@@ -0,0 +1,118 @@
+import { judgePrompt, parseJudgeOutput, sumUsage, untilAborted, usageOf, } from './metrics/llm-judge.js';
+import { conversationBlock, evidenceBlock, tagged, wrapped } from './metrics/judge-prompt.js';
+/** The instructions of a pairwise verdict */
+function pairwiseInstructions(criteria) {
+    const basis = criteria
+        ? `on these criteria:\n${criteria}`
+        : 'overall: which is more correct, relevant, complete and clear';
+    return `You are comparing two answers to the same request to decide which one is better ${basis}\n\nThe order of the answers says nothing about their quality, and a longer answer is better only for what its length adds.\n\nScore 0 when <answer_a> is better, 1 when <answer_b> is better, 0.5 when neither is better.\n\nRespond with JSON: {"score": <0, 0.5 or 1>, "reasoning": "<explanation>"}`;
+}
+/** The case and the two answers, A then B. A multi-turn case shows each side's whole conversation */
+function pairwiseMessage(a, b) {
+    const evidence = evidenceBlock(a, false);
+    const multiTurn = (a.case.turns?.length ?? 0) > 0;
+    const shared = [
+        ...(multiTurn
+            ? ['Each answer is a whole conversation, the user messages being the same in both.']
+            : [tagged('input', a.case.input)]),
+        ...(evidence === undefined ? [] : [evidence]),
+        ...(a.case.expected ? [tagged('expected', a.case.expected)] : []),
+    ];
+    const answer = (tag, result) => multiTurn ? wrapped(tag, conversationBlock(result)) : tagged(tag, result.output);
+    return [...shared, answer('answer_a', a), answer('answer_b', b)].join('\n');
+}
+const other = (side) => side === 'baseline' ? 'challenger' : 'baseline';
+async function ask(judge, instructions, first, a, b, signal) {
+    try {
+        const answered = await untilAborted(judge.cogitator.run({
+            input: judgePrompt(instructions, pairwiseMessage(a, b)),
+            ...(signal && { signal }),
+        }), signal);
+        const usage = usageOf(answered.usage);
+        const verdict = parseJudgeOutput(answered.output);
+        if (!verdict)
+            return { error: 'could not parse judge response', ...(usage && { usage }) };
+        const score = verdict.outOf === undefined ? verdict.score : verdict.score / verdict.outOf;
+        const preferred = score < 0.5 ? first : score > 0.5 ? other(first) : 'tie';
+        return {
+            verdict: {
+                first,
+                preferred,
+                ...(verdict.reasoning !== undefined && { reasoning: verdict.reasoning }),
+            },
+            ...(usage && { usage }),
+        };
+    }
+    catch (error) {
+        return { error: error instanceof Error ? error.message : String(error) };
+    }
+}
+/**
+ * A case judged both ways round: the baseline first, then the challenger first. Agreeing
+ * verdicts give the case to that side, disagreeing ones or a tie make it a tie
+ */
+async function judgeCase(judge, instructions, index, baseline, challenger, signal) {
+    const input = baseline.case.input;
+    if (baseline.error !== undefined || challenger.error !== undefined) {
+        const errored = baseline.error !== undefined ? 'baseline' : 'challenger';
+        return { result: { index, input, winner: other(errored), verdicts: [], errored } };
+    }
+    const asked = await Promise.all([
+        ask(judge, instructions, 'baseline', baseline, challenger, signal),
+        ask(judge, instructions, 'challenger', challenger, baseline, signal),
+    ]);
+    const usage = sumUsage(asked.map((one) => one.usage));
+    const verdicts = asked.flatMap((one) => (one.verdict ? [one.verdict] : []));
+    const errors = asked.flatMap((one) => (one.error === undefined ? [] : [one.error]));
+    const [first, second] = verdicts;
+    const winner = errors.length === 0 && first.preferred === second.preferred ? first.preferred : 'tie';
+    return {
+        result: {
+            index,
+            input,
+            winner,
+            verdicts,
+            ...(errors.length > 0 && { error: errors.join(', ') }),
+        },
+        ...(usage && { usage }),
+    };
+}
+/**
+ * Asks the judge which side answered each case better, see `EvalComparisonOptions.pairwise`.
+ * Cases both sides errored on are left out
+ */
+export async function judgePairwise(baseline, challenger, judge, options, concurrency, signal) {
+    const instructions = pairwiseInstructions(options.criteria);
+    const pairs = baseline.results.flatMap((b, index) => {
+        const c = challenger.results[index];
+        return c && (b.error === undefined || c.error === undefined) ? [{ index, b, c }] : [];
+    });
+    const judged = new Array(pairs.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < pairs.length && !signal?.aborted) {
+            const at = next++;
+            const { index, b, c } = pairs[at];
+            judged[at] = await judgeCase(judge, instructions, index, b, c, signal);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(concurrency, pairs.length) }, worker));
+    signal?.throwIfAborted();
+    const perCase = judged.map((one) => one.result);
+    const wins = {
+        baseline: perCase.filter((one) => one.winner === 'baseline').length,
+        challenger: perCase.filter((one) => one.winner === 'challenger').length,
+    };
+    const decided = wins.baseline + wins.challenger;
+    return {
+        name: options.name ?? 'pairwise',
+        wins,
+        ties: perCase.length - decided,
+        cases: perCase.length,
+        winRate: decided === 0 ? Number.NaN : wins.challenger / decided,
+        judgeErrors: perCase.filter((one) => one.error !== undefined).length,
+        cost: judged.reduce((sum, one) => sum + (one.usage?.cost ?? 0), 0),
+        perCase,
+    };
+}
+//# sourceMappingURL=pairwise.js.map

package/dist/pairwise.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.js","sourceRoot":"","sources":["../src/pairwise.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,QAAQ,EACR,YAAY,EACZ,OAAO,GACR,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAkD3F,6CAA6C;AAC7C,SAAS,oBAAoB,CAAC,QAA4B;IACxD,MAAM,KAAK,GAAG,QAAQ;QACpB,CAAC,CAAC,uBAAuB,QAAQ,EAAE;QACnC,CAAC,CAAC,8DAA8D,CAAC;IACnE,OAAO,mFAAmF,KAAK,ySAAyS,CAAC;AAC3Y,CAAC;AAED,qGAAqG;AACrG,SAAS,eAAe,CAAC,CAAmB,EAAE,CAAmB;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG;QACb,GAAG,CAAC,SAAS;YACX,CAAC,CAAC,CAAC,gFAAgF,CAAC;YACpF,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACpC,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC7C,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAClE,CAAC;IACF,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,MAAwB,EAAE,EAAE,CACvD,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9E,CAAC;AAQD,MAAM,KAAK,GAAG,CAAC,IAAkB,EAAgB,EAAE,CACjD,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;AAElD,KAAK,UAAU,GAAG,CAChB,KAAmB,EACnB,YAAoB,EACpB,KAAmB,EACnB,CAAmB,EACnB,CAAmB,EACnB,MAA+B;IAE/B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,YAAY,CACjC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC;YAClB,KAAK,EAAE,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvD,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC;SAC1B,CAAC,EACF,MAAM,CACP,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC1F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC1F,MAAM,SAAS,GAAoB,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5F,OAAO;YACL,OAAO,EAAE;gBACP,KAAK;gBACL,SAAS;gBACT,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;aACzE;YACD,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAC3E,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,SAAS,CACtB,KAAmB,EACnB,YAAoB,EACpB,KAAa,EACb,QAA0B,EAC1B,UAA4B,EAC5B,MAA+B;IAE/B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;IAClC,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACnE,MAAM,OAAO,GAAiB,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC;QACvF,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC9B,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC;QAClE,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC;KACrE,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACpF,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,QAAQ,CAAC;IACjC,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IACxF,OAAO;QACL,MAAM,EAAE;YACN,KAAK;YACL,KAAK;YACL,MAAM;YACN,QAAQ;YACR,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;SACvD;QACD,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;KACxB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAyB,EACzB,UAA2B,EAC3B,KAAmB,EACnB,OAAwB,EACxB,WAAmB,EACnB,MAAoB;IAEpB,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;QAClD,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,KAAK,CAAgD,KAAK,CAAC,MAAM,CAAC,CAAC;IACtF,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACzE,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IACvF,MAAM,EAAE,cAAc,EAAE,CAAC;IAEzB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG;QACX,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM;QACnE,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,MAAM;KACxE,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;IAChD,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,UAAU;QAChC,IAAI;QACJ,IAAI,EAAE,OAAO,CAAC,MAAM,GAAG,OAAO;QAC9B,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,OAAO,EAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO;QAC/D,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;QACpE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/schema.d.ts

@@ -30,6 +30,7 @@ export interface JudgeCogitator {
     run(agent: unknown, options: {
         input: string;
         useMemory?: boolean;
+        signal?: AbortSignal;
     }): Promise<{
         output: string;
         structured?: unknown;
@@ -54,12 +55,27 @@ export declare const JudgeConfigSchema: z.ZodObject<{
     cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
     cache: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<JudgeCache, JudgeCache>]>>;
     concurrency: z.ZodOptional<z.ZodNumber>;
+    samples: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const PairwiseOptionsSchema: z.ZodObject<{
+    criteria: z.ZodOptional<z.ZodString>;
+    name: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export declare const EvalComparisonConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
     trials: z.ZodDefault<z.ZodNumber>;
+    alpha: z.ZodDefault<z.ZodNumber>;
+    test: z.ZodDefault<z.ZodEnum<{
+        t: "t";
+        wilcoxon: "wilcoxon";
+    }>>;
+    seed: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
+    pairwise: z.ZodOptional<z.ZodObject<{
+        criteria: z.ZodOptional<z.ZodString>;
+        name: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
 }, z.core.$strip>;
 export type ExpectedToolCall = z.output<typeof ExpectedToolCallSchema>;
 export type EvalCase = z.output<typeof EvalCaseSchema>;

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,eAAO,MAAM,sBAAsB;;;iBAGjC,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;iBAWzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;iBAKhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;iBAKrC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACvE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,eAAO,MAAM,sBAAsB;;;iBAGjC,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;iBAWzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;iBAKhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACpE,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;;iBAsB5B,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;iBAGhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;iBAYrC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACvE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}

package/dist/schema.js

@@ -49,11 +49,28 @@ export const JudgeConfigSchema = z.object({
      * metrics. Default: no limit of its own
      */
     concurrency: z.number().int().positive().optional(),
+    /**
+     * Verdicts per case and judge metric, a positive integer, 1 by default. The score is the mean
+     * of the verdicts the judge gave, their spread is in the score's `details`. Each verdict is
+     * cached on its own, so use a `temperature` above 0 for verdicts that differ
+     */
+    samples: z.number().int().positive().optional(),
+});
+export const PairwiseOptionsSchema = z.object({
+    criteria: z.string().trim().min(1, 'criteria must not be empty').optional(),
+    name: z.string().trim().min(1, 'name must not be empty').optional(),
 });
 export const EvalComparisonConfigSchema = z.object({
     concurrency: z.number().int().min(1).default(5),
     timeout: z.number().int().min(1000).default(30000),
     retries: z.number().int().min(0).max(10).default(0),
     trials: z.number().int().min(1).default(1),
+    /** Significance level, after Holm's correction across the compared metrics */
+    alpha: z.number().gt(0).lt(1).default(0.05),
+    /** Test for metrics that are not binary, and for latency and cost */
+    test: z.enum(['t', 'wilcoxon']).default('t'),
+    /** Seeds the bootstrap of the confidence intervals, so they are the same on every run */
+    seed: z.union([z.number(), z.string()]).optional(),
+    pairwise: PairwiseOptionsSchema.optional(),
 });
 //# sourceMappingURL=schema.js.map

package/dist/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IAC7D,gEAAgE;IAChE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IAC7D,gEAAgE;IAChE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD;;;;OAIG;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAChD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4BAA4B,CAAC,CAAC,QAAQ,EAAE;IAC3E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC,QAAQ,EAAE;CACpE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3C,qEAAqE;IACrE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC5C,yFAAyF;IACzF,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClD,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC"}

package/dist/stats/bootstrap.d.ts

@@ -0,0 +1,22 @@
+import { z } from 'zod';
+export declare const BootstrapOptionsSchema: z.ZodObject<{
+    confidence: z.ZodDefault<z.ZodNumber>;
+    resamples: z.ZodDefault<z.ZodNumber>;
+    seed: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
+}, z.core.$strip>;
+export type BootstrapOptions = z.input<typeof BootstrapOptionsSchema>;
+export interface BootstrapInterval {
+    /** Lower bound of the interval */
+    low: number;
+    /** Upper bound of the interval */
+    high: number;
+    /** Mean of the values themselves */
+    mean: number;
+}
+/**
+ * Percentile bootstrap confidence interval of the mean: the values are resampled with
+ * replacement `resamples` times and the interval spans the middle `confidence` share of the
+ * resampled means. With `seed` the interval is the same on every run. All NaN for no values
+ */
+export declare function bootstrapCI(values: readonly number[], options?: BootstrapOptions): BootstrapInterval;
+//# sourceMappingURL=bootstrap.d.ts.map

package/dist/stats/bootstrap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.d.ts","sourceRoot":"","sources":["../../src/stats/bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,sBAAsB;;;;iBAIjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,OAAO,GAAE,gBAAqB,GAC7B,iBAAiB,CAiBnB"}

package/dist/stats/bootstrap.js

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+import { mean, percentile } from './percentiles.js';
+import { seededRandom } from './random.js';
+export const BootstrapOptionsSchema = z.object({
+    confidence: z.number().gt(0).lt(1).default(0.95),
+    resamples: z.number().int().positive().default(2000),
+    seed: z.union([z.number(), z.string()]).optional(),
+});
+/**
+ * Percentile bootstrap confidence interval of the mean: the values are resampled with
+ * replacement `resamples` times and the interval spans the middle `confidence` share of the
+ * resampled means. With `seed` the interval is the same on every run. All NaN for no values
+ */
+export function bootstrapCI(values, options = {}) {
+    const { confidence, resamples, seed } = BootstrapOptionsSchema.parse(options);
+    if (values.length === 0)
+        return { low: Number.NaN, high: Number.NaN, mean: Number.NaN };
+    const random = seededRandom(seed);
+    const n = values.length;
+    const means = new Array(resamples);
+    for (let r = 0; r < resamples; r++) {
+        let sum = 0;
+        for (let i = 0; i < n; i++)
+            sum += values[Math.floor(random() * n)];
+        means[r] = sum / n;
+    }
+    const tail = (1 - confidence) / 2;
+    return {
+        low: percentile(means, tail),
+        high: percentile(means, 1 - tail),
+        mean: mean([...values]),
+    };
+}
+//# sourceMappingURL=bootstrap.js.map

package/dist/stats/bootstrap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.js","sourceRoot":"","sources":["../../src/stats/bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAaH;;;;GAIG;AACH,MAAM,UAAU,WAAW,CACzB,MAAyB,EACzB,UAA4B,EAAE;IAE9B,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC9E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IACxF,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAS,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACpE,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO;QACL,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC;QAC5B,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;QACjC,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;KACxB,CAAC;AACJ,CAAC"}

package/dist/stats/correlation.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pearson's correlation of paired values, from -1 to 1. NaN when there are fewer than 2 pairs
+ * or either side does not vary
+ */
+export declare function pearson(x: readonly number[], y: readonly number[]): number;
+/**
+ * Spearman's rank correlation of paired values: Pearson's correlation of their ranks, ties
+ * sharing the mean of their ranks. NaN when Pearson's would be
+ */
+export declare function spearman(x: readonly number[], y: readonly number[]): number;
+//# sourceMappingURL=correlation.d.ts.map

package/dist/stats/correlation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"correlation.d.ts","sourceRoot":"","sources":["../../src/stats/correlation.ts"],"names":[],"mappings":"AAQA;;;GAGG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAkB1E;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAG3E"}

package/dist/stats/correlation.js

@@ -0,0 +1,42 @@
+import { mean } from './percentiles.js';
+import { averageRanks } from './ranks.js';
+import { isConstant, withoutNoise } from './precision.js';
+function assertPaired(x, y) {
+    if (x.length !== y.length)
+        throw new Error('Sample arrays must have equal length');
+}
+/**
+ * Pearson's correlation of paired values, from -1 to 1. NaN when there are fewer than 2 pairs
+ * or either side does not vary
+ */
+export function pearson(x, y) {
+    assertPaired(x, y);
+    if (x.length < 2)
+        return Number.NaN;
+    if (isConstant(x.map(withoutNoise)) || isConstant(y.map(withoutNoise)))
+        return Number.NaN;
+    const mx = mean([...x]);
+    const my = mean([...y]);
+    let sxy = 0;
+    let sxx = 0;
+    let syy = 0;
+    for (let i = 0; i < x.length; i++) {
+        const dx = x[i] - mx;
+        const dy = y[i] - my;
+        sxy += dx * dy;
+        sxx += dx * dx;
+        syy += dy * dy;
+    }
+    if (sxx === 0 || syy === 0)
+        return Number.NaN;
+    return Math.max(-1, Math.min(1, sxy / Math.sqrt(sxx * syy)));
+}
+/**
+ * Spearman's rank correlation of paired values: Pearson's correlation of their ranks, ties
+ * sharing the mean of their ranks. NaN when Pearson's would be
+ */
+export function spearman(x, y) {
+    assertPaired(x, y);
+    return pearson(averageRanks(x.map(withoutNoise)), averageRanks(y.map(withoutNoise)));
+}
+//# sourceMappingURL=correlation.js.map

package/dist/stats/correlation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"correlation.js","sourceRoot":"","sources":["../../src/stats/correlation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEvD,SAAS,YAAY,CAAC,CAAoB,EAAE,CAAoB;IAC9D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AACrF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,CAAoB,EAAE,CAAoB;IAChE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IACpC,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC1F,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACrB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACrB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;IACD,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAoB,EAAE,CAAoB;IACjE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AACvF,CAAC"}

package/dist/stats/effect-size.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Cohen's d for paired samples (dz): the mean of the paired differences over their standard
+ * deviation, the differences rounded to 12 significant digits. NaN with fewer than 2 differences,
+ * plus or minus Infinity when the differences are all the same and not 0, 0 when they are all 0
+ */
+export declare function cohensD(differences: readonly number[]): number;
+//# sourceMappingURL=effect-size.d.ts.map

package/dist/stats/effect-size.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"effect-size.d.ts","sourceRoot":"","sources":["../../src/stats/effect-size.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAO9D"}

package/dist/stats/effect-size.js

@@ -0,0 +1,17 @@
+import { mean, stdDev } from './percentiles.js';
+import { isConstant, withoutNoise } from './precision.js';
+/**
+ * Cohen's d for paired samples (dz): the mean of the paired differences over their standard
+ * deviation, the differences rounded to 12 significant digits. NaN with fewer than 2 differences,
+ * plus or minus Infinity when the differences are all the same and not 0, 0 when they are all 0
+ */
+export function cohensD(differences) {
+    if (differences.length < 2)
+        return Number.NaN;
+    const values = differences.map(withoutNoise);
+    if (isConstant(values)) {
+        return values[0] === 0 ? 0 : Math.sign(values[0]) * Number.POSITIVE_INFINITY;
+    }
+    return mean(values) / stdDev(values);
+}
+//# sourceMappingURL=effect-size.js.map

package/dist/stats/effect-size.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"effect-size.js","sourceRoot":"","sources":["../../src/stats/effect-size.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEvD;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,WAA8B;IACpD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC9C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC7C,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAC/E,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;AACvC,CAAC"}

package/dist/stats/holm.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Holm-Bonferroni adjusted p-values, in the order given: the i-th smallest of m p-values is
+ * multiplied by m - i + 1, kept from falling below the ones before it and capped at 1. Rejecting
+ * where an adjusted p-value is below alpha keeps the family-wise error rate at alpha
+ */
+export declare function holm(pValues: readonly number[]): number[];
+//# sourceMappingURL=holm.d.ts.map

package/dist/stats/holm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"holm.d.ts","sourceRoot":"","sources":["../../src/stats/holm.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,IAAI,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAazD"}

package/dist/stats/holm.js

@@ -0,0 +1,21 @@
+/**
+ * Holm-Bonferroni adjusted p-values, in the order given: the i-th smallest of m p-values is
+ * multiplied by m - i + 1, kept from falling below the ones before it and capped at 1. Rejecting
+ * where an adjusted p-value is below alpha keeps the family-wise error rate at alpha
+ */
+export function holm(pValues) {
+    for (const p of pValues) {
+        if (!(p >= 0 && p <= 1))
+            throw new Error(`p-values must be between 0 and 1, got ${p}`);
+    }
+    const m = pValues.length;
+    const order = pValues.map((p, index) => ({ p, index })).sort((a, b) => a.p - b.p);
+    const adjusted = new Array(m);
+    let running = 0;
+    for (const [rank, { p, index }] of order.entries()) {
+        running = Math.max(running, Math.min(1, (m - rank) * p));
+        adjusted[index] = running;
+    }
+    return adjusted;
+}
+//# sourceMappingURL=holm.js.map

package/dist/stats/holm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"holm.js","sourceRoot":"","sources":["../../src/stats/holm.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,UAAU,IAAI,CAAC,OAA0B;IAC7C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClF,MAAM,QAAQ,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAC;IACtC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzD,QAAQ,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;IAC5B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/stats/index.d.ts

@@ -1,6 +1,14 @@
 export { percentile, mean, median, stdDev, aggregate } from './percentiles.js';
 export { pairedTTest } from './t-test.js';
 export type { TTestResult } from './t-test.js';
-export { mcnemarsTest } from './mcnemar.js';
-export type { McNemarResult } from './mcnemar.js';
+export { mcnemarsTest, mcnemarExact } from './mcnemar.js';
+export type { McNemarResult, McNemarExactResult } from './mcnemar.js';
+export { wilcoxonSignedRank } from './wilcoxon.js';
+export type { WilcoxonResult } from './wilcoxon.js';
+export { bootstrapCI } from './bootstrap.js';
+export type { BootstrapOptions, BootstrapInterval } from './bootstrap.js';
+export { cohensD } from './effect-size.js';
+export { holm } from './holm.js';
+export { pearson, spearman } from './correlation.js';
+export { seededRandom } from './random.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/stats/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stats/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stats/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACvD,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/stats/index.js

@@ -1,4 +1,10 @@
 export { percentile, mean, median, stdDev, aggregate } from './percentiles.js';
 export { pairedTTest } from './t-test.js';
-export { mcnemarsTest } from './mcnemar.js';
+export { mcnemarsTest, mcnemarExact } from './mcnemar.js';
+export { wilcoxonSignedRank } from './wilcoxon.js';
+export { bootstrapCI } from './bootstrap.js';
+export { cohensD } from './effect-size.js';
+export { holm } from './holm.js';
+export { pearson, spearman } from './correlation.js';
+export { seededRandom } from './random.js';
 //# sourceMappingURL=index.js.map

package/dist/stats/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/stats/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/stats/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/stats/mcnemar.d.ts

@@ -4,4 +4,13 @@ export interface McNemarResult {
     significant: boolean;
 }
 export declare function mcnemarsTest(pairsACorrect_BIncorrect: number, pairsAIncorrect_BCorrect: number): McNemarResult;
+export interface McNemarExactResult {
+    pValue: number;
+    significant: boolean;
+}
+/**
+ * Exact McNemar test: the two-sided binomial test of `b` against `b + c` discordant pairs at
+ * one half. Preferred over `mcnemarsTest` when there are few discordant pairs (below 25)
+ */
+export declare function mcnemarExact(b: number, c: number): McNemarExactResult;
 //# sourceMappingURL=mcnemar.d.ts.map

package/dist/stats/mcnemar.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mcnemar.d.ts","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;CACtB;AAuBD,wBAAgB,YAAY,CAC1B,wBAAwB,EAAE,MAAM,EAChC,wBAAwB,EAAE,MAAM,GAC/B,aAAa,CAiBf"}
+{"version":3,"file":"mcnemar.d.ts","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;CACtB;AAOD,wBAAgB,YAAY,CAC1B,wBAAwB,EAAE,MAAM,EAChC,wBAAwB,EAAE,MAAM,GAC/B,aAAa,CAiBf;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAoBrE"}

package/dist/stats/mcnemar.js

@@ -1,16 +1,4 @@
-function erfc(x) {
-    const a1 = 0.254829592;
-    const a2 = -0.284496736;
-    const a3 = 1.421413741;
-    const a4 = -1.453152027;
-    const a5 = 1.061405429;
-    const p = 0.3275911;
-    const sign = x < 0 ? -1 : 1;
-    const absX = Math.abs(x);
-    const t = 1.0 / (1.0 + p * absX);
-    const y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX);
-    return 1.0 - sign * y;
-}
+import { erfc } from './normal.js';
 function chiSquareSurvival(x) {
     if (x <= 0)
         return 1;
@@ -31,4 +19,30 @@ export function mcnemarsTest(pairsACorrect_BIncorrect, pairsAIncorrect_BCorrect)
         significant: pValue < 0.05,
     };
 }
+/**
+ * Exact McNemar test: the two-sided binomial test of `b` against `b + c` discordant pairs at
+ * one half. Preferred over `mcnemarsTest` when there are few discordant pairs (below 25)
+ */
+export function mcnemarExact(b, c) {
+    for (const [label, count] of [
+        ['b', b],
+        ['c', c],
+    ]) {
+        if (!Number.isInteger(count) || count < 0) {
+            throw new Error(`${label} must be a non-negative integer, got ${count}`);
+        }
+    }
+    const n = b + c;
+    if (n === 0)
+        return { pValue: 1, significant: false };
+    const k = Math.min(b, c);
+    let logPmf = -n * Math.LN2;
+    let tail = 0;
+    for (let i = 0; i <= k; i++) {
+        tail += Math.exp(logPmf);
+        logPmf += Math.log((n - i) / (i + 1));
+    }
+    const pValue = Math.min(1, 2 * tail);
+    return { pValue, significant: pValue < 0.05 };
+}
 //# sourceMappingURL=mcnemar.js.map

package/dist/stats/mcnemar.js.map

@@ -1 +1 @@
-{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAMA,SAAS,IAAI,CAAC,CAAS;IACrB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC;IACxB,MAAM,EAAE,GAAG,WAAW,CAAC;IACvB,MAAM,CAAC,GAAG,SAAS,CAAC;IAEpB,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAE5F,OAAO,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,OAAO;QACL,SAAS;QACT,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI;KAC3B,CAAC;AACJ,CAAC"}
+{"version":3,"file":"mcnemar.js","sourceRoot":"","sources":["../../src/stats/mcnemar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAQhC,SAAS,iBAAiB,CAAC,CAAS;IAClC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,wBAAgC,EAChC,wBAAgC;IAEhC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IACnC,MAAM,CAAC,GAAG,wBAAwB,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE5C,OAAO;QACL,SAAS;QACT,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI;KAC3B,CAAC;AACJ,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI;QAC3B,CAAC,GAAG,EAAE,CAAC,CAAC;QACR,CAAC,GAAG,EAAE,CAAC,CAAC;KACA,EAAE,CAAC;QACX,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wCAAwC,KAAK,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACtD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;IAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzB,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI,EAAE,CAAC;AAChD,CAAC"}