@cogitator-ai/evals 0.11.0 → 0.13.0

package/dist/comparison-stats.js

@@ -0,0 +1,179 @@
+import { pairedTTest } from './stats/t-test.js';
+import { mcnemarExact, mcnemarsTest } from './stats/mcnemar.js';
+import { wilcoxonSignedRank } from './stats/wilcoxon.js';
+import { bootstrapCI } from './stats/bootstrap.js';
+import { cohensD } from './stats/effect-size.js';
+import { holm } from './stats/holm.js';
+import { mean } from './stats/percentiles.js';
+import { pairedDifference } from './stats/precision.js';
+import { truncateInput } from './reporters/format.js';
+/** A score moves when it changes by more than this */
+export const SCORE_EPSILON = 1e-9;
+function isBinary(scores) {
+    return scores.every((s) => s === 0 || s === 1);
+}
+/** Below this many discordant pairs McNemar's test is exact */
+const EXACT_MCNEMAR_BELOW = 25;
+function pairedTest(baseline, challenger, continuous, binaryAllowed) {
+    const direction = Math.sign(mean(differencesOf(baseline, challenger)));
+    if (baseline.length < 2)
+        return { test: 'none', pValue: 1, direction };
+    if (binaryAllowed && isBinary(baseline) && isBinary(challenger)) {
+        let b = 0;
+        let c = 0;
+        for (let i = 0; i < baseline.length; i++) {
+            if (baseline[i] === 1 && challenger[i] === 0)
+                b++;
+            if (baseline[i] === 0 && challenger[i] === 1)
+                c++;
+        }
+        return b + c < EXACT_MCNEMAR_BELOW
+            ? { test: 'mcnemar-exact', pValue: mcnemarExact(b, c).pValue, direction }
+            : { test: 'mcnemar', pValue: mcnemarsTest(b, c).pValue, direction };
+    }
+    if (continuous === 'wilcoxon') {
+        const { statistic, n, pValue } = wilcoxonSignedRank(baseline, challenger);
+        return { test: 'wilcoxon', pValue, direction: Math.sign((n * (n + 1)) / 4 - statistic) };
+    }
+    return { test: 't', pValue: pairedTTest(baseline, challenger).pValue, direction };
+}
+/** Challenger minus baseline per case, without float noise */
+function differencesOf(baseline, challenger) {
+    return challenger.map((score, i) => pairedDifference(score, baseline[i]));
+}
+function meanOrNaN(values) {
+    return values.length === 0 ? Number.NaN : mean(values);
+}
+function compareMetric(name, baselineScores, challengerScores, config) {
+    const differences = differencesOf(baselineScores, challengerScores);
+    const { low, high } = bootstrapCI(differences, {
+        confidence: 1 - config.alpha,
+        ...(config.seed !== undefined && { seed: `${String(config.seed)}:${name}` }),
+    });
+    return {
+        baseline: meanOrNaN(baselineScores),
+        challenger: meanOrNaN(challengerScores),
+        n: baselineScores.length,
+        difference: meanOrNaN(differences),
+        ci: { low, high },
+        effectSize: cohensD(differences),
+        ...pairedTest(baselineScores, challengerScores, config.test, true),
+    };
+}
+/**
+ * Compares each metric on its paired scores, then adjusts the p-values with Holm's correction
+ * across the metrics and decides each metric's winner
+ */
+export function compareMetrics(paired, config) {
+    const names = [...paired.keys()];
+    const compared = names.map((name) => {
+        const [baselineScores, challengerScores] = paired.get(name);
+        return compareMetric(name, baselineScores, challengerScores, config);
+    });
+    const adjusted = holm(compared.map((metric) => metric.pValue));
+    const decided = {};
+    for (const [at, name] of names.entries()) {
+        const { direction, ...metric } = compared[at];
+        const significant = adjusted[at] < config.alpha && direction !== 0;
+        decided[name] = {
+            ...metric,
+            adjustedPValue: adjusted[at],
+            significant,
+            winner: !significant ? 'tie' : direction > 0 ? 'challenger' : 'baseline',
+        };
+    }
+    return decided;
+}
+export function compareResource(baseline, challenger, config) {
+    const difference = meanOrNaN(differencesOf(baseline, challenger));
+    const { test, pValue, direction } = pairedTest(baseline, challenger, config.test, false);
+    const significant = pValue < config.alpha && direction !== 0;
+    return {
+        baseline: meanOrNaN(baseline),
+        challenger: meanOrNaN(challenger),
+        n: baseline.length,
+        difference,
+        test,
+        pValue,
+        significant,
+        winner: !significant ? 'tie' : direction < 0 ? 'challenger' : 'baseline',
+    };
+}
+/** The side more metrics name as their winner, a tie when as many name each */
+export function determineOverallWinner(metrics) {
+    let baselineWins = 0;
+    let challengerWins = 0;
+    for (const mc of Object.values(metrics)) {
+        if (mc.winner === 'baseline')
+            baselineWins++;
+        if (mc.winner === 'challenger')
+            challengerWins++;
+    }
+    if (challengerWins > baselineWins)
+        return 'challenger';
+    if (baselineWins > challengerWins)
+        return 'baseline';
+    return 'tie';
+}
+/** The names of the metrics any trial of the cases was scored on, in the order first seen */
+export function scoredMetricNames(results) {
+    const names = new Set();
+    for (const r of results) {
+        for (const trial of r.trials ?? [r]) {
+            for (const s of trial.scores)
+                names.add(s.name);
+        }
+    }
+    return names;
+}
+/** A case's score of a metric: the mean over its trials the metric scored, none when it scored none */
+export function caseScore(result, metricName) {
+    if (!result)
+        return undefined;
+    const trials = result.trials ?? [result];
+    if (trials.every((trial) => trial.error !== undefined))
+        return undefined;
+    const scores = trials.flatMap((trial) => trial.error !== undefined
+        ? [0]
+        : trial.scores.filter((s) => s.name === metricName).map((s) => s.score));
+    return scores.length === 0 ? undefined : mean(scores);
+}
+/**
+ * The scores of a metric for each pair, side by side, with the index of the pair. A case one side
+ * errored on counts as a 0 for that side, since a target that fails loses the case, and a case
+ * both sides errored on is left out, telling nothing about either
+ */
+function pairedCaseScores(pairs, metricName) {
+    const scored = [];
+    for (const [index, [first, second]] of pairs.entries()) {
+        const before = caseScore(first, metricName);
+        const after = caseScore(second, metricName);
+        if (before === undefined && after === undefined)
+            continue;
+        scored.push({ index, before: before ?? 0, after: after ?? 0 });
+    }
+    return scored;
+}
+/** The scores of a metric for each pair, side by side, as `pairedCaseScores` pairs them */
+export function pairedScores(pairs, metricName) {
+    const scored = pairedCaseScores(pairs, metricName);
+    return [scored.map((s) => s.before), scored.map((s) => s.after)];
+}
+/**
+ * The cases whose score of a metric rose or fell from the first side to the second by more than
+ * `SCORE_EPSILON`, the largest moves first
+ */
+export function caseChanges(pairs, metricName, inputOf) {
+    const improved = [];
+    const regressed = [];
+    for (const { index, before, after } of pairedCaseScores(pairs, metricName)) {
+        const delta = pairedDifference(after, before);
+        if (Math.abs(delta) <= SCORE_EPSILON)
+            continue;
+        const change = { index, input: truncateInput(inputOf(index)), before, after, delta };
+        (delta > 0 ? improved : regressed).push(change);
+    }
+    const byMove = (a, b) => Math.abs(b.delta) - Math.abs(a.delta) || a.index - b.index;
+    return { improved: improved.sort(byMove), regressed: regressed.sort(byMove) };
+}
+//# sourceMappingURL=comparison-stats.js.map

package/dist/comparison-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-stats.js","sourceRoot":"","sources":["../src/comparison-stats.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAkFnD,sDAAsD;AACtD,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC;AAElC,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,+DAA+D;AAC/D,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAY/B,SAAS,UAAU,CACjB,QAAkB,EAClB,UAAoB,EACpB,UAA4B,EAC5B,aAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;IACvE,IAAI,aAAa,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;YAClD,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;QACpD,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,mBAAmB;YAChC,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE;YACzE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,EAAE,CAAC;IAC3F,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;AACpF,CAAC;AAED,8DAA8D;AAC9D,SAAS,aAAa,CAAC,QAAkB,EAAE,UAAoB;IAC7D,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAMD,SAAS,SAAS,CAAC,MAAgB;IACjC,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,cAAwB,EACxB,gBAA0B,EAC1B,MAA6B;IAE7B,MAAM,WAAW,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IACpE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,WAAW,EAAE;QAC7C,UAAU,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK;QAC5B,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,cAAc,CAAC;QACnC,UAAU,EAAE,SAAS,CAAC,gBAAgB,CAAC;QACvC,CAAC,EAAE,cAAc,CAAC,MAAM;QACxB,UAAU,EAAE,SAAS,CAAC,WAAW,CAAC;QAClC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;QACjB,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC;QAChC,GAAG,UAAU,CAAC,cAAc,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC;KACnE,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,MAA0D,EAC1D,MAA6B;IAE7B,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAClC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAC7D,OAAO,aAAa,CAAC,IAAI,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,KAAK,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,GAAG;YACd,GAAG,MAAM;YACT,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC5B,WAAW;YACX,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU;SACzE,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,QAAkB,EAClB,UAAoB,EACpB,MAA6B;IAE7B,MAAM,UAAU,GAAG,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;IAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzF,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,KAAK,CAAC,CAAC;IAC7D,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC7B,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC;QACjC,CAAC,EAAE,QAAQ,CAAC,MAAM;QAClB,UAAU;QACV,IAAI;QACJ,MAAM;QACN,WAAW;QACX,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU;KACzE,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,sBAAsB,CACpC,OAAyD;IAEzD,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,iBAAiB,CAAC,OAA8B;IAC9D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,SAAS,CAAC,MAA+B,EAAE,UAAkB;IAC3E,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAC1E,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CACvB,KAA0B,EAC1B,UAAkB;IAElB,MAAM,MAAM,GAA4D,EAAE,CAAC;IAC3E,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAC5C,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAC1D,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,2FAA2F;AAC3F,MAAM,UAAU,YAAY,CAAC,KAA0B,EAAE,UAAkB;IACzE,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACnD,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AACnE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAA0B,EAC1B,UAAkB,EAClB,OAAkC;IAElC,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,MAAM,SAAS,GAAiB,EAAE,CAAC;IACnC,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,aAAa;YAAE,SAAS;QAC/C,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACrF,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,CAAa,EAAE,CAAa,EAAE,EAAE,CAC9C,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAC7D,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;AAChF,CAAC"}

package/dist/datasets/dataset.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,OAAO;IAIP,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO;WAI1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAKzC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKpD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,KAAK,IAAI,SAAS,QAAQ,EAAE,CAE/B;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;IAavC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO;IAI7C;;;OAGG;IACH,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GAAG,OAAO;IAYpE,yEAAyE;IACzE,OAAO,CAAC,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GAAG,OAAO;CAG3D"}
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAK1C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,OAAO;IAIP,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO;WAI1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAKzC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKpD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,KAAK,IAAI,SAAS,QAAQ,EAAE,CAE/B;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;IAavC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO;IAI7C;;;OAGG;IACH,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GAAG,OAAO;IAYpE,yEAAyE;IACzE,OAAO,CAAC,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GAAG,OAAO;CAG3D"}

package/dist/datasets/dataset.js

@@ -1,6 +1,7 @@
 import { EvalCaseSchema } from '../schema.js';
 import { loadJsonl } from './jsonl-loader.js';
 import { loadCsv } from './csv-loader.js';
+import { seededRandom } from '../stats/random.js';
 export class Dataset {
     _cases;
     constructor(cases) {
@@ -47,12 +48,12 @@ export class Dataset {
             return new Dataset([]);
         }
         const count = Math.min(n, this._cases.length);
-        const indices = shuffled(Array.from({ length: this._cases.length }, (_, i) => i), randomOf(options.seed));
+        const indices = shuffled(Array.from({ length: this._cases.length }, (_, i) => i), seededRandom(options.seed));
         return new Dataset(indices.slice(0, count).map((i) => this._cases[i]));
     }
     /** The cases in random order, the same order on every run with `seed` */
     shuffle(options = {}) {
-        return new Dataset(shuffled([...this._cases], randomOf(options.seed)));
+        return new Dataset(shuffled([...this._cases], seededRandom(options.seed)));
     }
 }
 /** Shuffles `items` in place (Fisher-Yates) with the random numbers `random` gives */
@@ -63,29 +64,4 @@ function shuffled(items, random) {
     }
     return items;
 }
-/**
- * `Math.random`, or a mulberry32 generator seeded by the hash of `seed`, so every number and
- * string seeds its own sequence (`1.2` and `1.7`, `-1` and `2 ** 32 - 1` alike)
- */
-function randomOf(seed) {
-    if (seed === undefined)
-        return Math.random;
-    let state = hashSeed(String(seed));
-    return () => {
-        state = (state + 0x6d2b79f5) >>> 0;
-        let t = state;
-        t = Math.imul(t ^ (t >>> 15), t | 1);
-        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
-        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-    };
-}
-/** FNV-1a of a string, to seed the generator */
-function hashSeed(seed) {
-    let hash = 0x811c9dc5;
-    for (let i = 0; i < seed.length; i++) {
-        hash ^= seed.charCodeAt(i);
-        hash = Math.imul(hash, 0x01000193) >>> 0;
-    }
-    return hash;
-}
 //# sourceMappingURL=dataset.js.map

package/dist/datasets/dataset.js.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,CAAS,EAAE,UAAsC,EAAE;QACxD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,QAAQ,CACtB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACvD,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CACvB,CAAC;QACF,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,yEAAyE;IACzE,OAAO,CAAC,UAAsC,EAAE;QAC9C,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;CACF;AAED,sFAAsF;AACtF,SAAS,QAAQ,CAAI,KAAU,EAAE,MAAoB;IACnD,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,IAAiC;IACjD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC3C,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,OAAO,GAAG,EAAE;QACV,KAAK,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,KAAK,CAAC;QACd,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAED,gDAAgD;AAChD,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,CAAS,EAAE,UAAsC,EAAE;QACxD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,QAAQ,CACtB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACvD,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAC3B,CAAC;QACF,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,yEAAyE;IACzE,OAAO,CAAC,UAAsC,EAAE;QAC9C,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;CACF;AAED,sFAAsF;AACtF,SAAS,QAAQ,CAAI,KAAU,EAAE,MAAoB;IACnD,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/eval-comparison.d.ts

@@ -2,6 +2,8 @@ import type { EvalTarget, EvalSuiteResult } from './eval-suite.js';
 import type { MetricFn, StatisticalMetricFn } from './metrics/types.js';
 import type { JudgeConfigInput } from './schema.js';
 import { Dataset } from './datasets/index.js';
+import type { PairwiseOptions, PairwiseResult } from './pairwise.js';
+import type { ComparisonWinner, MetricComparison, ResourceComparison } from './comparison-stats.js';
 export interface EvalComparisonOptions {
     dataset: Dataset;
     targets: {
@@ -10,6 +12,7 @@ export interface EvalComparisonOptions {
     };
     metrics?: MetricFn[];
     statisticalMetrics?: StatisticalMetricFn[];
+    /** The judge of LLM metrics and of `pairwise`, on both sides */
     judge?: JudgeConfigInput;
     concurrency?: number;
     timeout?: number;
@@ -19,6 +22,23 @@ export interface EvalComparisonOptions {
      * mean over its scored trials, and `passAtK()` statistical metrics see every trial
      */
     trials?: number;
+    /**
+     * Significance level, 0.05 by default. A metric is significant when its p-value, adjusted with
+     * Holm's correction across the compared metrics, is below it, and its `ci` covers `1 - alpha`
+     */
+    alpha?: number;
+    /**
+     * Test of metrics that are not binary, and of latency and cost: `t` (paired t-test, the
+     * default) or `wilcoxon` (Wilcoxon signed-rank). Binary metrics always take McNemar's test
+     */
+    test?: 't' | 'wilcoxon';
+    /** Seeds the bootstrap of each metric's `ci`, so the intervals are the same on every run */
+    seed?: number | string;
+    /**
+     * Asks the judge which side answered each case better, both ways round, into
+     * `result.pairwise`. Needs `judge`, run by `judge.cogitator` or an agent target's `cogitator`
+     */
+    pairwise?: PairwiseOptions;
     onProgress?: (progress: {
         target: string;
         completed: number;
@@ -29,25 +49,25 @@ export interface EvalComparisonRunOptions {
     /** Cancels both runs; `run()` rejects with the signal's reason */
     signal?: AbortSignal;
 }
-export interface MetricComparison {
-    baseline: number;
-    challenger: number;
-    /** Cases compared: those at least one side scored, an errored side counting as a 0 */
-    n: number;
-    pValue: number;
-    significant: boolean;
-    winner: 'baseline' | 'challenger' | 'tie';
-}
+export type { ComparisonTest, ComparisonWinner, MetricComparison, ResourceComparison, } from './comparison-stats.js';
 export interface ComparisonResult {
     summary: {
-        winner: 'baseline' | 'challenger' | 'tie';
+        winner: ComparisonWinner;
         metrics: Record<string, MetricComparison>;
+        /** Milliseconds per case, the mean over the answered trials of a case */
+        latency: ResourceComparison;
+        /** USD per case, over the cases both sides reported usage on. Absent when none did */
+        cost?: ResourceComparison;
     };
     baseline: EvalSuiteResult;
     challenger: EvalSuiteResult;
+    /** The judge's pairwise verdicts, with the `pairwise` option */
+    pairwise?: PairwiseResult;
 }
 export declare class EvalComparison {
     private readonly opts;
+    private readonly config;
+    private readonly pairwiseJudge?;
     constructor(opts: EvalComparisonOptions);
     run(options?: EvalComparisonRunOptions): Promise<ComparisonResult>;
 }

package/dist/eval-comparison.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAiC,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACpG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAmED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IAUjC,GAAG,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA2E7E"}
+{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAoB,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACvF,OAAO,KAAK,EAAwC,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEvF,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAIrC,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAUlE,OAAO,KAAK,EAEV,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,gEAAgE;IAChE,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC;IACxB,4FAA4F;IAC5F,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,gBAAgB,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAC1C,yEAAyE;QACzE,OAAO,EAAE,kBAAkB,CAAC;QAC5B,sFAAsF;QACtF,IAAI,CAAC,EAAE,kBAAkB,CAAC;KAC3B,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAe;gBAElC,IAAI,EAAE,qBAAqB;IAiCjC,GAAG,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA8F7E"}

package/dist/eval-comparison.js

@@ -1,77 +1,37 @@
 import { EvalSuite } from './eval-suite.js';
-import { EvalComparisonConfigSchema } from './schema.js';
-import { pairedTTest } from './stats/t-test.js';
-import { mcnemarsTest } from './stats/mcnemar.js';
+import { EvalComparisonConfigSchema, JudgeConfigSchema } from './schema.js';
+import { judgeContextFor } from './metrics/llm-judge.js';
+import { judgePairwise } from './pairwise.js';
+import { parseOptions } from './options.js';
 import { mean } from './stats/percentiles.js';
-function isBinary(scores) {
-    return scores.every((s) => s === 0 || s === 1);
-}
-function compareMetric(baselineScores, challengerScores) {
-    const baselineMean = mean(baselineScores);
-    const challengerMean = mean(challengerScores);
-    const n = baselineScores.length;
-    if (n < 2) {
-        return {
-            baseline: baselineMean,
-            challenger: challengerMean,
-            n,
-            pValue: 1,
-            significant: false,
-            winner: 'tie',
-        };
-    }
-    const binary = isBinary(baselineScores) && isBinary(challengerScores);
-    let pValue;
-    let significant;
-    if (binary) {
-        let bCorrect_cIncorrect = 0;
-        let bIncorrect_cCorrect = 0;
-        for (let i = 0; i < n; i++) {
-            if (baselineScores[i] === 1 && challengerScores[i] === 0)
-                bCorrect_cIncorrect++;
-            if (baselineScores[i] === 0 && challengerScores[i] === 1)
-                bIncorrect_cCorrect++;
-        }
-        const result = mcnemarsTest(bCorrect_cIncorrect, bIncorrect_cCorrect);
-        pValue = result.pValue;
-        significant = result.significant;
-    }
-    else {
-        const result = pairedTTest(baselineScores, challengerScores);
-        pValue = result.pValue;
-        significant = result.significant;
-    }
-    let winner = 'tie';
-    if (significant) {
-        winner = challengerMean > baselineMean ? 'challenger' : 'baseline';
-    }
-    return { baseline: baselineMean, challenger: challengerMean, n, pValue, significant, winner };
-}
-function determineOverallWinner(metrics) {
-    let baselineWins = 0;
-    let challengerWins = 0;
-    for (const mc of Object.values(metrics)) {
-        if (mc.winner === 'baseline')
-            baselineWins++;
-        if (mc.winner === 'challenger')
-            challengerWins++;
-    }
-    if (challengerWins > baselineWins)
-        return 'challenger';
-    if (baselineWins > challengerWins)
-        return 'baseline';
-    return 'tie';
-}
+import { compareMetrics, compareResource, determineOverallWinner, pairedScores, scoredMetricNames, } from './comparison-stats.js';
 export class EvalComparison {
     opts;
+    config;
+    pairwiseJudge;
     constructor(opts) {
-        EvalComparisonConfigSchema.parse({
+        this.config = parseOptions(EvalComparisonConfigSchema, {
             concurrency: opts.concurrency,
             timeout: opts.timeout,
             retries: opts.retries,
             trials: opts.trials,
-        });
+            alpha: opts.alpha,
+            test: opts.test,
+            seed: opts.seed,
+            pairwise: opts.pairwise,
+        }, 'EvalComparison');
         this.opts = opts;
+        if (this.config.pairwise) {
+            if (!opts.judge)
+                throw new Error('Pairwise judging requires a judge config');
+            const judgeConfig = JudgeConfigSchema.parse(opts.judge);
+            const cogitator = judgeConfig.cogitator ??
+                (opts.targets.baseline.cogitator ?? opts.targets.challenger.cogitator);
+            if (!cogitator) {
+                throw new Error('Pairwise judging needs a Cogitator to run the judge: pass judge.cogitator, or use an agent target');
+            }
+            this.pairwiseJudge = judgeContextFor(cogitator, judgeConfig);
+        }
     }
     async run(options = {}) {
         const { dataset, targets, metrics, statisticalMetrics, judge, concurrency, timeout, retries, trials, onProgress, } = this.opts;
@@ -103,59 +63,69 @@ export class EvalComparison {
             baselineSuite.run({ signal: options.signal }),
             challengerSuite.run({ signal: options.signal }),
         ]);
-        const metricNames = new Set();
-        for (const result of [baselineResult, challengerResult]) {
-            for (const r of result.results) {
-                for (const trial of r.trials ?? [r]) {
-                    for (const s of trial.scores)
-                        metricNames.add(s.name);
-                }
-            }
-        }
-        const metricComparisons = {};
-        for (const name of metricNames) {
-            const [baselineScores, challengerScores] = pairedScores(baselineResult, challengerResult, name);
-            metricComparisons[name] = compareMetric(baselineScores, challengerScores);
+        const pairs = casePairs(baselineResult, challengerResult);
+        const paired = new Map();
+        for (const name of scoredMetricNames([
+            ...baselineResult.results,
+            ...challengerResult.results,
+        ])) {
+            paired.set(name, pairedScores(pairs, name));
         }
+        const metricComparisons = compareMetrics(paired, this.config);
+        const latency = compareResource(...pairedResource(baselineResult, challengerResult, caseLatency), this.config);
+        const [baselineCost, challengerCost] = pairedResource(baselineResult, challengerResult, caseCost);
+        const cost = baselineCost.length > 0
+            ? compareResource(baselineCost, challengerCost, this.config)
+            : undefined;
+        const pairwise = this.pairwiseJudge && this.config.pairwise
+            ? await judgePairwise(baselineResult, challengerResult, this.pairwiseJudge, this.config.pairwise, this.config.concurrency, options.signal)
+            : undefined;
         return {
             summary: {
                 winner: determineOverallWinner(metricComparisons),
                 metrics: metricComparisons,
+                latency,
+                ...(cost && { cost }),
             },
             baseline: baselineResult,
             challenger: challengerResult,
+            ...(pairwise && { pairwise }),
         };
     }
 }
-/** A case's score of a metric: the mean over its trials the metric scored, none when it scored none */
-function caseScore(result, metricName) {
-    if (!result)
-        return undefined;
-    const trials = result.trials ?? [result];
-    if (trials.every((trial) => trial.error !== undefined))
-        return undefined;
-    const scores = trials.flatMap((trial) => trial.error !== undefined
-        ? [0]
-        : trial.scores.filter((s) => s.name === metricName).map((s) => s.score));
-    return scores.length === 0 ? undefined : mean(scores);
+/** The trials of a case that answered */
+function answeredTrials(result) {
+    return (result.trials ?? [result]).filter((trial) => trial.error === undefined);
+}
+/** A case's latency: the mean duration of its answered trials, none when none answered */
+function caseLatency(result) {
+    const answered = answeredTrials(result);
+    return answered.length === 0 ? undefined : mean(answered.map((trial) => trial.duration));
+}
+/** A case's cost: the mean cost of its answered trials that reported usage */
+function caseCost(result) {
+    const costs = answeredTrials(result).flatMap((trial) => trial.usage === undefined ? [] : [trial.usage.cost]);
+    return costs.length === 0 ? undefined : mean(costs);
 }
-/**
- * The scores of a metric for each case, side by side. A case one side errored on counts as a 0
- * for that side, since a target that fails loses the case, and a case both sides errored on is
- * left out, telling nothing about either
- */
-function pairedScores(baseline, challenger, metricName) {
-    const scoreOf = (result, at) => caseScore(result.results[at], metricName);
-    const baselineScores = [];
-    const challengerScores = [];
+/** A per-case figure of both sides, over the cases both have it for */
+function pairedResource(baseline, challenger, figure) {
+    const baselineValues = [];
+    const challengerValues = [];
     for (let at = 0; at < baseline.results.length; at++) {
-        const b = scoreOf(baseline, at);
-        const c = scoreOf(challenger, at);
-        if (b === undefined && c === undefined)
+        const challengerResult = challenger.results[at];
+        if (!challengerResult)
             continue;
-        baselineScores.push(b ?? 0);
-        challengerScores.push(c ?? 0);
+        const b = figure(baseline.results[at]);
+        const c = figure(challengerResult);
+        if (b === undefined || c === undefined)
+            continue;
+        baselineValues.push(b);
+        challengerValues.push(c);
     }
-    return [baselineScores, challengerScores];
+    return [baselineValues, challengerValues];
+}
+/** The cases of the two runs side by side, by position */
+function casePairs(baseline, challenger) {
+    return baseline.results.map((result, at) => [result, challenger.results[at]]);
 }
 //# sourceMappingURL=eval-comparison.js.map

package/dist/eval-comparison.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AA8C3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,CAAC;YACD,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAChG,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;wBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,uGAAuG;AACvG,SAAS,SAAS,CAAC,MAAoC,EAAE,UAAkB;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACvF,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGzE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EACL,cAAc,EACd,eAAe,EACf,sBAAsB,EACtB,YAAY,EACZ,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAyE5B,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAC5B,MAAM,CAAuB;IAC7B,aAAa,CAAgB;IAE9C,YAAY,IAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,YAAY,CACxB,0BAA0B,EAC1B;YACE,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,EACD,gBAAgB,CACjB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;YAC7E,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,SAAS,GACb,WAAW,CAAC,SAAS;gBACpB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CACzC,CAAC;YAChC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,mGAAmG,CACpG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,SAAS,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgC,CAAC;QACvD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc,CAAC,OAAO;YACzB,GAAG,gBAAgB,CAAC,OAAO;SAC5B,CAAC,EAAE,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAE9D,MAAM,OAAO,GAAG,eAAe,CAC7B,GAAG,cAAc,CAAC,cAAc,EAAE,gBAAgB,EAAE,WAAW,CAAC,EAChE,IAAI,CAAC,MAAM,CACZ,CAAC;QACF,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,cAAc,CACnD,cAAc,EACd,gBAAgB,EAChB,QAAQ,CACT,CAAC;QACF,MAAM,IAAI,GACR,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC;YAC5D,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,QAAQ,GACZ,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ;YACxC,CAAC,CAAC,MAAM,aAAa,CACjB,cAAc,EACd,gBAAgB,EAChB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CAAC,WAAW,EACvB,OAAO,CAAC,MAAM,CACf;YACH,CAAC,CAAC,SAAS,CAAC;QAEhB,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;gBAC1B,OAAO;gBACP,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC;aACtB;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;YAC5B,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;CACF;AAED,yCAAyC;AACzC,SAAS,cAAc,CAAC,MAAwB;IAC9C,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;AAClF,CAAC;AAED,0FAA0F;AAC1F,SAAS,WAAW,CAAC,MAAwB;IAC3C,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,MAAwB;IACxC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACrD,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CACpD,CAAC;IACF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,uEAAuE;AACvE,SAAS,cAAc,CACrB,QAAyB,EACzB,UAA2B,EAC3B,MAAwD;IAExD,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAChC,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC;AAED,0DAA0D;AAC1D,SAAS,SAAS,CAAC,QAAyB,EAAE,UAA2B;IACvE,OAAO,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAU,CAAC,CAAC;AACzF,CAAC"}