@cogitator-ai/evals 0.11.0 → 0.13.0

package/dist/reporters/markdown.js

@@ -0,0 +1,67 @@
+import { writeFileSync } from 'node:fs';
+import { caseLabel, caseProblems, cell, num, trialsPerCase } from './format.js';
+/** Most failures the summary lists before it counts the rest */
+export const MARKDOWN_MAX_FAILURES = 25;
+/** The failures of the cases, of every trial: errored runs, then metrics that failed */
+function failures(result) {
+    return result.results.flatMap((r, i) => {
+        const label = caseLabel(i, r.case.input);
+        return caseProblems(r).map((problem) => ({ label, problem }));
+    });
+}
+/**
+ * A compact Markdown summary of a suite run for a pull request comment: the totals, the mean of
+ * each metric with the cases it covers, the assertions and the failures of every case and trial.
+ * Every value from the run is escaped, so no answer can break the tables or hide the rest of the
+ * comment
+ */
+export function markdownSummary(result) {
+    const { stats } = result;
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.length - passed;
+    const trials = trialsPerCase(result);
+    const totals = [
+        `**${stats.total}** cases${trials > 1 ? ` x **${trials}** trials` : ''}`,
+        `**${stats.errors ?? 0}** errored`,
+        ...(stats.metricErrors ? [`**${stats.metricErrors}** metric errors`] : []),
+        `${stats.duration} ms`,
+        `${num(stats.cost)} USD`,
+        ...(result.assertions.length > 0
+            ? [`assertions: **${passed}** passed, **${failed}** failed`]
+            : []),
+    ];
+    const title = result.name === undefined ? 'Eval results' : `Eval results: ${cell(result.name)}`;
+    const lines = [`### ${title}`, '', totals.join(' | '), ''];
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        lines.push('| Metric | Mean | Cases |', '| --- | ---: | ---: |');
+        for (const m of metrics) {
+            lines.push(`| ${cell(m.name)} | ${num(m.mean)} | ${m.count === undefined ? '-' : m.count} |`);
+        }
+        lines.push('');
+    }
+    if (result.assertions.length > 0) {
+        lines.push('| Assertion | Result | Details |', '| --- | --- | --- |');
+        for (const a of result.assertions) {
+            lines.push(`| ${cell(a.name)} | ${a.passed ? 'pass' : '**fail**'} | ${cell(a.message)} |`);
+        }
+        lines.push('');
+    }
+    const failedCases = failures(result);
+    if (failedCases.length > 0) {
+        lines.push(`**Failures** (${failedCases.length})`, '', '| Case | Problem |', '| --- | --- |');
+        for (const { label, problem } of failedCases.slice(0, MARKDOWN_MAX_FAILURES)) {
+            lines.push(`| ${cell(label)} | ${cell(problem)} |`);
+        }
+        if (failedCases.length > MARKDOWN_MAX_FAILURES) {
+            lines.push('', `and ${failedCases.length - MARKDOWN_MAX_FAILURES} more`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+/** Writes the Markdown summary of a suite run */
+export function markdownReport(result, options) {
+    writeFileSync(options.path, markdownSummary(result), 'utf-8');
+}
+//# sourceMappingURL=markdown.js.map

package/dist/reporters/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/reporters/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE7E,gEAAgE;AAChE,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAExC,wFAAwF;AACxF,SAAS,QAAQ,CAAC,MAAuB;IACvC,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,MAAuB;IACrD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;QACxE,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,CAAC,iBAAiB,MAAM,gBAAgB,MAAM,WAAW,CAAC;YAC5D,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;IAChG,MAAM,KAAK,GAAG,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,uBAAuB,CAAC,CAAC;QACjE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QACtE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE,oBAAoB,EAAE,eAAe,CAAC,CAAC;QAC9F,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,EAAE,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,WAAW,CAAC,MAAM,GAAG,qBAAqB,OAAO,CAAC,CAAC;QAC3E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,cAAc,CAAC,MAAuB,EAAE,OAAyB;IAC/E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/runs/compare.d.ts

@@ -0,0 +1,103 @@
+import { z } from 'zod';
+import type { ComparisonTest } from '../comparison-stats.js';
+import type { EvalRunSource } from './run.js';
+export declare const CompareRunsOptionsSchema: z.ZodObject<{
+    alpha: z.ZodDefault<z.ZodNumber>;
+    test: z.ZodDefault<z.ZodEnum<{
+        t: "t";
+        wilcoxon: "wilcoxon";
+    }>>;
+    seed: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
+    metrics: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export interface CompareRunsOptions {
+    /**
+     * Significance level, 0.05 by default. A metric is significant when its p-value, adjusted with
+     * Holm's correction across the compared metrics, is below it, and its `ci` covers `1 - alpha`
+     */
+    alpha?: number;
+    /** Test of metrics that are not binary: `t` (paired t-test, the default) or `wilcoxon` */
+    test?: 't' | 'wilcoxon';
+    /** Seeds the bootstrap of each metric's `ci`, so the intervals are the same on every call */
+    seed?: number | string;
+    /**
+     * The metrics to compare. Default: every per-case metric both runs have and scored on at least
+     * one matched case, the others named in `warnings`, so a metric without data neither shows two
+     * means of 0 nor weakens the Holm correction of the others
+     */
+    metrics?: string[];
+}
+export type RunWinner = 'base' | 'head' | 'tie';
+/** How the score of a case moved from the base run to the head run */
+export interface RunCaseChange {
+    caseKey: string;
+    /** The case input on one line, shortened */
+    input: string;
+    base: number;
+    head: number;
+    /** `head` minus `base` */
+    delta: number;
+}
+export interface RunMetricComparison {
+    /** Mean score of the base run over the compared cases, NaN when `n` is 0 */
+    base: number;
+    head: number;
+    /** Cases compared: matched cases at least one run scored, an errored side counting as a 0 */
+    n: number;
+    /** Mean of the paired differences, head minus base */
+    difference: number;
+    /** Percentile bootstrap confidence interval of `difference` at `1 - alpha` */
+    ci: {
+        low: number;
+        high: number;
+    };
+    /** Cohen's d of the paired differences (dz), NaN with fewer than 2 cases */
+    effectSize: number;
+    test: ComparisonTest;
+    pValue: number;
+    /** `pValue` after Holm's correction across the compared metrics */
+    adjustedPValue: number;
+    significant: boolean;
+    /** The higher run when significant, by the test's direction */
+    winner: RunWinner;
+    /** Cases whose score rose, the largest rise first */
+    improved: RunCaseChange[];
+    /** Cases whose score fell, the largest fall first */
+    regressed: RunCaseChange[];
+}
+/** Which run a side of a comparison is */
+export interface RunRef {
+    runId: string;
+    name?: string;
+    startedAt: string;
+}
+/** A case only one of the runs has */
+export interface RunCaseRef {
+    caseKey: string;
+    input: string;
+}
+export interface RunComparison {
+    base: RunRef;
+    head: RunRef;
+    /** The run more metrics name as their winner, a tie when as many name each */
+    winner: RunWinner;
+    metrics: Record<string, RunMetricComparison>;
+    /** Cases both runs have, matched by `caseKey` */
+    matched: number;
+    /** Cases only the head run has */
+    added: RunCaseRef[];
+    /** Cases only the base run has */
+    removed: RunCaseRef[];
+    /**
+     * What makes the runs harder to compare: other cases, other prompts, other metrics or judge,
+     * and metrics left out because no matched case was scored on them
+     */
+    warnings: string[];
+}
+/**
+ * Compares two saved runs, or suite results, case by case: cases are matched by their `caseKey`,
+ * and each metric both runs have is compared as `EvalComparison` compares its sides, the base run
+ * in the baseline's place and the head run in the challenger's
+ */
+export declare function compareRuns(baseSource: EvalRunSource, headSource: EvalRunSource, options?: CompareRunsOptions): RunComparison;
+//# sourceMappingURL=compare.d.ts.map

package/dist/runs/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/runs/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,OAAO,KAAK,EAAE,cAAc,EAAoB,MAAM,qBAAqB,CAAC;AAI5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAG3C,eAAO,MAAM,wBAAwB;;;;;;;;iBAKnC,CAAC;AAEH,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0FAA0F;IAC1F,IAAI,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC;IACxB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;AAEhD,sEAAsE;AACtE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,4EAA4E;IAC5E,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,6FAA6F;IAC7F,CAAC,EAAE,MAAM,CAAC;IACV,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,EAAE,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAClC,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,+DAA+D;IAC/D,MAAM,EAAE,SAAS,CAAC;IAClB,qDAAqD;IACrD,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,qDAAqD;IACrD,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B;AAED,0CAA0C;AAC1C,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,sCAAsC;AACtC,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,EAAE,SAAS,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IAC7C,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,kCAAkC;IAClC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,kCAAkC;IAClC,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB;;;OAGG;IACH,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AA8JD;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,UAAU,EAAE,aAAa,EACzB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE,kBAAuB,GAC/B,aAAa,CAyDf"}

package/dist/runs/compare.js

@@ -0,0 +1,208 @@
+import { z } from 'zod';
+import { parseOptions } from '../options.js';
+import { caseChanges, compareMetrics, determineOverallWinner, pairedScores, scoredMetricNames, } from '../comparison-stats.js';
+import { truncateInput } from '../reporters/format.js';
+import { canonicalJson } from './json.js';
+import { runOf } from './run.js';
+export const CompareRunsOptionsSchema = z.object({
+    alpha: z.number().gt(0).lt(1).default(0.05),
+    test: z.enum(['t', 'wilcoxon']).default('t'),
+    seed: z.union([z.number(), z.string()]).optional(),
+    metrics: z.array(z.string().min(1)).min(1).optional(),
+});
+const WINNERS = {
+    baseline: 'base',
+    challenger: 'head',
+    tie: 'tie',
+};
+function refOf(run) {
+    return {
+        runId: run.runId,
+        ...(run.name !== undefined && { name: run.name }),
+        startedAt: run.startedAt,
+    };
+}
+function caseRef(result) {
+    return { caseKey: result.caseKey, input: truncateInput(result.case.input) };
+}
+/**
+ * Pairs the cases of the runs by `caseKey`, in the order of the base run. A key that repeats pairs
+ * its first case of one run with its first case of the other, its second with the second, and so on
+ */
+function matchCases(base, head) {
+    const headByKey = new Map();
+    for (const result of head.results) {
+        const list = headByKey.get(result.caseKey);
+        if (list)
+            list.push(result);
+        else
+            headByKey.set(result.caseKey, [result]);
+    }
+    const pairs = [];
+    const removed = [];
+    for (const result of base.results) {
+        const match = headByKey.get(result.caseKey)?.shift();
+        if (match)
+            pairs.push([result, match]);
+        else
+            removed.push(caseRef(result));
+    }
+    const added = [...headByKey.values()].flat();
+    const addedSet = new Set(added);
+    return {
+        pairs,
+        removed,
+        added: head.results.filter((result) => addedSet.has(result)).map(caseRef),
+    };
+}
+/** The per-case metrics of a run: those it was set up with and those its cases were scored on */
+function caseMetricNames(run) {
+    return new Set([...run.config.metrics, ...scoredMetricNames(run.results)]);
+}
+function metricsToCompare(base, head, requested) {
+    const baseNames = caseMetricNames(base);
+    const headNames = caseMetricNames(head);
+    const shared = [...baseNames].filter((name) => headNames.has(name));
+    if (requested === undefined)
+        return shared;
+    const missing = requested.filter((name) => !shared.includes(name));
+    if (missing.length > 0) {
+        throw new Error(`compareRuns: ${missing.map((name) => `'${name}'`).join(', ')} not in both runs, the base run has ${[...baseNames].join(', ') || 'no metrics'} and the head run ${[...headNames].join(', ') || 'no metrics'}`);
+    }
+    return [...new Set(requested)];
+}
+function promptLabel(prompt) {
+    if (prompt.abTest)
+        return `A/B test ${prompt.abTest.id} (${prompt.abTest.variant})`;
+    if (prompt.version !== undefined) {
+        return prompt.versionId !== undefined
+            ? `v${prompt.version} (${prompt.versionId})`
+            : `v${prompt.version}`;
+    }
+    return prompt.versionId ?? 'unversioned instructions';
+}
+function promptsByKey(run) {
+    const byKey = new Map();
+    for (const prompt of run.prompts) {
+        const labels = byKey.get(prompt.key) ?? new Map();
+        labels.set(canonicalJson(prompt), promptLabel(prompt));
+        byKey.set(prompt.key, labels);
+    }
+    return byKey;
+}
+function promptWarnings(base, head) {
+    const baseKeys = promptsByKey(base);
+    const headKeys = promptsByKey(head);
+    const warnings = [];
+    for (const key of new Set([...baseKeys.keys(), ...headKeys.keys()])) {
+        const before = baseKeys.get(key);
+        const after = headKeys.get(key);
+        const list = (labels) => [...labels.values()].join(', ');
+        if (!before) {
+            warnings.push(`Prompt '${key}' is used by the head run only (${list(after)})`);
+        }
+        else if (!after) {
+            warnings.push(`Prompt '${key}' is used by the base run only (${list(before)})`);
+        }
+        else if (before.size !== after.size ||
+            [...before.keys()].some((identity) => !after.has(identity))) {
+            warnings.push(`Prompt '${key}' differs: the base run used ${list(before)}, the head run ${list(after)}`);
+        }
+    }
+    return warnings;
+}
+function difference(label, base, head) {
+    const onlyBase = base.filter((name) => !head.includes(name));
+    const onlyHead = head.filter((name) => !base.includes(name));
+    if (onlyBase.length === 0 && onlyHead.length === 0)
+        return undefined;
+    const parts = [
+        ...(onlyBase.length > 0 ? [`only the base run has ${onlyBase.join(', ')}`] : []),
+        ...(onlyHead.length > 0 ? [`only the head run has ${onlyHead.join(', ')}`] : []),
+    ];
+    return `${label} differ: ${parts.join(', ')}`;
+}
+function judgeLabel(judge) {
+    if (!judge)
+        return 'no judge';
+    const samples = judge.samples !== undefined ? `, ${judge.samples} samples` : '';
+    return `${judge.model} at temperature ${judge.temperature}${samples}`;
+}
+function configWarnings(base, head) {
+    const warnings = [];
+    const metrics = difference('Metrics', [...base.config.metrics, ...base.config.statisticalMetrics], [...head.config.metrics, ...head.config.statisticalMetrics]);
+    if (metrics)
+        warnings.push(metrics);
+    if (canonicalJson(base.config.judge ?? null) !== canonicalJson(head.config.judge ?? null)) {
+        warnings.push(`Judge differs: the base run used ${judgeLabel(base.config.judge)}, the head run ${judgeLabel(head.config.judge)}`);
+    }
+    return warnings;
+}
+function datasetWarning(base, head, added, removed) {
+    if (base.dataset.fingerprint === head.dataset.fingerprint)
+        return undefined;
+    if (added === 0 && removed === 0) {
+        return 'The datasets differ (fingerprints differ) while every case matches by input, turns and expected answer: their order, context or metadata changed';
+    }
+    return `The datasets differ (fingerprints differ): ${added} cases only in the head run, ${removed} only in the base run`;
+}
+/**
+ * Compares two saved runs, or suite results, case by case: cases are matched by their `caseKey`,
+ * and each metric both runs have is compared as `EvalComparison` compares its sides, the base run
+ * in the baseline's place and the head run in the challenger's
+ */
+export function compareRuns(baseSource, headSource, options = {}) {
+    const config = parseOptions(CompareRunsOptionsSchema, options, 'compareRuns');
+    const base = runOf(baseSource);
+    const head = runOf(headSource);
+    const { pairs, added, removed } = matchCases(base, head);
+    const paired = new Map();
+    const unscored = [];
+    for (const name of metricsToCompare(base, head, config.metrics)) {
+        const scores = pairedScores(pairs, name);
+        if (scores[0].length === 0 && config.metrics === undefined)
+            unscored.push(name);
+        else
+            paired.set(name, scores);
+    }
+    const names = [...paired.keys()];
+    const compared = compareMetrics(paired, config);
+    const metrics = {};
+    for (const name of names) {
+        const { baseline, challenger, winner, ...stats } = compared[name];
+        const changes = caseChanges(pairs, name, (index) => pairs[index][0].case.input);
+        const toRunChange = (change) => ({
+            caseKey: pairs[change.index][0].caseKey,
+            input: change.input,
+            base: change.before,
+            head: change.after,
+            delta: change.delta,
+        });
+        metrics[name] = {
+            base: baseline,
+            head: challenger,
+            ...stats,
+            winner: WINNERS[winner],
+            improved: changes.improved.map(toRunChange),
+            regressed: changes.regressed.map(toRunChange),
+        };
+    }
+    const dataset = datasetWarning(base, head, added.length, removed.length);
+    const warnings = [
+        ...(dataset ? [dataset] : []),
+        ...promptWarnings(base, head),
+        ...configWarnings(base, head),
+        ...unscored.map((name) => `Metric '${name}' was scored on no case the runs share (every case errored or went unscored), so it is not compared`),
+    ];
+    return {
+        base: refOf(base),
+        head: refOf(head),
+        winner: WINNERS[determineOverallWinner(compared)],
+        metrics,
+        matched: pairs.length,
+        added,
+        removed,
+        warnings,
+    };
+}
+//# sourceMappingURL=compare.js.map

package/dist/runs/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/runs/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EACL,WAAW,EACX,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAI9B,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3C,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACtD,CAAC,CAAC;AA4FH,MAAM,OAAO,GAAwC;IACnD,QAAQ,EAAE,MAAM;IAChB,UAAU,EAAE,MAAM;IAClB,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,SAAS,KAAK,CAAC,GAAY;IACzB,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;QACjD,SAAS,EAAE,GAAG,CAAC,SAAS;KACzB,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,MAAmB;IAClC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,IAAa,EAAE,IAAa;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;IACnD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;YACvB,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAA+C,EAAE,CAAC;IAC7D,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC;QACrD,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;;YAClC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,OAAO;QACL,KAAK;QACL,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;KAC1E,CAAC;AACJ,CAAC;AAED,iGAAiG;AACjG,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa,EAAE,IAAa,EAAE,SAA+B;IACrF,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,gBAAgB,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,uCAAuC,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,qBAAqB,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,EAAE,CAC9M,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,MAAiB;IACpC,IAAI,MAAM,CAAC,MAAM;QAAE,OAAO,YAAY,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC;IACpF,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC,SAAS,KAAK,SAAS;YACnC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,SAAS,GAAG;YAC5C,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC,SAAS,IAAI,0BAA0B,CAAC;AACxD,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;QAClE,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAa,EAAE,IAAa;IAClD,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,CAAC,MAA2B,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,mCAAmC,IAAI,CAAC,KAAM,CAAC,GAAG,CAAC,CAAC;QAClF,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,mCAAmC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClF,CAAC;aAAM,IACL,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;YAC1B,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAC3D,CAAC;YACD,QAAQ,CAAC,IAAI,CACX,WAAW,GAAG,gCAAgC,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,EAAE,CAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,IAAc,EAAE,IAAc;IAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACrE,MAAM,KAAK,GAAG;QACZ,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChF,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACjF,CAAC;IACF,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,SAAS,UAAU,CAAC,KAAiC;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,UAAU,CAAC;IAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,OAAO,GAAG,KAAK,CAAC,KAAK,mBAAmB,KAAK,CAAC,WAAW,GAAG,OAAO,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,cAAc,CAAC,IAAa,EAAE,IAAa;IAClD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CACxB,SAAS,EACT,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAC3D,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAC5D,CAAC;IACF,IAAI,OAAO;QAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,CAAC;QAC1F,QAAQ,CAAC,IAAI,CACX,oCAAoC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACnH,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,IAAa,EACb,KAAa,EACb,OAAe;IAEf,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,IAAI,CAAC,OAAO,CAAC,WAAW;QAAE,OAAO,SAAS,CAAC;IAC5E,IAAI,KAAK,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,kJAAkJ,CAAC;IAC5J,CAAC;IACD,OAAO,8CAA8C,KAAK,gCAAgC,OAAO,uBAAuB,CAAC;AAC3H,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CACzB,UAAyB,EACzB,UAAyB,EACzB,UAA8B,EAAE;IAEhC,MAAM,MAAM,GAAG,YAAY,CAAC,wBAAwB,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgC,CAAC;IACvD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACzC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;YAC3E,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhD,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChF,MAAM,WAAW,GAAG,CAAC,MAAyC,EAAiB,EAAE,CAAC,CAAC;YACjF,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;YACvC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI,EAAE,MAAM,CAAC,MAAM;YACnB,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG;YACd,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,UAAU;YAChB,GAAG,KAAK;YACR,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC;YACvB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;YAC3C,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC;SAC9C,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG;QACf,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC;QAC7B,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC;QAC7B,GAAG,QAAQ,CAAC,GAAG,CACb,CAAC,IAAI,EAAE,EAAE,CACP,WAAW,IAAI,qGAAqG,CACvH;KACF,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QACjB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACjD,OAAO;QACP,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,KAAK;QACL,OAAO;QACP,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/runs/index.d.ts

@@ -0,0 +1,9 @@
+export { loadRun, caseKey } from './run.js';
+export type { EvalRunSource } from './run.js';
+export { RunStore } from './store.js';
+export type { RunSummary, RunFilter, RunStoreOptions, SkippedRunFile } from './store.js';
+export { compareRuns } from './compare.js';
+export type { CompareRunsOptions, RunComparison, RunMetricComparison, RunCaseChange, RunCaseRef, RunRef, RunWinner, } from './compare.js';
+export { EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION } from './schema.js';
+export type { EvalRun, EvalRunCase, EvalRunConfig, JsonMetadata } from './schema.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/runs/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/runs/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AACzC,YAAY,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,YAAY,EACV,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,aAAa,EACb,UAAU,EACV,MAAM,EACN,SAAS,GACV,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5E,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/runs/index.js

@@ -0,0 +1,5 @@
+export { loadRun, caseKey } from './run.js';
+export { RunStore } from './store.js';
+export { compareRuns } from './compare.js';
+export { EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION } from './schema.js';
+//# sourceMappingURL=index.js.map

package/dist/runs/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/runs/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAUxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC"}

package/dist/runs/json.d.ts

@@ -0,0 +1,11 @@
+/**
+ * A value as JSON keeps it, as `JSON.stringify` would write it (`toJSON` called once per value),
+ * except that a bigint becomes its
+ * decimal string and a circular reference the string `[Circular]` instead of throwing
+ */
+export declare function toJson(value: unknown): unknown;
+/** JSON of a value with the keys of every object sorted, the same text for the same data */
+export declare function canonicalJson(value: unknown): string;
+/** Hex SHA-256 of the canonical JSON of a value */
+export declare function fingerprint(value: unknown): string;
+//# sourceMappingURL=json.d.ts.map

package/dist/runs/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/runs/json.ts"],"names":[],"mappings":"AA8CA;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAE9C;AAED,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEpD;AAED,mDAAmD;AACnD,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAElD"}

package/dist/runs/json.js

@@ -0,0 +1,58 @@
+import { createHash } from 'node:crypto';
+function hasToJson(value) {
+    return typeof value.toJSON === 'function';
+}
+function convert(value, key, ancestors, sortKeys, calledToJson = false) {
+    if (value === null)
+        return null;
+    switch (typeof value) {
+        case 'string':
+        case 'boolean':
+            return value;
+        case 'number':
+            return Number.isFinite(value) ? value : null;
+        case 'bigint':
+            return value.toString();
+        case 'undefined':
+        case 'function':
+        case 'symbol':
+            return undefined;
+    }
+    const object = value;
+    if (!calledToJson && hasToJson(object)) {
+        return convert(object.toJSON(key), key, ancestors, sortKeys, true);
+    }
+    if (ancestors.includes(object))
+        return '[Circular]';
+    const nested = [...ancestors, object];
+    if (Array.isArray(object)) {
+        return object.map((item, at) => convert(item, String(at), nested, sortKeys) ?? null);
+    }
+    const keys = Object.keys(object);
+    if (sortKeys)
+        keys.sort();
+    const out = {};
+    for (const name of keys) {
+        const converted = convert(object[name], name, nested, sortKeys);
+        if (converted !== undefined)
+            out[name] = converted;
+    }
+    return out;
+}
+/**
+ * A value as JSON keeps it, as `JSON.stringify` would write it (`toJSON` called once per value),
+ * except that a bigint becomes its
+ * decimal string and a circular reference the string `[Circular]` instead of throwing
+ */
+export function toJson(value) {
+    return convert(value, '', [], false);
+}
+/** JSON of a value with the keys of every object sorted, the same text for the same data */
+export function canonicalJson(value) {
+    return JSON.stringify(convert(value, '', [], true) ?? null);
+}
+/** Hex SHA-256 of the canonical JSON of a value */
+export function fingerprint(value) {
+    return createHash('sha256').update(canonicalJson(value)).digest('hex');
+}
+//# sourceMappingURL=json.js.map

package/dist/runs/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/runs/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,OAAQ,KAA8B,CAAC,MAAM,KAAK,UAAU,CAAC;AACtE,CAAC;AAED,SAAS,OAAO,CACd,KAAc,EACd,GAAW,EACX,SAAmB,EACnB,QAAiB,EACjB,YAAY,GAAG,KAAK;IAEpB,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,QAAQ,OAAO,KAAK,EAAE,CAAC;QACrB,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,KAAK,CAAC;QACf,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/C,KAAK,QAAQ;YACX,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1B,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU,CAAC;QAChB,KAAK,QAAQ;YACX,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,MAAM,GAAG,KAAe,CAAC;IAC/B,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACpD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,EAAE,MAAM,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,QAAQ;QAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IAC1B,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,SAAS,GAAG,OAAO,CAAE,MAAkC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7F,IAAI,SAAS,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,KAAc;IACnC,OAAO,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;AACvC,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzE,CAAC"}

package/dist/runs/run.d.ts

@@ -0,0 +1,44 @@
+import type { EvalCase } from '../schema.js';
+import type { ScoredCaseResult } from '../metrics/types.js';
+import type { AggregatedMetric, AssertionResult } from '../assertions/index.js';
+import type { EvalRun, EvalRunConfig, JsonMetadata } from './schema.js';
+/** A saved run, or a suite result that turns into one */
+export type EvalRunSource = EvalRun | {
+    toRun(): EvalRun;
+};
+/** Everything a run file records of a finished suite run */
+export interface RunRecord {
+    runId: string;
+    startedAt: Date;
+    finishedAt: Date;
+    name?: string;
+    metadata?: JsonMetadata;
+    cases: readonly EvalCase[];
+    config: EvalRunConfig;
+    results: readonly ScoredCaseResult[];
+    aggregated: Record<string, AggregatedMetric>;
+    assertions: readonly AssertionResult[];
+    stats: EvalRun['stats'];
+}
+/** A run id that sorts by start time: `20261022T101530123Z-1a2b3c4d` */
+export declare function newRunId(startedAt: Date): string;
+/** The stable key a case is matched by across runs: its input, turns and expected answer */
+export declare function caseKey(evalCase: Pick<EvalCase, 'input' | 'turns' | 'expected'>): string;
+/** The run file of a finished suite run, every value as JSON keeps it */
+export declare function buildRun(record: RunRecord): EvalRun;
+/** The run of a source: the run itself, or a suite result's `toRun()` */
+export declare function runOf(source: EvalRunSource): EvalRun;
+/**
+ * Validates a parsed run file. Throws naming `origin` when it is not a run file, when it is of
+ * another format version, or when a field does not fit the format
+ */
+export declare function parseRun(value: unknown, origin: string): EvalRun;
+/** Reads and validates a run file written by `result.saveRun()` or a `RunStore` */
+export declare function loadRun(path: string): EvalRun;
+/**
+ * Validates a run, so its id can never lead outside the directory, then writes its run file
+ * and returns its path: `<target>/<runId>.json` when `target` is a directory
+ * or ends with a path separator, `target` itself otherwise. Missing directories are created
+ */
+export declare function writeRun(run: EvalRun, target: string): string;
+//# sourceMappingURL=run.d.ts.map

package/dist/runs/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/runs/run.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAc,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGvE,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGrE,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG;IAAE,KAAK,IAAI,OAAO,CAAA;CAAE,CAAC;AAE3D,4DAA4D;AAC5D,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC3B,MAAM,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IACvC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;CACzB;AAED,wEAAwE;AACxE,wBAAgB,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,CAGhD;AAED,4FAA4F;AAC5F,wBAAgB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,GAAG,UAAU,CAAC,GAAG,MAAM,CAExF;AA0BD,yEAAyE;AACzE,wBAAgB,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAuBnD;AAED,yEAAyE;AACzE,wBAAgB,KAAK,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAIpD;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAgBhE;AAED,mFAAmF;AACnF,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAc7C;AAUD;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAiB7D"}