@cogitator-ai/evals 0.12.0 → 0.13.0

package/dist/runs/compare.js

@@ -0,0 +1,208 @@
+import { z } from 'zod';
+import { parseOptions } from '../options.js';
+import { caseChanges, compareMetrics, determineOverallWinner, pairedScores, scoredMetricNames, } from '../comparison-stats.js';
+import { truncateInput } from '../reporters/format.js';
+import { canonicalJson } from './json.js';
+import { runOf } from './run.js';
+export const CompareRunsOptionsSchema = z.object({
+    alpha: z.number().gt(0).lt(1).default(0.05),
+    test: z.enum(['t', 'wilcoxon']).default('t'),
+    seed: z.union([z.number(), z.string()]).optional(),
+    metrics: z.array(z.string().min(1)).min(1).optional(),
+});
+const WINNERS = {
+    baseline: 'base',
+    challenger: 'head',
+    tie: 'tie',
+};
+function refOf(run) {
+    return {
+        runId: run.runId,
+        ...(run.name !== undefined && { name: run.name }),
+        startedAt: run.startedAt,
+    };
+}
+function caseRef(result) {
+    return { caseKey: result.caseKey, input: truncateInput(result.case.input) };
+}
+/**
+ * Pairs the cases of the runs by `caseKey`, in the order of the base run. A key that repeats pairs
+ * its first case of one run with its first case of the other, its second with the second, and so on
+ */
+function matchCases(base, head) {
+    const headByKey = new Map();
+    for (const result of head.results) {
+        const list = headByKey.get(result.caseKey);
+        if (list)
+            list.push(result);
+        else
+            headByKey.set(result.caseKey, [result]);
+    }
+    const pairs = [];
+    const removed = [];
+    for (const result of base.results) {
+        const match = headByKey.get(result.caseKey)?.shift();
+        if (match)
+            pairs.push([result, match]);
+        else
+            removed.push(caseRef(result));
+    }
+    const added = [...headByKey.values()].flat();
+    const addedSet = new Set(added);
+    return {
+        pairs,
+        removed,
+        added: head.results.filter((result) => addedSet.has(result)).map(caseRef),
+    };
+}
+/** The per-case metrics of a run: those it was set up with and those its cases were scored on */
+function caseMetricNames(run) {
+    return new Set([...run.config.metrics, ...scoredMetricNames(run.results)]);
+}
+function metricsToCompare(base, head, requested) {
+    const baseNames = caseMetricNames(base);
+    const headNames = caseMetricNames(head);
+    const shared = [...baseNames].filter((name) => headNames.has(name));
+    if (requested === undefined)
+        return shared;
+    const missing = requested.filter((name) => !shared.includes(name));
+    if (missing.length > 0) {
+        throw new Error(`compareRuns: ${missing.map((name) => `'${name}'`).join(', ')} not in both runs, the base run has ${[...baseNames].join(', ') || 'no metrics'} and the head run ${[...headNames].join(', ') || 'no metrics'}`);
+    }
+    return [...new Set(requested)];
+}
+function promptLabel(prompt) {
+    if (prompt.abTest)
+        return `A/B test ${prompt.abTest.id} (${prompt.abTest.variant})`;
+    if (prompt.version !== undefined) {
+        return prompt.versionId !== undefined
+            ? `v${prompt.version} (${prompt.versionId})`
+            : `v${prompt.version}`;
+    }
+    return prompt.versionId ?? 'unversioned instructions';
+}
+function promptsByKey(run) {
+    const byKey = new Map();
+    for (const prompt of run.prompts) {
+        const labels = byKey.get(prompt.key) ?? new Map();
+        labels.set(canonicalJson(prompt), promptLabel(prompt));
+        byKey.set(prompt.key, labels);
+    }
+    return byKey;
+}
+function promptWarnings(base, head) {
+    const baseKeys = promptsByKey(base);
+    const headKeys = promptsByKey(head);
+    const warnings = [];
+    for (const key of new Set([...baseKeys.keys(), ...headKeys.keys()])) {
+        const before = baseKeys.get(key);
+        const after = headKeys.get(key);
+        const list = (labels) => [...labels.values()].join(', ');
+        if (!before) {
+            warnings.push(`Prompt '${key}' is used by the head run only (${list(after)})`);
+        }
+        else if (!after) {
+            warnings.push(`Prompt '${key}' is used by the base run only (${list(before)})`);
+        }
+        else if (before.size !== after.size ||
+            [...before.keys()].some((identity) => !after.has(identity))) {
+            warnings.push(`Prompt '${key}' differs: the base run used ${list(before)}, the head run ${list(after)}`);
+        }
+    }
+    return warnings;
+}
+function difference(label, base, head) {
+    const onlyBase = base.filter((name) => !head.includes(name));
+    const onlyHead = head.filter((name) => !base.includes(name));
+    if (onlyBase.length === 0 && onlyHead.length === 0)
+        return undefined;
+    const parts = [
+        ...(onlyBase.length > 0 ? [`only the base run has ${onlyBase.join(', ')}`] : []),
+        ...(onlyHead.length > 0 ? [`only the head run has ${onlyHead.join(', ')}`] : []),
+    ];
+    return `${label} differ: ${parts.join(', ')}`;
+}
+function judgeLabel(judge) {
+    if (!judge)
+        return 'no judge';
+    const samples = judge.samples !== undefined ? `, ${judge.samples} samples` : '';
+    return `${judge.model} at temperature ${judge.temperature}${samples}`;
+}
+function configWarnings(base, head) {
+    const warnings = [];
+    const metrics = difference('Metrics', [...base.config.metrics, ...base.config.statisticalMetrics], [...head.config.metrics, ...head.config.statisticalMetrics]);
+    if (metrics)
+        warnings.push(metrics);
+    if (canonicalJson(base.config.judge ?? null) !== canonicalJson(head.config.judge ?? null)) {
+        warnings.push(`Judge differs: the base run used ${judgeLabel(base.config.judge)}, the head run ${judgeLabel(head.config.judge)}`);
+    }
+    return warnings;
+}
+function datasetWarning(base, head, added, removed) {
+    if (base.dataset.fingerprint === head.dataset.fingerprint)
+        return undefined;
+    if (added === 0 && removed === 0) {
+        return 'The datasets differ (fingerprints differ) while every case matches by input, turns and expected answer: their order, context or metadata changed';
+    }
+    return `The datasets differ (fingerprints differ): ${added} cases only in the head run, ${removed} only in the base run`;
+}
+/**
+ * Compares two saved runs, or suite results, case by case: cases are matched by their `caseKey`,
+ * and each metric both runs have is compared as `EvalComparison` compares its sides, the base run
+ * in the baseline's place and the head run in the challenger's
+ */
+export function compareRuns(baseSource, headSource, options = {}) {
+    const config = parseOptions(CompareRunsOptionsSchema, options, 'compareRuns');
+    const base = runOf(baseSource);
+    const head = runOf(headSource);
+    const { pairs, added, removed } = matchCases(base, head);
+    const paired = new Map();
+    const unscored = [];
+    for (const name of metricsToCompare(base, head, config.metrics)) {
+        const scores = pairedScores(pairs, name);
+        if (scores[0].length === 0 && config.metrics === undefined)
+            unscored.push(name);
+        else
+            paired.set(name, scores);
+    }
+    const names = [...paired.keys()];
+    const compared = compareMetrics(paired, config);
+    const metrics = {};
+    for (const name of names) {
+        const { baseline, challenger, winner, ...stats } = compared[name];
+        const changes = caseChanges(pairs, name, (index) => pairs[index][0].case.input);
+        const toRunChange = (change) => ({
+            caseKey: pairs[change.index][0].caseKey,
+            input: change.input,
+            base: change.before,
+            head: change.after,
+            delta: change.delta,
+        });
+        metrics[name] = {
+            base: baseline,
+            head: challenger,
+            ...stats,
+            winner: WINNERS[winner],
+            improved: changes.improved.map(toRunChange),
+            regressed: changes.regressed.map(toRunChange),
+        };
+    }
+    const dataset = datasetWarning(base, head, added.length, removed.length);
+    const warnings = [
+        ...(dataset ? [dataset] : []),
+        ...promptWarnings(base, head),
+        ...configWarnings(base, head),
+        ...unscored.map((name) => `Metric '${name}' was scored on no case the runs share (every case errored or went unscored), so it is not compared`),
+    ];
+    return {
+        base: refOf(base),
+        head: refOf(head),
+        winner: WINNERS[determineOverallWinner(compared)],
+        metrics,
+        matched: pairs.length,
+        added,
+        removed,
+        warnings,
+    };
+}
+//# sourceMappingURL=compare.js.map

package/dist/runs/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/runs/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EACL,WAAW,EACX,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAI9B,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3C,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACtD,CAAC,CAAC;AA4FH,MAAM,OAAO,GAAwC;IACnD,QAAQ,EAAE,MAAM;IAChB,UAAU,EAAE,MAAM;IAClB,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,SAAS,KAAK,CAAC,GAAY;IACzB,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;QACjD,SAAS,EAAE,GAAG,CAAC,SAAS;KACzB,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,MAAmB;IAClC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,IAAa,EAAE,IAAa;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;IACnD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;YACvB,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,KAAK,GAA+C,EAAE,CAAC;IAC7D,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC;QACrD,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;;YAClC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,OAAO;QACL,KAAK;QACL,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;KAC1E,CAAC;AACJ,CAAC;AAED,iGAAiG;AACjG,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAa,EAAE,IAAa,EAAE,SAA+B;IACrF,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACpE,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,gBAAgB,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,uCAAuC,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,qBAAqB,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,YAAY,EAAE,CAC9M,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,MAAiB;IACpC,IAAI,MAAM,CAAC,MAAM;QAAE,OAAO,YAAY,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC;IACpF,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC,SAAS,KAAK,SAAS;YACnC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,SAAS,GAAG;YAC5C,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC,SAAS,IAAI,0BAA0B,CAAC;AACxD,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrD,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;QAClE,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,IAAa,EAAE,IAAa;IAClD,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,CAAC,MAA2B,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,mCAAmC,IAAI,CAAC,KAAM,CAAC,GAAG,CAAC,CAAC;QAClF,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAClB,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,mCAAmC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClF,CAAC;aAAM,IACL,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;YAC1B,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAC3D,CAAC;YACD,QAAQ,CAAC,IAAI,CACX,WAAW,GAAG,gCAAgC,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,EAAE,CAC1F,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,IAAc,EAAE,IAAc;IAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACrE,MAAM,KAAK,GAAG;QACZ,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChF,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACjF,CAAC;IACF,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,SAAS,UAAU,CAAC,KAAiC;IACnD,IAAI,CAAC,KAAK;QAAE,OAAO,UAAU,CAAC;IAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAChF,OAAO,GAAG,KAAK,CAAC,KAAK,mBAAmB,KAAK,CAAC,WAAW,GAAG,OAAO,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,cAAc,CAAC,IAAa,EAAE,IAAa;IAClD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CACxB,SAAS,EACT,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAC3D,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAC5D,CAAC;IACF,IAAI,OAAO;QAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,CAAC;QAC1F,QAAQ,CAAC,IAAI,CACX,oCAAoC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACnH,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,IAAa,EACb,KAAa,EACb,OAAe;IAEf,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,IAAI,CAAC,OAAO,CAAC,WAAW;QAAE,OAAO,SAAS,CAAC;IAC5E,IAAI,KAAK,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,kJAAkJ,CAAC;IAC5J,CAAC;IACD,OAAO,8CAA8C,KAAK,gCAAgC,OAAO,uBAAuB,CAAC;AAC3H,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CACzB,UAAyB,EACzB,UAAyB,EACzB,UAA8B,EAAE;IAEhC,MAAM,MAAM,GAAG,YAAY,CAAC,wBAAwB,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgC,CAAC;IACvD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAChE,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACzC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;YAC3E,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhD,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChF,MAAM,WAAW,GAAG,CAAC,MAAyC,EAAiB,EAAE,CAAC,CAAC;YACjF,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;YACvC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI,EAAE,MAAM,CAAC,MAAM;YACnB,IAAI,EAAE,MAAM,CAAC,KAAK;YAClB,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG;YACd,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,UAAU;YAChB,GAAG,KAAK;YACR,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC;YACvB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;YAC3C,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC;SAC9C,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG;QACf,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC;QAC7B,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC;QAC7B,GAAG,QAAQ,CAAC,GAAG,CACb,CAAC,IAAI,EAAE,EAAE,CACP,WAAW,IAAI,qGAAqG,CACvH;KACF,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QACjB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC;QACjB,MAAM,EAAE,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QACjD,OAAO;QACP,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,KAAK;QACL,OAAO;QACP,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/runs/index.d.ts

@@ -0,0 +1,9 @@
+export { loadRun, caseKey } from './run.js';
+export type { EvalRunSource } from './run.js';
+export { RunStore } from './store.js';
+export type { RunSummary, RunFilter, RunStoreOptions, SkippedRunFile } from './store.js';
+export { compareRuns } from './compare.js';
+export type { CompareRunsOptions, RunComparison, RunMetricComparison, RunCaseChange, RunCaseRef, RunRef, RunWinner, } from './compare.js';
+export { EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION } from './schema.js';
+export type { EvalRun, EvalRunCase, EvalRunConfig, JsonMetadata } from './schema.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/runs/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/runs/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AACzC,YAAY,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,YAAY,EACV,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,aAAa,EACb,UAAU,EACV,MAAM,EACN,SAAS,GACV,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5E,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/runs/index.js

@@ -0,0 +1,5 @@
+export { loadRun, caseKey } from './run.js';
+export { RunStore } from './store.js';
+export { compareRuns } from './compare.js';
+export { EvalRunSchema, EVAL_RUN_FORMAT, EVAL_RUN_VERSION } from './schema.js';
+//# sourceMappingURL=index.js.map

package/dist/runs/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/runs/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAUxC,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC"}

package/dist/runs/json.d.ts

@@ -0,0 +1,11 @@
+/**
+ * A value as JSON keeps it, as `JSON.stringify` would write it (`toJSON` called once per value),
+ * except that a bigint becomes its
+ * decimal string and a circular reference the string `[Circular]` instead of throwing
+ */
+export declare function toJson(value: unknown): unknown;
+/** JSON of a value with the keys of every object sorted, the same text for the same data */
+export declare function canonicalJson(value: unknown): string;
+/** Hex SHA-256 of the canonical JSON of a value */
+export declare function fingerprint(value: unknown): string;
+//# sourceMappingURL=json.d.ts.map

package/dist/runs/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/runs/json.ts"],"names":[],"mappings":"AA8CA;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAE9C;AAED,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEpD;AAED,mDAAmD;AACnD,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAElD"}

package/dist/runs/json.js

@@ -0,0 +1,58 @@
+import { createHash } from 'node:crypto';
+function hasToJson(value) {
+    return typeof value.toJSON === 'function';
+}
+function convert(value, key, ancestors, sortKeys, calledToJson = false) {
+    if (value === null)
+        return null;
+    switch (typeof value) {
+        case 'string':
+        case 'boolean':
+            return value;
+        case 'number':
+            return Number.isFinite(value) ? value : null;
+        case 'bigint':
+            return value.toString();
+        case 'undefined':
+        case 'function':
+        case 'symbol':
+            return undefined;
+    }
+    const object = value;
+    if (!calledToJson && hasToJson(object)) {
+        return convert(object.toJSON(key), key, ancestors, sortKeys, true);
+    }
+    if (ancestors.includes(object))
+        return '[Circular]';
+    const nested = [...ancestors, object];
+    if (Array.isArray(object)) {
+        return object.map((item, at) => convert(item, String(at), nested, sortKeys) ?? null);
+    }
+    const keys = Object.keys(object);
+    if (sortKeys)
+        keys.sort();
+    const out = {};
+    for (const name of keys) {
+        const converted = convert(object[name], name, nested, sortKeys);
+        if (converted !== undefined)
+            out[name] = converted;
+    }
+    return out;
+}
+/**
+ * A value as JSON keeps it, as `JSON.stringify` would write it (`toJSON` called once per value),
+ * except that a bigint becomes its
+ * decimal string and a circular reference the string `[Circular]` instead of throwing
+ */
+export function toJson(value) {
+    return convert(value, '', [], false);
+}
+/** JSON of a value with the keys of every object sorted, the same text for the same data */
+export function canonicalJson(value) {
+    return JSON.stringify(convert(value, '', [], true) ?? null);
+}
+/** Hex SHA-256 of the canonical JSON of a value */
+export function fingerprint(value) {
+    return createHash('sha256').update(canonicalJson(value)).digest('hex');
+}
+//# sourceMappingURL=json.js.map

package/dist/runs/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/runs/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,OAAQ,KAA8B,CAAC,MAAM,KAAK,UAAU,CAAC;AACtE,CAAC;AAED,SAAS,OAAO,CACd,KAAc,EACd,GAAW,EACX,SAAmB,EACnB,QAAiB,EACjB,YAAY,GAAG,KAAK;IAEpB,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,QAAQ,OAAO,KAAK,EAAE,CAAC;QACrB,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,KAAK,CAAC;QACf,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/C,KAAK,QAAQ;YACX,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC1B,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU,CAAC;QAChB,KAAK,QAAQ;YACX,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,MAAM,GAAG,KAAe,CAAC;IAC/B,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,YAAY,CAAC;IACpD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,EAAE,MAAM,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,QAAQ;QAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IAC1B,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,SAAS,GAAG,OAAO,CAAE,MAAkC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC7F,IAAI,SAAS,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,KAAc;IACnC,OAAO,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;AACvC,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzE,CAAC"}

package/dist/runs/run.d.ts

@@ -0,0 +1,44 @@
+import type { EvalCase } from '../schema.js';
+import type { ScoredCaseResult } from '../metrics/types.js';
+import type { AggregatedMetric, AssertionResult } from '../assertions/index.js';
+import type { EvalRun, EvalRunConfig, JsonMetadata } from './schema.js';
+/** A saved run, or a suite result that turns into one */
+export type EvalRunSource = EvalRun | {
+    toRun(): EvalRun;
+};
+/** Everything a run file records of a finished suite run */
+export interface RunRecord {
+    runId: string;
+    startedAt: Date;
+    finishedAt: Date;
+    name?: string;
+    metadata?: JsonMetadata;
+    cases: readonly EvalCase[];
+    config: EvalRunConfig;
+    results: readonly ScoredCaseResult[];
+    aggregated: Record<string, AggregatedMetric>;
+    assertions: readonly AssertionResult[];
+    stats: EvalRun['stats'];
+}
+/** A run id that sorts by start time: `20261022T101530123Z-1a2b3c4d` */
+export declare function newRunId(startedAt: Date): string;
+/** The stable key a case is matched by across runs: its input, turns and expected answer */
+export declare function caseKey(evalCase: Pick<EvalCase, 'input' | 'turns' | 'expected'>): string;
+/** The run file of a finished suite run, every value as JSON keeps it */
+export declare function buildRun(record: RunRecord): EvalRun;
+/** The run of a source: the run itself, or a suite result's `toRun()` */
+export declare function runOf(source: EvalRunSource): EvalRun;
+/**
+ * Validates a parsed run file. Throws naming `origin` when it is not a run file, when it is of
+ * another format version, or when a field does not fit the format
+ */
+export declare function parseRun(value: unknown, origin: string): EvalRun;
+/** Reads and validates a run file written by `result.saveRun()` or a `RunStore` */
+export declare function loadRun(path: string): EvalRun;
+/**
+ * Validates a run, so its id can never lead outside the directory, then writes its run file
+ * and returns its path: `<target>/<runId>.json` when `target` is a directory
+ * or ends with a path separator, `target` itself otherwise. Missing directories are created
+ */
+export declare function writeRun(run: EvalRun, target: string): string;
+//# sourceMappingURL=run.d.ts.map

package/dist/runs/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/runs/run.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAc,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGvE,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAGrE,yDAAyD;AACzD,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG;IAAE,KAAK,IAAI,OAAO,CAAA;CAAE,CAAC;AAE3D,4DAA4D;AAC5D,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC3B,MAAM,EAAE,aAAa,CAAC;IACtB,OAAO,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IACvC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;CACzB;AAED,wEAAwE;AACxE,wBAAgB,QAAQ,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,CAGhD;AAED,4FAA4F;AAC5F,wBAAgB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,GAAG,UAAU,CAAC,GAAG,MAAM,CAExF;AA0BD,yEAAyE;AACzE,wBAAgB,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAuBnD;AAED,yEAAyE;AACzE,wBAAgB,KAAK,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAIpD;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAgBhE;AAED,mFAAmF;AACnF,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAc7C;AAUD;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAiB7D"}

package/dist/runs/run.js

@@ -0,0 +1,136 @@
+import { randomBytes } from 'node:crypto';
+import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { dirname, join, sep } from 'node:path';
+import { canonicalJson, fingerprint, toJson } from './json.js';
+import { EVAL_RUN_FORMAT, EVAL_RUN_VERSION, EvalRunSchema } from './schema.js';
+/** A run id that sorts by start time: `20261022T101530123Z-1a2b3c4d` */
+export function newRunId(startedAt) {
+    const stamp = startedAt.toISOString().replace(/[-:.]/g, '');
+    return `${stamp}-${randomBytes(4).toString('hex')}`;
+}
+/** The stable key a case is matched by across runs: its input, turns and expected answer */
+export function caseKey(evalCase) {
+    return fingerprint({ input: evalCase.input, turns: evalCase.turns, expected: evalCase.expected });
+}
+/** The distinct versioned instructions the runs of the cases used, in the order first seen */
+function promptsUsed(results) {
+    const seen = new Map();
+    const add = (prompt) => {
+        if (prompt)
+            seen.set(canonicalJson(prompt), prompt);
+    };
+    for (const result of results) {
+        for (const trial of result.trials ?? [result]) {
+            add(trial.prompt);
+            for (const turn of trial.turns ?? [])
+                add(turn.prompt);
+        }
+    }
+    return [...seen.values()];
+}
+function issuesOf(error) {
+    return error.issues
+        .slice(0, 5)
+        .map((issue) => issue.path.length > 0 ? `${issue.path.join('.')}: ${issue.message}` : issue.message)
+        .join(', ');
+}
+/** The run file of a finished suite run, every value as JSON keeps it */
+export function buildRun(record) {
+    const { results } = record;
+    const draft = {
+        format: EVAL_RUN_FORMAT,
+        version: EVAL_RUN_VERSION,
+        runId: record.runId,
+        startedAt: record.startedAt.toISOString(),
+        finishedAt: record.finishedAt.toISOString(),
+        name: record.name,
+        metadata: record.metadata,
+        dataset: { fingerprint: fingerprint(record.cases), cases: record.cases.length },
+        config: record.config,
+        prompts: promptsUsed(results),
+        results: results.map((result) => ({ ...result, caseKey: caseKey(result.case) })),
+        aggregated: record.aggregated,
+        assertions: record.assertions,
+        stats: record.stats,
+    };
+    const parsed = EvalRunSchema.safeParse(toJson(draft));
+    if (!parsed.success) {
+        throw new Error(`Cannot record the run: ${issuesOf(parsed.error)}`);
+    }
+    return parsed.data;
+}
+/** The run of a source: the run itself, or a suite result's `toRun()` */
+export function runOf(source) {
+    return 'toRun' in source && typeof source.toRun === 'function'
+        ? source.toRun()
+        : source;
+}
+/**
+ * Validates a parsed run file. Throws naming `origin` when it is not a run file, when it is of
+ * another format version, or when a field does not fit the format
+ */
+export function parseRun(value, origin) {
+    const head = typeof value === 'object' && value !== null ? value : {};
+    if (head.format !== EVAL_RUN_FORMAT) {
+        throw new Error(`${origin} is not an eval run file: its format is not '${EVAL_RUN_FORMAT}'`);
+    }
+    if (head.version !== EVAL_RUN_VERSION) {
+        throw new Error(`${origin} is an eval run file of version ${String(head.version)}, this version of @cogitator-ai/evals reads version ${EVAL_RUN_VERSION}`);
+    }
+    const parsed = EvalRunSchema.safeParse(value);
+    if (!parsed.success) {
+        throw new Error(`${origin} is not a valid eval run file: ${issuesOf(parsed.error)}`);
+    }
+    return parsed.data;
+}
+/** Reads and validates a run file written by `result.saveRun()` or a `RunStore` */
+export function loadRun(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf-8');
+    }
+    catch (error) {
+        throw new Error(`Cannot read the run file ${path}`, { cause: error });
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch (error) {
+        throw new Error(`${path} is not JSON`, { cause: error });
+    }
+    return parseRun(value, path);
+}
+function isDirectory(path) {
+    try {
+        return statSync(path).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Validates a run, so its id can never lead outside the directory, then writes its run file
+ * and returns its path: `<target>/<runId>.json` when `target` is a directory
+ * or ends with a path separator, `target` itself otherwise. Missing directories are created
+ */
+export function writeRun(run, target) {
+    const checked = EvalRunSchema.safeParse(toJson(run));
+    if (!checked.success) {
+        throw new Error(`Cannot write the run: ${issuesOf(checked.error)}`);
+    }
+    const intoDirectory = target.endsWith('/') || target.endsWith(sep) || isDirectory(target);
+    const path = intoDirectory ? join(target, `${checked.data.runId}.json`) : target;
+    mkdirSync(dirname(path), { recursive: true });
+    const partial = `${path}.${randomBytes(6).toString('hex')}.tmp`;
+    try {
+        writeFileSync(partial, `${JSON.stringify(checked.data, null, 2)}\n`, 'utf-8');
+        renameSync(partial, path);
+    }
+    catch (error) {
+        rmSync(partial, { force: true });
+        throw error;
+    }
+    return path;
+}
+//# sourceMappingURL=run.js.map

package/dist/runs/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/runs/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAI/C,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAsB5E,wEAAwE;AACxE,MAAM,UAAU,QAAQ,CAAC,SAAe;IACtC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,GAAG,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;AACtD,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,OAAO,CAAC,QAAwD;IAC9E,OAAO,WAAW,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;AACpG,CAAC;AAED,8FAA8F;AAC9F,SAAS,WAAW,CAAC,OAAoC;IACvD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC3C,MAAM,GAAG,GAAG,CAAC,MAA8B,EAAE,EAAE;QAC7C,IAAI,MAAM;YAAE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE;gBAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB;IACjC,OAAO,KAAK,CAAC,MAAM;SAChB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CACpF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,QAAQ,CAAC,MAAiB;IACxC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC3B,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE,eAAe;QACvB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE;QAC3C,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,OAAO,EAAE,EAAE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE;QAC/E,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;QAC7B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChF,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC;IACF,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,KAAK,CAAC,MAAqB;IACzC,OAAO,OAAO,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU;QAC5D,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE;QAChB,CAAC,CAAE,MAAkB,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAc,EAAE,MAAc;IACrD,MAAM,IAAI,GACR,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,IAAI,IAAI,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,gDAAgD,eAAe,GAAG,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,gBAAgB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,mCAAmC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,uDAAuD,gBAAgB,EAAE,CAC1I,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,kCAAkC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAY,EAAE,MAAc;IACnD,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACjF,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,IAAI,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;IAChE,IAAI,CAAC;QACH,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9E,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}