@cogitator-ai/evals 0.12.0 → 0.13.0

package/dist/runs/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/runs/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,0CAA0C;AAC1C,MAAM,CAAC,MAAM,eAAe,GAAG,oBAAoB,CAAC;AAEpD,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,8FAA8F;AAC9F,MAAM,MAAM,GAAG,CAAC;KACb,MAAM,EAAE;KACR,QAAQ,EAAE;KACV,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3F,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;CAC7C,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC7C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAE,cAAc;IACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC7C,WAAW,EAAE,CAAC;SACX,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAC9B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC,CACH;SACA,QAAQ,EAAE;IACb,SAAS,EAAE,CAAC;SACT,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC,CACH;SACA,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE;IACrC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC7C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;CAC7B,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAC5C,+EAA+E;IAC/E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;IAC3C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC5B,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACvC,KAAK,EAAE,CAAC;SACL,MAAM,CAAC;QACN,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,4CAA4C;QAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC3B,2DAA2D;QAC3D,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC;CACH,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,MAAM;IACd,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,MAAM,EAAE,MAAM;IACd,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAClC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE;IACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACxB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,kGAAkG;AAClG,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAEjG,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/F,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACpC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IACvD,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE;IAC3B,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE;IAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,+DAA+D;QAC/D,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAC/C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;KAC/B,CAAC;IACF,MAAM,EAAE,eAAe;IACvB,8DAA8D;IAC9D,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC;IAClD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;IACpC,KAAK,EAAE,WAAW;CACnB,CAAC,CAAC;AAWH,kEAAkE;AAClE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC,QAAQ,EAAE;IACnE,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC"}

package/dist/runs/store.d.ts

@@ -0,0 +1,57 @@
+import type { EvalRun, JsonMetadata } from './schema.js';
+import type { EvalRunSource } from './run.js';
+/** A run as `RunStore.list()` shows it */
+export interface RunSummary {
+    id: string;
+    name?: string;
+    metadata?: JsonMetadata;
+    startedAt: string;
+    finishedAt: string;
+    /** Cases run */
+    cases: number;
+    /** The mean of each aggregated metric, NaN for a metric no case was scored on */
+    metrics: Record<string, number>;
+    /** The run file */
+    path: string;
+}
+/** Picks runs: by `name`, or by a predicate over their summaries */
+export type RunFilter = {
+    name?: string;
+} | ((run: RunSummary) => boolean);
+/** A file `list()` and `latest()` passed over: what it is and why it was not read */
+export interface SkippedRunFile {
+    path: string;
+    reason: string;
+}
+export interface RunStoreOptions {
+    /**
+     * Called for each `.json` entry `list()` and `latest()` skip: a directory, a file that cannot
+     * be read or is not JSON, or a run file that is not valid (of another format version, or edited
+     * by hand). Default: a process warning with the code `COGITATOR_EVALS_RUN_SKIPPED`
+     */
+    onSkip?: (skipped: SkippedRunFile) => void;
+}
+/**
+ * Saved suite runs in a directory, one `<runId>.json` run file each. `list()` and `latest()` pass
+ * over JSON files that are not run files, such as a baseline, and report the `.json` entries they
+ * cannot read through `onSkip`, so one bad file never hides the other runs. `load(id)` is strict
+ * and throws for a run file it cannot read
+ */
+export declare class RunStore {
+    readonly directory: string;
+    private readonly onSkip;
+    constructor(directory: string, options?: RunStoreOptions);
+    /** Writes the run file of a suite result or a run, returning its path */
+    save(source: EvalRunSource): string;
+    /** The runs of the directory, newest first */
+    list(filter?: RunFilter): RunSummary[];
+    /**
+     * The run with this id. Throws when the directory has none, and when its file cannot be read
+     * or is not a valid run file
+     */
+    load(id: string): EvalRun;
+    /** The newest run, of those the filter picks, undefined when there is none */
+    latest(filter?: RunFilter): EvalRun | undefined;
+    private entries;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/runs/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/runs/store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAE3C,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oEAAoE;AACpE,MAAM,MAAM,SAAS,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,CAAC;AAyB3E,qFAAqF;AACrF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,IAAI,CAAC;CAC5C;AAyDD;;;;;GAKG;AACH,qBAAa,QAAQ;IAIjB,QAAQ,CAAC,SAAS,EAAE,MAAM;IAH5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;gBAGhD,SAAS,EAAE,MAAM,EAC1B,OAAO,GAAE,eAAoB;IAK/B,yEAAyE;IACzE,IAAI,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM;IAInC,8CAA8C;IAC9C,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,UAAU,EAAE;IAMtC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAOzB,8EAA8E;IAC9E,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS;IAO/C,OAAO,CAAC,OAAO;CAsBhB"}

package/dist/runs/store.js

@@ -0,0 +1,146 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { EVAL_RUN_FORMAT } from './schema.js';
+import { loadRun, parseRun, runOf, writeRun } from './run.js';
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+function summaryOf(run, path) {
+    const metrics = {};
+    for (const [name, aggregated] of Object.entries(run.aggregated))
+        metrics[name] = aggregated.mean;
+    return {
+        id: run.runId,
+        ...(run.name !== undefined && { name: run.name }),
+        ...(run.metadata !== undefined && { metadata: run.metadata }),
+        startedAt: run.startedAt,
+        finishedAt: run.finishedAt,
+        cases: run.dataset.cases,
+        metrics,
+        path,
+    };
+}
+function matches(summary, filter) {
+    if (filter === undefined)
+        return true;
+    if (typeof filter === 'function')
+        return filter(summary);
+    return filter.name === undefined || summary.name === filter.name;
+}
+function isGone(error) {
+    return error.code === 'ENOENT';
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function warnSkipped({ path, reason }) {
+    process.emitWarning(`RunStore skipped ${path}: ${reason}`, {
+        code: 'COGITATOR_EVALS_RUN_SKIPPED',
+    });
+}
+/** What a run file path holds: a valid run, JSON of another kind, or something unreadable */
+function readRunFile(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf-8');
+    }
+    catch (error) {
+        if (isGone(error))
+            return { kind: 'gone' };
+        return { kind: 'bad', reason: errorMessage(error) };
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        return { kind: 'bad', reason: 'not JSON' };
+    }
+    const format = typeof value === 'object' && value !== null ? value.format : null;
+    if (format !== EVAL_RUN_FORMAT)
+        return { kind: 'other' };
+    try {
+        return { kind: 'run', run: parseRun(value, path) };
+    }
+    catch (error) {
+        return { kind: 'bad', reason: errorMessage(error) };
+    }
+}
+/** Whether a listed entry is still a file: `null` when it is, the reason to skip it otherwise */
+function notAFile(path) {
+    try {
+        return statSync(path).isFile() ? null : { kind: 'bad', reason: 'not a file' };
+    }
+    catch (error) {
+        if (isGone(error))
+            return { kind: 'gone' };
+        return { kind: 'bad', reason: errorMessage(error) };
+    }
+}
+/**
+ * Saved suite runs in a directory, one `<runId>.json` run file each. `list()` and `latest()` pass
+ * over JSON files that are not run files, such as a baseline, and report the `.json` entries they
+ * cannot read through `onSkip`, so one bad file never hides the other runs. `load(id)` is strict
+ * and throws for a run file it cannot read
+ */
+export class RunStore {
+    directory;
+    onSkip;
+    constructor(directory, options = {}) {
+        this.directory = directory;
+        this.onSkip = options.onSkip ?? warnSkipped;
+    }
+    /** Writes the run file of a suite result or a run, returning its path */
+    save(source) {
+        return writeRun(runOf(source), `${this.directory}/`);
+    }
+    /** The runs of the directory, newest first */
+    list(filter) {
+        return this.entries()
+            .map(({ run, path }) => summaryOf(run, path))
+            .filter((summary) => matches(summary, filter));
+    }
+    /**
+     * The run with this id. Throws when the directory has none, and when its file cannot be read
+     * or is not a valid run file
+     */
+    load(id) {
+        if (!RUN_ID.test(id))
+            throw new Error(`'${id}' is not a run id`);
+        const path = join(this.directory, `${id}.json`);
+        if (!existsSync(path))
+            throw new Error(`No run '${id}' in ${this.directory}`);
+        return loadRun(path);
+    }
+    /** The newest run, of those the filter picks, undefined when there is none */
+    latest(filter) {
+        for (const { run, path } of this.entries()) {
+            if (matches(summaryOf(run, path), filter))
+                return run;
+        }
+        return undefined;
+    }
+    entries() {
+        let files;
+        try {
+            files = readdirSync(this.directory);
+        }
+        catch (error) {
+            if (isGone(error))
+                return [];
+            throw error;
+        }
+        const entries = [];
+        for (const file of files.sort()) {
+            if (!file.endsWith('.json'))
+                continue;
+            const path = join(this.directory, file);
+            const read = notAFile(path) ?? readRunFile(path);
+            if (read.kind === 'run')
+                entries.push({ run: read.run, path });
+            else if (read.kind === 'bad')
+                this.onSkip({ path, reason: read.reason });
+        }
+        return entries.sort((a, b) => Date.parse(b.run.startedAt) - Date.parse(a.run.startedAt) ||
+            b.run.runId.localeCompare(a.run.runId));
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/runs/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/runs/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAqB3D,MAAM,MAAM,GAAG,8BAA8B,CAAC;AAE9C,SAAS,SAAS,CAAC,GAAY,EAAE,IAAY;IAC3C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC;IACjG,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,KAAK;QACb,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;QACjD,GAAG,CAAC,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7D,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK;QACxB,OAAO;QACP,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,OAAmB,EAAE,MAA6B;IACjE,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,OAAO,MAAM,KAAK,UAAU;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC;IACzD,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;AACnE,CAAC;AAuBD,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAQ,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC5D,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAkB;IACnD,OAAO,CAAC,WAAW,CAAC,oBAAoB,IAAI,KAAK,MAAM,EAAE,EAAE;QACzD,IAAI,EAAE,6BAA6B;KACpC,CAAC,CAAC;AACL,CAAC;AAED,6FAA6F;AAC7F,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC3C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC;IACD,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GACV,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAA8B,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,IAAI,MAAM,KAAK,eAAe;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACzD,IAAI,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,iGAAiG;AACjG,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAChF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC3C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,QAAQ;IAIR;IAHM,MAAM,CAAoC;IAE3D,YACW,SAAiB,EAC1B,UAA2B,EAAE;QADpB,cAAS,GAAT,SAAS,CAAQ;QAG1B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,WAAW,CAAC;IAC9C,CAAC;IAED,yEAAyE;IACzE,IAAI,CAAC,MAAqB;QACxB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,MAAkB;QACrB,OAAO,IAAI,CAAC,OAAO,EAAE;aAClB,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;aAC5C,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,IAAI,CAAC,EAAU;QACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9E,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,8EAA8E;IAC9E,MAAM,CAAC,MAAkB;QACvB,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAE,OAAO,GAAG,CAAC;QACxD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,OAAO;QACb,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,CAAC,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAA0C,EAAE,CAAC;QAC1D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;iBAC1D,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;gBAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CACjB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;YACzD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CACzC,CAAC;IACJ,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/evals",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Evaluation framework for Cogitator AI agents",
   "keywords": [
     "cogitator",