@cogitator-ai/evals 0.11.0 → 0.13.0

package/dist/runs/run.js

@@ -0,0 +1,136 @@
+import { randomBytes } from 'node:crypto';
+import { mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { dirname, join, sep } from 'node:path';
+import { canonicalJson, fingerprint, toJson } from './json.js';
+import { EVAL_RUN_FORMAT, EVAL_RUN_VERSION, EvalRunSchema } from './schema.js';
+/** A run id that sorts by start time: `20261022T101530123Z-1a2b3c4d` */
+export function newRunId(startedAt) {
+    const stamp = startedAt.toISOString().replace(/[-:.]/g, '');
+    return `${stamp}-${randomBytes(4).toString('hex')}`;
+}
+/** The stable key a case is matched by across runs: its input, turns and expected answer */
+export function caseKey(evalCase) {
+    return fingerprint({ input: evalCase.input, turns: evalCase.turns, expected: evalCase.expected });
+}
+/** The distinct versioned instructions the runs of the cases used, in the order first seen */
+function promptsUsed(results) {
+    const seen = new Map();
+    const add = (prompt) => {
+        if (prompt)
+            seen.set(canonicalJson(prompt), prompt);
+    };
+    for (const result of results) {
+        for (const trial of result.trials ?? [result]) {
+            add(trial.prompt);
+            for (const turn of trial.turns ?? [])
+                add(turn.prompt);
+        }
+    }
+    return [...seen.values()];
+}
+function issuesOf(error) {
+    return error.issues
+        .slice(0, 5)
+        .map((issue) => issue.path.length > 0 ? `${issue.path.join('.')}: ${issue.message}` : issue.message)
+        .join(', ');
+}
+/** The run file of a finished suite run, every value as JSON keeps it */
+export function buildRun(record) {
+    const { results } = record;
+    const draft = {
+        format: EVAL_RUN_FORMAT,
+        version: EVAL_RUN_VERSION,
+        runId: record.runId,
+        startedAt: record.startedAt.toISOString(),
+        finishedAt: record.finishedAt.toISOString(),
+        name: record.name,
+        metadata: record.metadata,
+        dataset: { fingerprint: fingerprint(record.cases), cases: record.cases.length },
+        config: record.config,
+        prompts: promptsUsed(results),
+        results: results.map((result) => ({ ...result, caseKey: caseKey(result.case) })),
+        aggregated: record.aggregated,
+        assertions: record.assertions,
+        stats: record.stats,
+    };
+    const parsed = EvalRunSchema.safeParse(toJson(draft));
+    if (!parsed.success) {
+        throw new Error(`Cannot record the run: ${issuesOf(parsed.error)}`);
+    }
+    return parsed.data;
+}
+/** The run of a source: the run itself, or a suite result's `toRun()` */
+export function runOf(source) {
+    return 'toRun' in source && typeof source.toRun === 'function'
+        ? source.toRun()
+        : source;
+}
+/**
+ * Validates a parsed run file. Throws naming `origin` when it is not a run file, when it is of
+ * another format version, or when a field does not fit the format
+ */
+export function parseRun(value, origin) {
+    const head = typeof value === 'object' && value !== null ? value : {};
+    if (head.format !== EVAL_RUN_FORMAT) {
+        throw new Error(`${origin} is not an eval run file: its format is not '${EVAL_RUN_FORMAT}'`);
+    }
+    if (head.version !== EVAL_RUN_VERSION) {
+        throw new Error(`${origin} is an eval run file of version ${String(head.version)}, this version of @cogitator-ai/evals reads version ${EVAL_RUN_VERSION}`);
+    }
+    const parsed = EvalRunSchema.safeParse(value);
+    if (!parsed.success) {
+        throw new Error(`${origin} is not a valid eval run file: ${issuesOf(parsed.error)}`);
+    }
+    return parsed.data;
+}
+/** Reads and validates a run file written by `result.saveRun()` or a `RunStore` */
+export function loadRun(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf-8');
+    }
+    catch (error) {
+        throw new Error(`Cannot read the run file ${path}`, { cause: error });
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch (error) {
+        throw new Error(`${path} is not JSON`, { cause: error });
+    }
+    return parseRun(value, path);
+}
+function isDirectory(path) {
+    try {
+        return statSync(path).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Validates a run, so its id can never lead outside the directory, then writes its run file
+ * and returns its path: `<target>/<runId>.json` when `target` is a directory
+ * or ends with a path separator, `target` itself otherwise. Missing directories are created
+ */
+export function writeRun(run, target) {
+    const checked = EvalRunSchema.safeParse(toJson(run));
+    if (!checked.success) {
+        throw new Error(`Cannot write the run: ${issuesOf(checked.error)}`);
+    }
+    const intoDirectory = target.endsWith('/') || target.endsWith(sep) || isDirectory(target);
+    const path = intoDirectory ? join(target, `${checked.data.runId}.json`) : target;
+    mkdirSync(dirname(path), { recursive: true });
+    const partial = `${path}.${randomBytes(6).toString('hex')}.tmp`;
+    try {
+        writeFileSync(partial, `${JSON.stringify(checked.data, null, 2)}\n`, 'utf-8');
+        renameSync(partial, path);
+    }
+    catch (error) {
+        rmSync(partial, { force: true });
+        throw error;
+    }
+    return path;
+}
+//# sourceMappingURL=run.js.map

package/dist/runs/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/runs/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAI/C,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAsB5E,wEAAwE;AACxE,MAAM,UAAU,QAAQ,CAAC,SAAe;IACtC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,GAAG,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;AACtD,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,OAAO,CAAC,QAAwD;IAC9E,OAAO,WAAW,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;AACpG,CAAC;AAED,8FAA8F;AAC9F,SAAS,WAAW,CAAC,OAAoC;IACvD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC3C,MAAM,GAAG,GAAG,CAAC,MAA8B,EAAE,EAAE;QAC7C,IAAI,MAAM;YAAE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE;gBAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,QAAQ,CAAC,KAAiB;IACjC,OAAO,KAAK,CAAC,MAAM;SAChB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CACpF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,QAAQ,CAAC,MAAiB;IACxC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAC3B,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE,eAAe;QACvB,OAAO,EAAE,gBAAgB;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE;QAC3C,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,OAAO,EAAE,EAAE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE;QAC/E,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;QAC7B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChF,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC;IACF,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,KAAK,CAAC,MAAqB;IACzC,OAAO,OAAO,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU;QAC5D,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE;QAChB,CAAC,CAAE,MAAkB,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAc,EAAE,MAAc;IACrD,MAAM,IAAI,GACR,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,IAAI,IAAI,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,gDAAgD,eAAe,GAAG,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,gBAAgB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,mCAAmC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,uDAAuD,gBAAgB,EAAE,CAC1I,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,kCAAkC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAY,EAAE,MAAc;IACnD,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACjF,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,GAAG,IAAI,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;IAChE,IAAI,CAAC;QACH,aAAa,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9E,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/runs/schema.d.ts

@@ -0,0 +1,504 @@
+import { z } from 'zod';
+/** The `format` every run file carries */
+export declare const EVAL_RUN_FORMAT = "cogitator.eval-run";
+/** The version of the run file format this package writes and reads */
+export declare const EVAL_RUN_VERSION = 1;
+export declare const RunPromptSchema: z.ZodObject<{
+    key: z.ZodString;
+    versionId: z.ZodOptional<z.ZodString>;
+    version: z.ZodOptional<z.ZodNumber>;
+    abTest: z.ZodOptional<z.ZodObject<{
+        id: z.ZodString;
+        variant: z.ZodEnum<{
+            control: "control";
+            treatment: "treatment";
+        }>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const RunCaseSchema: z.ZodObject<{
+    case: z.ZodObject<{
+        input: z.ZodString;
+        expected: z.ZodOptional<z.ZodString>;
+        context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>>;
+        expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>;
+    output: z.ZodString;
+    duration: z.ZodNumber;
+    error: z.ZodOptional<z.ZodString>;
+    attempts: z.ZodOptional<z.ZodNumber>;
+    usage: z.ZodOptional<z.ZodObject<{
+        inputTokens: z.ZodNumber;
+        outputTokens: z.ZodNumber;
+        totalTokens: z.ZodNumber;
+        cost: z.ZodNumber;
+        duration: z.ZodNumber;
+    }, z.core.$strip>>;
+    toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strip>>>;
+    toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        callId: z.ZodString;
+        name: z.ZodString;
+        result: z.ZodOptional<z.ZodUnknown>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodOptional<z.ZodString>;
+        content: z.ZodString;
+        score: z.ZodOptional<z.ZodNumber>;
+        source: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        input: z.ZodString;
+        output: z.ZodString;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strip>>>;
+        traceId: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodOptional<z.ZodObject<{
+            key: z.ZodString;
+            versionId: z.ZodOptional<z.ZodString>;
+            version: z.ZodOptional<z.ZodNumber>;
+            abTest: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                variant: z.ZodEnum<{
+                    control: "control";
+                    treatment: "treatment";
+                }>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    traceId: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodOptional<z.ZodObject<{
+        key: z.ZodString;
+        versionId: z.ZodOptional<z.ZodString>;
+        version: z.ZodOptional<z.ZodNumber>;
+        abTest: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            variant: z.ZodEnum<{
+                control: "control";
+                treatment: "treatment";
+            }>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    truncated: z.ZodOptional<z.ZodBoolean>;
+    iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+    paused: z.ZodOptional<z.ZodBoolean>;
+    scores: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        score: z.ZodNumber;
+        details: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        error: z.ZodOptional<z.ZodString>;
+        usage: z.ZodOptional<z.ZodObject<{
+            inputTokens: z.ZodNumber;
+            outputTokens: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            cost: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    caseKey: z.ZodString;
+    trials: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        case: z.ZodObject<{
+            input: z.ZodString;
+            expected: z.ZodOptional<z.ZodString>;
+            context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, z.core.$strip>>>;
+            expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>;
+        output: z.ZodString;
+        duration: z.ZodNumber;
+        error: z.ZodOptional<z.ZodString>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        usage: z.ZodOptional<z.ZodObject<{
+            inputTokens: z.ZodNumber;
+            outputTokens: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            cost: z.ZodNumber;
+            duration: z.ZodNumber;
+        }, z.core.$strip>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strip>>>;
+        toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            callId: z.ZodString;
+            name: z.ZodString;
+            result: z.ZodOptional<z.ZodUnknown>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodOptional<z.ZodString>;
+            content: z.ZodString;
+            score: z.ZodOptional<z.ZodNumber>;
+            source: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            input: z.ZodString;
+            output: z.ZodString;
+            toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodString;
+                arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strip>>>;
+            traceId: z.ZodOptional<z.ZodString>;
+            prompt: z.ZodOptional<z.ZodObject<{
+                key: z.ZodString;
+                versionId: z.ZodOptional<z.ZodString>;
+                version: z.ZodOptional<z.ZodNumber>;
+                abTest: z.ZodOptional<z.ZodObject<{
+                    id: z.ZodString;
+                    variant: z.ZodEnum<{
+                        control: "control";
+                        treatment: "treatment";
+                    }>;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        traceId: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodOptional<z.ZodObject<{
+            key: z.ZodString;
+            versionId: z.ZodOptional<z.ZodString>;
+            version: z.ZodOptional<z.ZodNumber>;
+            abTest: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                variant: z.ZodEnum<{
+                    control: "control";
+                    treatment: "treatment";
+                }>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        truncated: z.ZodOptional<z.ZodBoolean>;
+        iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+        paused: z.ZodOptional<z.ZodBoolean>;
+        scores: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            score: z.ZodNumber;
+            details: z.ZodOptional<z.ZodString>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            error: z.ZodOptional<z.ZodString>;
+            usage: z.ZodOptional<z.ZodObject<{
+                inputTokens: z.ZodNumber;
+                outputTokens: z.ZodNumber;
+                totalTokens: z.ZodNumber;
+                cost: z.ZodNumber;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+declare const RunConfigSchema: z.ZodObject<{
+    metrics: z.ZodArray<z.ZodString>;
+    statisticalMetrics: z.ZodArray<z.ZodString>;
+    judge: z.ZodOptional<z.ZodObject<{
+        model: z.ZodString;
+        temperature: z.ZodNumber;
+        samples: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    trials: z.ZodNumber;
+    concurrency: z.ZodNumber;
+    target: z.ZodObject<{
+        kind: z.ZodEnum<{
+            fn: "fn";
+            agent: "agent";
+        }>;
+        name: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+/** JSON-serializable metadata: object keys holding `undefined` are dropped, as JSON drops them */
+export declare const JsonMetadataSchema: z.ZodPreprocess<z.ZodRecord<z.ZodString, z.ZodJSONSchema>, unknown>;
+export declare const EvalRunSchema: z.ZodObject<{
+    format: z.ZodLiteral<"cogitator.eval-run">;
+    version: z.ZodLiteral<1>;
+    runId: z.ZodString;
+    startedAt: z.ZodISODateTime;
+    finishedAt: z.ZodISODateTime;
+    name: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodPreprocess<z.ZodRecord<z.ZodString, z.ZodJSONSchema>, unknown>>;
+    dataset: z.ZodObject<{
+        fingerprint: z.ZodString;
+        cases: z.ZodNumber;
+    }, z.core.$strip>;
+    config: z.ZodObject<{
+        metrics: z.ZodArray<z.ZodString>;
+        statisticalMetrics: z.ZodArray<z.ZodString>;
+        judge: z.ZodOptional<z.ZodObject<{
+            model: z.ZodString;
+            temperature: z.ZodNumber;
+            samples: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+        trials: z.ZodNumber;
+        concurrency: z.ZodNumber;
+        target: z.ZodObject<{
+            kind: z.ZodEnum<{
+                fn: "fn";
+                agent: "agent";
+            }>;
+            name: z.ZodOptional<z.ZodString>;
+            model: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+    }, z.core.$strip>;
+    prompts: z.ZodArray<z.ZodObject<{
+        key: z.ZodString;
+        versionId: z.ZodOptional<z.ZodString>;
+        version: z.ZodOptional<z.ZodNumber>;
+        abTest: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            variant: z.ZodEnum<{
+                control: "control";
+                treatment: "treatment";
+            }>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    results: z.ZodArray<z.ZodObject<{
+        case: z.ZodObject<{
+            input: z.ZodString;
+            expected: z.ZodOptional<z.ZodString>;
+            context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, z.core.$strip>>>;
+            expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>;
+        output: z.ZodString;
+        duration: z.ZodNumber;
+        error: z.ZodOptional<z.ZodString>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        usage: z.ZodOptional<z.ZodObject<{
+            inputTokens: z.ZodNumber;
+            outputTokens: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            cost: z.ZodNumber;
+            duration: z.ZodNumber;
+        }, z.core.$strip>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strip>>>;
+        toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            callId: z.ZodString;
+            name: z.ZodString;
+            result: z.ZodOptional<z.ZodUnknown>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodOptional<z.ZodString>;
+            content: z.ZodString;
+            score: z.ZodOptional<z.ZodNumber>;
+            source: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            input: z.ZodString;
+            output: z.ZodString;
+            toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodString;
+                arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strip>>>;
+            traceId: z.ZodOptional<z.ZodString>;
+            prompt: z.ZodOptional<z.ZodObject<{
+                key: z.ZodString;
+                versionId: z.ZodOptional<z.ZodString>;
+                version: z.ZodOptional<z.ZodNumber>;
+                abTest: z.ZodOptional<z.ZodObject<{
+                    id: z.ZodString;
+                    variant: z.ZodEnum<{
+                        control: "control";
+                        treatment: "treatment";
+                    }>;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        traceId: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodOptional<z.ZodObject<{
+            key: z.ZodString;
+            versionId: z.ZodOptional<z.ZodString>;
+            version: z.ZodOptional<z.ZodNumber>;
+            abTest: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                variant: z.ZodEnum<{
+                    control: "control";
+                    treatment: "treatment";
+                }>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        truncated: z.ZodOptional<z.ZodBoolean>;
+        iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+        paused: z.ZodOptional<z.ZodBoolean>;
+        scores: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            score: z.ZodNumber;
+            details: z.ZodOptional<z.ZodString>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            error: z.ZodOptional<z.ZodString>;
+            usage: z.ZodOptional<z.ZodObject<{
+                inputTokens: z.ZodNumber;
+                outputTokens: z.ZodNumber;
+                totalTokens: z.ZodNumber;
+                cost: z.ZodNumber;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        caseKey: z.ZodString;
+        trials: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            case: z.ZodObject<{
+                input: z.ZodString;
+                expected: z.ZodOptional<z.ZodString>;
+                context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodString;
+                    arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                }, z.core.$strip>>>;
+                expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            }, z.core.$strip>;
+            output: z.ZodString;
+            duration: z.ZodNumber;
+            error: z.ZodOptional<z.ZodString>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            usage: z.ZodOptional<z.ZodObject<{
+                inputTokens: z.ZodNumber;
+                outputTokens: z.ZodNumber;
+                totalTokens: z.ZodNumber;
+                cost: z.ZodNumber;
+                duration: z.ZodNumber;
+            }, z.core.$strip>>;
+            toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodString;
+                arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strip>>>;
+            toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                callId: z.ZodString;
+                name: z.ZodString;
+                result: z.ZodOptional<z.ZodUnknown>;
+                error: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+            retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodOptional<z.ZodString>;
+                content: z.ZodString;
+                score: z.ZodOptional<z.ZodNumber>;
+                source: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+            turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                input: z.ZodString;
+                output: z.ZodString;
+                toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    id: z.ZodString;
+                    name: z.ZodString;
+                    arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                }, z.core.$strip>>>;
+                traceId: z.ZodOptional<z.ZodString>;
+                prompt: z.ZodOptional<z.ZodObject<{
+                    key: z.ZodString;
+                    versionId: z.ZodOptional<z.ZodString>;
+                    version: z.ZodOptional<z.ZodNumber>;
+                    abTest: z.ZodOptional<z.ZodObject<{
+                        id: z.ZodString;
+                        variant: z.ZodEnum<{
+                            control: "control";
+                            treatment: "treatment";
+                        }>;
+                    }, z.core.$strip>>;
+                }, z.core.$strip>>;
+                error: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+            traceId: z.ZodOptional<z.ZodString>;
+            prompt: z.ZodOptional<z.ZodObject<{
+                key: z.ZodString;
+                versionId: z.ZodOptional<z.ZodString>;
+                version: z.ZodOptional<z.ZodNumber>;
+                abTest: z.ZodOptional<z.ZodObject<{
+                    id: z.ZodString;
+                    variant: z.ZodEnum<{
+                        control: "control";
+                        treatment: "treatment";
+                    }>;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+            truncated: z.ZodOptional<z.ZodBoolean>;
+            iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+            paused: z.ZodOptional<z.ZodBoolean>;
+            scores: z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                score: z.ZodNumber;
+                details: z.ZodOptional<z.ZodString>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                error: z.ZodOptional<z.ZodString>;
+                usage: z.ZodOptional<z.ZodObject<{
+                    inputTokens: z.ZodNumber;
+                    outputTokens: z.ZodNumber;
+                    totalTokens: z.ZodNumber;
+                    cost: z.ZodNumber;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>;
+    aggregated: z.ZodRecord<z.ZodString, z.ZodObject<{
+        name: z.ZodString;
+        mean: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        median: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        min: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        max: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        stdDev: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        p50: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        p95: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        p99: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        count: z.ZodOptional<z.ZodNumber>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>;
+    assertions: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        passed: z.ZodBoolean;
+        message: z.ZodString;
+        actual: z.ZodOptional<z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>>;
+        expected: z.ZodOptional<z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>>;
+    }, z.core.$strip>>;
+    stats: z.ZodObject<{
+        total: z.ZodNumber;
+        errors: z.ZodNumber;
+        metricErrors: z.ZodNumber;
+        duration: z.ZodNumber;
+        cost: z.ZodNumber;
+        targetCost: z.ZodNumber;
+        judgeCost: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+/** A saved suite run, as `loadRun()` returns it */
+export type EvalRun = z.output<typeof EvalRunSchema>;
+/** A case result of a saved run, with its `caseKey` */
+export type EvalRunCase = z.output<typeof RunCaseSchema>;
+/** What a saved run records of the suite's configuration */
+export type EvalRunConfig = z.output<typeof RunConfigSchema>;
+/** JSON-serializable metadata of an experiment */
+export type JsonMetadata = z.output<typeof JsonMetadataSchema>;
+/** The options of a suite that describe the experiment it runs */
+export declare const ExperimentOptionsSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodPreprocess<z.ZodRecord<z.ZodString, z.ZodJSONSchema>, unknown>>;
+}, z.core.$strip>;
+export {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/runs/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/runs/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,0CAA0C;AAC1C,eAAO,MAAM,eAAe,uBAAuB,CAAC;AAEpD,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAQlC,eAAO,MAAM,eAAe;;;;;;;;;;;iBAK1B,CAAC;AAsEH,QAAA,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIjB,CAAC;AAEH,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;iBAmBnB,CAAC;AAkCH,kGAAkG;AAClG,eAAO,MAAM,kBAAkB,qEAAiE,CAAC;AAejG,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoBxB,CAAC;AAEH,mDAAmD;AACnD,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,aAAa,CAAC,CAAC;AACrD,uDAAuD;AACvD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,aAAa,CAAC,CAAC;AACzD,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,eAAe,CAAC,CAAC;AAC7D,kDAAkD;AAClD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE/D,kEAAkE;AAClE,eAAO,MAAM,uBAAuB;;;iBAGlC,CAAC"}