@cogitator-ai/evals 0.12.0 → 0.13.0

package/dist/runs/schema.d.ts

@@ -0,0 +1,504 @@
+import { z } from 'zod';
+/** The `format` every run file carries */
+export declare const EVAL_RUN_FORMAT = "cogitator.eval-run";
+/** The version of the run file format this package writes and reads */
+export declare const EVAL_RUN_VERSION = 1;
+export declare const RunPromptSchema: z.ZodObject<{
+    key: z.ZodString;
+    versionId: z.ZodOptional<z.ZodString>;
+    version: z.ZodOptional<z.ZodNumber>;
+    abTest: z.ZodOptional<z.ZodObject<{
+        id: z.ZodString;
+        variant: z.ZodEnum<{
+            control: "control";
+            treatment: "treatment";
+        }>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+declare const RunCaseSchema: z.ZodObject<{
+    case: z.ZodObject<{
+        input: z.ZodString;
+        expected: z.ZodOptional<z.ZodString>;
+        context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strip>>>;
+        expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>;
+    output: z.ZodString;
+    duration: z.ZodNumber;
+    error: z.ZodOptional<z.ZodString>;
+    attempts: z.ZodOptional<z.ZodNumber>;
+    usage: z.ZodOptional<z.ZodObject<{
+        inputTokens: z.ZodNumber;
+        outputTokens: z.ZodNumber;
+        totalTokens: z.ZodNumber;
+        cost: z.ZodNumber;
+        duration: z.ZodNumber;
+    }, z.core.$strip>>;
+    toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strip>>>;
+    toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        callId: z.ZodString;
+        name: z.ZodString;
+        result: z.ZodOptional<z.ZodUnknown>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodOptional<z.ZodString>;
+        content: z.ZodString;
+        score: z.ZodOptional<z.ZodNumber>;
+        source: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        input: z.ZodString;
+        output: z.ZodString;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strip>>>;
+        traceId: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodOptional<z.ZodObject<{
+            key: z.ZodString;
+            versionId: z.ZodOptional<z.ZodString>;
+            version: z.ZodOptional<z.ZodNumber>;
+            abTest: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                variant: z.ZodEnum<{
+                    control: "control";
+                    treatment: "treatment";
+                }>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
+    traceId: z.ZodOptional<z.ZodString>;
+    prompt: z.ZodOptional<z.ZodObject<{
+        key: z.ZodString;
+        versionId: z.ZodOptional<z.ZodString>;
+        version: z.ZodOptional<z.ZodNumber>;
+        abTest: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            variant: z.ZodEnum<{
+                control: "control";
+                treatment: "treatment";
+            }>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    truncated: z.ZodOptional<z.ZodBoolean>;
+    iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+    paused: z.ZodOptional<z.ZodBoolean>;
+    scores: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        score: z.ZodNumber;
+        details: z.ZodOptional<z.ZodString>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        error: z.ZodOptional<z.ZodString>;
+        usage: z.ZodOptional<z.ZodObject<{
+            inputTokens: z.ZodNumber;
+            outputTokens: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            cost: z.ZodNumber;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    caseKey: z.ZodString;
+    trials: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        case: z.ZodObject<{
+            input: z.ZodString;
+            expected: z.ZodOptional<z.ZodString>;
+            context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, z.core.$strip>>>;
+            expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>;
+        output: z.ZodString;
+        duration: z.ZodNumber;
+        error: z.ZodOptional<z.ZodString>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        usage: z.ZodOptional<z.ZodObject<{
+            inputTokens: z.ZodNumber;
+            outputTokens: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            cost: z.ZodNumber;
+            duration: z.ZodNumber;
+        }, z.core.$strip>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strip>>>;
+        toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            callId: z.ZodString;
+            name: z.ZodString;
+            result: z.ZodOptional<z.ZodUnknown>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodOptional<z.ZodString>;
+            content: z.ZodString;
+            score: z.ZodOptional<z.ZodNumber>;
+            source: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            input: z.ZodString;
+            output: z.ZodString;
+            toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodString;
+                arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strip>>>;
+            traceId: z.ZodOptional<z.ZodString>;
+            prompt: z.ZodOptional<z.ZodObject<{
+                key: z.ZodString;
+                versionId: z.ZodOptional<z.ZodString>;
+                version: z.ZodOptional<z.ZodNumber>;
+                abTest: z.ZodOptional<z.ZodObject<{
+                    id: z.ZodString;
+                    variant: z.ZodEnum<{
+                        control: "control";
+                        treatment: "treatment";
+                    }>;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        traceId: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodOptional<z.ZodObject<{
+            key: z.ZodString;
+            versionId: z.ZodOptional<z.ZodString>;
+            version: z.ZodOptional<z.ZodNumber>;
+            abTest: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                variant: z.ZodEnum<{
+                    control: "control";
+                    treatment: "treatment";
+                }>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        truncated: z.ZodOptional<z.ZodBoolean>;
+        iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+        paused: z.ZodOptional<z.ZodBoolean>;
+        scores: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            score: z.ZodNumber;
+            details: z.ZodOptional<z.ZodString>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            error: z.ZodOptional<z.ZodString>;
+            usage: z.ZodOptional<z.ZodObject<{
+                inputTokens: z.ZodNumber;
+                outputTokens: z.ZodNumber;
+                totalTokens: z.ZodNumber;
+                cost: z.ZodNumber;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>>;
+}, z.core.$strip>;
+declare const RunConfigSchema: z.ZodObject<{
+    metrics: z.ZodArray<z.ZodString>;
+    statisticalMetrics: z.ZodArray<z.ZodString>;
+    judge: z.ZodOptional<z.ZodObject<{
+        model: z.ZodString;
+        temperature: z.ZodNumber;
+        samples: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    trials: z.ZodNumber;
+    concurrency: z.ZodNumber;
+    target: z.ZodObject<{
+        kind: z.ZodEnum<{
+            fn: "fn";
+            agent: "agent";
+        }>;
+        name: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+/** JSON-serializable metadata: object keys holding `undefined` are dropped, as JSON drops them */
+export declare const JsonMetadataSchema: z.ZodPreprocess<z.ZodRecord<z.ZodString, z.ZodJSONSchema>, unknown>;
+export declare const EvalRunSchema: z.ZodObject<{
+    format: z.ZodLiteral<"cogitator.eval-run">;
+    version: z.ZodLiteral<1>;
+    runId: z.ZodString;
+    startedAt: z.ZodISODateTime;
+    finishedAt: z.ZodISODateTime;
+    name: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodPreprocess<z.ZodRecord<z.ZodString, z.ZodJSONSchema>, unknown>>;
+    dataset: z.ZodObject<{
+        fingerprint: z.ZodString;
+        cases: z.ZodNumber;
+    }, z.core.$strip>;
+    config: z.ZodObject<{
+        metrics: z.ZodArray<z.ZodString>;
+        statisticalMetrics: z.ZodArray<z.ZodString>;
+        judge: z.ZodOptional<z.ZodObject<{
+            model: z.ZodString;
+            temperature: z.ZodNumber;
+            samples: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+        trials: z.ZodNumber;
+        concurrency: z.ZodNumber;
+        target: z.ZodObject<{
+            kind: z.ZodEnum<{
+                fn: "fn";
+                agent: "agent";
+            }>;
+            name: z.ZodOptional<z.ZodString>;
+            model: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>;
+    }, z.core.$strip>;
+    prompts: z.ZodArray<z.ZodObject<{
+        key: z.ZodString;
+        versionId: z.ZodOptional<z.ZodString>;
+        version: z.ZodOptional<z.ZodNumber>;
+        abTest: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            variant: z.ZodEnum<{
+                control: "control";
+                treatment: "treatment";
+            }>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+    results: z.ZodArray<z.ZodObject<{
+        case: z.ZodObject<{
+            input: z.ZodString;
+            expected: z.ZodOptional<z.ZodString>;
+            context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, z.core.$strip>>>;
+            expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        }, z.core.$strip>;
+        output: z.ZodString;
+        duration: z.ZodNumber;
+        error: z.ZodOptional<z.ZodString>;
+        attempts: z.ZodOptional<z.ZodNumber>;
+        usage: z.ZodOptional<z.ZodObject<{
+            inputTokens: z.ZodNumber;
+            outputTokens: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            cost: z.ZodNumber;
+            duration: z.ZodNumber;
+        }, z.core.$strip>>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strip>>>;
+        toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            callId: z.ZodString;
+            name: z.ZodString;
+            result: z.ZodOptional<z.ZodUnknown>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodOptional<z.ZodString>;
+            content: z.ZodString;
+            score: z.ZodOptional<z.ZodNumber>;
+            source: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            input: z.ZodString;
+            output: z.ZodString;
+            toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodString;
+                arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strip>>>;
+            traceId: z.ZodOptional<z.ZodString>;
+            prompt: z.ZodOptional<z.ZodObject<{
+                key: z.ZodString;
+                versionId: z.ZodOptional<z.ZodString>;
+                version: z.ZodOptional<z.ZodNumber>;
+                abTest: z.ZodOptional<z.ZodObject<{
+                    id: z.ZodString;
+                    variant: z.ZodEnum<{
+                        control: "control";
+                        treatment: "treatment";
+                    }>;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>>;
+        traceId: z.ZodOptional<z.ZodString>;
+        prompt: z.ZodOptional<z.ZodObject<{
+            key: z.ZodString;
+            versionId: z.ZodOptional<z.ZodString>;
+            version: z.ZodOptional<z.ZodNumber>;
+            abTest: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                variant: z.ZodEnum<{
+                    control: "control";
+                    treatment: "treatment";
+                }>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        truncated: z.ZodOptional<z.ZodBoolean>;
+        iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+        paused: z.ZodOptional<z.ZodBoolean>;
+        scores: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            score: z.ZodNumber;
+            details: z.ZodOptional<z.ZodString>;
+            metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            error: z.ZodOptional<z.ZodString>;
+            usage: z.ZodOptional<z.ZodObject<{
+                inputTokens: z.ZodNumber;
+                outputTokens: z.ZodNumber;
+                totalTokens: z.ZodNumber;
+                cost: z.ZodNumber;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        caseKey: z.ZodString;
+        trials: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            case: z.ZodObject<{
+                input: z.ZodString;
+                expected: z.ZodOptional<z.ZodString>;
+                context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    name: z.ZodString;
+                    arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                }, z.core.$strip>>>;
+                expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            }, z.core.$strip>;
+            output: z.ZodString;
+            duration: z.ZodNumber;
+            error: z.ZodOptional<z.ZodString>;
+            attempts: z.ZodOptional<z.ZodNumber>;
+            usage: z.ZodOptional<z.ZodObject<{
+                inputTokens: z.ZodNumber;
+                outputTokens: z.ZodNumber;
+                totalTokens: z.ZodNumber;
+                cost: z.ZodNumber;
+                duration: z.ZodNumber;
+            }, z.core.$strip>>;
+            toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodString;
+                arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strip>>>;
+            toolResults: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                callId: z.ZodString;
+                name: z.ZodString;
+                result: z.ZodOptional<z.ZodUnknown>;
+                error: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+            retrieved: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                id: z.ZodOptional<z.ZodString>;
+                content: z.ZodString;
+                score: z.ZodOptional<z.ZodNumber>;
+                source: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+            turns: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                input: z.ZodString;
+                output: z.ZodString;
+                toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    id: z.ZodString;
+                    name: z.ZodString;
+                    arguments: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                }, z.core.$strip>>>;
+                traceId: z.ZodOptional<z.ZodString>;
+                prompt: z.ZodOptional<z.ZodObject<{
+                    key: z.ZodString;
+                    versionId: z.ZodOptional<z.ZodString>;
+                    version: z.ZodOptional<z.ZodNumber>;
+                    abTest: z.ZodOptional<z.ZodObject<{
+                        id: z.ZodString;
+                        variant: z.ZodEnum<{
+                            control: "control";
+                            treatment: "treatment";
+                        }>;
+                    }, z.core.$strip>>;
+                }, z.core.$strip>>;
+                error: z.ZodOptional<z.ZodString>;
+            }, z.core.$strip>>>;
+            traceId: z.ZodOptional<z.ZodString>;
+            prompt: z.ZodOptional<z.ZodObject<{
+                key: z.ZodString;
+                versionId: z.ZodOptional<z.ZodString>;
+                version: z.ZodOptional<z.ZodNumber>;
+                abTest: z.ZodOptional<z.ZodObject<{
+                    id: z.ZodString;
+                    variant: z.ZodEnum<{
+                        control: "control";
+                        treatment: "treatment";
+                    }>;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+            truncated: z.ZodOptional<z.ZodBoolean>;
+            iterationLimitReached: z.ZodOptional<z.ZodBoolean>;
+            paused: z.ZodOptional<z.ZodBoolean>;
+            scores: z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                score: z.ZodNumber;
+                details: z.ZodOptional<z.ZodString>;
+                metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+                error: z.ZodOptional<z.ZodString>;
+                usage: z.ZodOptional<z.ZodObject<{
+                    inputTokens: z.ZodNumber;
+                    outputTokens: z.ZodNumber;
+                    totalTokens: z.ZodNumber;
+                    cost: z.ZodNumber;
+                }, z.core.$strip>>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>>;
+    }, z.core.$strip>>;
+    aggregated: z.ZodRecord<z.ZodString, z.ZodObject<{
+        name: z.ZodString;
+        mean: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        median: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        min: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        max: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        stdDev: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        p50: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        p95: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        p99: z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>;
+        count: z.ZodOptional<z.ZodNumber>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>;
+    assertions: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        passed: z.ZodBoolean;
+        message: z.ZodString;
+        actual: z.ZodOptional<z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>>;
+        expected: z.ZodOptional<z.ZodPipe<z.ZodNullable<z.ZodNumber>, z.ZodTransform<number, number | null>>>;
+    }, z.core.$strip>>;
+    stats: z.ZodObject<{
+        total: z.ZodNumber;
+        errors: z.ZodNumber;
+        metricErrors: z.ZodNumber;
+        duration: z.ZodNumber;
+        cost: z.ZodNumber;
+        targetCost: z.ZodNumber;
+        judgeCost: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+/** A saved suite run, as `loadRun()` returns it */
+export type EvalRun = z.output<typeof EvalRunSchema>;
+/** A case result of a saved run, with its `caseKey` */
+export type EvalRunCase = z.output<typeof RunCaseSchema>;
+/** What a saved run records of the suite's configuration */
+export type EvalRunConfig = z.output<typeof RunConfigSchema>;
+/** JSON-serializable metadata of an experiment */
+export type JsonMetadata = z.output<typeof JsonMetadataSchema>;
+/** The options of a suite that describe the experiment it runs */
+export declare const ExperimentOptionsSchema: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    metadata: z.ZodOptional<z.ZodPreprocess<z.ZodRecord<z.ZodString, z.ZodJSONSchema>, unknown>>;
+}, z.core.$strip>;
+export {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/runs/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/runs/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,0CAA0C;AAC1C,eAAO,MAAM,eAAe,uBAAuB,CAAC;AAEpD,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAQlC,eAAO,MAAM,eAAe;;;;;;;;;;;iBAK1B,CAAC;AAsEH,QAAA,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIjB,CAAC;AAEH,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;iBAmBnB,CAAC;AAkCH,kGAAkG;AAClG,eAAO,MAAM,kBAAkB,qEAAiE,CAAC;AAejG,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoBxB,CAAC;AAEH,mDAAmD;AACnD,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,aAAa,CAAC,CAAC;AACrD,uDAAuD;AACvD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,aAAa,CAAC,CAAC;AACzD,4DAA4D;AAC5D,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,eAAe,CAAC,CAAC;AAC7D,kDAAkD;AAClD,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE/D,kEAAkE;AAClE,eAAO,MAAM,uBAAuB;;;iBAGlC,CAAC"}

package/dist/runs/schema.js

@@ -0,0 +1,171 @@
+import { z } from 'zod';
+import { EvalCaseSchema } from '../schema.js';
+/** The `format` every run file carries */
+export const EVAL_RUN_FORMAT = 'cogitator.eval-run';
+/** The version of the run file format this package writes and reads */
+export const EVAL_RUN_VERSION = 1;
+/** A figure JSON writes as `null` when it is not a finite number, such as an unscored mean */
+const figure = z
+    .number()
+    .nullable()
+    .transform((value) => value ?? Number.NaN);
+export const RunPromptSchema = z.object({
+    key: z.string().min(1),
+    versionId: z.string().optional(),
+    version: z.number().optional(),
+    abTest: z.object({ id: z.string(), variant: z.enum(['control', 'treatment']) }).optional(),
+});
+const UsageSchema = z.object({
+    inputTokens: z.number(),
+    outputTokens: z.number(),
+    totalTokens: z.number(),
+    cost: z.number(),
+});
+const ScoreSchema = z.object({
+    name: z.string(),
+    score: z.number(),
+    details: z.string().optional(),
+    metadata: z.record(z.string(), z.unknown()).optional(),
+    error: z.string().optional(),
+    usage: UsageSchema.optional(),
+});
+const ToolCallSchema = z.object({
+    id: z.string(),
+    name: z.string(),
+    arguments: z.record(z.string(), z.unknown()),
+});
+const TurnSchema = z.object({
+    input: z.string(),
+    output: z.string(),
+    toolCalls: z.array(ToolCallSchema).optional(),
+    traceId: z.string().optional(),
+    prompt: RunPromptSchema.optional(),
+    error: z.string().optional(),
+});
+const CaseResultSchema = z.object({
+    case: EvalCaseSchema,
+    output: z.string(),
+    duration: z.number(),
+    error: z.string().optional(),
+    attempts: z.number().int().optional(),
+    usage: UsageSchema.extend({ duration: z.number() }).optional(),
+    toolCalls: z.array(ToolCallSchema).optional(),
+    toolResults: z
+        .array(z.object({
+        callId: z.string(),
+        name: z.string(),
+        result: z.unknown().optional(),
+        error: z.string().optional(),
+    }))
+        .optional(),
+    retrieved: z
+        .array(z.object({
+        id: z.string().optional(),
+        content: z.string(),
+        score: z.number().optional(),
+        source: z.string().optional(),
+    }))
+        .optional(),
+    turns: z.array(TurnSchema).optional(),
+    traceId: z.string().optional(),
+    prompt: RunPromptSchema.optional(),
+    truncated: z.boolean().optional(),
+    iterationLimitReached: z.boolean().optional(),
+    paused: z.boolean().optional(),
+    scores: z.array(ScoreSchema),
+});
+const RunCaseSchema = CaseResultSchema.extend({
+    /** SHA-256 of the canonical JSON of the case's `{ input, turns, expected }` */
+    caseKey: z.string().regex(/^[0-9a-f]{64}$/),
+    trials: z.array(CaseResultSchema).optional(),
+});
+const RunConfigSchema = z.object({
+    metrics: z.array(z.string()),
+    statisticalMetrics: z.array(z.string()),
+    judge: z
+        .object({
+        model: z.string(),
+        temperature: z.number(),
+        samples: z.number().int().positive().optional(),
+    })
+        .optional(),
+    trials: z.number().int().min(1),
+    concurrency: z.number().int().min(1),
+    target: z.object({
+        kind: z.enum(['fn', 'agent']),
+        /** The agent's name, for an agent target */
+        name: z.string().optional(),
+        /** The agent's model, for an agent target that sets one */
+        model: z.string().optional(),
+    }),
+});
+const AggregatedSchema = z.object({
+    name: z.string(),
+    mean: figure,
+    median: figure,
+    min: figure,
+    max: figure,
+    stdDev: figure,
+    p50: figure,
+    p95: figure,
+    p99: figure,
+    count: z.number().int().optional(),
+    metadata: z.record(z.string(), z.unknown()).optional(),
+});
+const AssertionSchema = z.object({
+    name: z.string(),
+    passed: z.boolean(),
+    message: z.string(),
+    actual: figure.optional(),
+    expected: figure.optional(),
+});
+const StatsSchema = z.object({
+    total: z.number().int(),
+    errors: z.number().int(),
+    metricErrors: z.number().int(),
+    duration: z.number(),
+    cost: z.number(),
+    targetCost: z.number(),
+    judgeCost: z.number(),
+});
+/** JSON-serializable metadata: object keys holding `undefined` are dropped, as JSON drops them */
+export const JsonMetadataSchema = z.preprocess(withoutUndefined, z.record(z.string(), z.json()));
+function withoutUndefined(value) {
+    if (Array.isArray(value))
+        return value.map(withoutUndefined);
+    if (typeof value !== 'object' || value === null)
+        return value;
+    if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) {
+        return value;
+    }
+    return Object.fromEntries(Object.entries(value)
+        .filter(([, item]) => item !== undefined)
+        .map(([key, item]) => [key, withoutUndefined(item)]));
+}
+export const EvalRunSchema = z.object({
+    format: z.literal(EVAL_RUN_FORMAT),
+    version: z.literal(EVAL_RUN_VERSION),
+    runId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/),
+    startedAt: z.iso.datetime(),
+    finishedAt: z.iso.datetime(),
+    name: z.string().optional(),
+    metadata: JsonMetadataSchema.optional(),
+    dataset: z.object({
+        /** SHA-256 of the canonical JSON of the cases run, in order */
+        fingerprint: z.string().regex(/^[0-9a-f]{64}$/),
+        cases: z.number().int().min(0),
+    }),
+    config: RunConfigSchema,
+    /** The distinct versioned instructions the agent runs used */
+    prompts: z.array(RunPromptSchema),
+    results: z.array(RunCaseSchema),
+    aggregated: z.record(z.string(), AggregatedSchema),
+    assertions: z.array(AssertionSchema),
+    stats: StatsSchema,
+});
+/** The options of a suite that describe the experiment it runs */
+export const ExperimentOptionsSchema = z.object({
+    name: z.string().trim().min(1, 'name must not be empty').optional(),
+    metadata: JsonMetadataSchema.optional(),
+});
+//# sourceMappingURL=schema.js.map