@cef-ai/eval 0.0.0-stage → 0.2.0

package/dist/pool.js

@@ -0,0 +1,34 @@
+/**
+ * Bounded-concurrency for-each, shared by `runExperiment` and the store
+ * conformance suite. Dependency-free and browser-safe.
+ *
+ * IT FAILS SAFELY. When one item throws, no further item is started, every
+ * item already in flight is awaited, and only then does the pool reject, with
+ * the first error. `Promise.all` over the workers would reject at the first
+ * throw while its siblings kept running — so a caller's cleanup (releasing a
+ * version pin, writing the final summary) would run under work that was still
+ * writing, and land before it.
+ */
+export async function pool(items, limit, fn) {
+    let next = 0;
+    let failed = false;
+    let first;
+    const worker = async () => {
+        while (!failed && next < items.length) {
+            const item = items[next++];
+            try {
+                await fn(item);
+            }
+            catch (e) {
+                if (!failed) {
+                    failed = true;
+                    first = e;
+                }
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
+    if (failed)
+        throw first;
+}
+//# sourceMappingURL=pool.js.map

package/dist/pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pool.js","sourceRoot":"","sources":["../src/pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAI,KAAmB,EAAE,KAAa,EAAE,EAA8B;IAC9F,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,IAAI,KAAc,CAAC;IACnB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,CAAC,MAAM,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,GAAG,IAAI,CAAC;oBACd,KAAK,GAAG,CAAC,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9F,IAAI,MAAM;QAAE,MAAM,KAAK,CAAC;AAC1B,CAAC"}

package/dist/result.d.ts

@@ -0,0 +1,11 @@
+import type { ResultField } from "./types.js";
+/**
+ * The fields the graph's `output` node(s) declare. A graph with several
+ * outputs (branches ending differently) yields the union; a field two outputs
+ * type differently is reported untyped, since either type can come back.
+ *
+ * undefined when no output declares a result — the Result is unknown, and
+ * scoring then judges every expected field as before.
+ */
+export declare function resultFieldsFromGraph(graph: unknown): ResultField[] | undefined;
+//# sourceMappingURL=result.d.ts.map

package/dist/result.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AA0B9C;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,EAAE,GAAG,SAAS,CAmB/E"}

package/dist/result.js

@@ -0,0 +1,72 @@
+/**
+ * Reading a workflow version's declared Result off its graph — what ROC and
+ * the CLI record as `Experiment.resultFields`.
+ *
+ * Mirrors the runner's own reading (`@cef-ai/agent-sdk` `readResultSpec`)
+ * without depending on it: this package is zero-dependency and browser-safe.
+ * An `output` node declares `params.result` (`{ field: value }`, or that object
+ * as JSON text — what a builder text field saves) and optionally
+ * `params.resultTypes` (`{ field: type }`, object or JSON text).
+ */
+import { isObject } from "./contract.js";
+/** An object, or JSON text that parses to one; anything else is undefined. */
+function objectOf(raw) {
+    if (isObject(raw))
+        return raw;
+    if (typeof raw !== "string" || !raw.trim().startsWith("{"))
+        return undefined;
+    try {
+        const parsed = JSON.parse(raw);
+        return isObject(parsed) ? parsed : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * The graph itself, from any of the forms it travels in: the object, its JSON
+ * text, or a manifest param (`params.graph` = `{ type: "string", default: "<json>" }`).
+ */
+function graphOf(raw) {
+    const g = objectOf(raw);
+    if (!g)
+        return undefined;
+    if (Array.isArray(g.nodes))
+        return g;
+    if ("default" in g)
+        return graphOf(g.default);
+    return undefined;
+}
+/**
+ * The fields the graph's `output` node(s) declare. A graph with several
+ * outputs (branches ending differently) yields the union; a field two outputs
+ * type differently is reported untyped, since either type can come back.
+ *
+ * undefined when no output declares a result — the Result is unknown, and
+ * scoring then judges every expected field as before.
+ */
+export function resultFieldsFromGraph(graph) {
+    const g = graphOf(graph);
+    if (!g)
+        return undefined;
+    const types = new Map();
+    let declared = false;
+    for (const node of g.nodes) {
+        if (!isObject(node) || (node.kind ?? node.type) !== "output" || !isObject(node.params))
+            continue;
+        const result = objectOf(node.params.result);
+        if (!result)
+            continue;
+        declared = true;
+        const typed = objectOf(node.params.resultTypes) ?? {};
+        for (const name of Object.keys(result)) {
+            const t = typeof typed[name] === "string" && typed[name] ? typed[name] : undefined;
+            // First sighting sets the type; any disagreement (typed vs untyped included) drops it.
+            types.set(name, types.has(name) && types.get(name) !== t ? undefined : t);
+        }
+    }
+    if (!declared)
+        return undefined;
+    return [...types].sort(([a], [b]) => a.localeCompare(b)).map(([name, type]) => (type ? { name, type } : { name }));
+}
+//# sourceMappingURL=result.js.map

package/dist/result.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"result.js","sourceRoot":"","sources":["../src/result.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,GAAY;IAC5B,IAAI,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IAC9B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7E,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CAAC,GAAY;IAC3B,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IACxB,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACrC,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACzB,IAAI,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACzB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;IACpD,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,KAAkB,EAAE,CAAC;QACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS;QACjG,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,QAAQ,GAAG,IAAI,CAAC;QAChB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QACtD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,KAAK,CAAC,IAAI,CAAY,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/F,uFAAuF;YACvF,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChC,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACrH,CAAC"}

package/dist/run.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Running an experiment: one dataset version × one workflow version × N
+ * repeats, against an injected {@link WorkflowTarget}.
+ *
+ * WHAT IS WRITTEN, AND WHEN. The meta first, `running`, so an experiment that
+ * dies half way is still visible as one. Each run record as its run finishes,
+ * write-once, so a long experiment is resumable: rerun with the same
+ * `experimentId` and every run already on file is reused rather than repeated.
+ * The summary last, with the final status.
+ *
+ * A NON-LIVE VERSION is run through `target.pinVersion`, which routes only this
+ * experiment's contexts (`eval-<expId>-…`) to the candidate and is released
+ * however the experiment ends.
+ */
+import { type EvalStore } from "./storage.js";
+import type { Case, Experiment, Header, ResultField, RunRecord, WorkflowTarget } from "./types.js";
+export interface ExperimentProgress {
+    /** Runs finished (or reused) so far, including this one. */
+    done: number;
+    total: number;
+    run: RunRecord;
+    /** True when the run was already on file from an earlier attempt. */
+    reused: boolean;
+}
+export interface RunExperimentOptions {
+    store: EvalStore;
+    dataset: string;
+    /**
+     * The dataset version to run. Absent: `ensureDatasetVersion` — the latest,
+     * or a new one cut automatically when the cases changed since.
+     */
+    datasetVersion?: number;
+    workflowId: string;
+    /** The version being evaluated. With `live`, the version the live deployment runs (recorded only). */
+    workflowVersion: string;
+    /** Run against the live deployment (no pin). False requires `target.pinVersion`. */
+    live: boolean;
+    /** How many times each case runs. Default 1. */
+    repeats?: number;
+    /**
+     * The Result the workflow version declares (`resultFieldsFromGraph`).
+     * Recorded on the experiment; expected fields it rules out score n/a.
+     */
+    resultFields?: ResultField[];
+    /** Runs in flight at once. Default 4. */
+    concurrency?: number;
+    /** How long one run may take to end. Default 300 000 ms. */
+    timeoutMs?: number;
+    target: WorkflowTarget;
+    onProgress?: (p: ExperimentProgress) => void;
+    baselineId?: string;
+    /** Resume this experiment instead of starting a new one. */
+    experimentId?: string;
+    origin?: "roc" | "cli";
+    createdBy?: string;
+    note?: string;
+    /** Stops dispatching new runs; the experiment ends `cancelled`. */
+    signal?: AbortSignal;
+    /** Clock, for tests. */
+    now?: () => number;
+    /** Waits between retries of a run record's write; for tests. */
+    sleep?: (ms: number) => Promise<void>;
+}
+export declare const DEFAULT_TIMEOUT_MS = 300000;
+export declare const DEFAULT_CONCURRENCY = 4;
+/** Backoff before each retry of a run record whose write-once lost to nothing. */
+export declare const RECORD_WRITE_BACKOFF_MS: readonly [250, 1000, 4000];
+/** Run one case × repeat against the target, and score it. Never throws. */
+export declare function runOne(opts: {
+    target: WorkflowTarget;
+    header?: Pick<Header, "limits">;
+    case: Case;
+    caseRev: number;
+    /** 1-based. */
+    repeat: number;
+    /** Recorded on the run; the context already carries it. */
+    attempt?: number;
+    context: string;
+    timeoutMs: number;
+    now?: () => number;
+    /** The version's declared Result; see {@link scoreFields}. */
+    resultFields?: readonly ResultField[];
+}): Promise<RunRecord>;
+/** Run an experiment end to end. Resolves with the final meta (summary included). */
+export declare function runExperiment(opts: RunExperimentOptions): Promise<Experiment>;
+//# sourceMappingURL=run.d.ts.map

package/dist/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EASL,KAAK,SAAS,EACf,MAAM,cAAc,CAAC;AAMtB,OAAO,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAiB,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAElH,MAAM,WAAW,kBAAkB;IACjC,4DAA4D;IAC5D,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,SAAS,CAAC;IACf,qEAAqE;IACrE,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,SAAS,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,sGAAsG;IACtG,eAAe,EAAE,MAAM,CAAC;IACxB,oFAAoF;IACpF,IAAI,EAAE,OAAO,CAAC;IACd,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,yCAAyC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wBAAwB;IACxB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,gEAAgE;IAChE,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,eAAO,MAAM,kBAAkB,SAAU,CAAC;AAC1C,eAAO,MAAM,mBAAmB,IAAI,CAAC;AACrC,kFAAkF;AAClF,eAAO,MAAM,uBAAuB,4BAA6B,CAAC;AAKlE,4EAA4E;AAC5E,wBAAsB,MAAM,CAAC,IAAI,EAAE;IACjC,MAAM,EAAE,cAAc,CAAC;IACvB,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe;IACf,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACvC,GAAG,OAAO,CAAC,SAAS,CAAC,CAuErB;AAED,qFAAqF;AACrF,wBAAsB,aAAa,CAAC,IAAI,EAAE,oBAAoB,GAAG,OAAO,CAAC,UAAU,CAAC,CA0InF"}

package/dist/run.js

@@ -0,0 +1,240 @@
+/**
+ * Running an experiment: one dataset version × one workflow version × N
+ * repeats, against an injected {@link WorkflowTarget}.
+ *
+ * WHAT IS WRITTEN, AND WHEN. The meta first, `running`, so an experiment that
+ * dies half way is still visible as one. Each run record as its run finishes,
+ * write-once, so a long experiment is resumable: rerun with the same
+ * `experimentId` and every run already on file is reused rather than repeated.
+ * The summary last, with the final status.
+ *
+ * A NON-LIVE VERSION is run through `target.pinVersion`, which routes only this
+ * experiment's contexts (`eval-<expId>-…`) to the candidate and is released
+ * however the experiment ends.
+ */
+import { claimAttempt, ensureDatasetVersion, getExperiment, getRunRecord, loadVersion, putExperiment, putRunRecord, requireHeader, } from "./storage.js";
+import { experimentContextPrefix, newExperimentId, runContext, runIdOf } from "./layout.js";
+import { casePass, effectiveLimits, scoreFields, scoreLimits } from "./score.js";
+import { emptyMetrics, summarize } from "./summary.js";
+import { EvalContractError } from "./contract.js";
+import { pool } from "./pool.js";
+export const DEFAULT_TIMEOUT_MS = 300_000;
+export const DEFAULT_CONCURRENCY = 4;
+/** Backoff before each retry of a run record whose write-once lost to nothing. */
+export const RECORD_WRITE_BACKOFF_MS = [250, 1000, 4000];
+const iso = (ms) => new Date(ms).toISOString();
+const message = (e) => (e instanceof Error ? e.message : String(e));
+/** Run one case × repeat against the target, and score it. Never throws. */
+export async function runOne(opts) {
+    const now = opts.now ?? Date.now;
+    const c = opts.case;
+    const started = now();
+    const base = {
+        caseId: c.id,
+        caseRev: opts.caseRev,
+        repeat: opts.repeat,
+        ...(opts.attempt !== undefined ? { attempt: opts.attempt } : {}),
+        context: opts.context,
+        runId: runIdOf(opts.context),
+        startedAt: iso(started),
+    };
+    try {
+        await opts.target.start(c.input, opts.context);
+    }
+    catch (e) {
+        return {
+            ...base, status: "error", output: null, error: `could not start the run: ${message(e)}`,
+            endedAt: null, metrics: emptyMetrics(), scores: [], pass: null,
+        };
+    }
+    let end;
+    try {
+        end = await opts.target.waitForEnd(opts.context, opts.timeoutMs);
+    }
+    catch (e) {
+        return {
+            ...base, status: "error", output: null, error: `could not read how the run ended: ${message(e)}`,
+            endedAt: null, metrics: emptyMetrics(), scores: [], pass: null,
+        };
+    }
+    const endedMs = end.endedAt ? Date.parse(end.endedAt) : NaN;
+    const startedMs = end.startedAt ? Date.parse(end.startedAt) : NaN;
+    let metrics;
+    let metricsError;
+    try {
+        metrics = await opts.target.metrics(opts.context);
+    }
+    catch (e) {
+        metrics = emptyMetrics();
+        metricsError = `metrics unavailable: ${message(e)}`;
+    }
+    /*
+     * ONE CLOCK. Duration is the end event's timestamp minus the start event's,
+     * both stamped by the server. When the target could not read both, whatever
+     * it measured itself (also server-side — the CLI target uses the jobs'
+     * createdAt/completedAt) stands, and otherwise there is no duration. This
+     * machine's clock is never subtracted from the server's: their skew would
+     * land in every number and differ between the CLI and ROC.
+     */
+    if (Number.isFinite(endedMs) && Number.isFinite(startedMs)) {
+        metrics = { ...metrics, durationMs: Math.max(0, endedMs - startedMs) };
+    }
+    const output = end.status === "done" ? (end.output ?? null) : null;
+    const fieldScores = scoreFields(c.expected, end.status === "done" ? (end.output ?? null) : undefined, opts.resultFields);
+    const scores = [...fieldScores, ...scoreLimits(effectiveLimits(opts.header, c), metrics)];
+    // A run that did not complete has no Result to judge; it is a failure of the
+    // workflow, whether or not the case states expectations.
+    const pass = end.status === "done" ? casePass(scores) : false;
+    const error = [end.error, metricsError].filter(Boolean).join("; ");
+    return {
+        ...base,
+        ...(Number.isFinite(startedMs) ? { startedAt: iso(startedMs) } : {}),
+        status: end.status,
+        output,
+        ...(end.outcome ? { outcome: end.outcome } : {}),
+        ...(error ? { error } : {}),
+        endedAt: Number.isFinite(endedMs) ? iso(endedMs) : null,
+        metrics,
+        scores,
+        pass,
+    };
+}
+/** Run an experiment end to end. Resolves with the final meta (summary included). */
+export async function runExperiment(opts) {
+    const now = opts.now ?? Date.now;
+    const repeats = opts.repeats ?? 1;
+    const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
+    const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    if (!Number.isInteger(repeats) || repeats < 1)
+        throw new EvalContractError("repeats must be a whole number ≥ 1");
+    if (!Number.isInteger(concurrency) || concurrency < 1)
+        throw new EvalContractError("concurrency must be a whole number ≥ 1");
+    if (!opts.live && !opts.target.pinVersion) {
+        throw new EvalContractError(`running workflow version ${opts.workflowVersion} that is not live needs a target that can pin a version`);
+    }
+    await requireHeader(opts.store, opts.workflowId, opts.dataset);
+    const existing = opts.experimentId ? await getExperiment(opts.store, opts.workflowId, opts.experimentId) : null;
+    if (opts.experimentId && (!existing || existing.dataset !== opts.dataset)) {
+        throw new EvalContractError(`experiment "${opts.experimentId}" not found for dataset "${opts.dataset}"`);
+    }
+    // A resume runs the version it started with; a new experiment the chosen one, or the working set's.
+    const datasetVersion = opts.datasetVersion
+        ?? existing?.datasetVersion
+        ?? (await ensureDatasetVersion(opts.store, opts.workflowId, opts.dataset, { now: new Date(now()) })).version;
+    const { manifest, cases } = await loadVersion(opts.store, opts.workflowId, opts.dataset, datasetVersion);
+    let experiment;
+    if (existing) {
+        if (existing.datasetVersion !== datasetVersion || existing.workflowVersion !== opts.workflowVersion
+            || existing.repeats !== repeats) {
+            throw new EvalContractError(`experiment "${existing.id}" ran dataset v${existing.datasetVersion} × workflow ${existing.workflowVersion} × ${existing.repeats}; `
+                + "resume it with the same settings");
+        }
+        // A new attempt, claimed write-once before anything is published: runs the
+        // dead one may have started — or a concurrent resumer is starting — keep
+        // their contexts to themselves.
+        const attempt = await claimAttempt(opts.store, opts.workflowId, existing.id, {
+            atLeast: (existing.attempt ?? 1) + 1, claimedAt: iso(now()),
+        });
+        experiment = { ...existing, status: "running", attempt };
+        delete experiment.finishedAt;
+        delete experiment.error;
+    }
+    else {
+        const id = newExperimentId(now());
+        await claimAttempt(opts.store, opts.workflowId, id, { atLeast: 1, claimedAt: iso(now()) });
+        experiment = {
+            id,
+            dataset: opts.dataset,
+            datasetVersion: manifest.version,
+            workflowId: opts.workflowId,
+            workflowVersion: opts.workflowVersion,
+            live: opts.live,
+            repeats,
+            ...(opts.resultFields ? { resultFields: opts.resultFields } : {}),
+            attempt: 1,
+            startedAt: iso(now()),
+            status: "running",
+            origin: opts.origin ?? "cli",
+            ...(opts.createdBy ? { createdBy: opts.createdBy } : {}),
+            ...(opts.baselineId ? { baselineId: opts.baselineId } : {}),
+            ...(opts.note ? { note: opts.note } : {}),
+        };
+    }
+    await putExperiment(opts.store, experiment);
+    const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    /*
+     * Write-once, and honest about what happened. `putIfAbsent` answering false
+     * means "not written", which is two different things: the key EXISTS (a
+     * concurrent resumer got there first — its record is the one kept, and this
+     * run is reported as reused) or it does NOT (the store refused for another
+     * reason, e.g. a 409 from a colliding write). The second is retried with
+     * backoff and, if it never sticks, fails the experiment loudly — a record
+     * that was never stored must not be reported as reused.
+     */
+    const storeRecord = async (record) => {
+        for (let attempt = 0;; attempt++) {
+            if (await putRunRecord(opts.store, opts.workflowId, experiment.id, record))
+                return { kept: record, reused: false };
+            const there = await getRunRecord(opts.store, opts.workflowId, experiment.id, record.caseId, record.repeat);
+            if (there)
+                return { kept: there, reused: true };
+            const wait = RECORD_WRITE_BACKOFF_MS[attempt];
+            if (wait === undefined) {
+                throw new EvalContractError(`could not store the run record for ${record.caseId}#${record.repeat}: the store refused the write `
+                    + `${attempt + 1} times and the key is still empty`);
+            }
+            await sleep(wait);
+        }
+    };
+    const work = cases.flatMap(({ case: c, rev }) => Array.from({ length: repeats }, (_, i) => ({ c, rev, repeat: i + 1 })));
+    const records = [];
+    let done = 0;
+    let release;
+    let failure;
+    try {
+        if (!opts.live)
+            release = await opts.target.pinVersion(opts.workflowVersion, experimentContextPrefix(experiment.id));
+        await pool(work, concurrency, async ({ c, rev, repeat }) => {
+            const onFile = await getRunRecord(opts.store, opts.workflowId, experiment.id, c.id, repeat);
+            if (onFile) {
+                records.push(onFile);
+                opts.onProgress?.({ done: ++done, total: work.length, run: onFile, reused: true });
+                return;
+            }
+            if (opts.signal?.aborted)
+                return;
+            const record = await runOne({
+                target: opts.target, header: manifest.header, case: c, caseRev: rev, repeat,
+                attempt: experiment.attempt ?? 1,
+                context: runContext(experiment.id, c.id, repeat, experiment.attempt ?? 1), timeoutMs, now,
+                ...(experiment.resultFields ? { resultFields: experiment.resultFields } : {}),
+            });
+            const { kept, reused } = await storeRecord(record);
+            records.push(kept);
+            opts.onProgress?.({ done: ++done, total: work.length, run: kept, reused });
+        });
+    }
+    catch (e) {
+        failure = e;
+    }
+    finally {
+        if (release) {
+            try {
+                await release();
+            }
+            catch (e) {
+                failure ??= new Error(`could not release the version pin: ${message(e)}`);
+            }
+        }
+    }
+    const finished = {
+        ...experiment,
+        finishedAt: iso(now()),
+        status: failure ? "error" : opts.signal?.aborted ? "cancelled" : "done",
+        ...(failure ? { error: message(failure) } : {}),
+        summary: summarize(records),
+    };
+    await putExperiment(opts.store, finished);
+    return finished;
+}
+//# sourceMappingURL=run.js.map

package/dist/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EACL,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,aAAa,GAEd,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAC5F,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAoDjC,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AAC1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AACrC,kFAAkF;AAClF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAU,CAAC;AAElE,MAAM,GAAG,GAAG,CAAC,EAAU,EAAU,EAAE,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAC/D,MAAM,OAAO,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAErF,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAc5B;IACC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IACpB,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IACtB,MAAM,IAAI,GAAG;QACX,MAAM,EAAE,CAAC,CAAC,EAAE;QACZ,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,GAAG,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAC5B,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC;KACxB,CAAC;IACF,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,4BAA4B,OAAO,CAAC,CAAC,CAAC,EAAE;YACvF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI;SAC/D,CAAC;IACJ,CAAC;IACD,IAAI,GAAsD,CAAC;IAC3D,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,qCAAqC,OAAO,CAAC,CAAC,CAAC,EAAE;YAChG,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI;SAC/D,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,IAAI,OAAgB,CAAC;IACrB,IAAI,YAAgC,CAAC;IACrC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,GAAG,YAAY,EAAE,CAAC;QACzB,YAAY,GAAG,wBAAwB,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IACtD,CAAC;IACD;;;;;;;OAOG;IACH,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC,EAAE,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAgB,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,MAAM,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACzH,MAAM,MAAM,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1F,6EAA6E;IAC7E,yDAAyD;IACzD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC9D,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnE,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM;QACN,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChD,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;QACvD,OAAO;QACP,MAAM;QACN,IAAI;KACL,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAA0B;IAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IAClC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;IACvD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,oCAAoC,CAAC,CAAC;IACjH,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,wCAAwC,CAAC,CAAC;IAC7H,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,iBAAiB,CACzB,4BAA4B,IAAI,CAAC,eAAe,yDAAyD,CAC1G,CAAC;IACJ,CAAC;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChH,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,iBAAiB,CAAC,eAAe,IAAI,CAAC,YAAY,4BAA4B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;IAC3G,CAAC;IACD,oGAAoG;IACpG,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc;WACrC,QAAQ,EAAE,cAAc;WACxB,CAAC,MAAM,oBAAoB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/G,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAEzG,IAAI,UAAsB,CAAC;IAC3B,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,QAAQ,CAAC,cAAc,KAAK,cAAc,IAAI,QAAQ,CAAC,eAAe,KAAK,IAAI,CAAC,eAAe;eAC9F,QAAQ,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAClC,MAAM,IAAI,iBAAiB,CACzB,eAAe,QAAQ,CAAC,EAAE,kBAAkB,QAAQ,CAAC,cAAc,eAAe,QAAQ,CAAC,eAAe,MAAM,QAAQ,CAAC,OAAO,IAAI;kBAChI,kCAAkC,CACvC,CAAC;QACJ,CAAC;QACD,2EAA2E;QAC3E,yEAAyE;QACzE,gCAAgC;QAChC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,EAAE;YAC3E,OAAO,EAAE,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;SAC5D,CAAC,CAAC;QACH,UAAU,GAAG,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;QACzD,OAAO,UAAU,CAAC,UAAU,CAAC;QAC7B,OAAO,UAAU,CAAC,KAAK,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,GAAG,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC;QAClC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3F,UAAU,GAAG;YACX,EAAE;YACF,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,cAAc,EAAE,QAAQ,CAAC,OAAO;YAChC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO;YACP,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;YACrB,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK;YAC5B,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1C,CAAC;IACJ,CAAC;IACD,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1F;;;;;;;;OAQG;IACH,MAAM,WAAW,GAAG,KAAK,EAAE,MAAiB,EAAiD,EAAE;QAC7F,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;YAClC,IAAI,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACnH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC3G,IAAI,KAAK;gBAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,MAAM,IAAI,iBAAiB,CACzB,sCAAsC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,gCAAgC;sBAChG,GAAG,OAAO,GAAG,CAAC,mCAAmC,CACtD,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAC9C,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAA0C,CAAC;IAC/C,IAAI,OAAgB,CAAC;IACrB,IAAI,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,IAAI;YAAE,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAW,CAAC,IAAI,CAAC,eAAe,EAAE,uBAAuB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;QACtH,MAAM,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE;YACzD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAC5F,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnF,OAAO;YACT,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;gBAAE,OAAO;YACjC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;gBAC1B,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM;gBAC3E,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC;gBAChC,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG;gBACzF,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9E,CAAC,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,GAAG,CAAC,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,MAAM,OAAO,EAAE,CAAC;YAClB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,KAAK,IAAI,KAAK,CAAC,sCAAsC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAe;QAC3B,GAAG,UAAU;QACb,UAAU,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM;QACvE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;KAC5B,CAAC;IACF,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC1C,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/score.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Deterministic scoring of a run's Result against a case's expectations.
+ *
+ * One score per expected field (`eval/field@1`). Field access is by dot path
+ * (`"forecast.amount"`, `"items.0.id"`). An expected field the output does not
+ * have FAILS — silently skipping a missing field is how a broken run goes green.
+ *
+ * Per field, the expectation is:
+ *   - a matcher object (`{ $oneOf: [...] }`, see {@link Matcher});
+ *   - a numeric band string (`">= 0.8"`) when the actual value is a number;
+ *   - a plain object: SUBSET match — every key it names must match (leaves
+ *     recursively by these same rules), keys it does not name are ignored;
+ *   - anything else: deep equality (arrays ordered and length-sensitive).
+ *
+ * Limits (`eval/limit@1`) are scored too, and only WARN: a breach is counted
+ * and shown, and never flips a case's pass.
+ *
+ * AGAINST A DECLARED RESULT. When the experiment knows the Result the workflow
+ * version declares (`Experiment.resultFields`), an expectation that version
+ * cannot meet by construction is not a failure of the run: a field it does
+ * not declare, or declares with a type the expectation cannot match, is scored
+ * `pass: null` (n/a) with the reason. The dataset is older than the version;
+ * that is a finding about the dataset, shown as one, not a regression.
+ */
+import type { Case, Header, Json, Limits, Matcher, Metrics, ResultField, Score } from "./types.js";
+export declare const FIELD_SCORER = "eval/field@1";
+export declare const LIMIT_SCORER = "eval/limit@1";
+declare const MISSING: unique symbol;
+type Actual = Json | typeof MISSING;
+/** Read a dot path out of a value. `MISSING` when any segment is absent. */
+export declare function readField(value: Json | null | undefined, path: string): Json | undefined;
+export declare function deepEqual(a: Json, b: Json): boolean;
+export type BandOp = ">=" | "<=" | ">" | "<";
+export declare function parseBand(expr: string): {
+    op: BandOp;
+    threshold: number;
+} | null;
+/** Whether an expectation is a matcher object (every key starts with `$`). */
+export declare function isMatcher(v: unknown): v is Matcher;
+/** Longest `$regex` pattern scoring will compile. */
+export declare const MAX_REGEX_LENGTH = 256;
+/**
+ * Why a `$regex` pattern is refused before it is compiled, or null.
+ *
+ * TRUST MODEL. Cases come from a shared bucket, and JavaScript's regex engine
+ * backtracks: a pattern like `(a+)+$` runs for exponential time on a short
+ * non-matching string, synchronously, inside scoring — no timeout can
+ * interrupt it, and the CLI or the ROC tab freezes. So a pattern is refused
+ * when it is longer than {@link MAX_REGEX_LENGTH} or quantifies a group that
+ * itself contains a quantifier (`(a+)+`, `(a*)*`, `((ab)+){2,}`; a fixed
+ * `{n}` does not count). This is a heuristic, not a proof: overlapping
+ * alternations (`(a|a)*`) still pass it.
+ * Anyone who can write cases can make scoring slow; datasets are trusted the
+ * way the workflow code they test is.
+ */
+export declare function unsafeRegex(pattern: string): string | null;
+/** null when `actual` satisfies `expected`, otherwise why not. */
+export declare function matchValue(expected: Json | Matcher, actual: Actual | Json): string | null;
+/**
+ * The value types an expectation can match: `["number"]` for `{ $gte: 1 }`,
+ * `["number", "string"]` for a band string such as `">= 0.8"` (a band against
+ * a number, plain equality against a string), null when it can match any
+ * (`null`, `{ $exists }`, an empty or unknown matcher).
+ */
+export declare function expectedTypesOf(want: Json | Matcher): string[] | null;
+/**
+ * One `eval/field@1` score per expected field. With `resultFields` (the
+ * version's declared Result), an expectation the declaration rules out is
+ * n/a (`pass: null`) rather than a failure — see the file comment.
+ */
+export declare function scoreFields(expected: Case["expected"], output: Json | null | undefined, resultFields?: readonly ResultField[]): Score[];
+/** The limits in force for a case: the header's, each overridden by the case's own. */
+export declare function effectiveLimits(header: Pick<Header, "limits"> | undefined, c: Pick<Case, "limits">): Limits;
+/** One `eval/limit@1` score per limit in force. A metric that was not measured is not judged. */
+export declare function scoreLimits(limits: Limits, metrics: Metrics): Score[];
+/** A case passes when every field score passes; null when there were none. Limits never count. */
+export declare function casePass(scores: Score[]): boolean | null;
+export interface ResultCompatibility {
+    /** Expected fields the Result does not declare, with the cases expecting each. */
+    missing: Array<{
+        field: string;
+        caseIds: string[];
+    }>;
+    /** Expected fields whose expectation cannot match the declared type. */
+    typeChanged: Array<{
+        field: string;
+        expectedType: string;
+        declaredType: string;
+        caseIds: string[];
+    }>;
+    /** Declared fields no case states an expectation for. */
+    unexpectedNew: string[];
+}
+/**
+ * Whether a dataset's expectations fit a workflow version's declared Result —
+ * the check behind "2 dataset cases expect `priority`, which this version no
+ * longer returns". Fields are reported by their top-level name; each list is
+ * sorted.
+ */
+export declare function checkResultCompatibility(resultFields: readonly ResultField[], cases: ReadonlyArray<Pick<Case, "id" | "expected">>): ResultCompatibility;
+export {};
+//# sourceMappingURL=score.d.ts.map

package/dist/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnG,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAC3C,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAE3C,QAAA,MAAM,OAAO,eAAoB,CAAC;AAClC,KAAK,MAAM,GAAG,IAAI,GAAG,OAAO,OAAO,CAAC;AAKpC,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAGxF;AAqBD,wBAAgB,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,GAAG,OAAO,CAYnD;AAID,MAAM,MAAM,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC;AAE7C,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAGhF;AAQD,8EAA8E;AAC9E,wBAAgB,SAAS,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,IAAI,OAAO,CAIlD;AAWD,qDAAqD;AACrD,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA8B1D;AAoFD,kEAAkE;AAClE,wBAAgB,UAAU,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAkBzF;AAYD;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,GAAG,MAAM,EAAE,GAAG,IAAI,CA2BrE;AA8BD;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,EAC1B,MAAM,EAAE,IAAI,GAAG,IAAI,GAAG,SAAS,EAC/B,YAAY,CAAC,EAAE,SAAS,WAAW,EAAE,GACpC,KAAK,EAAE,CA2BT;AASD,uFAAuF;AACvF,wBAAgB,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,MAAM,CAE3G;AAED,iGAAiG;AACjG,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,KAAK,EAAE,CAerE;AAED,kGAAkG;AAClG,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,GAAG,IAAI,CAIxD;AAED,MAAM,WAAW,mBAAmB;IAClC,kFAAkF;IAClF,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IACrD,wEAAwE;IACxE,WAAW,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IACrG,yDAAyD;IACzD,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACtC,YAAY,EAAE,SAAS,WAAW,EAAE,EACpC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,GAAG,UAAU,CAAC,CAAC,GAClD,mBAAmB,CA8BrB"}