@cef-ai/eval 0.0.0-stage → 0.1.0

package/dist/contract.js

@@ -0,0 +1,176 @@
+/**
+ * Parsers for what crosses the store boundary. A case or header comes from a
+ * person — a repo file, a ROC form — so it is checked field by field and a bad
+ * one is refused with the field named. Records the library writes itself are
+ * read back leniently.
+ */
+import { assertCaseId, assertDatasetName, isCaseId } from "./layout.js";
+export class EvalContractError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "EvalContractError";
+    }
+}
+/** A plain JSON object: not null, not an array. */
+export const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+function fail(where, what) {
+    throw new EvalContractError(`${where}: ${what}`);
+}
+function isJson(v, depth = 0) {
+    if (depth > 64)
+        return false;
+    if (v === null || typeof v === "string" || typeof v === "boolean")
+        return true;
+    if (typeof v === "number")
+        return Number.isFinite(v);
+    if (Array.isArray(v))
+        return v.every((x) => isJson(x, depth + 1));
+    if (isObject(v))
+        return Object.values(v).every((x) => isJson(x, depth + 1));
+    return false;
+}
+const LIMIT_KEYS = ["maxDurationMs", "maxTokens", "maxSteps", "maxCost"];
+export function parseLimits(raw, where) {
+    if (raw === undefined || raw === null)
+        return undefined;
+    if (!isObject(raw))
+        fail(where, "limits must be an object");
+    const out = {};
+    for (const [k, v] of Object.entries(raw)) {
+        if (!LIMIT_KEYS.includes(k))
+            fail(where, `unknown limit "${k}" (known: ${LIMIT_KEYS.join(", ")})`);
+        if (typeof v !== "number" || !Number.isFinite(v) || v < 0)
+            fail(where, `limits.${k} must be a number ≥ 0`);
+        out[k] = v;
+    }
+    return out;
+}
+export function parseHeader(raw) {
+    const where = "header";
+    if (!isObject(raw))
+        fail(where, "must be an object");
+    if (typeof raw.dataset !== "string")
+        fail(where, "dataset must be a string");
+    try {
+        assertDatasetName(raw.dataset);
+    }
+    catch (e) {
+        fail(where, e.message);
+    }
+    if (typeof raw.workflowId !== "string" || !raw.workflowId)
+        fail(where, "workflowId must be a non-empty string");
+    if (raw.description !== undefined && typeof raw.description !== "string")
+        fail(where, "description must be a string");
+    if (typeof raw.createdAt !== "string")
+        fail(where, "createdAt must be an ISO timestamp string");
+    if (raw.baselineId !== undefined && raw.baselineId !== null && typeof raw.baselineId !== "string") {
+        fail(where, "baselineId must be a string");
+    }
+    const limits = parseLimits(raw.limits, where);
+    return {
+        dataset: raw.dataset,
+        workflowId: raw.workflowId,
+        ...(raw.description !== undefined ? { description: raw.description } : {}),
+        ...(limits ? { limits } : {}),
+        createdAt: raw.createdAt,
+        ...(typeof raw.baselineId === "string" && raw.baselineId ? { baselineId: raw.baselineId } : {}),
+    };
+}
+export function parseCase(raw) {
+    if (!isObject(raw))
+        fail("case", "must be an object");
+    if (typeof raw.id !== "string")
+        fail("case", "id must be a string");
+    const where = `case "${raw.id}"`;
+    try {
+        assertCaseId(raw.id);
+    }
+    catch (e) {
+        fail(where, e.message);
+    }
+    if (!isObject(raw.input) || !isJson(raw.input))
+        fail(where, "input must be a JSON object (the workflow.start payload)");
+    if (raw.expected !== undefined) {
+        if (!isObject(raw.expected) || !isJson(raw.expected))
+            fail(where, "expected must be a JSON object keyed by Result field");
+    }
+    if (raw.tags !== undefined && (!Array.isArray(raw.tags) || raw.tags.some((t) => typeof t !== "string"))) {
+        fail(where, "tags must be an array of strings");
+    }
+    if (raw.notes !== undefined && typeof raw.notes !== "string")
+        fail(where, "notes must be a string");
+    let source;
+    if (raw.source !== undefined) {
+        const s = raw.source;
+        if (!isObject(s) || typeof s.context !== "string" || typeof s.capturedAt !== "string") {
+            fail(where, "source must be { context, capturedAt, workflowVersion? }");
+        }
+        if (s.workflowVersion !== undefined && typeof s.workflowVersion !== "string") {
+            fail(where, "source.workflowVersion must be a string");
+        }
+        source = {
+            context: s.context,
+            ...(s.workflowVersion !== undefined ? { workflowVersion: s.workflowVersion } : {}),
+            capturedAt: s.capturedAt,
+        };
+    }
+    const limits = parseLimits(raw.limits, where);
+    return {
+        id: raw.id,
+        input: raw.input,
+        ...(raw.expected !== undefined ? { expected: raw.expected } : {}),
+        ...(limits ? { limits } : {}),
+        ...(raw.tags !== undefined ? { tags: raw.tags } : {}),
+        ...(raw.notes !== undefined ? { notes: raw.notes } : {}),
+        ...(source ? { source } : {}),
+    };
+}
+export function parseVersionManifest(raw, dataset, version) {
+    const where = `dataset "${dataset}" v${version}`;
+    if (!isObject(raw))
+        fail(where, "manifest must be an object");
+    if (raw.version !== version)
+        fail(where, `manifest says version ${String(raw.version)}`);
+    if (!Array.isArray(raw.cases))
+        fail(where, "manifest has no cases array");
+    const cases = raw.cases.map((c, i) => {
+        if (!isObject(c) || typeof c.id !== "string" || !isCaseId(c.id)
+            || typeof c.rev !== "number" || !Number.isInteger(c.rev) || c.rev < 1
+            || typeof c.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(c.sha256)) {
+            fail(where, `cases[${i}] must be { id, rev, sha256 }`);
+        }
+        return { id: c.id, rev: c.rev, sha256: c.sha256 };
+    });
+    return {
+        version,
+        header: parseHeader(raw.header),
+        cases,
+        createdAt: typeof raw.createdAt === "string" ? raw.createdAt : "",
+        ...(typeof raw.note === "string" ? { note: raw.note } : {}),
+    };
+}
+/**
+ * A run record read back. Lenient: the library wrote it. A record written
+ * before v2 numbers its repeat `trial`; it reads as `repeat`.
+ */
+export function parseRunRecord(raw, where) {
+    const repeat = isObject(raw) ? (typeof raw.repeat === "number" ? raw.repeat : raw.trial) : undefined;
+    if (!isObject(raw) || typeof raw.caseId !== "string" || typeof repeat !== "number"
+        || typeof raw.status !== "string" || !isObject(raw.metrics) || !Array.isArray(raw.scores)) {
+        fail(where, "not a run record");
+    }
+    const { trial: _trial, ...rest } = raw;
+    return { ...rest, repeat };
+}
+/** An experiment's meta read back. Lenient: the library wrote it. */
+export function parseExperiment(raw, where) {
+    if (!isObject(raw) || typeof raw.id !== "string" || typeof raw.dataset !== "string"
+        || typeof raw.datasetVersion !== "number" || typeof raw.status !== "string") {
+        fail(where, "not an experiment");
+    }
+    // Summaries written before v2 have no per-field accuracy.
+    if (isObject(raw.summary) && !isObject(raw.summary.fields))
+        return { ...raw, summary: { ...raw.summary, fields: {} } };
+    return raw;
+}
+//# sourceMappingURL=contract.js.map

package/dist/contract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contract.js","sourceRoot":"","sources":["../src/contract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGxE,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,mDAAmD;AACnD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CACnE,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAE3D,SAAS,IAAI,CAAC,KAAa,EAAE,IAAY;IACvC,MAAM,IAAI,iBAAiB,CAAC,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,MAAM,CAAC,CAAU,EAAE,KAAK,GAAG,CAAC;IACnC,IAAI,KAAK,GAAG,EAAE;QAAE,OAAO,KAAK,CAAC;IAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC/E,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAClE,IAAI,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,GAAwB,CAAC,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;AAE9F,MAAM,UAAU,WAAW,CAAC,GAAY,EAAE,KAAa;IACrD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IACxD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;IAC5D,MAAM,GAAG,GAAW,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAiB,CAAC;YAAE,IAAI,CAAC,KAAK,EAAE,kBAAkB,CAAC,aAAa,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnH,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,uBAAuB,CAAC,CAAC;QAC3G,GAAG,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAY;IACtC,MAAM,KAAK,GAAG,QAAQ,CAAC;IACvB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAAE,IAAI,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;IAC7E,IAAI,CAAC;QACH,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,KAAK,EAAG,CAAW,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,UAAU;QAAE,IAAI,CAAC,KAAK,EAAE,uCAAuC,CAAC,CAAC;IAChH,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ;QAAE,IAAI,CAAC,KAAK,EAAE,8BAA8B,CAAC,CAAC;IACtH,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ;QAAE,IAAI,CAAC,KAAK,EAAE,2CAA2C,CAAC,CAAC;IAChG,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClG,IAAI,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9C,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,GAAG,CAAC,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAqB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,GAAG,CAAC,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChG,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAY;IACpC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IACtD,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ;QAAE,IAAI,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,EAAE,GAAG,CAAC;IACjC,IAAI,CAAC;QACH,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,KAAK,EAAG,CAAW,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,0DAA0D,CAAC,CAAC;IACxH,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,IAAI,CAAC,KAAK,EAAE,sDAAsD,CAAC,CAAC;IAC5H,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxG,IAAI,CAAC,KAAK,EAAE,kCAAkC,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;QAAE,IAAI,CAAC,KAAK,EAAE,wBAAwB,CAAC,CAAC;IACpG,IAAI,MAAsB,CAAC;IAC3B,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YACtF,IAAI,CAAC,KAAK,EAAE,0DAA0D,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;YAC7E,IAAI,CAAC,KAAK,EAAE,yCAAyC,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,GAAG;YACP,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,GAAG,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAyB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5F,UAAU,EAAE,CAAC,CAAC,UAAU;SACzB,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,KAAK,EAAE,GAAG,CAAC,KAA6B;QACxC,GAAG,CAAC,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAA4B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,GAAY,EAAE,OAAe,EAAE,OAAe;IACjF,MAAM,KAAK,GAAG,YAAY,OAAO,MAAM,OAAO,EAAE,CAAC;IACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;IAC9D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO;QAAE,IAAI,CAAC,KAAK,EAAE,yBAAyB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACzF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;eAC1D,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;eAClE,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,+BAA+B,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IACpD,CAAC,CAAC,CAAC;IACH,OAAO;QACL,OAAO;QACP,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;QAC/B,KAAK;QACL,SAAS,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;QACjE,GAAG,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5D,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,GAAY,EAAE,KAAa;IACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACrG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM,KAAK,QAAQ;WAC7E,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5F,IAAI,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IACvC,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAA0B,CAAC;AACrD,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,eAAe,CAAC,GAAY,EAAE,KAAa;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;WAC9E,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC9E,IAAI,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC;IACnC,CAAC;IACD,0DAA0D;IAC1D,IAAI,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,EAA2B,CAAC;IAChJ,OAAO,GAA4B,CAAC;AACtC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,18 @@
+export type { Json, Limits, Header, Matcher, Case, VersionEntry, VersionManifest, Metrics, Score, RunStatus, RunRecord, ExperimentStatus, Experiment, Summary, FieldAccuracy, DatasetDiff, ResultField, ResultFieldType, RunEnd, WorkflowTarget, } from "./types.js";
+export { WORKFLOWS_PREFIX, LEGACY_EVALS_PREFIX, MIGRATION_MARKER_KEY, isWorkflowId, isDatasetName, isCaseId, isExperimentId, assertWorkflowId, assertDatasetName, assertCaseId, assertExperimentId, workflowPrefix, datasetsPrefix, datasetPrefix, headerKey, casesPrefix, caseDirPrefix, caseRevKey, tombstoneKey, archivedPrefix, archiveKey, versionsPrefix, versionKey, experimentsPrefix, experimentPrefix, experimentMetaKey, runsPrefix, runKey, attemptsPrefix, attemptKey, attemptFromKey, revFromKey, caseIdFromKey, archivedIdFromKey, versionFromKey, experimentIdFromMetaKey, runFromKey, canonicalJson, sha256Hex, newExperimentId, runContext, experimentContextPrefix, runIdOf, } from "./layout.js";
+export { EvalContractError, isObject, parseHeader, parseCase, parseLimits, parseVersionManifest, parseRunRecord, parseExperiment, } from "./contract.js";
+export { memoryStore, createDataset, getHeader, requireHeader, updateHeader, setBaseline, listWorkflows, listDatasets, listCaseIds, listArchivedCaseIds, listDeletedCaseIds, listCaseRevisions, getCaseRevision, getCase, saveCase, addCase, archiveCase, unarchiveCase, deleteCase, undeleteCase, loadWorkingSet, listVersions, latestVersionNumber, getVersion, cutVersion, diffVersionEntries, autoVersionNote, ensureDatasetVersion, versionHistory, loadVersion, putExperiment, claimAttempt, getExperiment, listExperiments, putRunRecord, getRunRecord, listRunRecords, loadExperiment, findExperiment, childPrefixes, } from "./storage.js";
+export type { EvalStore, StoreAtomicity, CreateDatasetInput, DatasetSummary, VersionHistoryEntry } from "./storage.js";
+export { migrateLegacyLayout, needsLegacyMigration } from "./migrate.js";
+export type { MigrationReport } from "./migrate.js";
+export { resultFieldsFromGraph } from "./result.js";
+export { storeConformance } from "./conformance.js";
+export { FIELD_SCORER, LIMIT_SCORER, readField, deepEqual, parseBand, isMatcher, matchValue, scoreFields, effectiveLimits, scoreLimits, casePass, expectedTypesOf, checkResultCompatibility, unsafeRegex, MAX_REGEX_LENGTH, } from "./score.js";
+export type { BandOp, ResultCompatibility } from "./score.js";
+export { emptyMetrics, percentile, summarize, compareExperiments } from "./summary.js";
+export type { Change, MetricDelta, RunComparison, CaseComparison, ExperimentComparison, ComparisonSide, } from "./summary.js";
+export { runExperiment, runOne, DEFAULT_TIMEOUT_MS, DEFAULT_CONCURRENCY } from "./run.js";
+export type { RunExperimentOptions, ExperimentProgress } from "./run.js";
+export { caseFromRun, expectationFrom, isFreeText, slugCaseId, uniqueCaseId } from "./capture.js";
+export type { CapturedRun } from "./capture.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,IAAI,EACJ,MAAM,EACN,MAAM,EACN,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,eAAe,EACf,OAAO,EACP,KAAK,EACL,SAAS,EACT,SAAS,EACT,gBAAgB,EAChB,UAAU,EACV,OAAO,EACP,aAAa,EACb,WAAW,EACX,WAAW,EACX,eAAe,EACf,MAAM,EACN,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,aAAa,EACb,SAAS,EACT,WAAW,EACX,aAAa,EACb,UAAU,EACV,YAAY,EACZ,cAAc,EACd,UAAU,EACV,cAAc,EACd,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EACV,MAAM,EACN,cAAc,EACd,UAAU,EACV,cAAc,EACd,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,uBAAuB,EACvB,UAAU,EACV,aAAa,EACb,SAAS,EACT,eAAe,EACf,UAAU,EACV,uBAAuB,EACvB,OAAO,GACR,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,iBAAiB,EACjB,QAAQ,EACR,WAAW,EACX,SAAS,EACT,WAAW,EACX,oBAAoB,EACpB,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,WAAW,EACX,aAAa,EACb,SAAS,EACT,aAAa,EACb,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,OAAO,EACP,QAAQ,EACR,OAAO,EACP,WAAW,EACX,aAAa,EACb,UAAU,EACV,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,cAAc,EACd,WAAW,EACX,aAAa,EACb,YAAY,EACZ,aAAa,EACb,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,cAAc,EACd,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEvH,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACzE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EACL,YAAY,EACZ,YAAY,EACZ,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,QAAQ,EACR,eAAe,EACf,wBAAwB,EACxB,WAAW,EACX,gBAAgB,GACjB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAE9D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvF,YAAY,EACV,MAAM,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,oBAAoB,EAAE,cAAc,GACzF,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAC1F,YAAY,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEzE,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAClG,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,11 @@
+export { WORKFLOWS_PREFIX, LEGACY_EVALS_PREFIX, MIGRATION_MARKER_KEY, isWorkflowId, isDatasetName, isCaseId, isExperimentId, assertWorkflowId, assertDatasetName, assertCaseId, assertExperimentId, workflowPrefix, datasetsPrefix, datasetPrefix, headerKey, casesPrefix, caseDirPrefix, caseRevKey, tombstoneKey, archivedPrefix, archiveKey, versionsPrefix, versionKey, experimentsPrefix, experimentPrefix, experimentMetaKey, runsPrefix, runKey, attemptsPrefix, attemptKey, attemptFromKey, revFromKey, caseIdFromKey, archivedIdFromKey, versionFromKey, experimentIdFromMetaKey, runFromKey, canonicalJson, sha256Hex, newExperimentId, runContext, experimentContextPrefix, runIdOf, } from "./layout.js";
+export { EvalContractError, isObject, parseHeader, parseCase, parseLimits, parseVersionManifest, parseRunRecord, parseExperiment, } from "./contract.js";
+export { memoryStore, createDataset, getHeader, requireHeader, updateHeader, setBaseline, listWorkflows, listDatasets, listCaseIds, listArchivedCaseIds, listDeletedCaseIds, listCaseRevisions, getCaseRevision, getCase, saveCase, addCase, archiveCase, unarchiveCase, deleteCase, undeleteCase, loadWorkingSet, listVersions, latestVersionNumber, getVersion, cutVersion, diffVersionEntries, autoVersionNote, ensureDatasetVersion, versionHistory, loadVersion, putExperiment, claimAttempt, getExperiment, listExperiments, putRunRecord, getRunRecord, listRunRecords, loadExperiment, findExperiment, childPrefixes, } from "./storage.js";
+export { migrateLegacyLayout, needsLegacyMigration } from "./migrate.js";
+export { resultFieldsFromGraph } from "./result.js";
+export { storeConformance } from "./conformance.js";
+export { FIELD_SCORER, LIMIT_SCORER, readField, deepEqual, parseBand, isMatcher, matchValue, scoreFields, effectiveLimits, scoreLimits, casePass, expectedTypesOf, checkResultCompatibility, unsafeRegex, MAX_REGEX_LENGTH, } from "./score.js";
+export { emptyMetrics, percentile, summarize, compareExperiments } from "./summary.js";
+export { runExperiment, runOne, DEFAULT_TIMEOUT_MS, DEFAULT_CONCURRENCY } from "./run.js";
+export { caseFromRun, expectationFrom, isFreeText, slugCaseId, uniqueCaseId } from "./capture.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAuBA,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,aAAa,EACb,SAAS,EACT,WAAW,EACX,aAAa,EACb,UAAU,EACV,YAAY,EACZ,cAAc,EACd,UAAU,EACV,cAAc,EACd,UAAU,EACV,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EACV,MAAM,EACN,cAAc,EACd,UAAU,EACV,cAAc,EACd,UAAU,EACV,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,uBAAuB,EACvB,UAAU,EACV,aAAa,EACb,SAAS,EACT,eAAe,EACf,UAAU,EACV,uBAAuB,EACvB,OAAO,GACR,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,iBAAiB,EACjB,QAAQ,EACR,WAAW,EACX,SAAS,EACT,WAAW,EACX,oBAAoB,EACpB,cAAc,EACd,eAAe,GAChB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,WAAW,EACX,aAAa,EACb,SAAS,EACT,aAAa,EACb,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,EACf,OAAO,EACP,QAAQ,EACR,OAAO,EACP,WAAW,EACX,aAAa,EACb,UAAU,EACV,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,cAAc,EACd,WAAW,EACX,aAAa,EACb,YAAY,EACZ,aAAa,EACb,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,cAAc,EACd,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AAGtB,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGzE,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,EACL,YAAY,EACZ,YAAY,EACZ,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,QAAQ,EACR,eAAe,EACf,wBAAwB,EACxB,WAAW,EACX,gBAAgB,GACjB,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAKvF,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAG1F,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}

package/dist/layout.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Storage layout — one layout for ROC and the CLI, so either reads what the
+ * other wrote. Grouped by workflow: a dataset belongs to one workflow, and an
+ * experiment is a sibling of the datasets (it names `dataset@version`), not a
+ * child of one.
+ *
+ *   workflows/<wf>/datasets/<dataset>/header.json                  Header (mutable)
+ *   workflows/<wf>/datasets/<dataset>/cases/<caseId>/r<n>.json     Case revision (write-once)
+ *   workflows/<wf>/datasets/<dataset>/cases/<caseId>/deleted.json  tombstone
+ *   workflows/<wf>/datasets/<dataset>/archived/<caseId>.json       archive flag
+ *   workflows/<wf>/datasets/<dataset>/versions/v<n>.json           VersionManifest (write-once)
+ *   workflows/<wf>/experiments/<expId>/meta.json                   Experiment (mutable: status)
+ *   workflows/<wf>/experiments/<expId>/runs/<caseId>.<repeat>.json RunRecord (write-once)
+ *   workflows/<wf>/experiments/<expId>/attempts/a<n>.json          attempt claim (write-once)
+ *
+ * The v1 layout (`evals/datasets/<dataset>/…`, experiments nested in their
+ * dataset) is read only by `migrateLegacyLayout`.
+ *
+ * WRITE-ONCE REVISIONS, NOT ONE FILE. Editing a case writes `r<n+1>`; nothing
+ * is rewritten, so a version that pinned `r3` by sha256 keeps loading exactly
+ * what it pinned however often the case is edited. Revision and version numbers
+ * are derived by listing — there is no mutable "latest" pointer to lose a write
+ * to — and the store's `putIfAbsent` decides who holds a number two writers
+ * computed at once.
+ *
+ * FLAGS ARE THEIR OWN OBJECTS. An archived/deleted marker inside the case would
+ * change its bytes and so its hash, and every version pinning it would refuse
+ * to load. A marker beside it leaves history intact.
+ */
+export declare const WORKFLOWS_PREFIX = "workflows/";
+/** The v1 layout's root; only `migrateLegacyLayout` reads it. */
+export declare const LEGACY_EVALS_PREFIX = "evals/datasets/";
+/** Written once `migrateLegacyLayout` has copied every legacy dataset. */
+export declare const MIGRATION_MARKER_KEY = "evals/MIGRATED.json";
+/** A workflow id is the workflow agent's alias; it is a key segment here. */
+export declare function isWorkflowId(id: string): boolean;
+export declare function isDatasetName(name: string): boolean;
+export declare function isCaseId(id: string): boolean;
+export declare function isExperimentId(id: string): boolean;
+export declare function assertWorkflowId(id: string): void;
+export declare function assertDatasetName(name: string): void;
+export declare function assertCaseId(id: string): void;
+export declare function assertExperimentId(id: string): void;
+export declare const workflowPrefix: (workflowId: string) => string;
+export declare const datasetsPrefix: (workflowId: string) => string;
+export declare const datasetPrefix: (workflowId: string, dataset: string) => string;
+export declare const headerKey: (workflowId: string, dataset: string) => string;
+export declare const casesPrefix: (workflowId: string, dataset: string) => string;
+export declare const caseDirPrefix: (workflowId: string, dataset: string, caseId: string) => string;
+export declare const caseRevKey: (workflowId: string, dataset: string, caseId: string, rev: number) => string;
+export declare const tombstoneKey: (workflowId: string, dataset: string, caseId: string) => string;
+export declare const archivedPrefix: (workflowId: string, dataset: string) => string;
+export declare const archiveKey: (workflowId: string, dataset: string, caseId: string) => string;
+export declare const versionsPrefix: (workflowId: string, dataset: string) => string;
+export declare const versionKey: (workflowId: string, dataset: string, version: number) => string;
+export declare const experimentsPrefix: (workflowId: string) => string;
+export declare const experimentPrefix: (workflowId: string, expId: string) => string;
+export declare const experimentMetaKey: (workflowId: string, expId: string) => string;
+export declare const runsPrefix: (workflowId: string, expId: string) => string;
+export declare const runKey: (workflowId: string, expId: string, caseId: string, repeat: number) => string;
+export declare const attemptsPrefix: (workflowId: string, expId: string) => string;
+export declare const attemptKey: (workflowId: string, expId: string, attempt: number) => string;
+/** An attempt number from an attempt claim key, or null. `a01` is not `a1`. */
+export declare function attemptFromKey(workflowId: string, expId: string, key: string): number | null;
+/** A revision number from a key inside a case's directory, or null. `r01` is not `r1`. */
+export declare function revFromKey(workflowId: string, dataset: string, caseId: string, key: string): number | null;
+/** The case id from any key inside a case's directory, or null. */
+export declare function caseIdFromKey(workflowId: string, dataset: string, key: string): string | null;
+export declare function archivedIdFromKey(workflowId: string, dataset: string, key: string): string | null;
+export declare function versionFromKey(workflowId: string, dataset: string, key: string): number | null;
+export declare function experimentIdFromMetaKey(workflowId: string, key: string): string | null;
+/** `{caseId, repeat}` from a run record key, or null. */
+export declare function runFromKey(workflowId: string, expId: string, key: string): {
+    caseId: string;
+    repeat: number;
+} | null;
+/** Highest number `parse` finds in `keys`, 0 when none. */
+export declare function highest(keys: string[], parse: (key: string) => number | null): number;
+/**
+ * Canonical JSON: sorted keys, two-space indent, trailing newline. A case's
+ * sha256 is over these bytes, so it depends on content and not on the key order
+ * some serialiser happened to emit.
+ */
+export declare function canonicalJson(value: unknown): string;
+/** SHA-256 hex of a UTF-8 string, via Web Crypto (browser-safe; no node:crypto). */
+export declare function sha256Hex(text: string): Promise<string>;
+/**
+ * `exp-<base36 ms>-<4 rand>`: sorts by creation time. The time is padded to 8
+ * base36 digits — its natural width from 1973 to 2059 — so a lexical sort is a
+ * time sort.
+ */
+export declare function newExperimentId(now?: number): string;
+/**
+ * The context one run is published into: `eval-<expId>-<caseId>-<repeat>-a<attempt>`.
+ *
+ * WHY AN ATTEMPT. A resumed experiment re-runs every case × repeat that has no
+ * record yet — and one of those may have been STARTED by the attempt that
+ * died. Publishing a second `workflow.start` into that context would land a
+ * second run beside the first (same runId, same context), and whichever end
+ * event arrives first would be scored. Each attempt of an experiment gets
+ * fresh contexts instead; they all still share the experiment's prefix, so a
+ * version pin covers them.
+ */
+export declare const runContext: (expId: string, caseId: string, repeat: number, attempt: number) => string;
+/**
+ * The prefix every run of an experiment shares — what a version pin targets.
+ * Ends in `-` so no other experiment's contexts can share it.
+ */
+export declare const experimentContextPrefix: (expId: string) => string;
+/** The id the runner knows a run by (`runIdFor` in the runner). */
+export declare const runIdOf: (context: string) => string;
+//# sourceMappingURL=layout.d.ts.map

package/dist/layout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"layout.d.ts","sourceRoot":"","sources":["../src/layout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,eAAO,MAAM,gBAAgB,eAAe,CAAC;AAC7C,iEAAiE;AACjE,eAAO,MAAM,mBAAmB,oBAAoB,CAAC;AACrD,0EAA0E;AAC1E,eAAO,MAAM,oBAAoB,wBAAwB,CAAC;AAO1D,6EAA6E;AAC7E,wBAAgB,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAEhD;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE5C;AAED,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAElD;AAED,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAEjD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAIpD;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAEnD;AAED,eAAO,MAAM,cAAc,GAAI,YAAY,MAAM,KAAG,MAA6C,CAAC;AAClG,eAAO,MAAM,cAAc,GAAI,YAAY,MAAM,KAAG,MAAkD,CAAC;AACvG,eAAO,MAAM,aAAa,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,KAAG,MAAoD,CAAC;AACzH,eAAO,MAAM,SAAS,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,KAAG,MAA4D,CAAC;AAC7H,eAAO,MAAM,WAAW,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,KAAG,MAAuD,CAAC;AAC1H,eAAO,MAAM,aAAa,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,EAAE,QAAQ,MAAM,KAAG,MACnC,CAAC;AAClD,eAAO,MAAM,UAAU,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,EAAE,QAAQ,MAAM,EAAE,KAAK,MAAM,KAAG,MACjC,CAAC;AAC9D,eAAO,MAAM,YAAY,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,EAAE,QAAQ,MAAM,KAAG,MACtB,CAAC;AAC9D,eAAO,MAAM,cAAc,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,KAAG,MAA0D,CAAC;AAChI,eAAO,MAAM,UAAU,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,EAAE,QAAQ,MAAM,KAAG,MACzB,CAAC;AACzD,eAAO,MAAM,cAAc,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,KAAG,MAA0D,CAAC;AAChI,eAAO,MAAM,UAAU,GAAI,YAAY,MAAM,EAAE,SAAS,MAAM,EAAE,SAAS,MAAM,KAAG,MACxB,CAAC;AAC3D,eAAO,MAAM,iBAAiB,GAAI,YAAY,MAAM,KAAG,MAAqD,CAAC;AAC7G,eAAO,MAAM,gBAAgB,GAAI,YAAY,MAAM,EAAE,OAAO,MAAM,KAAG,MAAqD,CAAC;AAC3H,eAAO,MAAM,iBAAiB,GAAI,YAAY,MAAM,EAAE,OAAO,MAAM,KAAG,MACnB,CAAC;AACpD,eAAO,MAAM,UAAU,GAAI,YAAY,MAAM,EAAE,OAAO,MAAM,KAAG,MAAuD,CAAC;AACvH,eAAO,MAAM,MAAM,GAAI,YAAY,MAAM,EAAE,OAAO,MAAM,EAAE,QAAQ,MAAM,EAAE,QAAQ,MAAM,KAAG,MAC/B,CAAC;AAC7D,eAAO,MAAM,cAAc,GAAI,YAAY,MAAM,EAAE,OAAO,MAAM,KAAG,MAA2D,CAAC;AAC/H,eAAO,MAAM,UAAU,GAAI,YAAY,MAAM,EAAE,OAAO,MAAM,EAAE,SAAS,MAAM,KAAG,MACxB,CAAC;AAEzD,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK5F;AAED,0FAA0F;AAC1F,wBAAgB,UAAU,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK1G;AAED,mEAAmE;AACnE,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAQ7F;AAED,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKjG;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAK9F;AAED,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAKtF;AAED,yDAAyD;AACzD,wBAAgB,UAAU,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAUpH;AAED,2DAA2D;AAC3D,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,GAAG,MAAM,CAOrF;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEpD;AAiBD,oFAAoF;AACpF,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG7D;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,GAAG,GAAE,MAAmB,GAAG,MAAM,CAKhE;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU,GAAI,OAAO,MAAM,EAAE,QAAQ,MAAM,EAAE,QAAQ,MAAM,EAAE,SAAS,MAAM,KAAG,MAC3C,CAAC;AAElD;;;GAGG;AACH,eAAO,MAAM,uBAAuB,GAAI,OAAO,MAAM,KAAG,MAA0B,CAAC;AAEnF,mEAAmE;AACnE,eAAO,MAAM,OAAO,GAAI,SAAS,MAAM,KAAG,MAA2C,CAAC"}

package/dist/layout.js

@@ -0,0 +1,222 @@
+/**
+ * Storage layout — one layout for ROC and the CLI, so either reads what the
+ * other wrote. Grouped by workflow: a dataset belongs to one workflow, and an
+ * experiment is a sibling of the datasets (it names `dataset@version`), not a
+ * child of one.
+ *
+ *   workflows/<wf>/datasets/<dataset>/header.json                  Header (mutable)
+ *   workflows/<wf>/datasets/<dataset>/cases/<caseId>/r<n>.json     Case revision (write-once)
+ *   workflows/<wf>/datasets/<dataset>/cases/<caseId>/deleted.json  tombstone
+ *   workflows/<wf>/datasets/<dataset>/archived/<caseId>.json       archive flag
+ *   workflows/<wf>/datasets/<dataset>/versions/v<n>.json           VersionManifest (write-once)
+ *   workflows/<wf>/experiments/<expId>/meta.json                   Experiment (mutable: status)
+ *   workflows/<wf>/experiments/<expId>/runs/<caseId>.<repeat>.json RunRecord (write-once)
+ *   workflows/<wf>/experiments/<expId>/attempts/a<n>.json          attempt claim (write-once)
+ *
+ * The v1 layout (`evals/datasets/<dataset>/…`, experiments nested in their
+ * dataset) is read only by `migrateLegacyLayout`.
+ *
+ * WRITE-ONCE REVISIONS, NOT ONE FILE. Editing a case writes `r<n+1>`; nothing
+ * is rewritten, so a version that pinned `r3` by sha256 keeps loading exactly
+ * what it pinned however often the case is edited. Revision and version numbers
+ * are derived by listing — there is no mutable "latest" pointer to lose a write
+ * to — and the store's `putIfAbsent` decides who holds a number two writers
+ * computed at once.
+ *
+ * FLAGS ARE THEIR OWN OBJECTS. An archived/deleted marker inside the case would
+ * change its bytes and so its hash, and every version pinning it would refuse
+ * to load. A marker beside it leaves history intact.
+ */
+export const WORKFLOWS_PREFIX = "workflows/";
+/** The v1 layout's root; only `migrateLegacyLayout` reads it. */
+export const LEGACY_EVALS_PREFIX = "evals/datasets/";
+/** Written once `migrateLegacyLayout` has copied every legacy dataset. */
+export const MIGRATION_MARKER_KEY = "evals/MIGRATED.json";
+const WORKFLOW_ID_RE = /^[A-Za-z0-9._-]{1,128}$/;
+const DATASET_RE = /^[a-z0-9][a-z0-9-]{0,62}$/;
+const CASE_ID_RE = /^[A-Za-z0-9._-]{1,80}$/;
+const EXP_ID_RE = /^exp-[a-z0-9]+-[a-z0-9]{4}$/;
+/** A workflow id is the workflow agent's alias; it is a key segment here. */
+export function isWorkflowId(id) {
+    return WORKFLOW_ID_RE.test(id) && id !== "." && id !== "..";
+}
+export function isDatasetName(name) {
+    return DATASET_RE.test(name);
+}
+export function isCaseId(id) {
+    return CASE_ID_RE.test(id) && id !== "." && id !== "..";
+}
+export function isExperimentId(id) {
+    return EXP_ID_RE.test(id);
+}
+export function assertWorkflowId(id) {
+    if (!isWorkflowId(id))
+        throw new Error(`workflow id "${id}" must match ${WORKFLOW_ID_RE} — it is a storage key prefix`);
+}
+export function assertDatasetName(name) {
+    if (!isDatasetName(name)) {
+        throw new Error(`dataset name "${name}" must match ${DATASET_RE} — it is a storage key prefix`);
+    }
+}
+export function assertCaseId(id) {
+    if (!isCaseId(id))
+        throw new Error(`case id "${id}" must match ${CASE_ID_RE}`);
+}
+export function assertExperimentId(id) {
+    if (!isExperimentId(id))
+        throw new Error(`experiment id "${id}" must look like exp-<base36 ms>-<4 chars>`);
+}
+export const workflowPrefix = (workflowId) => `${WORKFLOWS_PREFIX}${workflowId}/`;
+export const datasetsPrefix = (workflowId) => `${workflowPrefix(workflowId)}datasets/`;
+export const datasetPrefix = (workflowId, dataset) => `${datasetsPrefix(workflowId)}${dataset}/`;
+export const headerKey = (workflowId, dataset) => `${datasetPrefix(workflowId, dataset)}header.json`;
+export const casesPrefix = (workflowId, dataset) => `${datasetPrefix(workflowId, dataset)}cases/`;
+export const caseDirPrefix = (workflowId, dataset, caseId) => `${casesPrefix(workflowId, dataset)}${caseId}/`;
+export const caseRevKey = (workflowId, dataset, caseId, rev) => `${caseDirPrefix(workflowId, dataset, caseId)}r${rev}.json`;
+export const tombstoneKey = (workflowId, dataset, caseId) => `${caseDirPrefix(workflowId, dataset, caseId)}deleted.json`;
+export const archivedPrefix = (workflowId, dataset) => `${datasetPrefix(workflowId, dataset)}archived/`;
+export const archiveKey = (workflowId, dataset, caseId) => `${archivedPrefix(workflowId, dataset)}${caseId}.json`;
+export const versionsPrefix = (workflowId, dataset) => `${datasetPrefix(workflowId, dataset)}versions/`;
+export const versionKey = (workflowId, dataset, version) => `${versionsPrefix(workflowId, dataset)}v${version}.json`;
+export const experimentsPrefix = (workflowId) => `${workflowPrefix(workflowId)}experiments/`;
+export const experimentPrefix = (workflowId, expId) => `${experimentsPrefix(workflowId)}${expId}/`;
+export const experimentMetaKey = (workflowId, expId) => `${experimentPrefix(workflowId, expId)}meta.json`;
+export const runsPrefix = (workflowId, expId) => `${experimentPrefix(workflowId, expId)}runs/`;
+export const runKey = (workflowId, expId, caseId, repeat) => `${runsPrefix(workflowId, expId)}${caseId}.${repeat}.json`;
+export const attemptsPrefix = (workflowId, expId) => `${experimentPrefix(workflowId, expId)}attempts/`;
+export const attemptKey = (workflowId, expId, attempt) => `${attemptsPrefix(workflowId, expId)}a${attempt}.json`;
+/** An attempt number from an attempt claim key, or null. `a01` is not `a1`. */
+export function attemptFromKey(workflowId, expId, key) {
+    const prefix = attemptsPrefix(workflowId, expId);
+    if (!key.startsWith(prefix) || !key.endsWith(".json"))
+        return null;
+    const m = /^a([1-9]\d*)$/.exec(key.slice(prefix.length, -".json".length));
+    return m ? Number.parseInt(m[1], 10) : null;
+}
+/** A revision number from a key inside a case's directory, or null. `r01` is not `r1`. */
+export function revFromKey(workflowId, dataset, caseId, key) {
+    const prefix = caseDirPrefix(workflowId, dataset, caseId);
+    if (!key.startsWith(prefix) || !key.endsWith(".json"))
+        return null;
+    const m = /^r([1-9]\d*)$/.exec(key.slice(prefix.length, -".json".length));
+    return m ? Number.parseInt(m[1], 10) : null;
+}
+/** The case id from any key inside a case's directory, or null. */
+export function caseIdFromKey(workflowId, dataset, key) {
+    const prefix = casesPrefix(workflowId, dataset);
+    if (!key.startsWith(prefix))
+        return null;
+    const rest = key.slice(prefix.length);
+    const slash = rest.indexOf("/");
+    if (slash < 0)
+        return null;
+    const id = rest.slice(0, slash);
+    return isCaseId(id) ? id : null;
+}
+export function archivedIdFromKey(workflowId, dataset, key) {
+    const prefix = archivedPrefix(workflowId, dataset);
+    if (!key.startsWith(prefix) || !key.endsWith(".json"))
+        return null;
+    const id = key.slice(prefix.length, -".json".length);
+    return isCaseId(id) ? id : null;
+}
+export function versionFromKey(workflowId, dataset, key) {
+    const prefix = versionsPrefix(workflowId, dataset);
+    if (!key.startsWith(prefix) || !key.endsWith(".json"))
+        return null;
+    const m = /^v([1-9]\d*)$/.exec(key.slice(prefix.length, -".json".length));
+    return m ? Number.parseInt(m[1], 10) : null;
+}
+export function experimentIdFromMetaKey(workflowId, key) {
+    const prefix = experimentsPrefix(workflowId);
+    if (!key.startsWith(prefix) || !key.endsWith("/meta.json"))
+        return null;
+    const id = key.slice(prefix.length, -"/meta.json".length);
+    return isExperimentId(id) ? id : null;
+}
+/** `{caseId, repeat}` from a run record key, or null. */
+export function runFromKey(workflowId, expId, key) {
+    const prefix = runsPrefix(workflowId, expId);
+    if (!key.startsWith(prefix) || !key.endsWith(".json"))
+        return null;
+    const stem = key.slice(prefix.length, -".json".length);
+    const dot = stem.lastIndexOf(".");
+    if (dot <= 0)
+        return null;
+    const caseId = stem.slice(0, dot);
+    const repeat = Number(stem.slice(dot + 1));
+    if (!isCaseId(caseId) || !Number.isInteger(repeat) || repeat < 1)
+        return null;
+    return { caseId, repeat };
+}
+/** Highest number `parse` finds in `keys`, 0 when none. */
+export function highest(keys, parse) {
+    let max = 0;
+    for (const key of keys) {
+        const n = parse(key);
+        if (n !== null && n > max)
+            max = n;
+    }
+    return max;
+}
+/**
+ * Canonical JSON: sorted keys, two-space indent, trailing newline. A case's
+ * sha256 is over these bytes, so it depends on content and not on the key order
+ * some serialiser happened to emit.
+ */
+export function canonicalJson(value) {
+    return JSON.stringify(sortKeys(value), null, 2) + "\n";
+}
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (typeof value === "object" && value !== null) {
+        // Null prototype: a plain `{}` drops a `__proto__` key on assignment, which
+        // is silent data loss in a write-once store.
+        const out = Object.create(null);
+        for (const k of Object.keys(value).sort()) {
+            const v = value[k];
+            if (v !== undefined)
+                out[k] = sortKeys(v);
+        }
+        return out;
+    }
+    return value;
+}
+/** SHA-256 hex of a UTF-8 string, via Web Crypto (browser-safe; no node:crypto). */
+export async function sha256Hex(text) {
+    const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+    return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+/**
+ * `exp-<base36 ms>-<4 rand>`: sorts by creation time. The time is padded to 8
+ * base36 digits — its natural width from 1973 to 2059 — so a lexical sort is a
+ * time sort.
+ */
+export function newExperimentId(now = Date.now()) {
+    const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789";
+    let rand = "";
+    for (let i = 0; i < 4; i++)
+        rand += alphabet[Math.floor(Math.random() * alphabet.length)];
+    return `exp-${now.toString(36).padStart(8, "0")}-${rand}`;
+}
+/**
+ * The context one run is published into: `eval-<expId>-<caseId>-<repeat>-a<attempt>`.
+ *
+ * WHY AN ATTEMPT. A resumed experiment re-runs every case × repeat that has no
+ * record yet — and one of those may have been STARTED by the attempt that
+ * died. Publishing a second `workflow.start` into that context would land a
+ * second run beside the first (same runId, same context), and whichever end
+ * event arrives first would be scored. Each attempt of an experiment gets
+ * fresh contexts instead; they all still share the experiment's prefix, so a
+ * version pin covers them.
+ */
+export const runContext = (expId, caseId, repeat, attempt) => `eval-${expId}-${caseId}-${repeat}-a${attempt}`;
+/**
+ * The prefix every run of an experiment shares — what a version pin targets.
+ * Ends in `-` so no other experiment's contexts can share it.
+ */
+export const experimentContextPrefix = (expId) => `eval-${expId}-`;
+/** The id the runner knows a run by (`runIdFor` in the runner). */
+export const runIdOf = (context) => (context ? `run-${context}` : "");
+//# sourceMappingURL=layout.js.map

package/dist/layout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"layout.js","sourceRoot":"","sources":["../src/layout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAC7C,iEAAiE;AACjE,MAAM,CAAC,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;AACrD,0EAA0E;AAC1E,MAAM,CAAC,MAAM,oBAAoB,GAAG,qBAAqB,CAAC;AAE1D,MAAM,cAAc,GAAG,yBAAyB,CAAC;AACjD,MAAM,UAAU,GAAG,2BAA2B,CAAC;AAC/C,MAAM,UAAU,GAAG,wBAAwB,CAAC;AAC5C,MAAM,SAAS,GAAG,6BAA6B,CAAC;AAEhD,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAAC,EAAU;IACrC,OAAO,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,OAAO,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,EAAU;IACvC,OAAO,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAU;IACzC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,EAAE,gBAAgB,cAAc,+BAA+B,CAAC,CAAC;AAC1H,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,gBAAgB,UAAU,+BAA+B,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,EAAU;IACrC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,EAAE,gBAAgB,UAAU,EAAE,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,EAAU;IAC3C,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,4CAA4C,CAAC,CAAC;AAC7G,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,UAAkB,EAAU,EAAE,CAAC,GAAG,gBAAgB,GAAG,UAAU,GAAG,CAAC;AAClG,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,UAAkB,EAAU,EAAE,CAAC,GAAG,cAAc,CAAC,UAAU,CAAC,WAAW,CAAC;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CAAC,GAAG,cAAc,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,CAAC;AACzH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CAAC,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC;AAC7H,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CAAC,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC;AAC1H,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAE,MAAc,EAAU,EAAE,CAC3F,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,CAAC;AAClD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAE,MAAc,EAAE,GAAW,EAAU,EAAE,CACrG,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC;AAC9D,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAE,MAAc,EAAU,EAAE,CAC1F,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,cAAc,CAAC;AAC9D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CAAC,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC;AAChI,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAE,MAAc,EAAU,EAAE,CACxF,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC;AACzD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAU,EAAE,CAAC,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC;AAChI,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAkB,EAAE,OAAe,EAAE,OAAe,EAAU,EAAE,CACzF,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC;AAC3D,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,UAAkB,EAAU,EAAE,CAAC,GAAG,cAAc,CAAC,UAAU,CAAC,cAAc,CAAC;AAC7G,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAU,EAAE,CAAC,GAAG,iBAAiB,CAAC,UAAU,CAAC,GAAG,KAAK,GAAG,CAAC;AAC3H,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAU,EAAE,CAC7E,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC;AACpD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAU,EAAE,CAAC,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC;AACvH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAE,MAAc,EAAE,MAAc,EAAU,EAAE,CAClG,GAAG,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,MAAM,OAAO,CAAC;AAC7D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAU,EAAE,CAAC,GAAG,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC;AAC/H,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,UAAkB,EAAE,KAAa,EAAE,OAAe,EAAU,EAAE,CACvF,GAAG,cAAc,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,OAAO,OAAO,CAAC;AAEzD,+EAA+E;AAC/E,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,KAAa,EAAE,GAAW;IAC3E,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACjD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,UAAU,CAAC,UAAkB,EAAE,OAAe,EAAE,MAAc,EAAE,GAAW;IACzF,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,aAAa,CAAC,UAAkB,EAAE,OAAe,EAAE,GAAW;IAC5E,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAChC,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,UAAkB,EAAE,OAAe,EAAE,GAAW;IAChF,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrD,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,UAAkB,EAAE,OAAe,EAAE,GAAW;IAC7E,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,UAAkB,EAAE,GAAW;IACrE,MAAM,MAAM,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IACxE,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1D,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACxC,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,UAAU,CAAC,UAAkB,EAAE,KAAa,EAAE,GAAW;IACvE,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,OAAO,CAAC,IAAc,EAAE,KAAqC;IAC3E,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AACzD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,4EAA4E;QAC5E,6CAA6C;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAA4B,CAAC;QAC3D,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAgC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACrE,MAAM,CAAC,GAAI,KAAiC,CAAC,CAAC,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,SAAS;gBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC1F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc,IAAI,CAAC,GAAG,EAAE;IACtD,MAAM,QAAQ,GAAG,sCAAsC,CAAC;IACxD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1F,OAAO,OAAO,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,MAAc,EAAE,MAAc,EAAE,OAAe,EAAU,EAAE,CACnG,QAAQ,KAAK,IAAI,MAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,QAAQ,KAAK,GAAG,CAAC;AAEnF,mEAAmE;AACnE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAe,EAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC"}

package/dist/migrate.d.ts

@@ -0,0 +1,30 @@
+import { type EvalStore } from "./storage.js";
+export interface MigrationReport {
+    /** The marker was already there; nothing was read or written. */
+    alreadyMigrated: boolean;
+    datasets: Array<{
+        dataset: string;
+        workflowId: string;
+        keys: number;
+        experiments: number;
+    }>;
+    /** Keys this call wrote. */
+    copied: number;
+    /** Keys already at their target (an earlier run, or a v2 writer); left as they were. */
+    existing: number;
+    /** Legacy datasets not copied, and why. Any entry keeps the marker unwritten. */
+    skipped: Array<{
+        dataset: string;
+        reason: string;
+    }>;
+}
+/** Whether this store has v1 keys and no marker saying they were migrated. */
+export declare function needsLegacyMigration(store: EvalStore): Promise<boolean>;
+/**
+ * Copy every v1 dataset — header, cases, flags, versions and its experiments —
+ * into `workflows/<header.workflowId>/…`. Idempotent; see the file comment.
+ */
+export declare function migrateLegacyLayout(store: EvalStore, opts?: {
+    now?: Date;
+}): Promise<MigrationReport>;
+//# sourceMappingURL=migrate.d.ts.map