@cef-ai/cli 2.7.5 → 2.8.0

package/dist/commands/eval.js

@@ -0,0 +1,860 @@
+/**
+ * `cef eval` — workflow evaluations from the command line.
+ *
+ *   cef eval datasets                                    list the workflow's datasets
+ *   cef eval create <dataset>                            create one
+ *   cef eval pull <dataset> [--dir datasets] [--force]   bucket → repo dir (refuses to overwrite unpushed edits)
+ *   cef eval push <dataset> [--dir datasets] [--cut note] repo dir → bucket (new revisions for changed cases)
+ *   cef eval run <dataset> [--dataset-version N] [--workflow-version v] [--repeats 3] [--concurrency 4] [--baseline exp]
+ *   cef eval experiments [dataset]                       experiment history
+ *   cef eval compare <a> <b>                             how experiment b differs from a
+ *   cef eval migrate                                     copy v1 storage (evals/datasets/…) into the v2 layout
+ *
+ * Every command is about one workflow: `--workflow <id>`, by default the one
+ * `cef.config.ts` (or the build output) in the current directory declares.
+ *
+ * REPO DIR, one directory per dataset, in the workflow's folder:
+ *
+ *   datasets/<dataset>/header.json          the Header (without the baseline, which is chosen on the platform)
+ *   datasets/<dataset>/cases/<caseId>.json  each case at its latest revision
+ *   datasets/<dataset>/.versions.json       what was pulled: each case's revision and content hash, and the versions cut so far
+ *
+ * Experiments are not in the repo; their results stay on the platform.
+ *
+ * The bucket keeps every revision; the repo holds the latest, so a git diff of a
+ * case reads as the edit it is. `push` writes a new revision only for a case
+ * whose content changed, and refuses one that changed in the bucket since it was
+ * pulled (someone edited it in ROC) unless `--force`, and one that is deleted or
+ * archived there unless `--restore`. `pull` and `push --prune` only ever remove
+ * what the last pull wrote (`.versions.json`), never a case the other side has
+ * not seen. `pull` refuses to overwrite or remove a case file edited since the
+ * last pull/push (its content matches neither what was pulled nor what the
+ * bucket now holds) unless `--force`.
+ *
+ * The functions below take an `EvalStore` (and a `WorkflowTarget` for `run`) so
+ * they run unchanged against a memory store in tests.
+ */
+import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { canonicalJson, checkResultCompatibility, compareExperiments, createDataset, cutVersion, deleteCase, findExperiment, getCaseRevision, getHeader, getVersion, listArchivedCaseIds, listCaseRevisions, listDatasets, listDeletedCaseIds, listExperiments, listRunRecords, listVersions, listWorkflows, loadWorkingSet, migrateLegacyLayout, parseCase, parseHeader, resultFieldsFromGraph, runExperiment, saveCase, sha256Hex, unarchiveCase, undeleteCase, updateHeader, } from "@cef-ai/eval";
+import { DelegationAuthProvider, WalletAuthProvider } from "@cef-ai/vault-sdk";
+import { Browse, Events, VaultHttpClient, Vaults } from "@cef-ai/vault-sdk/internal";
+import { loadWallet, resolveGatewayAccess } from "../lib/eval-auth.js";
+import { gatewayStore } from "../lib/eval-store.js";
+import { fetchOrchestrator, getDeploymentSet, nodeWorkflowTarget, workflowAgentId, } from "../lib/eval-target.js";
+import { localGraph, resolveWorkflowId } from "../lib/eval-workflow.js";
+import { envConfig } from "../lib/env.js";
+/* ------------------------------------------------------------------------ */
+/* Repo dir                                                                  */
+/* ------------------------------------------------------------------------ */
+export const VERSIONS_FILE = ".versions.json";
+/** Where datasets live in a workflow's folder. */
+export const DEFAULT_DATASETS_DIR = "datasets";
+const datasetDir = (dir, dataset) => path.join(dir, dataset);
+async function readJsonFile(file) {
+    try {
+        return JSON.parse(await readFile(file, "utf8"));
+    }
+    catch (e) {
+        if (e.code === "ENOENT")
+            return undefined;
+        throw new Error(`[cef eval] ${file} is not valid JSON: ${e.message}`);
+    }
+}
+async function writeState(store, workflowId, dir, dataset, cases, hashes) {
+    const state = {
+        dataset,
+        pulledAt: new Date().toISOString(),
+        cases,
+        hashes,
+        versions: (await listVersions(store, workflowId, dataset)).map((m) => ({
+            version: m.version, createdAt: m.createdAt, ...(m.note ? { note: m.note } : {}), cases: m.cases.length,
+        })),
+    };
+    await writeFile(path.join(datasetDir(dir, dataset), VERSIONS_FILE), canonicalJson(state));
+    return state;
+}
+/**
+ * Read the repo dir: header and every case file, each checked. A header written
+ * by hand need not carry `createdAt` (the bucket's wins on push) nor
+ * `workflowId` (the folder's workflow is meant); a `baselineId` in it is
+ * ignored — the baseline is chosen on the platform.
+ */
+export async function readRepoDataset(dir, dataset, workflowId) {
+    const root = datasetDir(dir, dataset);
+    const rawHeader = await readJsonFile(path.join(root, "header.json"));
+    const header = rawHeader === undefined
+        ? undefined
+        : withoutBaseline(parseHeader({ createdAt: new Date().toISOString(), workflowId, ...rawHeader }));
+    if (header && header.dataset !== dataset) {
+        throw new Error(`[cef eval] ${root}/header.json names dataset "${header.dataset}", not "${dataset}"`);
+    }
+    const cases = [];
+    let files = [];
+    try {
+        files = (await readdir(path.join(root, "cases"))).filter((f) => f.endsWith(".json")).sort();
+    }
+    catch (e) {
+        if (e.code !== "ENOENT")
+            throw e;
+    }
+    for (const f of files) {
+        const c = parseCase(await readJsonFile(path.join(root, "cases", f)));
+        if (`${c.id}.json` !== f)
+            throw new Error(`[cef eval] cases/${f} holds case "${c.id}" — the file name must be <id>.json`);
+        cases.push(c);
+    }
+    const state = (await readJsonFile(path.join(root, VERSIONS_FILE)));
+    return { header, cases, state };
+}
+/* ------------------------------------------------------------------------ */
+/* Commands, over an injected store                                          */
+/* ------------------------------------------------------------------------ */
+const withoutBaseline = ({ baselineId: _b, ...h }) => h;
+export async function evalCreate(opts) {
+    return createDataset(opts.store, {
+        dataset: opts.dataset,
+        workflowId: opts.workflowId,
+        ...(opts.description ? { description: opts.description } : {}),
+        ...(opts.limits ? { limits: opts.limits } : {}),
+    });
+}
+/** The pulled state in the repo dir, when it is this dataset's. */
+async function readState(dir, dataset) {
+    const raw = (await readJsonFile(path.join(datasetDir(dir, dataset), VERSIONS_FILE)));
+    return raw?.dataset === dataset && raw.cases && typeof raw.cases === "object" ? raw : undefined;
+}
+const has = (o, id) => Object.prototype.hasOwnProperty.call(o, id);
+/** sha256 of a case's canonical JSON — what a pulled case file holds byte for byte. */
+const caseHash = (c) => sha256Hex(canonicalJson(c));
+/**
+ * sha256 of a case file's content, canonicalised so a reformat alone is not an
+ * edit; of the raw text when it does not parse (an edit in progress). undefined
+ * when there is no file.
+ */
+async function caseFileHash(file) {
+    let raw;
+    try {
+        raw = await readFile(file, "utf8");
+    }
+    catch (e) {
+        if (e.code === "ENOENT")
+            return undefined;
+        throw e;
+    }
+    try {
+        return await sha256Hex(canonicalJson(JSON.parse(raw)));
+    }
+    catch {
+        return sha256Hex(raw);
+    }
+}
+/**
+ * Bucket → repo dir.
+ *
+ * A case file is removed only when the LAST PULL wrote it (its id is in
+ * `.versions.json`) and it is no longer live in the bucket — deleted or
+ * archived there since. A file the last pull did not write is a case someone
+ * is adding in the repo and has not pushed yet; it is reported, never touched.
+ *
+ * UNPUSHED EDITS ARE NOT OVERWRITTEN. Before writing anything, every case file
+ * pull would rewrite or remove is compared with what the last pull/push left
+ * there (the hash in `.versions.json`; for an older file without hashes, the
+ * pulled revision's content) and with what the bucket now holds. A file that
+ * matches neither was edited locally and not pushed: the pull is refused
+ * unless `force`.
+ */
+export async function evalPull(opts) {
+    const root = datasetDir(opts.dir, opts.dataset);
+    // The same guard as push: a repo dir holding another workflow's dataset of this name is not overwritten.
+    const localHeader = await readJsonFile(path.join(root, "header.json"));
+    const localWorkflow = localHeader?.workflowId;
+    if (typeof localWorkflow === "string" && localWorkflow !== opts.workflowId) {
+        throw new Error(`[cef eval] ${root}/header.json is for workflow "${localWorkflow}", not "${opts.workflowId}". `
+            + `A dataset belongs to one workflow — pull it with --workflow ${localWorkflow}, or into another --dir.`);
+    }
+    const { header, cases } = await loadWorkingSet(opts.store, opts.workflowId, opts.dataset);
+    const before = await readState(opts.dir, opts.dataset);
+    const pulledBefore = before?.cases ?? {};
+    const hashesBefore = before?.hashes ?? {};
+    await mkdir(path.join(root, "cases"), { recursive: true });
+    const live = new Map(cases.map(({ case: c }) => [c.id, c]));
+    const toRemove = [];
+    const localOnly = [];
+    for (const f of (await readdir(path.join(root, "cases"))).sort()) {
+        if (!f.endsWith(".json"))
+            continue;
+        const id = f.slice(0, -".json".length);
+        if (live.has(id))
+            continue;
+        if (has(pulledBefore, id))
+            toRemove.push(id);
+        else
+            localOnly.push(id);
+    }
+    /** What the repo file held after the last pull/push, as a hash; undefined when unknown. */
+    const lastSynced = async (id) => {
+        if (has(hashesBefore, id))
+            return hashesBefore[id];
+        if (!has(pulledBefore, id))
+            return undefined;
+        const c = await getCaseRevision(opts.store, opts.workflowId, opts.dataset, id, pulledBefore[id]);
+        return c ? caseHash(c) : undefined;
+    };
+    const edited = [];
+    if (!opts.force) {
+        for (const [id, c] of [...live, ...toRemove.map((x) => [x, undefined])]) {
+            const local = await caseFileHash(path.join(root, "cases", `${id}.json`));
+            if (local === undefined)
+                continue;
+            if (c && local === (await caseHash(c)))
+                continue;
+            if (local === (await lastSynced(id)))
+                continue;
+            edited.push(c ? id : `${id} (removed in the bucket)`);
+        }
+    }
+    if (edited.length) {
+        throw new Error(`[cef eval] these case files were edited since the last pull/push and differ from the bucket:\n  ${edited.join("\n  ")}\n`
+            + "Nothing was written. Push them first (cef eval push), move the edits aside, "
+            + "or pull with --force to overwrite them with the bucket's version.");
+    }
+    // The baseline stays on the platform: in git it would be stale the moment someone picks another in ROC.
+    await writeFile(path.join(root, "header.json"), canonicalJson(withoutBaseline(header)));
+    for (const id of toRemove)
+        await rm(path.join(root, "cases", `${id}.json`));
+    const hashes = {};
+    for (const { case: c } of cases) {
+        const body = canonicalJson(c);
+        await writeFile(path.join(root, "cases", `${c.id}.json`), body);
+        hashes[c.id] = await sha256Hex(body);
+    }
+    const state = await writeState(opts.store, opts.workflowId, opts.dir, opts.dataset, Object.fromEntries(cases.map(({ case: c, rev }) => [c.id, rev])), hashes);
+    return { dir: root, cases: cases.length, removed: toRemove, localOnly, state };
+}
+/** The newest revision of a case whatever its flags say, or undefined when it has none. */
+async function latestRevision(store, workflowId, dataset, id) {
+    const revs = await listCaseRevisions(store, workflowId, dataset, id);
+    const rev = revs[revs.length - 1];
+    if (rev === undefined)
+        return undefined;
+    const c = await getCaseRevision(store, workflowId, dataset, id, rev);
+    return c ? { case: c, rev } : undefined;
+}
+/**
+ * Repo dir → bucket.
+ *
+ * EVERY CHECK BEFORE ANY WRITE, so a refused push leaves the bucket exactly as
+ * it was — header included:
+ *   - a repo case that is deleted or archived in the bucket is refused unless
+ *     `restore` (an explicit undelete/unarchive, then the repo's version is
+ *     written as a new revision). `force` does NOT imply it: overwriting an
+ *     edit and resurrecting a case someone removed are different decisions.
+ *   - a case edited in the bucket since the pull, whose content differs from
+ *     the repo's, is a conflict unless `force`.
+ *   - `prune` deletes only cases the last pull wrote and the repo has since
+ *     dropped; a case edited in the bucket since that pull is a conflict
+ *     unless `force`. Cases added in the bucket after the pull are never
+ *     pruned — the repo never knew them.
+ */
+export async function evalPush(opts) {
+    const wf = opts.workflowId;
+    const local = await readRepoDataset(opts.dir, opts.dataset, wf);
+    if (!local.header) {
+        throw new Error(`[cef eval] ${datasetDir(opts.dir, opts.dataset)}/header.json is missing — pull the dataset or create it first`);
+    }
+    if (local.header.workflowId !== wf) {
+        throw new Error(`[cef eval] ${datasetDir(opts.dir, opts.dataset)}/header.json is for workflow "${local.header.workflowId}", `
+            + `not "${wf}". A dataset belongs to one workflow — push it with --workflow ${local.header.workflowId}.`);
+    }
+    const remoteHeader = await getHeader(opts.store, wf, opts.dataset);
+    // The baseline is the platform's; the repo neither carries nor overwrites it.
+    const headerChanged = !!remoteHeader
+        && canonicalJson({ ...withoutBaseline(remoteHeader), createdAt: "" }) !== canonicalJson({ ...local.header, createdAt: "" });
+    const remote = new Map(remoteHeader ? (await loadWorkingSet(opts.store, wf, opts.dataset)).cases.map((x) => [x.case.id, x]) : []);
+    const deleted = new Set(remoteHeader ? await listDeletedCaseIds(opts.store, wf, opts.dataset) : []);
+    const archived = new Set(remoteHeader ? await listArchivedCaseIds(opts.store, wf, opts.dataset) : []);
+    const pulled = local.state?.dataset === opts.dataset ? local.state.cases : {};
+    // 1. Repo cases the bucket's working set hides.
+    const hidden = local.cases.filter((c) => deleted.has(c.id) || archived.has(c.id));
+    if (hidden.length && !opts.restore) {
+        throw new Error(`[cef eval] these cases are ${hidden.length === 1 ? "" : "each "}deleted or archived in the bucket, and still in the repo dir:\n  `
+            + hidden.map((c) => `${c.id} (${deleted.has(c.id) ? "deleted" : "archived"})`).join("\n  ")
+            + "\nNothing was written. Pull to drop them from the repo dir, give a case a new id to add it as a new case, "
+            + "or push with --restore to undelete/unarchive them and write the repo's version.");
+    }
+    // 2. Edits in the bucket since the pull.
+    const conflicts = [];
+    for (const c of local.cases) {
+        if (opts.force)
+            break;
+        const r = remote.get(c.id) ?? (hidden.includes(c) ? await latestRevision(opts.store, wf, opts.dataset, c.id) : undefined);
+        if (!r)
+            continue;
+        const base = has(pulled, c.id) ? pulled[c.id] : undefined;
+        const editedRemotely = base === undefined || r.rev !== base;
+        if (editedRemotely && canonicalJson(r.case) !== canonicalJson(c))
+            conflicts.push(`${c.id} (bucket r${r.rev}, pulled ${base === undefined ? "never" : `r${base}`})`);
+    }
+    // 3. What prune would delete.
+    const localIds = new Set(local.cases.map((c) => c.id));
+    const pruneConflicts = [];
+    const missingLocally = [];
+    const notPulled = [];
+    for (const [id, r] of remote) {
+        if (localIds.has(id))
+            continue;
+        if (!has(pulled, id)) {
+            notPulled.push(id);
+            continue;
+        }
+        missingLocally.push(id);
+        if (opts.prune && !opts.force && r.rev !== pulled[id]) {
+            pruneConflicts.push(`${id} (removed from the repo, but bucket r${r.rev} was edited since the pull of r${pulled[id]})`);
+        }
+    }
+    if (conflicts.length || pruneConflicts.length) {
+        throw new Error([
+            "[cef eval] Nothing was written.",
+            ...(conflicts.length
+                ? [
+                    `These cases changed in the bucket since they were pulled, and differ from the repo:\n  ${conflicts.join("\n  ")}`,
+                    "Pull first and redo the edit, or push with --force to write the repo's version over them as a new revision.",
+                ]
+                : []),
+            ...(pruneConflicts.length
+                ? [
+                    `--prune would delete these cases, edited in the bucket since they were pulled:\n  ${pruneConflicts.join("\n  ")}`,
+                    "Pull to get the edited case back into the repo dir, or push with --prune --force to delete it, edit and all "
+                        + "(--restore brings a deleted case back).",
+                ]
+                : []),
+        ].join("\n"));
+    }
+    // Writes.
+    const result = {
+        created: false, headerUpdated: false, added: [], updated: [], unchanged: [], restored: [],
+        missingLocally: [], notPulled, deleted: [],
+    };
+    if (!remoteHeader) {
+        await createDataset(opts.store, local.header);
+        result.created = true;
+    }
+    else if (headerChanged) {
+        await updateHeader(opts.store, {
+            ...local.header,
+            createdAt: remoteHeader.createdAt,
+            ...(remoteHeader.baselineId ? { baselineId: remoteHeader.baselineId } : {}),
+        });
+        result.headerUpdated = true;
+    }
+    for (const c of hidden) {
+        if (deleted.has(c.id))
+            await undeleteCase(opts.store, wf, opts.dataset, c.id);
+        if (archived.has(c.id))
+            await unarchiveCase(opts.store, wf, opts.dataset, c.id);
+        result.restored.push(c.id);
+    }
+    const revs = {};
+    const hashes = {};
+    const hashesBefore = local.state?.dataset === opts.dataset ? local.state.hashes ?? {} : {};
+    for (const c of local.cases) {
+        const saved = await saveCase(opts.store, wf, opts.dataset, c);
+        revs[c.id] = saved.rev;
+        hashes[c.id] = await caseHash(c);
+        if (saved.created)
+            result.added.push(c.id);
+        else if (saved.changed)
+            result.updated.push({ id: c.id, rev: saved.rev });
+        else
+            result.unchanged.push(c.id);
+    }
+    for (const id of missingLocally) {
+        if (opts.prune) {
+            await deleteCase(opts.store, wf, opts.dataset, id);
+            result.deleted.push(id);
+        }
+        else {
+            result.missingLocally.push(id);
+            // Still "pulled" — the repo dropped it on purpose, so a later --prune may delete it.
+            revs[id] = pulled[id];
+            if (has(hashesBefore, id))
+                hashes[id] = hashesBefore[id];
+        }
+    }
+    if (opts.cut !== undefined && opts.cut !== false) {
+        const m = await cutVersion(opts.store, wf, opts.dataset, typeof opts.cut === "string" && opts.cut ? { note: opts.cut } : {});
+        result.cut = { version: m.version, cases: m.cases.length };
+    }
+    await writeState(opts.store, wf, opts.dir, opts.dataset, revs, hashes);
+    return result;
+}
+export async function evalRun(opts) {
+    const header = await getHeader(opts.store, opts.workflowId, opts.dataset);
+    if (!header)
+        throw new Error(`[cef eval] workflow "${opts.workflowId}" has no dataset "${opts.dataset}"`);
+    return runExperiment({
+        store: opts.store,
+        workflowId: opts.workflowId,
+        dataset: opts.dataset,
+        ...(opts.version !== undefined ? { datasetVersion: opts.version } : {}),
+        workflowVersion: opts.workflowVersion,
+        live: opts.live,
+        target: opts.target,
+        origin: "cli",
+        ...(opts.resultFields ? { resultFields: opts.resultFields } : {}),
+        ...(opts.repeats !== undefined ? { repeats: opts.repeats } : {}),
+        ...(opts.concurrency !== undefined ? { concurrency: opts.concurrency } : {}),
+        ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
+        ...(opts.baselineId ? { baselineId: opts.baselineId } : {}),
+        ...(opts.experimentId ? { experimentId: opts.experimentId } : {}),
+        ...(opts.note ? { note: opts.note } : {}),
+        ...(opts.createdBy ? { createdBy: opts.createdBy } : {}),
+        ...(opts.onProgress ? { onProgress: opts.onProgress } : {}),
+    });
+}
+/** Whether the dataset's live cases fit the Result `resultFields` declares. */
+export async function evalCheckResult(opts) {
+    const { cases } = await loadWorkingSet(opts.store, opts.workflowId, opts.dataset);
+    return checkResultCompatibility(opts.resultFields, cases.map((x) => x.case));
+}
+/** The Result pre-check of `cef eval run`, for both output formats. */
+export async function evalResultCheck(opts) {
+    const { fields, from, why } = opts.result;
+    if (!fields)
+        return { workflowVersion: opts.workflowVersion, ...(why ? { unknown: why } : {}) };
+    const compatibility = await evalCheckResult({ store: opts.store, workflowId: opts.workflowId, dataset: opts.dataset, resultFields: fields });
+    return { workflowVersion: opts.workflowVersion, fields, ...(from ? { from } : {}), compatibility };
+}
+export async function evalCompare(opts) {
+    const load = async (id) => {
+        const experiment = await findExperiment(opts.store, id);
+        if (!experiment)
+            throw new Error(`[cef eval] experiment "${id}" not found`);
+        // The version's own case list, so a case with no run still counts as present.
+        const cases = await getVersion(opts.store, experiment.workflowId, experiment.dataset, experiment.datasetVersion)
+            .then((m) => m.cases, () => undefined);
+        return { experiment, runs: await listRunRecords(opts.store, experiment.workflowId, id), ...(cases ? { cases } : {}) };
+    };
+    return compareExperiments(await load(opts.base), await load(opts.candidate));
+}
+export async function evalMigrate(opts) {
+    return migrateLegacyLayout(opts.store);
+}
+/**
+ * The Result a workflow version declares, read off its graph:
+ *   - the live deployment row's `params.graph` when the version is the live
+ *     one (a deployment may override the graph — that one is what runs);
+ *   - else the project's own graph of exactly that version (`cef.config.ts`
+ *     or `dist/<id>/manifest.json`).
+ * undefined (with the reason) when neither is available: scoring then judges
+ * every expected field, as before Result declarations.
+ */
+export async function resultFieldsFor(opts) {
+    const rowGraph = opts.liveRow?.version === opts.workflowVersion ? opts.liveRow.params?.graph : undefined;
+    if (rowGraph !== undefined) {
+        const fields = resultFieldsFromGraph(rowGraph);
+        return fields ? { fields, from: "the live deployment" } : { why: "the live deployment's graph declares no Result" };
+    }
+    const local = await localGraph(opts.workflowId, opts.workflowVersion, opts.local);
+    if (local) {
+        const fields = resultFieldsFromGraph(local.graph);
+        return fields ? { fields, from: local.from } : { why: `${local.from} declares no Result` };
+    }
+    return { why: `no graph of ${opts.workflowId}@${opts.workflowVersion} here (cef.config.ts or dist/) or on the live deployment` };
+}
+/* ------------------------------------------------------------------------ */
+/* Output                                                                    */
+/* ------------------------------------------------------------------------ */
+/** A plain fixed-width table. */
+export function table(headers, rows) {
+    const cells = rows.map((r) => r.map((c) => (c === null || c === undefined ? "–" : String(c))));
+    const widths = headers.map((h, i) => Math.max(h.length, ...cells.map((r) => (r[i] ?? "").length)));
+    const line = (r) => r.map((c, i) => c.padEnd(widths[i])).join("  ").trimEnd();
+    return [line(headers), line(widths.map((w) => "-".repeat(w))), ...cells.map(line)].join("\n");
+}
+const pct = (n) => (n === null || n === undefined ? null : `${Math.round(n * 1000) / 10}%`);
+const ms = (n) => (n === null || n === undefined ? null : n >= 1000 ? `${(n / 1000).toFixed(1)}s` : `${n}ms`);
+const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
+export function formatDatasets(list) {
+    if (!list.length)
+        return "No datasets.";
+    return table(["DATASET", "WORKFLOW", "CASES", "VERSION", "EXPERIMENTS", "BASELINE"], list.map((d) => [
+        d.dataset, d.workflowId, d.cases, d.latestVersion === null ? null : `v${d.latestVersion}`, d.experiments, d.baselineId,
+    ]));
+}
+export function formatExperiments(list, baselineId) {
+    if (!list.length)
+        return "No experiments.";
+    return table(["EXPERIMENT", "STARTED", "DATASET", "RUN ON", "STATUS", "ACCURACY", "P50", "TOKENS/RUN", "COST/RUN", "RUNS"], list.map((e) => [
+        `${e.id === baselineId ? "★ " : ""}${e.id}`, e.startedAt.replace("T", " ").slice(0, 16), e.dataset,
+        `workflow ${e.workflowVersion} · dataset v${e.datasetVersion}`, e.status,
+        pct(e.summary?.passRate), ms(e.summary?.durationMs.p50), e.summary?.tokens.meanPerRun, e.summary?.cost.meanPerRun,
+        e.summary?.runs,
+    ]));
+}
+export function formatExperiment(e) {
+    const s = e.summary;
+    const fields = Object.entries(s?.fields ?? {});
+    return [
+        `${e.id}  ${e.status}  dataset ${e.dataset} v${e.datasetVersion} × workflow ${e.workflowVersion}${e.live ? " (live)" : ""}`
+            + ` × ${plural(e.repeats, "repeat")}`,
+        ...(e.error ? [`error: ${e.error}`] : []),
+        ...(s
+            ? [
+                `accuracy ${pct(s.passRate) ?? "–"}  (${s.passed} passed, ${s.failed} failed, ${s.errored} errored, ${s.unscored} unscored of ${s.runs} runs)`,
+                ...fields.map(([name, f]) => `  ${name}: ${pct(f.rate) ?? "–"} (${f.passed}/${f.passed + f.failed}${f.notApplicable ? `, ${f.notApplicable} n/a` : ""})`),
+                `duration p50 ${ms(s.durationMs.p50) ?? "–"}  p95 ${ms(s.durationMs.p95) ?? "–"}  ·  tokens ${s.tokens.total} (${s.tokens.meanPerRun}/run)`
+                    + `  ·  cost ${s.cost.total} (${s.cost.meanPerRun}/run)  ·  steps ${s.steps.meanPerRun}/run  ·  limit breaches ${s.limitBreaches}`,
+            ]
+            : []),
+    ].join("\n");
+}
+/** "Dataset changed between these experiments: …" — or nothing when it did not. */
+export function formatDatasetDiff(c) {
+    const d = c.datasetDiff;
+    if (!d)
+        return null;
+    const parts = [
+        d.added.length ? `${plural(d.added.length, "case")} added` : "",
+        d.removed.length ? `${plural(d.removed.length, "case")} removed` : "",
+        d.edited.length ? `${plural(d.edited.length, "expectation")} edited` : "",
+    ].filter(Boolean);
+    if (!parts.length)
+        return null;
+    return `Dataset changed between these experiments: ${parts.join(", ")}. `
+        + `Compared on the ${plural(c.comparedOn, "unchanged case")}.`;
+}
+export function formatResultCheck(r) {
+    if (!("fields" in r))
+        return [`Result of ${r.workflowVersion} unknown (${r.unknown ?? "no graph"}); every expected field is judged`];
+    return [
+        `Result of ${r.workflowVersion}${r.from ? ` (from ${r.from})` : ""}: ${r.fields.map((f) => f.name).join(", ")}`,
+        ...formatCompatibility(r.compatibility, r.workflowVersion).map((l) => `! ${l}`),
+    ];
+}
+export function formatCompatibility(r, version) {
+    return [
+        ...r.missing.map((m) => `${plural(m.caseIds.length, "dataset case")} expect \`${m.field}\`, which ${version} does not return`
+            + ` — scored n/a (${m.caseIds.join(", ")})`),
+        ...r.typeChanged.map((t) => `${plural(t.caseIds.length, "dataset case")} expect \`${t.field}\` as ${t.expectedType}; `
+            + `${version} declares ${t.declaredType} — scored n/a (${t.caseIds.join(", ")})`),
+        ...(r.unexpectedNew.length ? [`${version} returns ${r.unexpectedNew.map((f) => `\`${f}\``).join(", ")}, which no case expects yet`] : []),
+    ];
+}
+const signed = (d, f = String) => d.delta === null ? "–" : `${d.delta > 0 ? "+" : ""}${f(d.delta)}${d.relative === null ? "" : ` (${d.relative > 0 ? "+" : ""}${Math.round(d.relative * 100)}%)`}`;
+export function formatComparison(c) {
+    const m = c.metrics;
+    const head = `${c.base.id} (${c.base.workflowVersion ?? "?"}) → ${c.candidate.id} (${c.candidate.workflowVersion ?? "?"})`;
+    const counts = Object.entries(c.counts).filter(([, n]) => n > 0).map(([k, n]) => `${n} ${k}`).join(", ") || "no runs";
+    const diff = formatDatasetDiff(c);
+    const metrics = table(["METRIC", "BASE", "CANDIDATE", "DELTA"], [
+        ["accuracy", pct(m.passRate.base), pct(m.passRate.candidate), signed(m.passRate, (n) => pct(n))],
+        ["duration p50", ms(m.durationP50.base), ms(m.durationP50.candidate), signed(m.durationP50, (n) => ms(n))],
+        ["duration p95", ms(m.durationP95.base), ms(m.durationP95.candidate), signed(m.durationP95, (n) => ms(n))],
+        ["tokens/run", m.tokensPerRun.base, m.tokensPerRun.candidate, signed(m.tokensPerRun)],
+        ["cost/run", m.costPerRun.base, m.costPerRun.candidate, signed(m.costPerRun)],
+        ["steps/run", m.stepsPerRun.base, m.stepsPerRun.candidate, signed(m.stepsPerRun)],
+        ["limit breaches", m.limitBreaches.base, m.limitBreaches.candidate, signed(m.limitBreaches)],
+    ]);
+    const changed = c.runs.filter((r) => r.change !== "unchanged");
+    const runs = changed.length
+        ? table(["CASE", "REPEAT", "CHANGE", "BASE", "CANDIDATE", "DURATION Δ", "TOKENS Δ"], changed.map((r) => [
+            r.caseId, r.repeat, r.change,
+            r.base ? `${r.base.status}/${r.base.pass === null ? "–" : r.base.pass ? "pass" : "fail"}` : null,
+            r.candidate ? `${r.candidate.status}/${r.candidate.pass === null ? "–" : r.candidate.pass ? "pass" : "fail"}` : null,
+            signed(r.durationMs, (n) => ms(n)), signed(r.totalTokens),
+        ]))
+        : "Every run unchanged.";
+    return [head, ...(diff ? [diff] : []), counts, "", metrics, "", runs].join("\n");
+}
+export function formatMigration(r) {
+    if (r.alreadyMigrated)
+        return "Already migrated (evals/MIGRATED.json is there); nothing to do.";
+    return [
+        `✓ ${r.datasets.length ? `Copied ${plural(r.datasets.length, "dataset")}` : "No legacy datasets"}: `
+            + `${r.copied} keys written, ${r.existing} already in place`,
+        ...r.datasets.map((d) => `  ${d.dataset} → workflows/${d.workflowId}/ (${d.keys} keys, ${plural(d.experiments, "experiment")})`),
+        ...r.skipped.map((x) => `  skipped ${x.dataset}: ${x.reason}`),
+        ...(r.skipped.length ? ["Not marked migrated: fix the skipped datasets and run it again."] : []),
+    ].join("\n");
+}
+const int = (v, flag) => {
+    if (v === undefined)
+        return undefined;
+    const n = Number(v);
+    if (!Number.isInteger(n) || n < 1)
+        throw new Error(`[cef eval] ${flag} must be a whole number ≥ 1 (got "${v}")`);
+    return n;
+};
+/** A finite number ≥ 0 (a cost limit may be fractional). */
+const number = (v, flag) => {
+    if (v === undefined)
+        return undefined;
+    const n = v.trim() === "" ? NaN : Number(v);
+    if (!Number.isFinite(n) || n < 0)
+        throw new Error(`[cef eval] ${flag} must be a number ≥ 0 (got "${v}")`);
+    return n;
+};
+function withStoreOptions(cmd) {
+    return cmd
+        .option("--env <env>", "Environment: dev | stage | prod (default dev; overrides $CEF_ENV)")
+        .option("--as-pubkey <hex>", "Agent service pubkey — names the bucket ($CEF_AS_PUBKEY)")
+        .option("--keystore <file>", "Owner wallet keystore JSON, ed25519 ($CEF_KEYSTORE)")
+        .option("--password <pw>", "Keystore password ($CEF_KEYSTORE_PASSWORD)")
+        .option("--secret-phrase <phrase>", "Owner wallet secret phrase, ed25519 ($CEF_SECRET_PHRASE)")
+        .option("--gateway <url>", "S3 gateway base URL ($CEF_EVAL_ENDPOINT; default from --env)")
+        .option("--bucket <name>", "Bucket override ($CEF_EVAL_BUCKET)")
+        .option("--access-key <id>", "Explicit S3 access key ($CEF_EVAL_KEY)")
+        .option("--secret <key>", "Explicit S3 secret ($CEF_EVAL_SECRET)")
+        .option("--json", "Print JSON instead of a table");
+}
+const WORKFLOW_HELP = "Workflow id (agent alias); default: the workflow cef.config.ts (or dist/) in this directory declares";
+function withWorkflow(cmd) {
+    return withStoreOptions(cmd).option("--workflow <id>", WORKFLOW_HELP);
+}
+function withDir(cmd) {
+    return cmd.option("--dir <dir>", "Directory holding the workflow's datasets", DEFAULT_DATASETS_DIR);
+}
+async function openStore(o, wallet) {
+    return gatewayStore(await resolveGatewayAccess({ ...o, ...(o.gateway ? { endpoint: o.gateway } : {}) }, wallet));
+}
+/** The Node target: vault-api for events, the orchestrator for jobs and deployments. */
+async function openTarget(o, wallet, workflowId, fetchImpl) {
+    const env = envConfig(o.env);
+    const asPubKey = o.asPubkey ?? process.env.CEF_AS_PUBKEY;
+    if (!asPubKey)
+        throw new Error("[cef eval] --as-pubkey (or $CEF_AS_PUBKEY) is required");
+    const accessToken = o.accessToken ?? process.env.CEF_ACCESS_TOKEN;
+    const auth = wallet
+        ? new WalletAuthProvider(wallet)
+        : accessToken
+            ? new DelegationAuthProvider(accessToken)
+            : undefined;
+    if (!auth)
+        throw new Error("[cef eval] run needs a wallet (--keystore/--secret-phrase) or --access-token to reach the vault");
+    const http = new VaultHttpClient({ url: o.vaultApi ?? env.vault, auth, ...(fetchImpl ? { fetch: fetchImpl } : {}) });
+    const vaultId = o.vault ?? process.env.CEF_VAULT_ID ?? (await new Vaults(http).current()).vaultId;
+    const events = new Events(http);
+    const browse = new Browse(http);
+    const orchestrator = fetchOrchestrator({
+        endpoint: o.orchestrator ?? process.env.CEF_ENDPOINT ?? env.orchestrator,
+        ...(accessToken ? { accessToken } : {}),
+        ...(fetchImpl ? { fetchImpl } : {}),
+    });
+    const target = nodeWorkflowTarget({
+        vaultId,
+        asPubKey,
+        workflowId,
+        orchestrator,
+        vault: {
+            async publish(ev) {
+                const res = await events.publish(vaultId, o.scope, ev);
+                const rejected = res.rejected ?? [];
+                if (rejected.length)
+                    throw new Error(`the vault rejected ${ev.type}: ${rejected[0].reason}`);
+            },
+            events: (context, cursor) => browse.listStreamEvents(vaultId, o.scope, context, { limit: 200, ...(cursor ? { cursor } : {}) }),
+        },
+    });
+    return { target, orchestrator, agentId: workflowAgentId(asPubKey, workflowId), vaultId };
+}
+const print = (json, data, text) => console.log(json ? JSON.stringify(data, null, 2) : text());
+export function addEvalCommands(program, opts) {
+    const local = () => ({ cwd: opts.cwd?.() ?? process.cwd() });
+    const workflowOf = (o) => resolveWorkflowId(o.workflow, local());
+    const ev = program
+        .command("eval")
+        .description("Workflow evaluations: datasets, experiments, comparisons");
+    withWorkflow(ev.command("datasets").description("List the workflow's datasets (every workflow's, when none is named or found)"))
+        .action(async (o) => {
+        try {
+            const workflowId = await workflowOf(o).catch(() => undefined);
+            const store = await openStore(o);
+            const list = [];
+            for (const wf of workflowId ? [workflowId] : await listWorkflows(store))
+                list.push(...await listDatasets(store, wf));
+            print(o.json, list, () => formatDatasets(list));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("create <dataset>").description("Create a dataset for the workflow"))
+        .option("--description <text>", "What the dataset covers")
+        .option("--max-duration-ms <n>", "Warn when a run takes longer")
+        .option("--max-tokens <n>", "Warn when a run uses more tokens")
+        .option("--max-steps <n>", "Warn when a run takes more steps")
+        .option("--max-cost <n>", "Warn when a run costs more")
+        .action(async (dataset, o) => {
+        try {
+            // Validated first, so a typo fails before any credential is minted; NaN would serialize as null.
+            const limits = {};
+            const set = (key, n) => {
+                if (n !== undefined)
+                    limits[key] = n;
+            };
+            set("maxDurationMs", int(o.maxDurationMs, "--max-duration-ms"));
+            set("maxTokens", int(o.maxTokens, "--max-tokens"));
+            set("maxSteps", int(o.maxSteps, "--max-steps"));
+            set("maxCost", number(o.maxCost, "--max-cost"));
+            const workflowId = await workflowOf(o);
+            const header = await evalCreate({
+                store: await openStore(o), dataset, workflowId,
+                ...(o.description ? { description: o.description } : {}),
+                ...(Object.keys(limits).length ? { limits } : {}),
+            });
+            print(o.json, header, () => `✓ Created dataset ${header.dataset} for workflow ${header.workflowId}`);
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withDir(withWorkflow(ev.command("pull <dataset>").description("Write a dataset's live cases to the repo dir")))
+        .option("--force", "Overwrite case files edited locally since the last pull/push")
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const r = await evalPull({ store: await openStore(o), workflowId, dataset, dir: o.dir, force: !!o.force });
+            print(o.json, r, () => [
+                `✓ Pulled ${r.cases} case(s) into ${r.dir}`,
+                ...(r.removed.length ? [`  removed (no longer live): ${r.removed.join(", ")}`] : []),
+                ...(r.localOnly.length ? [`  local only (never pulled; kept — push adds them): ${r.localOnly.join(", ")}`] : []),
+                ...(r.state.versions.length ? [`  versions: v${r.state.versions.map((v) => v.version).join(", v")}`] : ["  no versions cut yet"]),
+            ].join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withDir(withWorkflow(ev.command("push <dataset>").description("Write the repo dir's cases to the bucket as new revisions")))
+        .option("--cut [note]", "Cut a new version after pushing (runs cut one automatically when the cases changed)")
+        .option("--force", "Write over cases that changed in the bucket since the last pull")
+        .option("--prune", "Delete cases the last pull wrote and the repo dir has since removed")
+        .option("--restore", "Undelete/unarchive cases that are deleted or archived in the bucket but present in the repo dir")
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const r = await evalPush({
+                store: await openStore(o), workflowId, dataset, dir: o.dir,
+                ...(o.cut !== undefined ? { cut: o.cut } : {}), force: !!o.force, prune: !!o.prune, restore: !!o.restore,
+            });
+            print(o.json, r, () => [
+                r.created ? `✓ Created dataset ${dataset}` : r.headerUpdated ? "✓ Updated the header" : null,
+                `✓ ${r.added.length} added, ${r.updated.length} updated, ${r.unchanged.length} unchanged`,
+                ...r.updated.map((u) => `  ${u.id} → r${u.rev}`),
+                ...(r.added.length ? [`  added: ${r.added.join(", ")}`] : []),
+                ...(r.restored.length ? [`  restored (were deleted/archived): ${r.restored.join(", ")}`] : []),
+                ...(r.missingLocally.length ? [`  removed from the repo, still in the bucket (kept; --prune deletes): ${r.missingLocally.join(", ")}`] : []),
+                ...(r.notPulled.length ? [`  added in the bucket since the last pull (kept; pull to get them): ${r.notPulled.join(", ")}`] : []),
+                ...(r.deleted.length ? [`  deleted: ${r.deleted.join(", ")}`] : []),
+                r.cut ? `✓ Cut v${r.cut.version} (${r.cut.cases} cases)` : null,
+            ].filter((l) => l !== null).join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("run <dataset>").description("Run a dataset version against a workflow version"))
+        // Not `--version`: the root program's `.version()` claims that flag on every
+        // subcommand, printing the CLI's version and exiting before the action runs.
+        .option("--dataset-version <n>", "Dataset version (default: the latest, cut automatically when the cases changed)")
+        .option("--workflow-version <v>", "Workflow version (default: the live one); a non-live version is pinned for this experiment only")
+        .option("--repeats <n>", "How many times each case runs", "1")
+        .option("--concurrency <n>", "Runs in flight at once", "4")
+        .option("--timeout <ms>", "How long one run may take", "300000")
+        .option("--baseline <exp>", "Experiment to compare against when done (default: the dataset's baseline)")
+        .option("--resume <exp>", "Resume an experiment that did not finish")
+        .option("--note <text>", "Note recorded on the experiment")
+        .option("--vault <id>", "Vault to run in ($CEF_VAULT_ID; default: the wallet's own)")
+        .option("--scope <scope>", "Vault scope to publish into", "default")
+        .option("--orchestrator <url>", "Orchestrator base URL ($CEF_ENDPOINT; default from --env)")
+        .option("--vault-api <url>", "vault-api base URL (default from --env)")
+        .option("--access-token <token>", "CLI access token for the orchestrator ($CEF_ACCESS_TOKEN)")
+        .action(async (dataset, o) => {
+        try {
+            // Numbers first, so a typo fails before any credential is minted or run started.
+            const numbers = {
+                ...(o.datasetVersion ? { version: int(o.datasetVersion, "--dataset-version") } : {}),
+                repeats: int(o.repeats, "--repeats"),
+                concurrency: int(o.concurrency, "--concurrency"),
+                timeoutMs: int(o.timeout, "--timeout"),
+            };
+            const workflowId = await workflowOf(o);
+            const wallet = await loadWallet(o);
+            const store = await openStore(o, wallet);
+            const header = await getHeader(store, workflowId, dataset);
+            if (!header)
+                throw new Error(`[cef eval] workflow "${workflowId}" has no dataset "${dataset}"`);
+            const { target, orchestrator, agentId } = await openTarget(o, wallet, workflowId, opts.fetchImpl);
+            const liveRow = (await getDeploymentSet(orchestrator, agentId))?.deployments
+                .find((d) => d.targeting === "" && d.enabled !== false);
+            const live = liveRow?.version ?? null;
+            const workflowVersion = o.workflowVersion ?? live ?? "latest";
+            const isLive = o.workflowVersion === undefined || o.workflowVersion === live;
+            const result = await resultFieldsFor({
+                workflowId, workflowVersion, ...(liveRow ? { liveRow } : {}), local: local(),
+            });
+            // Checked whatever the output format: --json carries it instead of printing it.
+            const resultCheck = await evalResultCheck({ store, workflowId, dataset, workflowVersion, result });
+            if (!o.json)
+                for (const line of formatResultCheck(resultCheck))
+                    console.error(`  ${line}`);
+            const baseline = o.baseline ?? header.baselineId;
+            const exp = await evalRun({
+                store, target, workflowId, dataset, workflowVersion, live: isLive, ...numbers,
+                ...(result.fields ? { resultFields: result.fields } : {}),
+                ...(baseline ? { baselineId: baseline } : {}),
+                ...(o.resume ? { experimentId: o.resume } : {}),
+                ...(o.note ? { note: o.note } : {}),
+                onProgress: (p) => {
+                    if (!o.json) {
+                        const verdict = p.run.pass === null ? p.run.status : p.run.pass ? "pass" : "FAIL";
+                        console.error(`  [${p.done}/${p.total}] ${p.run.caseId}#${p.run.repeat} ${verdict}${p.reused ? " (reused)" : ""}`);
+                    }
+                },
+            });
+            const comparison = baseline && baseline !== exp.id
+                ? await evalCompare({ store, base: baseline, candidate: exp.id })
+                : undefined;
+            print(o.json, { experiment: exp, result: resultCheck, ...(comparison ? { comparison } : {}) }, () => [formatExperiment(exp), ...(comparison ? ["", formatComparison(comparison)] : [])].join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("experiments [dataset]").description("The workflow's experiment history (one dataset's, when named)"))
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o);
+            const list = await listExperiments(store, workflowId, dataset ? { dataset } : {});
+            const baselineId = dataset ? (await getHeader(store, workflowId, dataset))?.baselineId : undefined;
+            print(o.json, list, () => formatExperiments(list, baselineId));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withStoreOptions(ev.command("compare <base> <candidate>").description("How one experiment differs from another"))
+        .action(async (base, candidate, o) => {
+        try {
+            const c = await evalCompare({ store: await openStore(o), base, candidate });
+            print(o.json, c, () => formatComparison(c));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withStoreOptions(ev.command("migrate").description("Copy v1 storage (evals/datasets/…) into the per-workflow layout"))
+        .action(async (o) => {
+        try {
+            const r = await evalMigrate({ store: await openStore(o) });
+            print(o.json, r, () => formatMigration(r));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+}
+//# sourceMappingURL=eval.js.map