@cef-ai/cli 4.2.0 → 4.3.0

package/dist/commands/eval.js

@@ -8,6 +8,14 @@
  *   cef eval run <dataset> [--dataset-version N] [--workflow-version v] [--repeats 3] [--concurrency 4] [--baseline exp]
  *   cef eval experiments [dataset]                       experiment history
  *   cef eval compare <a> <b>                             how experiment b differs from a
+ *   cef eval show <experiment>                           one experiment's results, per case
+ *   cef eval cancel <experiment> --yes                   mark a stuck/running experiment cancelled (ROC's "Mark cancelled")
+ *   cef eval baseline <dataset> [exp] [--reset]          set (or clear) the dataset's baseline
+ *   cef eval versions <dataset>                          the dataset's version history
+ *   cef eval archive <dataset> <caseId> [--undo]         hide a case from new versions
+ *   cef eval check <dataset> [--workflow-version v]      Result-compatibility check (what run prints first)
+ *   cef eval capture <dataset> <context...> [--id c] [--workflow-version v] [--dry-run]
+ *                                                        build cases from real runs (ROC's "Add from executions")
  *
  * Every command is about one workflow: `--workflow <id>`, by default the one
  * `cef.config.ts` (or the build output) in the current directory declares.
@@ -35,11 +43,11 @@
  */
 import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
-import { canonicalJson, casesFromTable, checkResultCompatibility, compareExperiments, createDataset, cutVersion, deleteCase, findExperiment, getCaseRevision, getHeader, getVersion, listArchivedCaseIds, listCaseRevisions, listDatasets, listDeletedCaseIds, listExperiments, listRunRecords, listVersions, listWorkflows, loadWorkingSet, parseCase, parseDelimited, parseHeader, resultFieldsFromGraph, runExperiment, saveCase, sha256Hex, unarchiveCase, undeleteCase, updateHeader, } from "@cef-ai/eval";
+import { addCase, archiveCase, canonicalJson, caseFromRun, capturedFromEvents, casesFromTable, checkResultCompatibility, compareExperiments, createDataset, cutVersion, deleteCase, findExperiment, getCaseRevision, getHeader, isCaseId, listCaseIds, slugCaseId, uniqueCaseId, getVersion, listArchivedCaseIds, listCaseRevisions, listDatasets, listDeletedCaseIds, listExperiments, listRunRecords, listVersions, listWorkflows, loadWorkingSet, markCancelled, parseCase, parseDelimited, parseHeader, resultFieldsFromGraph, runExperiment, saveCase, setBaseline, sha256Hex, unarchiveCase, undeleteCase, updateHeader, versionHistory, } from "@cef-ai/eval";
 import { Browse, Events } from "@cef-ai/vault-sdk/internal";
 import { loadWallet } from "../lib/eval-auth.js";
 import { openVault, vaultEvalStore } from "../lib/eval-vault.js";
-import { fetchOrchestrator, getDeploymentSet, liveOf, nodeWorkflowTarget, workflowAgentId, } from "../lib/eval-target.js";
+import { allEvents, fetchOrchestrator, getDeploymentSet, liveOf, nodeWorkflowTarget, workflowAgentId, } from "../lib/eval-target.js";
 import { DEFAULT_WORKFLOW_SCOPE, localGraph, localWorkflowScope, resolveWorkflowId, } from "../lib/eval-workflow.js";
 import { envConfig } from "../lib/env.js";
 /* ------------------------------------------------------------------------ */
@@ -147,6 +155,88 @@ export async function evalImport(opts) {
     }
     return { created, added, skipped, warnings, ...(cut ? { cut } : {}) };
 }
+/**
+ * Build eval cases from real runs — the CLI equivalent of ROC's "Add from
+ * executions". For each context it reads the run's event stream from the vault
+ * (`vaultEvents`; no orchestrator, no `--as-pubkey`) and WRITES the resulting
+ * cases:
+ *   - input  = the run's `workflow.start` event payload (an inline `graph` dropped);
+ *   - output = the `workflow.completed` event whose `runId` matches this run
+ *              (older runners that stamp no `runId` are not matched), turned into
+ *              the expectation to be edited afterwards.
+ * `--workflow-version` is stamped on each case's `source` but does NOT change the
+ * read scope. Unlike ROC's `captureRun`, which reads the runner's first task,
+ * this reads the `workflow.start` event.
+ *
+ * IDEMPOTENT: a context already recorded as a case's `source.context` is skipped,
+ * so re-running the same contexts writes nothing; the same context named twice in
+ * one call yields one case (the first occurrence).
+ *
+ * PLAN THEN WRITE, like ROC's `planCases`: every case is built and its id
+ * resolved before anything is saved, and `--id` is validated up front (format,
+ * single context, not already taken) before any event is read, so a refusal never
+ * leaves half the runs written. A run with no `workflow.start` cannot become a
+ * case — it is skipped with a note, not fatal. `dryRun` stops after planning.
+ * Auto ids come from the context (`slugCaseId`) and are made unique against the
+ * dataset's live, archived and deleted ids and the batch so far; the write itself
+ * uses `addCase`, which refuses an id that already exists.
+ */
+export async function evalCapture(opts) {
+    if (opts.id !== undefined && opts.contexts.length !== 1) {
+        throw new Error("[cef eval] --id names one case; pass a single context");
+    }
+    if (!(await getHeader(opts.store, opts.workflowId, opts.dataset))) {
+        throw new Error(`[cef eval] workflow "${opts.workflowId}" has no dataset "${opts.dataset}"`);
+    }
+    // Every id a new case may not take: live, archived and deleted alike; plus the
+    // working set, to skip contexts already captured as a case's source.
+    const [live, deleted, working] = await Promise.all([
+        listCaseIds(opts.store, opts.workflowId, opts.dataset, { includeArchived: true }),
+        listDeletedCaseIds(opts.store, opts.workflowId, opts.dataset),
+        loadWorkingSet(opts.store, opts.workflowId, opts.dataset),
+    ]);
+    const taken = new Set([...live, ...deleted]);
+    // `--id` up front, before any event is read: format, and not already taken.
+    if (opts.id !== undefined) {
+        if (!isCaseId(opts.id))
+            throw new Error(`[cef eval] "${opts.id}" is not a case id`);
+        if (taken.has(opts.id))
+            throw new Error(`[cef eval] "${opts.id}" already exists in ${opts.dataset}`);
+    }
+    const captured = new Set(working.cases.map(({ case: c }) => c.source?.context).filter((c) => !!c));
+    const planned = [];
+    const skipped = [];
+    const seen = new Set();
+    for (const context of opts.contexts) {
+        if (seen.has(context))
+            continue; // the same context twice in one call → one case
+        seen.add(context);
+        if (captured.has(context)) {
+            skipped.push(`${context}: already captured as a case — nothing to add`);
+            continue;
+        }
+        const run = capturedFromEvents(await opts.vaultEvents(context), context, opts.workflowVersion);
+        if (run.problem) {
+            skipped.push(`${context}: ${run.problem}`);
+            continue;
+        }
+        const id = opts.id ?? uniqueCaseId(slugCaseId(context), taken);
+        taken.add(id);
+        planned.push(caseFromRun({
+            input: run.input,
+            context,
+            ...(run.output != null ? { output: run.output } : {}),
+            ...(run.workflowVersion ? { workflowVersion: run.workflowVersion } : {}),
+            id,
+        }));
+    }
+    // `addCase` refuses an id that already exists, so a derived or `--id` id that
+    // collides errors instead of appending a revision to someone else's case.
+    if (!opts.dryRun)
+        for (const c of planned)
+            await addCase(opts.store, opts.workflowId, opts.dataset, c);
+    return { dataset: opts.dataset, added: planned, skipped };
+}
 /** The pulled state in the repo dir, when it is this dataset's. */
 async function readState(dir, dataset) {
     const raw = (await readJsonFile(path.join(datasetDir(dir, dataset), VERSIONS_FILE)));
@@ -466,6 +556,73 @@ export async function evalCompare(opts) {
     };
     return compareExperiments(await load(opts.base), await load(opts.candidate));
 }
+/** One experiment with its per-case run records. Found by id alone (no workflow needed), like `compare`. */
+export async function evalShow(opts) {
+    const experiment = await findExperiment(opts.store, opts.id);
+    if (!experiment)
+        throw new Error(`[cef eval] experiment "${opts.id}" not found`);
+    return { experiment, runs: await listRunRecords(opts.store, experiment.workflowId, experiment.id) };
+}
+/**
+ * Mark a stuck/running experiment cancelled — the CLI equivalent of ROC's
+ * "Mark cancelled". Found by id alone (no workflow needed), like `show`. A vault
+ * write: any workflow-version pin the experiment holds is left to expire on its
+ * own, so no pin is released here. Returns both the experiment as it was and as
+ * it is now, so the caller can tell "we cancelled it" from "it was already
+ * terminal" or "it turned out complete and was recorded done".
+ */
+export async function evalCancel(opts) {
+    const before = await findExperiment(opts.store, opts.experiment);
+    if (!before)
+        throw new Error(`[cef eval] experiment "${opts.experiment}" not found`);
+    // The default no-op releasePin: the CLI does not hold or release pins.
+    const after = await markCancelled(opts.store, before.workflowId, opts.experiment, undefined);
+    return { before, after };
+}
+/** Make `experiment` the dataset's baseline, or clear it with `reset`. */
+export async function evalBaseline(opts) {
+    if (opts.experiment && opts.reset) {
+        throw new Error("[cef eval] pass an experiment or --reset, not both");
+    }
+    if (!opts.reset && !opts.experiment) {
+        throw new Error("[cef eval] baseline <experiment> is required (or --reset to clear the baseline)");
+    }
+    if (!opts.reset) {
+        // A baseline is this dataset's own experiment — a typo or another dataset's id is refused.
+        const exp = await findExperiment(opts.store, opts.experiment);
+        if (!exp)
+            throw new Error(`[cef eval] experiment "${opts.experiment}" not found`);
+        if (exp.workflowId !== opts.workflowId || exp.dataset !== opts.dataset) {
+            throw new Error(`[cef eval] experiment "${opts.experiment}" is not an experiment of ${opts.workflowId}/${opts.dataset}`);
+        }
+    }
+    // `null` clears it; `setBaseline` also checks the id's format.
+    return setBaseline(opts.store, opts.workflowId, opts.dataset, opts.reset ? null : opts.experiment);
+}
+/** The dataset's versions, newest first, each with what changed and how many experiments ran it. */
+export async function evalVersions(opts) {
+    // A missing dataset has no versions either way; say so, rather than "No versions." as if it existed.
+    if (!(await getHeader(opts.store, opts.workflowId, opts.dataset))) {
+        throw new Error(`[cef eval] workflow "${opts.workflowId}" has no dataset "${opts.dataset}"`);
+    }
+    return versionHistory(opts.store, opts.workflowId, opts.dataset);
+}
+/** Archive a case (hide it from new versions) or, with `undo`, unarchive it. */
+export async function evalArchive(opts) {
+    // The case must exist in the dataset — live, archived or deleted — so --undo on a
+    // typo is refused rather than printing "✓ Unarchived" for a case that never was.
+    const [ids, deleted] = await Promise.all([
+        listCaseIds(opts.store, opts.workflowId, opts.dataset, { includeArchived: true }),
+        listDeletedCaseIds(opts.store, opts.workflowId, opts.dataset),
+    ]);
+    if (!ids.includes(opts.caseId) && !deleted.includes(opts.caseId)) {
+        throw new Error(`[cef eval] dataset "${opts.dataset}" has no case "${opts.caseId}"`);
+    }
+    if (opts.undo)
+        await unarchiveCase(opts.store, opts.workflowId, opts.dataset, opts.caseId);
+    else
+        await archiveCase(opts.store, opts.workflowId, opts.dataset, opts.caseId);
+}
 /**
  * The Result a workflow version declares, read off its graph:
  *   - the live deployment row's `params.graph` when the version is the live
@@ -547,6 +704,23 @@ export function formatExperiment(e) {
             : []),
     ].join("\n");
 }
+/** An experiment's per-case runs, one row each (case, repeat, status, verdict, cost). */
+export function formatRunRecords(runs) {
+    if (!runs.length)
+        return "No runs.";
+    const verdict = (r) => (r.pass === null ? "–" : r.pass ? "pass" : "FAIL");
+    return table(["CASE", "REPEAT", "STATUS", "VERDICT", "OUTCOME", "DURATION", "TOKENS"], [...runs]
+        .sort((a, b) => a.caseId.localeCompare(b.caseId) || a.repeat - b.repeat)
+        .map((r) => [r.caseId, r.repeat, r.status, verdict(r), r.outcome, ms(r.metrics.durationMs), r.metrics.totalTokens]));
+}
+/** A dataset's version history, newest first — the columns of ROC's History table. */
+export function formatVersionHistory(list) {
+    if (!list.length)
+        return "No versions.";
+    // Compact change counts against the previous version: added / edited / removed.
+    const changes = (v) => `+${v.diff.added.length} ~${v.diff.edited.length} -${v.diff.removed.length}`;
+    return table(["VERSION", "CREATED", "CASES", "CHANGES", "EXPERIMENTS", "NOTE"], list.map((v) => [`v${v.version}`, v.createdAt.replace("T", " ").slice(0, 16), v.cases, changes(v), v.experiments, v.note]));
+}
 /** "Dataset changed between these experiments: …" — or nothing when it did not. */
 export function formatDatasetDiff(c) {
     const d = c.datasetDiff;
@@ -766,6 +940,40 @@ export function addEvalCommands(program, opts) {
             opts.onError(e);
         }
     });
+    withWorkflow(ev.command("capture <dataset> <context...>")
+        .description("Build cases from real runs' input and output, and add them to the dataset (ROC's \"Add from executions\")"))
+        .option("--id <caseId>", "Name the case (only with a single context); otherwise derived from the context")
+        .option("--workflow-version <v>", "Stamp this workflow version on each case's source")
+        .option("--dry-run", "Build and print what would be added, write nothing")
+        .action(async (dataset, contexts, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            // Reads the run's events from the vault and writes cases: no wallet, no orchestrator, no --as-pubkey.
+            const vault = await openVault(o, undefined, opts.fetchImpl);
+            const scope = await scopeOf(o, workflowId);
+            const store = vaultEvalStore(vault, scope);
+            const browse = new Browse(vault.http);
+            const reader = {
+                events: (context, cursor) => browse.listStreamEvents(vault.vaultId, scope, context, { limit: 200, ...(cursor ? { cursor } : {}) }),
+            };
+            const r = await evalCapture({
+                store, vaultEvents: (context) => allEvents(reader, context),
+                workflowId, dataset, contexts,
+                ...(o.id !== undefined ? { id: o.id } : {}),
+                ...(o.workflowVersion ? { workflowVersion: o.workflowVersion } : {}),
+                dryRun: !!o.dryRun,
+            });
+            print(o.json, r, () => [
+                o.dryRun ? "• Dry run — nothing written" : null,
+                `${o.dryRun ? "Would add" : "✓ Added"} ${plural(r.added.length, "case")}`
+                    + (r.added.length ? `: ${r.added.map((c) => c.id).join(", ")}` : ""),
+                ...(r.skipped.length ? ["  skipped:", ...r.skipped.map((s) => `    ${s}`)] : []),
+            ].filter((l) => l !== null).join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
     withDir(withWorkflow(ev.command("pull <dataset>").description("Write a dataset's live cases to the repo dir")))
         .option("--force", "Overwrite case files edited locally since the last pull/push")
         .action(async (dataset, o) => {
@@ -919,5 +1127,124 @@ export function addEvalCommands(program, opts) {
             opts.onError(e);
         }
     });
+    // An experiment id is global, so the workflow is optional here (as in `compare`).
+    withWorkflow(ev.command("show <experiment>").description("One experiment's results: summary, per-field accuracy and per-case runs"))
+        .action(async (experiment, o) => {
+        try {
+            const workflowId = await workflowOf(o).catch(() => undefined);
+            const { experiment: exp, runs } = await evalShow({ store: await openStore(o, workflowId), id: experiment });
+            print(o.json, { experiment: exp, runs }, () => [formatExperiment(exp), "", formatRunRecords(runs)].join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    // An experiment id is global, so the workflow is optional here (as in `show`).
+    withWorkflow(ev.command("cancel <experiment>")
+        .description("Mark a stuck/running experiment cancelled (the CLI's \"Mark cancelled\")"))
+        .option("--yes", "Confirm: write the experiment record cancelled now")
+        .addHelpText("after", "\ncancel writes the experiment record now. Only do this if nothing is still running it\n"
+        + "(a ROC tab or another \"cef eval run --resume\") — that would flip it back to done/error.\n"
+        + "Pass --yes to confirm.\n"
+        + "\nWrites only the experiment record. A workflow-version pin it may hold is not released here;\n"
+        + "it expires on its own (~6h).")
+        .action(async (experiment, o) => {
+        try {
+            // Refuse until confirmed: cancelling a record that is still being run flips it back.
+            if (!o.yes) {
+                throw new Error("[cef eval] cancel writes the experiment record now. Only do this if nothing is still running it "
+                    + "(a ROC tab or another \"cef eval run --resume\") — that would flip it back to done/error. "
+                    + "Pass --yes to confirm.");
+            }
+            const workflowId = await workflowOf(o).catch(() => undefined);
+            const store = await openStore(o, workflowId);
+            const { before, after } = await evalCancel({ store, experiment });
+            // The message turns on what it was before: --json carries the resulting record.
+            print(o.json, after, () => before.status !== "running"
+                ? `• Experiment ${experiment} is ${before.status} — nothing to cancel`
+                : after.status === "done"
+                    ? `✓ Experiment ${experiment} was already complete — recorded as done`
+                    : `✓ Experiment ${experiment} cancelled`);
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    // `[experiment]` is optional so `--reset` needs no placeholder; it is required otherwise.
+    withWorkflow(ev.command("baseline <dataset> [experiment]").description("Set the dataset's baseline experiment (--reset to clear it)"))
+        .option("--reset", "Clear the dataset's baseline")
+        .action(async (dataset, experiment, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o, workflowId);
+            const header = await evalBaseline({ store, workflowId, dataset, ...(experiment ? { experiment } : {}), reset: !!o.reset });
+            print(o.json, header, () => o.reset ? `✓ Baseline for ${dataset} reset` : `✓ Baseline for ${dataset} set to ${experiment}`);
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("versions <dataset>").description("The dataset's version history"))
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o, workflowId);
+            const list = await evalVersions({ store, workflowId, dataset });
+            print(o.json, list, () => formatVersionHistory(list));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("archive <dataset> <caseId>")
+        .description("Archive a case: hide it from new versions (cef eval push --restore also unarchives from the repo dir)"))
+        .option("--undo", "Unarchive the case instead")
+        .action(async (dataset, caseId, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o, workflowId);
+            await evalArchive({ store, workflowId, dataset, caseId, undo: !!o.undo });
+            print(o.json, { dataset, caseId, archived: !o.undo }, () => (o.undo ? `✓ Unarchived ${caseId}` : `✓ Archived ${caseId}`));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("check <dataset>").description("Check the dataset's live cases against the workflow version's declared Result"))
+        .option("--workflow-version <v>", "Workflow version (default: the live one)")
+        .option("--as-pubkey <hex>", "Agent service pubkey the workflow belongs to ($CEF_AS_PUBKEY); needed only to resolve the live version (not with --workflow-version)")
+        .option("--orchestrator <url>", "Orchestrator base URL ($CEF_ENDPOINT; default from --env)")
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            // No wallet needed to read: the vault takes an access token.
+            const vault = await openVault(o, undefined, opts.fetchImpl);
+            const scope = await scopeOf(o, workflowId);
+            const store = vaultEvalStore(vault, scope);
+            const header = await getHeader(store, workflowId, dataset);
+            if (!header)
+                throw new Error(`[cef eval] workflow "${workflowId}" has no dataset "${dataset}"`);
+            let workflowVersion;
+            let result;
+            if (o.workflowVersion) {
+                // A named version resolves from the LOCAL graph (cef.config.ts or dist/) — no
+                // orchestrator, so no --as-pubkey.
+                workflowVersion = o.workflowVersion;
+                result = await resultFieldsFor({ workflowId, workflowVersion, local: local() });
+            }
+            else {
+                // The live version: through the orchestrator (needs --as-pubkey), as `run` does.
+                const { orchestrator, agentId } = openTarget(o, vault, scope, workflowId, opts.fetchImpl);
+                const liveRow = liveOf((await getDeploymentSet(orchestrator, agentId))?.deployments).row;
+                workflowVersion = liveRow?.version ?? "latest";
+                result = await resultFieldsFor({ workflowId, workflowVersion, ...(liveRow ? { liveRow } : {}), local: local() });
+            }
+            const check = await evalResultCheck({ store, workflowId, dataset, workflowVersion, result });
+            print(o.json, check, () => formatResultCheck(check).join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
 }
 //# sourceMappingURL=eval.js.map