@cef-ai/cli 4.1.1 → 4.3.0

package/dist/commands/eval.d.ts

@@ -1,5 +1,7 @@
 import type { Command } from "commander";
-import { type Case, type DatasetSummary, type EvalStore, type Experiment, type ExperimentComparison, type ExperimentProgress, type Header, type Limits, type ResultCompatibility, type ResultField, type WorkflowTarget } from "@cef-ai/eval";
+import { type Case, type DatasetSummary, type EvalStore, type Experiment, type ExperimentComparison, type ExperimentProgress, type Header, type Limits, type ResultCompatibility, type ResultField, type RunRecord, type RunStreamEvent, type VersionHistoryEntry, type WorkflowTarget } from "@cef-ai/eval";
+import { type VaultAccess } from "../lib/eval-vault.js";
+import { type OrchestratorPort } from "../lib/eval-target.js";
 import { type LocalWorkflowOptions } from "../lib/eval-workflow.js";
 export declare const VERSIONS_FILE = ".versions.json";
 /** Where datasets live in a workflow's folder. */
@@ -67,6 +69,52 @@ export declare function evalImport(opts: {
     cut?: string | boolean;
     dryRun?: boolean;
 }): Promise<ImportSummary>;
+export interface CaptureResult {
+    dataset: string;
+    /** The cases that were (or, under `dryRun`, would be) added. */
+    added: Case[];
+    /**
+     * Contexts that produced no case, each with why: no `workflow.start`, or
+     * already captured as an existing case's `source.context` (a re-run).
+     */
+    skipped: string[];
+}
+/**
+ * Build eval cases from real runs — the CLI equivalent of ROC's "Add from
+ * executions". For each context it reads the run's event stream from the vault
+ * (`vaultEvents`; no orchestrator, no `--as-pubkey`) and WRITES the resulting
+ * cases:
+ *   - input  = the run's `workflow.start` event payload (an inline `graph` dropped);
+ *   - output = the `workflow.completed` event whose `runId` matches this run
+ *              (older runners that stamp no `runId` are not matched), turned into
+ *              the expectation to be edited afterwards.
+ * `--workflow-version` is stamped on each case's `source` but does NOT change the
+ * read scope. Unlike ROC's `captureRun`, which reads the runner's first task,
+ * this reads the `workflow.start` event.
+ *
+ * IDEMPOTENT: a context already recorded as a case's `source.context` is skipped,
+ * so re-running the same contexts writes nothing; the same context named twice in
+ * one call yields one case (the first occurrence).
+ *
+ * PLAN THEN WRITE, like ROC's `planCases`: every case is built and its id
+ * resolved before anything is saved, and `--id` is validated up front (format,
+ * single context, not already taken) before any event is read, so a refusal never
+ * leaves half the runs written. A run with no `workflow.start` cannot become a
+ * case — it is skipped with a note, not fatal. `dryRun` stops after planning.
+ * Auto ids come from the context (`slugCaseId`) and are made unique against the
+ * dataset's live, archived and deleted ids and the batch so far; the write itself
+ * uses `addCase`, which refuses an id that already exists.
+ */
+export declare function evalCapture(opts: {
+    store: EvalStore;
+    vaultEvents: (context: string) => Promise<RunStreamEvent[]>;
+    workflowId: string;
+    dataset: string;
+    contexts: string[];
+    id?: string;
+    workflowVersion?: string;
+    dryRun?: boolean;
+}): Promise<CaptureResult>;
 export interface PullResult {
     dir: string;
     cases: number;
@@ -199,6 +247,51 @@ export declare function evalCompare(opts: {
     base: string;
     candidate: string;
 }): Promise<ExperimentComparison>;
+/** One experiment with its per-case run records. Found by id alone (no workflow needed), like `compare`. */
+export declare function evalShow(opts: {
+    store: EvalStore;
+    id: string;
+}): Promise<{
+    experiment: Experiment;
+    runs: RunRecord[];
+}>;
+/**
+ * Mark a stuck/running experiment cancelled — the CLI equivalent of ROC's
+ * "Mark cancelled". Found by id alone (no workflow needed), like `show`. A vault
+ * write: any workflow-version pin the experiment holds is left to expire on its
+ * own, so no pin is released here. Returns both the experiment as it was and as
+ * it is now, so the caller can tell "we cancelled it" from "it was already
+ * terminal" or "it turned out complete and was recorded done".
+ */
+export declare function evalCancel(opts: {
+    store: EvalStore;
+    experiment: string;
+}): Promise<{
+    before: Experiment;
+    after: Experiment;
+}>;
+/** Make `experiment` the dataset's baseline, or clear it with `reset`. */
+export declare function evalBaseline(opts: {
+    store: EvalStore;
+    workflowId: string;
+    dataset: string;
+    experiment?: string;
+    reset?: boolean;
+}): Promise<Header>;
+/** The dataset's versions, newest first, each with what changed and how many experiments ran it. */
+export declare function evalVersions(opts: {
+    store: EvalStore;
+    workflowId: string;
+    dataset: string;
+}): Promise<VersionHistoryEntry[]>;
+/** Archive a case (hide it from new versions) or, with `undo`, unarchive it. */
+export declare function evalArchive(opts: {
+    store: EvalStore;
+    workflowId: string;
+    dataset: string;
+    caseId: string;
+    undo?: boolean;
+}): Promise<void>;
 /**
  * The Result a workflow version declares, read off its graph:
  *   - the live deployment row's `params.graph` when the version is the live
@@ -223,14 +316,33 @@ export declare function resultFieldsFor(opts: {
 }>;
 /** A plain fixed-width table. */
 export declare function table(headers: string[], rows: Array<Array<string | number | null | undefined>>): string;
+/** One line per run: case, repeat, verdict, and the failing fields or the error. */
+export declare function formatCases(runs: RunRecord[]): string;
 export declare function formatDatasets(list: DatasetSummary[]): string;
 export declare function formatExperiments(list: Experiment[], baselineId?: string): string;
 export declare function formatExperiment(e: Experiment): string;
+/** An experiment's per-case runs, one row each (case, repeat, status, verdict, cost). */
+export declare function formatRunRecords(runs: RunRecord[]): string;
+/** A dataset's version history, newest first — the columns of ROC's History table. */
+export declare function formatVersionHistory(list: VersionHistoryEntry[]): string;
 /** "Dataset changed between these experiments: …" — or nothing when it did not. */
 export declare function formatDatasetDiff(c: Pick<ExperimentComparison, "datasetDiff" | "comparedOn">): string | null;
 export declare function formatResultCheck(r: ResultCheck): string[];
 export declare function formatCompatibility(r: ResultCompatibility, version: string): string[];
 export declare function formatComparison(c: ExperimentComparison): string;
+/** What {@link openTarget} reads of the options. */
+export interface TargetOptions {
+    env?: string;
+    asPubkey?: string;
+    accessToken?: string;
+    orchestrator?: string;
+}
+/** The Node target: vault-api for events, the orchestrator for jobs and deployments. */
+export declare function openTarget(o: TargetOptions, vault: VaultAccess, scope: string, workflowId: string, fetchImpl?: typeof fetch): {
+    target: WorkflowTarget;
+    orchestrator: OrchestratorPort;
+    agentId: string;
+};
 export declare function addEvalCommands(program: Command, opts: {
     onError: (e: unknown) => never;
     fetchImpl?: typeof fetch;

package/dist/commands/eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/commands/eval.ts"],"names":[],"mappings":"AAsCA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EA+BL,KAAK,IAAI,EAET,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,UAAU,EACf,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,MAAM,EACX,KAAK,MAAM,EAEX,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAYtB,OAAO,EAKL,KAAK,oBAAoB,EAC1B,MAAM,yBAAyB,CAAC;AAOjC,eAAO,MAAM,aAAa,mBAAmB,CAAC;AAC9C,kDAAkD;AAClD,eAAO,MAAM,oBAAoB,aAAa,CAAC;AAE/C,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,QAAQ,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvF;AAkCD;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/F,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,CAAC,CAuBD;AAQD,wBAAsB,UAAU,CAAC,IAAI,EAAE;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,MAAM,CAAC,CAOlB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,GAAG,GAAG,IAAI,GAAG,MAAM,CAAC;IAChC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,GAAG,OAAO,CAAC,aAAa,CAAC,CAyBzB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,mFAAmF;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,WAAW,CAAC;CACpB;AAiCD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiEtB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qGAAqG;IACrG,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,6FAA6F;IAC7F,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gGAAgG;IAChG,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C;AAgBD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,UAAU,CAAC,CAkItB;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,OAAO,CAAC;IACd,+EAA+E;IAC/E,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,CAAC;CAC9C,GAAG,OAAO,CAAC,UAAU,CAAC,CAsBtB;AAED,+EAA+E;AAC/E,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,WAAW,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAG/B;AAED,gHAAgH;AAChH,MAAM,MAAM,WAAW,GACnB;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,mBAAmB,CAAA;CAAE,GACrG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD,uEAAuE;AACvE,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACjE,GAAG,OAAO,CAAC,WAAW,CAAC,CAKvB;AAED,wBAAsB,WAAW,CAAC,IAAI,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAU5H;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;KAAE,CAAC;IACvE,KAAK,EAAE,oBAAoB,CAAC;CAC7B,GAAG,OAAO,CAAC;IAAE,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAYnE;AAMD,iCAAiC;AACjC,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC,GAAG,MAAM,CAKvG;AAMD,wBAAgB,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,MAAM,CAQ7D;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAWjF;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,CAiBtD;AAED,mFAAmF;AACnF,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,CAAC,oBAAoB,EAAE,aAAa,GAAG,YAAY,CAAC,GAAG,MAAM,GAAG,IAAI,CAW5G;AAED,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,WAAW,GAAG,MAAM,EAAE,CAM1D;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAQrF;AAKD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,oBAAoB,GAAG,MAAM,CA8BhE;AAqGD,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE;IAAE,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,KAAK,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAAE,GACrF,IAAI,CAyON"}
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/commands/eval.ts"],"names":[],"mappings":"AA8CA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EA0CL,KAAK,IAAI,EAET,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,UAAU,EACf,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,MAAM,EACX,KAAK,MAAM,EAEX,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAItB,OAAO,EAA6B,KAAK,WAAW,EAA2B,MAAM,sBAAsB,CAAC;AAC5G,OAAO,EAOL,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAKL,KAAK,oBAAoB,EAC1B,MAAM,yBAAyB,CAAC;AAOjC,eAAO,MAAM,aAAa,mBAAmB,CAAC;AAC9C,kDAAkD;AAClD,eAAO,MAAM,oBAAoB,aAAa,CAAC;AAE/C,gEAAgE;AAChE,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,QAAQ,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvF;AAkCD;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAC/F,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,CAAC,CAuBD;AAQD,wBAAsB,UAAU,CAAC,IAAI,EAAE;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,MAAM,CAAC,CAOlB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,GAAG,GAAG,IAAI,GAAG,MAAM,CAAC;IAChC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,GAAG,OAAO,CAAC,aAAa,CAAC,CAyBzB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,KAAK,EAAE,IAAI,EAAE,CAAC;IACd;;;OAGG;IACH,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE;IACtC,KAAK,EAAE,SAAS,CAAC;IACjB,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAC5D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,GAAG,OAAO,CAAC,aAAa,CAAC,CAqDzB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,mFAAmF;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,WAAW,CAAC;CACpB;AAiCD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiEtB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qGAAqG;IACrG,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,6FAA6F;IAC7F,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,gGAAgG;IAChG,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1C;AAgBD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,UAAU,CAAC,CAkItB;AAED,wBAAsB,OAAO,CAAC,IAAI,EAAE;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,OAAO,CAAC;IACd,+EAA+E;IAC/E,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,CAAC;CAC9C,GAAG,OAAO,CAAC,UAAU,CAAC,CAsBtB;AAED,+EAA+E;AAC/E,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,WAAW,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAG/B;AAED,gHAAgH;AAChH,MAAM,MAAM,WAAW,GACnB;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,mBAAmB,CAAA;CAAE,GACrG;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD,uEAAuE;AACvE,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACjE,GAAG,OAAO,CAAC,WAAW,CAAC,CAKvB;AAED,wBAAsB,WAAW,CAAC,IAAI,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAU5H;AAED,4GAA4G;AAC5G,wBAAsB,QAAQ,CAAC,IAAI,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC;IAAE,UAAU,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,SAAS,EAAE,CAAA;CAAE,CAAC,CAI7H;AAED;;;;;;;GAOG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAAC,CAMnI;AAED,0EAA0E;AAC1E,wBAAsB,YAAY,CAAC,IAAI,EAAE;IACvC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,CAAC,MAAM,CAAC,CAiBlB;AAED,oGAAoG;AACpG,wBAAsB,YAAY,CAAC,IAAI,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAMlI;AAED,gFAAgF;AAChF,wBAAsB,WAAW,CAAC,IAAI,EAAE;IACtC,KAAK,EAAE,SAAS,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB,GAAG,OAAO,CAAC,IAAI,CAAC,CAYhB;AAED;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;KAAE,CAAC;IACvE,KAAK,EAAE,oBAAoB,CAAC;CAC7B,GAAG,OAAO,CAAC;IAAE,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAYnE;AAMD,iCAAiC;AACjC,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC,GAAG,MAAM,CAKvG;AAMD,oFAAoF;AACpF,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CASrD;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,MAAM,CAQ7D;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAWjF;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,CAiBtD;AAED,yFAAyF;AACzF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAS1D;AAED,sFAAsF;AACtF,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,mBAAmB,EAAE,GAAG,MAAM,CAQxE;AAED,mFAAmF;AACnF,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,CAAC,oBAAoB,EAAE,aAAa,GAAG,YAAY,CAAC,GAAG,MAAM,GAAG,IAAI,CAW5G;AAED,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,WAAW,GAAG,MAAM,EAAE,CAM1D;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAQrF;AAKD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,oBAAoB,GAAG,MAAM,CA8BhE;AA+DD,oDAAoD;AACpD,MAAM,WAAW,aAAa;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wFAAwF;AACxF,wBAAgB,UAAU,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,OAAO,KAAK,GAAG;IAC7H,MAAM,EAAE,cAAc,CAAC;IACvB,YAAY,EAAE,gBAAgB,CAAC;IAC/B,OAAO,EAAE,MAAM,CAAC;CACjB,CA4BA;AAKD,wBAAgB,eAAe,CAC7B,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE;IAAE,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,KAAK,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CAAE,GACrF,IAAI,CAsZN"}

package/dist/commands/eval.js

@@ -8,6 +8,14 @@
  *   cef eval run <dataset> [--dataset-version N] [--workflow-version v] [--repeats 3] [--concurrency 4] [--baseline exp]
  *   cef eval experiments [dataset]                       experiment history
  *   cef eval compare <a> <b>                             how experiment b differs from a
+ *   cef eval show <experiment>                           one experiment's results, per case
+ *   cef eval cancel <experiment> --yes                   mark a stuck/running experiment cancelled (ROC's "Mark cancelled")
+ *   cef eval baseline <dataset> [exp] [--reset]          set (or clear) the dataset's baseline
+ *   cef eval versions <dataset>                          the dataset's version history
+ *   cef eval archive <dataset> <caseId> [--undo]         hide a case from new versions
+ *   cef eval check <dataset> [--workflow-version v]      Result-compatibility check (what run prints first)
+ *   cef eval capture <dataset> <context...> [--id c] [--workflow-version v] [--dry-run]
+ *                                                        build cases from real runs (ROC's "Add from executions")
  *
  * Every command is about one workflow: `--workflow <id>`, by default the one
  * `cef.config.ts` (or the build output) in the current directory declares.
@@ -35,11 +43,11 @@
  */
 import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
-import { canonicalJson, casesFromTable, checkResultCompatibility, compareExperiments, createDataset, cutVersion, deleteCase, findExperiment, getCaseRevision, getHeader, getVersion, listArchivedCaseIds, listCaseRevisions, listDatasets, listDeletedCaseIds, listExperiments, listRunRecords, listVersions, listWorkflows, loadWorkingSet, parseCase, parseDelimited, parseHeader, resultFieldsFromGraph, runExperiment, saveCase, sha256Hex, unarchiveCase, undeleteCase, updateHeader, } from "@cef-ai/eval";
+import { addCase, archiveCase, canonicalJson, caseFromRun, capturedFromEvents, casesFromTable, checkResultCompatibility, compareExperiments, createDataset, cutVersion, deleteCase, findExperiment, getCaseRevision, getHeader, isCaseId, listCaseIds, slugCaseId, uniqueCaseId, getVersion, listArchivedCaseIds, listCaseRevisions, listDatasets, listDeletedCaseIds, listExperiments, listRunRecords, listVersions, listWorkflows, loadWorkingSet, markCancelled, parseCase, parseDelimited, parseHeader, resultFieldsFromGraph, runExperiment, saveCase, setBaseline, sha256Hex, unarchiveCase, undeleteCase, updateHeader, versionHistory, } from "@cef-ai/eval";
 import { Browse, Events } from "@cef-ai/vault-sdk/internal";
 import { loadWallet } from "../lib/eval-auth.js";
 import { openVault, vaultEvalStore } from "../lib/eval-vault.js";
-import { fetchOrchestrator, getDeploymentSet, nodeWorkflowTarget, workflowAgentId, } from "../lib/eval-target.js";
+import { allEvents, fetchOrchestrator, getDeploymentSet, liveOf, nodeWorkflowTarget, workflowAgentId, } from "../lib/eval-target.js";
 import { DEFAULT_WORKFLOW_SCOPE, localGraph, localWorkflowScope, resolveWorkflowId, } from "../lib/eval-workflow.js";
 import { envConfig } from "../lib/env.js";
 /* ------------------------------------------------------------------------ */
@@ -147,6 +155,88 @@ export async function evalImport(opts) {
     }
     return { created, added, skipped, warnings, ...(cut ? { cut } : {}) };
 }
+/**
+ * Build eval cases from real runs — the CLI equivalent of ROC's "Add from
+ * executions". For each context it reads the run's event stream from the vault
+ * (`vaultEvents`; no orchestrator, no `--as-pubkey`) and WRITES the resulting
+ * cases:
+ *   - input  = the run's `workflow.start` event payload (an inline `graph` dropped);
+ *   - output = the `workflow.completed` event whose `runId` matches this run
+ *              (older runners that stamp no `runId` are not matched), turned into
+ *              the expectation to be edited afterwards.
+ * `--workflow-version` is stamped on each case's `source` but does NOT change the
+ * read scope. Unlike ROC's `captureRun`, which reads the runner's first task,
+ * this reads the `workflow.start` event.
+ *
+ * IDEMPOTENT: a context already recorded as a case's `source.context` is skipped,
+ * so re-running the same contexts writes nothing; the same context named twice in
+ * one call yields one case (the first occurrence).
+ *
+ * PLAN THEN WRITE, like ROC's `planCases`: every case is built and its id
+ * resolved before anything is saved, and `--id` is validated up front (format,
+ * single context, not already taken) before any event is read, so a refusal never
+ * leaves half the runs written. A run with no `workflow.start` cannot become a
+ * case — it is skipped with a note, not fatal. `dryRun` stops after planning.
+ * Auto ids come from the context (`slugCaseId`) and are made unique against the
+ * dataset's live, archived and deleted ids and the batch so far; the write itself
+ * uses `addCase`, which refuses an id that already exists.
+ */
+export async function evalCapture(opts) {
+    if (opts.id !== undefined && opts.contexts.length !== 1) {
+        throw new Error("[cef eval] --id names one case; pass a single context");
+    }
+    if (!(await getHeader(opts.store, opts.workflowId, opts.dataset))) {
+        throw new Error(`[cef eval] workflow "${opts.workflowId}" has no dataset "${opts.dataset}"`);
+    }
+    // Every id a new case may not take: live, archived and deleted alike; plus the
+    // working set, to skip contexts already captured as a case's source.
+    const [live, deleted, working] = await Promise.all([
+        listCaseIds(opts.store, opts.workflowId, opts.dataset, { includeArchived: true }),
+        listDeletedCaseIds(opts.store, opts.workflowId, opts.dataset),
+        loadWorkingSet(opts.store, opts.workflowId, opts.dataset),
+    ]);
+    const taken = new Set([...live, ...deleted]);
+    // `--id` up front, before any event is read: format, and not already taken.
+    if (opts.id !== undefined) {
+        if (!isCaseId(opts.id))
+            throw new Error(`[cef eval] "${opts.id}" is not a case id`);
+        if (taken.has(opts.id))
+            throw new Error(`[cef eval] "${opts.id}" already exists in ${opts.dataset}`);
+    }
+    const captured = new Set(working.cases.map(({ case: c }) => c.source?.context).filter((c) => !!c));
+    const planned = [];
+    const skipped = [];
+    const seen = new Set();
+    for (const context of opts.contexts) {
+        if (seen.has(context))
+            continue; // the same context twice in one call → one case
+        seen.add(context);
+        if (captured.has(context)) {
+            skipped.push(`${context}: already captured as a case — nothing to add`);
+            continue;
+        }
+        const run = capturedFromEvents(await opts.vaultEvents(context), context, opts.workflowVersion);
+        if (run.problem) {
+            skipped.push(`${context}: ${run.problem}`);
+            continue;
+        }
+        const id = opts.id ?? uniqueCaseId(slugCaseId(context), taken);
+        taken.add(id);
+        planned.push(caseFromRun({
+            input: run.input,
+            context,
+            ...(run.output != null ? { output: run.output } : {}),
+            ...(run.workflowVersion ? { workflowVersion: run.workflowVersion } : {}),
+            id,
+        }));
+    }
+    // `addCase` refuses an id that already exists, so a derived or `--id` id that
+    // collides errors instead of appending a revision to someone else's case.
+    if (!opts.dryRun)
+        for (const c of planned)
+            await addCase(opts.store, opts.workflowId, opts.dataset, c);
+    return { dataset: opts.dataset, added: planned, skipped };
+}
 /** The pulled state in the repo dir, when it is this dataset's. */
 async function readState(dir, dataset) {
     const raw = (await readJsonFile(path.join(datasetDir(dir, dataset), VERSIONS_FILE)));
@@ -466,6 +556,73 @@ export async function evalCompare(opts) {
     };
     return compareExperiments(await load(opts.base), await load(opts.candidate));
 }
+/** One experiment with its per-case run records. Found by id alone (no workflow needed), like `compare`. */
+export async function evalShow(opts) {
+    const experiment = await findExperiment(opts.store, opts.id);
+    if (!experiment)
+        throw new Error(`[cef eval] experiment "${opts.id}" not found`);
+    return { experiment, runs: await listRunRecords(opts.store, experiment.workflowId, experiment.id) };
+}
+/**
+ * Mark a stuck/running experiment cancelled — the CLI equivalent of ROC's
+ * "Mark cancelled". Found by id alone (no workflow needed), like `show`. A vault
+ * write: any workflow-version pin the experiment holds is left to expire on its
+ * own, so no pin is released here. Returns both the experiment as it was and as
+ * it is now, so the caller can tell "we cancelled it" from "it was already
+ * terminal" or "it turned out complete and was recorded done".
+ */
+export async function evalCancel(opts) {
+    const before = await findExperiment(opts.store, opts.experiment);
+    if (!before)
+        throw new Error(`[cef eval] experiment "${opts.experiment}" not found`);
+    // The default no-op releasePin: the CLI does not hold or release pins.
+    const after = await markCancelled(opts.store, before.workflowId, opts.experiment, undefined);
+    return { before, after };
+}
+/** Make `experiment` the dataset's baseline, or clear it with `reset`. */
+export async function evalBaseline(opts) {
+    if (opts.experiment && opts.reset) {
+        throw new Error("[cef eval] pass an experiment or --reset, not both");
+    }
+    if (!opts.reset && !opts.experiment) {
+        throw new Error("[cef eval] baseline <experiment> is required (or --reset to clear the baseline)");
+    }
+    if (!opts.reset) {
+        // A baseline is this dataset's own experiment — a typo or another dataset's id is refused.
+        const exp = await findExperiment(opts.store, opts.experiment);
+        if (!exp)
+            throw new Error(`[cef eval] experiment "${opts.experiment}" not found`);
+        if (exp.workflowId !== opts.workflowId || exp.dataset !== opts.dataset) {
+            throw new Error(`[cef eval] experiment "${opts.experiment}" is not an experiment of ${opts.workflowId}/${opts.dataset}`);
+        }
+    }
+    // `null` clears it; `setBaseline` also checks the id's format.
+    return setBaseline(opts.store, opts.workflowId, opts.dataset, opts.reset ? null : opts.experiment);
+}
+/** The dataset's versions, newest first, each with what changed and how many experiments ran it. */
+export async function evalVersions(opts) {
+    // A missing dataset has no versions either way; say so, rather than "No versions." as if it existed.
+    if (!(await getHeader(opts.store, opts.workflowId, opts.dataset))) {
+        throw new Error(`[cef eval] workflow "${opts.workflowId}" has no dataset "${opts.dataset}"`);
+    }
+    return versionHistory(opts.store, opts.workflowId, opts.dataset);
+}
+/** Archive a case (hide it from new versions) or, with `undo`, unarchive it. */
+export async function evalArchive(opts) {
+    // The case must exist in the dataset — live, archived or deleted — so --undo on a
+    // typo is refused rather than printing "✓ Unarchived" for a case that never was.
+    const [ids, deleted] = await Promise.all([
+        listCaseIds(opts.store, opts.workflowId, opts.dataset, { includeArchived: true }),
+        listDeletedCaseIds(opts.store, opts.workflowId, opts.dataset),
+    ]);
+    if (!ids.includes(opts.caseId) && !deleted.includes(opts.caseId)) {
+        throw new Error(`[cef eval] dataset "${opts.dataset}" has no case "${opts.caseId}"`);
+    }
+    if (opts.undo)
+        await unarchiveCase(opts.store, opts.workflowId, opts.dataset, opts.caseId);
+    else
+        await archiveCase(opts.store, opts.workflowId, opts.dataset, opts.caseId);
+}
 /**
  * The Result a workflow version declares, read off its graph:
  *   - the live deployment row's `params.graph` when the version is the live
@@ -501,6 +658,18 @@ export function table(headers, rows) {
 const pct = (n) => (n === null || n === undefined ? null : `${Math.round(n * 1000) / 10}%`);
 const ms = (n) => (n === null || n === undefined ? null : n >= 1000 ? `${(n / 1000).toFixed(1)}s` : `${n}ms`);
 const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
+/** One line per run: case, repeat, verdict, and the failing fields or the error. */
+export function formatCases(runs) {
+    if (runs.length === 0)
+        return "No runs.";
+    return table(["case", "#", "status", "verdict", "why"], runs.map((r) => [
+        r.caseId,
+        r.repeat,
+        r.status,
+        r.pass === null ? "n/a" : r.pass ? "pass" : "FAIL",
+        r.error ?? r.scores.filter((s) => s.pass === false).map((s) => `${s.name}=${JSON.stringify(s.value)}`).join(", "),
+    ]));
+}
 export function formatDatasets(list) {
     if (!list.length)
         return "No datasets.";
@@ -535,6 +704,23 @@ export function formatExperiment(e) {
             : []),
     ].join("\n");
 }
+/** An experiment's per-case runs, one row each (case, repeat, status, verdict, cost). */
+export function formatRunRecords(runs) {
+    if (!runs.length)
+        return "No runs.";
+    const verdict = (r) => (r.pass === null ? "–" : r.pass ? "pass" : "FAIL");
+    return table(["CASE", "REPEAT", "STATUS", "VERDICT", "OUTCOME", "DURATION", "TOKENS"], [...runs]
+        .sort((a, b) => a.caseId.localeCompare(b.caseId) || a.repeat - b.repeat)
+        .map((r) => [r.caseId, r.repeat, r.status, verdict(r), r.outcome, ms(r.metrics.durationMs), r.metrics.totalTokens]));
+}
+/** A dataset's version history, newest first — the columns of ROC's History table. */
+export function formatVersionHistory(list) {
+    if (!list.length)
+        return "No versions.";
+    // Compact change counts against the previous version: added / edited / removed.
+    const changes = (v) => `+${v.diff.added.length} ~${v.diff.edited.length} -${v.diff.removed.length}`;
+    return table(["VERSION", "CREATED", "CASES", "CHANGES", "EXPERIMENTS", "NOTE"], list.map((v) => [`v${v.version}`, v.createdAt.replace("T", " ").slice(0, 16), v.cases, changes(v), v.experiments, v.note]));
+}
 /** "Dataset changed between these experiments: …" — or nothing when it did not. */
 export function formatDatasetDiff(c) {
     const d = c.datasetDiff;
@@ -630,7 +816,7 @@ function withDir(cmd) {
     return cmd.option("--dir <dir>", "Directory holding the workflow's datasets", DEFAULT_DATASETS_DIR);
 }
 /** The Node target: vault-api for events, the orchestrator for jobs and deployments. */
-function openTarget(o, vault, scope, workflowId, fetchImpl) {
+export function openTarget(o, vault, scope, workflowId, fetchImpl) {
     const env = envConfig(o.env);
     const asPubKey = o.asPubkey ?? process.env.CEF_AS_PUBKEY;
     if (!asPubKey)
@@ -754,6 +940,40 @@ export function addEvalCommands(program, opts) {
             opts.onError(e);
         }
     });
+    withWorkflow(ev.command("capture <dataset> <context...>")
+        .description("Build cases from real runs' input and output, and add them to the dataset (ROC's \"Add from executions\")"))
+        .option("--id <caseId>", "Name the case (only with a single context); otherwise derived from the context")
+        .option("--workflow-version <v>", "Stamp this workflow version on each case's source")
+        .option("--dry-run", "Build and print what would be added, write nothing")
+        .action(async (dataset, contexts, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            // Reads the run's events from the vault and writes cases: no wallet, no orchestrator, no --as-pubkey.
+            const vault = await openVault(o, undefined, opts.fetchImpl);
+            const scope = await scopeOf(o, workflowId);
+            const store = vaultEvalStore(vault, scope);
+            const browse = new Browse(vault.http);
+            const reader = {
+                events: (context, cursor) => browse.listStreamEvents(vault.vaultId, scope, context, { limit: 200, ...(cursor ? { cursor } : {}) }),
+            };
+            const r = await evalCapture({
+                store, vaultEvents: (context) => allEvents(reader, context),
+                workflowId, dataset, contexts,
+                ...(o.id !== undefined ? { id: o.id } : {}),
+                ...(o.workflowVersion ? { workflowVersion: o.workflowVersion } : {}),
+                dryRun: !!o.dryRun,
+            });
+            print(o.json, r, () => [
+                o.dryRun ? "• Dry run — nothing written" : null,
+                `${o.dryRun ? "Would add" : "✓ Added"} ${plural(r.added.length, "case")}`
+                    + (r.added.length ? `: ${r.added.map((c) => c.id).join(", ")}` : ""),
+                ...(r.skipped.length ? ["  skipped:", ...r.skipped.map((s) => `    ${s}`)] : []),
+            ].filter((l) => l !== null).join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
     withDir(withWorkflow(ev.command("pull <dataset>").description("Write a dataset's live cases to the repo dir")))
         .option("--force", "Overwrite case files edited locally since the last pull/push")
         .action(async (dataset, o) => {
@@ -831,8 +1051,7 @@ export function addEvalCommands(program, opts) {
             if (!header)
                 throw new Error(`[cef eval] workflow "${workflowId}" has no dataset "${dataset}"`);
             const { target, orchestrator, agentId } = openTarget(o, vault, scope, workflowId, opts.fetchImpl);
-            const liveRow = (await getDeploymentSet(orchestrator, agentId))?.deployments
-                .find((d) => d.targeting === "" && d.enabled !== false);
+            const liveRow = liveOf((await getDeploymentSet(orchestrator, agentId))?.deployments).row;
             const live = liveRow?.version ?? null;
             const workflowVersion = o.workflowVersion ?? live ?? "latest";
             const isLive = o.workflowVersion === undefined || o.workflowVersion === live;
@@ -867,6 +1086,23 @@ export function addEvalCommands(program, opts) {
             opts.onError(e);
         }
     });
+    withWorkflow(ev.command("cases <experiment>").description("An experiment's runs, case by case (the failed ones with --failed)"))
+        .option("--failed", "Only the runs that failed or errored")
+        .action(async (experiment, o) => {
+        try {
+            const workflowId = await workflowOf(o).catch(() => undefined);
+            const store = await openStore(o, workflowId);
+            const exp = await findExperiment(store, experiment);
+            if (!exp)
+                throw new Error(`[cef eval] experiment "${experiment}" not found`);
+            const runs = (await listRunRecords(store, exp.workflowId, exp.id))
+                .filter((r) => !o.failed || r.pass === false || r.status !== "done");
+            print(o.json, runs, () => formatCases(runs));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
     withWorkflow(ev.command("experiments [dataset]").description("The workflow's experiment history (one dataset's, when named)"))
         .action(async (dataset, o) => {
         try {
@@ -891,5 +1127,124 @@ export function addEvalCommands(program, opts) {
             opts.onError(e);
         }
     });
+    // An experiment id is global, so the workflow is optional here (as in `compare`).
+    withWorkflow(ev.command("show <experiment>").description("One experiment's results: summary, per-field accuracy and per-case runs"))
+        .action(async (experiment, o) => {
+        try {
+            const workflowId = await workflowOf(o).catch(() => undefined);
+            const { experiment: exp, runs } = await evalShow({ store: await openStore(o, workflowId), id: experiment });
+            print(o.json, { experiment: exp, runs }, () => [formatExperiment(exp), "", formatRunRecords(runs)].join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    // An experiment id is global, so the workflow is optional here (as in `show`).
+    withWorkflow(ev.command("cancel <experiment>")
+        .description("Mark a stuck/running experiment cancelled (the CLI's \"Mark cancelled\")"))
+        .option("--yes", "Confirm: write the experiment record cancelled now")
+        .addHelpText("after", "\ncancel writes the experiment record now. Only do this if nothing is still running it\n"
+        + "(a ROC tab or another \"cef eval run --resume\") — that would flip it back to done/error.\n"
+        + "Pass --yes to confirm.\n"
+        + "\nWrites only the experiment record. A workflow-version pin it may hold is not released here;\n"
+        + "it expires on its own (~6h).")
+        .action(async (experiment, o) => {
+        try {
+            // Refuse until confirmed: cancelling a record that is still being run flips it back.
+            if (!o.yes) {
+                throw new Error("[cef eval] cancel writes the experiment record now. Only do this if nothing is still running it "
+                    + "(a ROC tab or another \"cef eval run --resume\") — that would flip it back to done/error. "
+                    + "Pass --yes to confirm.");
+            }
+            const workflowId = await workflowOf(o).catch(() => undefined);
+            const store = await openStore(o, workflowId);
+            const { before, after } = await evalCancel({ store, experiment });
+            // The message turns on what it was before: --json carries the resulting record.
+            print(o.json, after, () => before.status !== "running"
+                ? `• Experiment ${experiment} is ${before.status} — nothing to cancel`
+                : after.status === "done"
+                    ? `✓ Experiment ${experiment} was already complete — recorded as done`
+                    : `✓ Experiment ${experiment} cancelled`);
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    // `[experiment]` is optional so `--reset` needs no placeholder; it is required otherwise.
+    withWorkflow(ev.command("baseline <dataset> [experiment]").description("Set the dataset's baseline experiment (--reset to clear it)"))
+        .option("--reset", "Clear the dataset's baseline")
+        .action(async (dataset, experiment, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o, workflowId);
+            const header = await evalBaseline({ store, workflowId, dataset, ...(experiment ? { experiment } : {}), reset: !!o.reset });
+            print(o.json, header, () => o.reset ? `✓ Baseline for ${dataset} reset` : `✓ Baseline for ${dataset} set to ${experiment}`);
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("versions <dataset>").description("The dataset's version history"))
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o, workflowId);
+            const list = await evalVersions({ store, workflowId, dataset });
+            print(o.json, list, () => formatVersionHistory(list));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("archive <dataset> <caseId>")
+        .description("Archive a case: hide it from new versions (cef eval push --restore also unarchives from the repo dir)"))
+        .option("--undo", "Unarchive the case instead")
+        .action(async (dataset, caseId, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            const store = await openStore(o, workflowId);
+            await evalArchive({ store, workflowId, dataset, caseId, undo: !!o.undo });
+            print(o.json, { dataset, caseId, archived: !o.undo }, () => (o.undo ? `✓ Unarchived ${caseId}` : `✓ Archived ${caseId}`));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
+    withWorkflow(ev.command("check <dataset>").description("Check the dataset's live cases against the workflow version's declared Result"))
+        .option("--workflow-version <v>", "Workflow version (default: the live one)")
+        .option("--as-pubkey <hex>", "Agent service pubkey the workflow belongs to ($CEF_AS_PUBKEY); needed only to resolve the live version (not with --workflow-version)")
+        .option("--orchestrator <url>", "Orchestrator base URL ($CEF_ENDPOINT; default from --env)")
+        .action(async (dataset, o) => {
+        try {
+            const workflowId = await workflowOf(o);
+            // No wallet needed to read: the vault takes an access token.
+            const vault = await openVault(o, undefined, opts.fetchImpl);
+            const scope = await scopeOf(o, workflowId);
+            const store = vaultEvalStore(vault, scope);
+            const header = await getHeader(store, workflowId, dataset);
+            if (!header)
+                throw new Error(`[cef eval] workflow "${workflowId}" has no dataset "${dataset}"`);
+            let workflowVersion;
+            let result;
+            if (o.workflowVersion) {
+                // A named version resolves from the LOCAL graph (cef.config.ts or dist/) — no
+                // orchestrator, so no --as-pubkey.
+                workflowVersion = o.workflowVersion;
+                result = await resultFieldsFor({ workflowId, workflowVersion, local: local() });
+            }
+            else {
+                // The live version: through the orchestrator (needs --as-pubkey), as `run` does.
+                const { orchestrator, agentId } = openTarget(o, vault, scope, workflowId, opts.fetchImpl);
+                const liveRow = liveOf((await getDeploymentSet(orchestrator, agentId))?.deployments).row;
+                workflowVersion = liveRow?.version ?? "latest";
+                result = await resultFieldsFor({ workflowId, workflowVersion, ...(liveRow ? { liveRow } : {}), local: local() });
+            }
+            const check = await evalResultCheck({ store, workflowId, dataset, workflowVersion, result });
+            print(o.json, check, () => formatResultCheck(check).join("\n"));
+        }
+        catch (e) {
+            opts.onError(e);
+        }
+    });
 }
 //# sourceMappingURL=eval.js.map