@eigenpal/cli 0.10.15 → 0.10.16

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @eigenpal/cli
 
+## 0.10.16
+
+### Patch Changes
+
+- b1f0bca: `eigenpal run <workflow> --example <name>` now actually grades the example
+  instead of printing `PASS` whenever the run merely finished. It runs the
+  workflow's current published version on the server's stored dataset example, so
+  configured evaluators run automatically and the CLI surfaces the real weighted
+  score and per-evaluator pass/fail. Workflows with no evaluators fall back to a
+  structural diff of the output against the example's stored expected output. The
+  summary reports two separate signals: execution health (`ok`/`errored`) and
+  accuracy. Add `--fail-on-mismatch` to exit non-zero when a graded example fails.
+
+  Because the run uses the server-side example, push your latest dataset and
+  evaluators (`eigenpal workflow dataset push`, `eigenpal workflow evaluators
+push`) before running `--example`.
+
 ## 0.10.9
 
 ### Patch Changes

package/dist/cli.js

@@ -183722,7 +183722,7 @@ import { fileURLToPath as fileURLToPath8 } from "url";
 // package.json
 var package_default = {
   name: "@eigenpal/cli",
-  version: "0.10.15",
+  version: "0.10.16",
   description: "Eigenpal CLI — author and run document workflows from your terminal. Agent-ready.",
   keywords: [
     "eigenpal",
@@ -184205,9 +184205,6 @@ function resolveProfileKey(name, profiles) {
 
 // src/lib/ui.ts
 var import_picocolors = __toESM(require_picocolors(), 1);
-function isTTY() {
-  return Boolean(process.stdout.isTTY);
-}
 var ui = {
   ok: import_picocolors.default.green,
   err: import_picocolors.default.red,
@@ -190289,10 +190286,6 @@ function formatEigenpalDirIfAvailable(dir) {
   });
 }
 
-// src/commands/workflow/exec.ts
-init_src();
-import { join as join9 } from "path";
-
 // src/lib/execution-artifacts.ts
 import { existsSync as existsSync13, mkdirSync as mkdirSync5, readdirSync as readdirSync3, readFileSync as readFileSync12, writeFileSync as writeFileSync7 } from "fs";
 import { join as join7 } from "path";
@@ -190418,363 +190411,318 @@ async function writeExecutionArtifacts(client, exampleDir, payload) {
   return executionsDir;
 }
 
-// src/lib/payload.ts
-init_src();
-import { existsSync as existsSync14, readdirSync as readdirSync4, readFileSync as readFileSync13, statSync as statSync4 } from "fs";
-import { join as join8 } from "path";
-var DATASET_EXAMPLES_DIR = ["dataset", "examples"];
-function resolveEvalBaseDir(dir) {
-  const candidates = [join8(dir, ...DATASET_EXAMPLES_DIR), join8(...DATASET_EXAMPLES_DIR)];
-  for (const candidate of candidates) {
-    if (existsSync14(candidate))
-      return candidate;
+// src/lib/grade-example.ts
+var MAX_VALUE_LEN = 60;
+function preview(value) {
+  let text;
+  try {
+    text = typeof value === "string" ? value : JSON.stringify(value);
+  } catch {
+    text = String(value);
   }
-  return null;
+  if (text === undefined)
+    text = String(value);
+  return text.length > MAX_VALUE_LEN ? `${text.slice(0, MAX_VALUE_LEN)}…` : text;
 }
-function getExampleNames(dir, exampleNamesFilter) {
-  const baseDir = resolveEvalBaseDir(dir);
-  if (!baseDir) {
-    throw new Error(`No dataset/examples directory found at ${join8(dir, ...DATASET_EXAMPLES_DIR)}. Run \`eigenpal init workflow <name>\` to scaffold one.`);
-  }
-  const all = readdirSync4(baseDir).filter((name) => statSync4(join8(baseDir, name)).isDirectory() && existsSync14(join8(baseDir, name, "input.json"))).sort((a5, b8) => a5.localeCompare(b8, undefined, { numeric: true }));
-  if (!exampleNamesFilter?.length)
-    return all;
-  return all.filter((n5) => exampleNamesFilter.includes(n5));
+function isPlainObject3(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
 }
-function readInput(exampleDir) {
-  const canonicalPath = join8(exampleDir, "input.json");
-  return readCanonicalInput(exampleDir, canonicalPath);
+function joinPath(path15, key2) {
+  return path15 ? `${path15}.${key2}` : key2;
 }
-function readCanonicalInput(exampleDir, inputPath) {
-  const raw = JSON.parse(readFileSync13(inputPath, "utf-8"));
-  if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
-    throw new Error(`${inputPath} must contain a JSON object.`);
-  }
-  const scalars = {};
-  const files = [];
-  for (const [key2, value] of Object.entries(raw)) {
-    const refs = topLevelFileRefs(value);
-    if (refs.length === 0) {
-      scalars[key2] = materializeNestedFileRefs(exampleDir, inputPath, value);
-      continue;
+function walk2(expected, actual, path15, diffs) {
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) {
+      diffs.push(`${path15 || "(root)"}: expected array, got ${preview(actual)}`);
+      return;
     }
-    for (const ref of refs) {
-      const validation2 = validateScopedArtifactPath(ref, { allowedRoots: ["input"] });
-      if (!validation2.ok) {
-        throw new Error(`${inputPath}: ${key2} must reference files under input/ without path traversal.`);
-      }
-      const relative3 = validation2.value.segments.slice(1).join("/");
-      const filePath = join8(exampleDir, validation2.value.path);
-      const filename = validation2.value.segments.at(-1) ?? relative3;
-      files.push({
-        argument: key2,
-        filename,
-        content: readFileSync13(filePath),
-        mimeType: guessMimeType(filename)
-      });
+    if (expected.length !== actual.length) {
+      diffs.push(`${path15 || "(root)"}: length ${expected.length} != ${actual.length}`);
     }
-  }
-  return { scalars, files };
-}
-function materializeNestedFileRefs(exampleDir, inputPath, value) {
-  if (isScopedFileRef2(value)) {
-    const validation2 = validateScopedArtifactPath(value.$file, { allowedRoots: ["input"] });
-    if (!validation2.ok) {
-      throw new Error(`${inputPath}: $file refs must reference files under input/.`);
+    const overlap = Math.min(expected.length, actual.length);
+    for (let i5 = 0;i5 < overlap; i5++) {
+      walk2(expected[i5], actual[i5], `${path15}[${i5}]`, diffs);
     }
-    const filename = validation2.value.segments.at(-1) ?? "file";
-    const filePath = join8(exampleDir, validation2.value.path);
-    return {
-      $inline: {
-        filename,
-        mimeType: guessMimeType(filename) ?? "application/octet-stream",
-        base64: readFileSync13(filePath).toString("base64")
-      }
-    };
-  }
-  if (Array.isArray(value)) {
-    return value.map((item) => materializeNestedFileRefs(exampleDir, inputPath, item));
+    return;
   }
-  if (value && typeof value === "object") {
-    if (hasFileKey(value)) {
-      throw new Error('file refs must be exact { "$file": "input/..." } objects.');
+  if (isPlainObject3(expected)) {
+    if (!isPlainObject3(actual)) {
+      diffs.push(`${path15 || "(root)"}: expected object, got ${preview(actual)}`);
+      return;
     }
-    const out = {};
-    for (const [key2, item] of Object.entries(value)) {
-      out[key2] = materializeNestedFileRefs(exampleDir, inputPath, item);
+    for (const [key2, value] of Object.entries(expected)) {
+      const childPath = joinPath(path15, key2);
+      if (!(key2 in actual)) {
+        diffs.push(`${childPath}: missing in output`);
+        continue;
+      }
+      walk2(value, actual[key2], childPath, diffs);
     }
-    return out;
-  }
-  return value;
-}
-function topLevelFileRefs(value) {
-  if (isScopedFileRef2(value))
-    return [value.$file];
-  if (hasFileKey(value)) {
-    throw new Error('file refs must be exact { "$file": "input/..." } objects.');
+    return;
   }
-  if (Array.isArray(value) && value.every(isScopedFileRef2))
-    return value.map((item) => item.$file);
-  if (Array.isArray(value) && value.some(hasFileKey)) {
-    throw new Error('file arrays must contain only exact { "$file": "input/..." } entries.');
+  if (!Object.is(expected, actual)) {
+    diffs.push(`${path15 || "(root)"}: ${preview(expected)} != ${preview(actual)}`);
   }
-  return [];
 }
-function isScopedFileRef2(value) {
-  return value !== null && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 1 && typeof value.$file === "string";
-}
-function hasFileKey(value) {
-  return value !== null && typeof value === "object" && !Array.isArray(value) && "$file" in value;
+function gradeAgainstExpected(expected, actual) {
+  const diffs = [];
+  walk2(expected, actual, "", diffs);
+  return { matched: diffs.length === 0, diffs };
 }
-function readOverrides(exampleDir) {
-  const metaPath = join8(exampleDir, "meta.json");
-  if (!existsSync14(metaPath))
-    return null;
-  const raw = JSON.parse(readFileSync13(metaPath, "utf-8"));
-  const ov2 = raw.overrides;
-  if (ov2 && typeof ov2 === "object" && "steps" in ov2 && typeof ov2.steps === "object") {
-    return ov2;
+
+// src/lib/payload.ts
+import { existsSync as existsSync14, readdirSync as readdirSync4, readFileSync as readFileSync13, statSync as statSync4 } from "fs";
+import { join as join8 } from "path";
+var DATASET_EXAMPLES_DIR = ["dataset", "examples"];
+function resolveEvalBaseDir(dir) {
+  const candidates = [join8(dir, ...DATASET_EXAMPLES_DIR), join8(...DATASET_EXAMPLES_DIR)];
+  for (const candidate of candidates) {
+    if (existsSync14(candidate))
+      return candidate;
   }
   return null;
 }
-function buildExamplePayload(exampleDir) {
-  const { scalars, files } = readInput(exampleDir);
-  return {
-    scalars,
-    files,
-    overrides: readOverrides(exampleDir)
-  };
-}
 
-// src/lib/progress-lines.ts
-var SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
-var SPINNER_INTERVAL_MS = 120;
-function writeLineInPlace(text) {
-  process.stdout.write(`\r\x1B[K${text}`);
+// src/commands/workflow/eval-example.ts
+var RUN_POLL_INTERVAL_MS = 2000;
+var RUN_MAX_WAIT_MS = 5 * 60 * 1000;
+var EVAL_GRACE_MS = 90 * 1000;
+var EVAL_POLL_INTERVAL_MS = 2000;
+var EXAMPLE_PAGE_SIZE = 100;
+var TERMINAL = new Set(["completed", "failed", "cancelled", "rejected", "timeout"]);
+function fmtScore(score) {
+  return typeof score === "number" ? score.toFixed(2) : "n/a";
 }
-function cursorUp(n5) {
-  if (n5 > 0)
-    process.stdout.write(`\x1B[${n5}A`);
+function toNumberOrNull(value) {
+  if (typeof value === "number")
+    return Number.isFinite(value) ? value : null;
+  if (typeof value === "string") {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : null;
+  }
+  return null;
 }
-function cursorDown(n5) {
-  if (n5 > 0)
-    process.stdout.write(`\x1B[${n5}B`);
+function runOutput(run) {
+  return run.output !== undefined ? run.output : run.result?.output;
 }
-function updateLineAt(lineIndex, totalLines, text) {
-  const up2 = totalLines - lineIndex;
-  cursorUp(up2);
-  writeLineInPlace(text);
-  cursorDown(up2);
+async function fetchHasEvaluators(client, workflowId) {
+  const cfg = await client.get(`/api/v1/automations/${workflowId}/evaluators`);
+  return Array.isArray(cfg.config?.evaluators) && cfg.config.evaluators.length > 0;
 }
-function createProgressLines(options8) {
-  const {
-    interactive,
-    lineLabels,
-    waitingLabel = "waiting",
-    runningLabel = "running",
-    dim: dim2 = (s5) => s5,
-    spinnerStyle = (s5) => s5
-  } = options8;
-  const totalLines = lineLabels.length;
-  const runningLabels = new Map;
-  let updateLock = Promise.resolve();
-  let spinnerInterval = null;
-  let spinnerFrame = 0;
-  const scheduleUpdate = (fn9) => {
-    const run = updateLock.then(() => fn9());
-    updateLock = run;
-    return run;
-  };
-  const start = () => {
-    if (!interactive || totalLines === 0)
-      return;
-    const wait = dim2(waitingLabel);
-    for (const label2 of lineLabels) {
-      process.stdout.write(`${label2}  ${wait}
-`);
-    }
-  };
-  const setRunning = (lineIndex, customLabel) => {
-    if (!interactive)
-      return;
-    runningLabels.set(lineIndex, customLabel ?? runningLabel);
-    if (spinnerInterval === null) {
-      spinnerInterval = setInterval(() => {
-        updateLock = updateLock.then(() => {
-          if (runningLabels.size === 0) {
-            if (spinnerInterval) {
-              clearInterval(spinnerInterval);
-              spinnerInterval = null;
-            }
-            return;
-          }
-          const frame = SPINNER_FRAMES[spinnerFrame++ % SPINNER_FRAMES.length];
-          for (const [i5, label2] of runningLabels) {
-            const lineLabel = lineLabels[i5] ?? "";
-            updateLineAt(i5, totalLines, `${lineLabel}  ${label2} ${spinnerStyle(frame)}`);
-          }
-        });
-      }, SPINNER_INTERVAL_MS);
+async function resolveExample(client, workflowId, name) {
+  let offset = 0;
+  for (;; ) {
+    const page = await client.get(`/api/v1/automations/${workflowId}/examples`, {
+      limit: String(EXAMPLE_PAGE_SIZE),
+      offset: String(offset)
+    });
+    const rows = page.data ?? [];
+    const hit = rows.find((r5) => r5.name === name);
+    if (hit)
+      return { id: hit.id, expected: hit.expected };
+    if (rows.length < EXAMPLE_PAGE_SIZE)
+      return null;
+    offset += rows.length;
+  }
+}
+async function pollRunTerminal(client, runId) {
+  const start = Date.now();
+  const base = `/api/v1/runs/${encodeURIComponent(runId)}`;
+  for (;; ) {
+    const run = await client.get(base);
+    const status = run.execution?.status ?? run.status;
+    if (run.finished || status && TERMINAL.has(status)) {
+      return await client.get(`${base}?expand=execution`);
     }
-  };
-  const setDone = async (lineIndex, text, extraLines = []) => {
-    if (interactive) {
-      runningLabels.delete(lineIndex);
-      await scheduleUpdate(() => updateLineAt(lineIndex, totalLines, text));
-      for (const line3 of extraLines)
-        console.log(line3);
-    } else {
-      console.log(text);
-      for (const line3 of extraLines)
-        console.log(line3);
+    if (Date.now() - start > RUN_MAX_WAIT_MS) {
+      return { ...run, status: "timeout", execution: { status: "timeout" } };
     }
-  };
-  return {
-    totalLines,
-    start,
-    setRunning,
-    setDone
-  };
-}
-
-// src/commands/workflow/exec.ts
-var POLL_INTERVAL_MS = 2000;
-var MAX_POLL_MS = 5 * 60 * 1000;
-var DEFAULT_CONCURRENCY = 3;
-function runTerminalOutput(run) {
-  return run.output !== undefined ? run.output : run.result?.output;
-}
-function runTerminalError(run) {
-  const error52 = run.error !== undefined ? run.error : run.result?.error;
-  return error52 ?? undefined;
+    await new Promise((r5) => setTimeout(r5, RUN_POLL_INTERVAL_MS));
+  }
 }
-async function pollExecution(client, executionId) {
+async function pollEvalRollup(client, runId) {
   const start = Date.now();
-  while (Date.now() - start < MAX_POLL_MS) {
-    const run = await client.get(`/api/v1/runs/${executionId}`);
-    const status = run.execution?.status ?? run.status;
-    if (run.finished || isTerminalExecutionStatus(status)) {
-      return {
-        executionId: run.id,
-        status: status ?? (run.error ? "failed" : "completed"),
-        output: runTerminalOutput(run),
-        error: runTerminalError(run)
-      };
+  for (;; ) {
+    const run = await client.get(`/api/v1/runs/${encodeURIComponent(runId)}`);
+    if (run.eval && run.eval.passed != null) {
+      return { score: run.eval.score ?? null, passed: run.eval.passed };
     }
-    await new Promise((r5) => setTimeout(r5, POLL_INTERVAL_MS));
+    if (Date.now() - start > EVAL_GRACE_MS)
+      return null;
+    await new Promise((r5) => setTimeout(r5, EVAL_POLL_INTERVAL_MS));
   }
-  return { executionId, status: "timeout", error: "Execution timed out" };
 }
-function toExecutionArtifactPayload(run) {
-  const status = run.execution?.status ?? run.status ?? "unknown";
+async function fetchEvalRows(client, workflowId, batchId, runId) {
+  const detail = await client.get(`/api/v1/automations/${workflowId}/experiments/${batchId}`);
+  return detail.resultsByRun?.[runId] ?? [];
+}
+function toArtifactPayload(run, runId) {
   return {
-    executionId: run.id,
-    status,
+    executionId: runId,
+    status: run.execution?.status ?? run.status ?? "unknown",
     createdAt: run.timing?.createdAt ?? new Date().toISOString(),
     completedAt: run.timing?.completedAt ?? null,
     error: run.error !== undefined ? run.error : run.result?.error ?? null,
-    output: runTerminalOutput(run),
-    stepExecutions: run.execution?.steps ?? []
+    output: runOutput(run),
+    stepExecutions: []
   };
 }
-async function runExec(client, dir, workflowIdOrSlug, exampleNames, options8 = {}) {
-  const workflowId = await resolveWorkflowId(client, workflowIdOrSlug);
-  const names = getExampleNames(dir, exampleNames.length ? exampleNames : undefined);
-  if (names.length === 0)
-    throw new Error("No matching eval examples found.");
-  const requested = options8.concurrencyOverride ?? DEFAULT_CONCURRENCY;
-  const requestedValid = Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_CONCURRENCY;
-  const concurrency = Math.max(1, Math.min(requestedValid, names.length));
+async function runWorkflowExamplesWithEval(client, dir, workflowIdOrSlug, workflowId, exampleNames) {
+  const hasEvaluators = await fetchHasEvaluators(client, workflowId);
+  const mode = hasEvaluators ? "evaluators" : "diff";
+  info(`Running ${ui.bold(`"${workflowIdOrSlug}"`)} ${ui.dim(`(${workflowId}, ${exampleNames.length} example(s))`)}`);
+  dim(hasEvaluators ? "Server example run; grading with configured evaluators." : "Server example run; no evaluators configured, grading by structural diff vs expected.");
   const evalBaseDir = resolveEvalBaseDir(dir);
-  if (!evalBaseDir)
-    throw new Error("No dataset/examples directory found.");
-  info(`Running workflow ${ui.bold(`"${workflowIdOrSlug}"`)} ${ui.dim(`(${workflowId}, ${names.length} example(s), concurrency ${concurrency})`)}`);
-  dim("Using unified run endpoint.");
-  const interactive = isTTY();
-  const progress = createProgressLines({
-    interactive,
-    lineLabels: names,
-    waitingLabel: "waiting",
-    runningLabel: "running",
-    dim: ui.dim,
-    spinnerStyle: ui.info
-  });
-  progress.start();
-  let passed = 0;
-  let failed = 0;
-  let nextIndex = 0;
-  const reportDone = (index, label2) => {
-    if (interactive)
-      progress.setDone(index, label2);
-    else
-      console.log(label2);
-  };
-  const runOne = async () => {
-    while (true) {
-      const index = nextIndex++;
-      if (index >= names.length)
-        return;
-      const name = names[index];
-      const exampleDir = join9(evalBaseDir, name);
-      try {
-        const example = buildExamplePayload(exampleDir);
-        if (interactive)
-          progress.setRunning(index);
-        else
-          console.log(`${ui.dim("→")} ${name}`);
-        const form = await buildRunFormData({
-          target: runStartMultipartTarget({
-            type: "workflow",
-            id: workflowId,
-            version: options8.version
-          }),
-          input: example.scalars,
-          overrides: example.overrides,
-          files: example.files.map((file2) => ({
-            fieldName: file2.argument,
-            content: file2.content,
-            filename: file2.filename,
-            mimeType: file2.mimeType
-          }))
-        });
-        const res = await client.postFormData(runTargetApiPath({ type: "workflow", id: workflowId, version: options8.version }), form);
-        if (typeof res?.id !== "string") {
-          throw new Error("Run API did not return a run id");
-        }
-        const executionId = res.id;
-        process.stderr.write(`  ${ui.dim(`→ ${name}: ${executionId}`)}
+  const results = [];
+  for (const name of exampleNames) {
+    process.stderr.write(`${ui.dim("→")} ${name}
 `);
-        const result = await pollExecution(client, executionId);
-        reportDone(index, result.status === "completed" ? `${name} ${ui.ok("PASS")}` : `${name} ${ui.err("FAIL")} ${result.error ?? result.status}`);
-        if (result.status === "completed")
-          passed++;
-        else
-          failed++;
-        try {
-          const run = await client.get(`/api/v1/runs/${executionId}?expand=execution`);
-          const artifactDir = await writeExecutionArtifacts(client, exampleDir, toExecutionArtifactPayload(run));
-          if (!interactive)
-            console.log(ui.dim(`  Artifacts: ${artifactDir}`));
-        } catch (artErr) {
-          if (!interactive)
-            console.warn(ui.dim(`  Warning: could not write artifacts: ${formatCliError(artErr)}`));
+    try {
+      const example = await resolveExample(client, workflowId, name);
+      if (!example) {
+        throw new Error(`example "${name}" not found in the server dataset for ${workflowId}. ` + `Push it first: eigenpal workflow dataset push ${workflowId}`);
+      }
+      const started = await client.post(`/api/v1/automations/${workflowId}/examples/${example.id}/run`, {});
+      const runId = started.id;
+      if (typeof runId !== "string")
+        throw new Error("Example run did not return a run id");
+      process.stderr.write(`  ${ui.dim(`run: ${runId}`)}
+`);
+      const run = await pollRunTerminal(client, runId);
+      const status = run.execution?.status ?? run.status;
+      const runOk = status === "completed";
+      const result = {
+        name,
+        exampleId: example.id,
+        executionId: runId,
+        run: runOk ? "ok" : "error",
+        runError: runOk ? undefined : run.error ?? status ?? "unknown",
+        mode: "none"
+      };
+      if (runOk && hasEvaluators && started.batchId) {
+        const rollup = await pollEvalRollup(client, runId);
+        if (rollup) {
+          const rows = await fetchEvalRows(client, workflowId, started.batchId, runId);
+          result.mode = "evaluators";
+          result.score = rollup.score;
+          result.passed = rollup.passed;
+          result.evaluators = rows.map((r5) => ({
+            name: r5.evaluatorName ?? "evaluator",
+            score: toNumberOrNull(r5.score),
+            passed: r5.passed ?? null,
+            weight: toNumberOrNull(r5.weight),
+            error: r5.error ?? null
+          }));
+        } else {
+          warn(`  ${name}: no evaluator score after ${EVAL_GRACE_MS / 1000}s; grading by diff.`);
+          gradeByDiff(result, example.expected, runOutput(run));
         }
-      } catch (err2) {
-        reportDone(index, `${name} ${ui.err("FAIL")} ${formatCliError(err2)}`);
-        failed++;
+      } else if (runOk) {
+        gradeByDiff(result, example.expected, runOutput(run));
+      }
+      results.push(result);
+      reportLine(result);
+      if (evalBaseDir) {
+        try {
+          await writeExecutionArtifacts(client, `${evalBaseDir}/${name}`, toArtifactPayload(run, runId));
+        } catch {}
       }
+    } catch (err2) {
+      const result = {
+        name,
+        run: "error",
+        runError: formatCliError(err2),
+        mode: "none"
+      };
+      results.push(result);
+      reportLine(result);
     }
-  };
-  const workers = Array.from({ length: concurrency }, () => runOne());
-  await Promise.all(workers);
-  console.log(`
-${ui.bold("Results:")} ${ui.ok(String(passed))} passed, ${failed > 0 ? ui.err(String(failed)) : ui.ok(String(failed))} failed`);
+  }
+  const summary = summarize(workflowIdOrSlug, mode, results);
+  process.stderr.write(`
+${formatEvalSummary(summary)}
+`);
+  return summary;
+}
+function gradeByDiff(result, expected, output) {
+  if (expected === undefined || expected === null) {
+    result.mode = "none";
+    return;
+  }
+  const graded = gradeAgainstExpected(expected, output);
+  result.mode = "diff";
+  result.matched = graded.matched;
+  result.diffCount = graded.diffs.length;
+  result.diffs = graded.diffs.length ? graded.diffs : undefined;
+}
+function formatExampleLines(result) {
+  if (result.run === "error") {
+    return `${result.name}  ${ui.err(`err ${result.runError ?? ""}`.trimEnd())}`;
+  }
+  if (result.mode === "evaluators") {
+    const verdict = result.passed === true ? ui.ok("PASS") : ui.err("FAIL");
+    const head = `${result.name}  ${ui.ok("ok")}  score ${fmtScore(result.score)}  ${verdict}`;
+    const detail = (result.evaluators ?? []).map((e8) => {
+      const mark = e8.error ? ui.warn("error") : e8.passed === true ? ui.ok("pass") : e8.passed === false ? ui.err("fail") : ui.dim("n/a");
+      return `    ${e8.name}  ${fmtScore(e8.score)}  ${mark}`;
+    });
+    return [head, ...detail].join(`
+`);
+  }
+  if (result.mode === "diff") {
+    const verdict = result.matched ? ui.ok("PASS") : ui.err(`FAIL (${result.diffCount} diff)`);
+    return `${result.name}  ${ui.ok("ok")}  ${verdict}`;
+  }
+  return `${result.name}  ${ui.ok("ok")}  ${ui.dim("-")}`;
+}
+function reportLine(result) {
+  process.stderr.write(`${formatExampleLines(result)}
+`);
+}
+function summarize(workflow2, mode, examples) {
+  const ok2 = examples.filter((r5) => r5.run === "ok").length;
+  const errored = examples.length - ok2;
+  const gradedResults = examples.filter((r5) => r5.mode === "evaluators" || r5.mode === "diff");
+  const passedCases = gradedResults.filter((r5) => r5.mode === "evaluators" ? r5.passed === true : r5.matched === true).length;
+  const scores = gradedResults.filter((r5) => r5.mode === "evaluators" && typeof r5.score === "number").map((r5) => r5.score);
   return {
-    workflow: workflowIdOrSlug,
-    passed,
-    failed,
-    total: names.length
-  };
+    workflow: workflow2,
+    mode,
+    total: examples.length,
+    ok: ok2,
+    errored,
+    graded: gradedResults.length,
+    passedCases,
+    failedCases: gradedResults.length - passedCases,
+    ungraded: ok2 - gradedResults.length,
+    weightedAvg: scores.length ? scores.reduce((s5, n5) => s5 + n5, 0) / scores.length : null,
+    examples,
+    passed: ok2,
+    failed: errored
+  };
+}
+function formatEvalSummary(summary) {
+  const lines = [];
+  let line3 = `${ui.bold("Ran")} ${summary.total}: ${ui.ok(`${summary.ok} ok`)}`;
+  if (summary.errored > 0)
+    line3 += `, ${ui.err(`${summary.errored} errored`)}`;
+  if (summary.graded > 0) {
+    const allPass = summary.failedCases === 0;
+    if (summary.mode === "evaluators") {
+      const cases = `${summary.passedCases}/${summary.graded} cases pass`;
+      const weighted = summary.weightedAvg != null ? `weighted ${summary.weightedAvg.toFixed(2)}, ` : "";
+      line3 += `  ${ui.dim("/")}  ${weighted}${allPass ? ui.ok(cases) : ui.err(cases)}`;
+    } else {
+      const acc = `${summary.passedCases}/${summary.graded} matched`;
+      line3 += `  ${ui.dim("/")}  accuracy ${allPass ? ui.ok(acc) : ui.err(acc)}`;
+    }
+  }
+  lines.push(line3);
+  if (summary.ungraded > 0) {
+    lines.push(ui.dim(`  ${summary.ungraded} ungraded (no stored expected output)`));
+  }
+  return lines.join(`
+`);
 }
 
 // src/commands/workflow/execution.ts
@@ -190783,13 +190731,16 @@ async function runSavedWorkflowExamples(workflow2, examples, opts) {
   try {
     requireApiKey(config2);
     const client = new ApiClient(config2);
-    const summary = await runExec(client, config2.dir, workflow2, examples, {
-      concurrencyOverride: opts.concurrency,
-      version: opts.version
-    });
+    const workflowId = await resolveWorkflowId(client, workflow2);
+    if (opts.version && !["latest", "current", "undefined"].includes(opts.version)) {
+      warn(`--example runs the current published version; ignoring @${opts.version}.`);
+    }
+    const summary = await runWorkflowExamplesWithEval(client, config2.dir, workflow2, workflowId, examples);
     if (opts.json)
       console.log(JSON.stringify(summary, null, 2));
-    if (summary.failed > 0)
+    if (summary.errored > 0)
+      process.exit(1);
+    if (opts.failOnMismatch && summary.failedCases > 0)
       process.exit(1);
   } catch (err2) {
     const msg = err2 instanceof Error ? err2.message : String(err2);
@@ -190802,7 +190753,23 @@ async function runSavedWorkflowExamples(workflow2, examples, opts) {
 
 // src/commands/run.ts
 function registerRunCommands(program2) {
-  addJsonFlag(withBaseUrl(program2.command("run <target>"))).description("Start a workflow or agent run, e.g. workflows.extract-invoice.").option("--input-json <json>", "JSON input object").option("--input-file <field=path>", 'Input file to upload as multipart form-data. Repeat for multiple files; bare paths use field "file".', collectRepeated, []).option("--example <name>", "Run one persisted example by name").option("--dir <dir>", "Local eigenpal directory for workflow examples", undefined).option("--wait", "Poll until the run reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(runTarget));
+  addJsonFlag(withBaseUrl(program2.command("run <target>"))).description("Start a workflow or agent run, e.g. workflows.extract-invoice.").option("--input-json <json>", "JSON input object").option("--input-file <field=path>", 'Input file to upload as multipart form-data. Repeat for multiple files; bare paths use field "file".', collectRepeated, []).option("--example <name>", "Run one persisted example by name").option("--dir <dir>", "Local eigenpal directory for workflow examples", undefined).option("--fail-on-mismatch", "For --example runs, exit non-zero when a graded example fails (evaluator fail, or output mismatch)").option("--wait", "Poll until the run reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(runTarget)).addHelpText("after", `
+Grading --example runs
+  --example runs the workflow's CURRENT published version on the SERVER's stored
+  dataset example, so push your latest dataset/evaluators first
+  (eigenpal workflow dataset push / evaluators push).
+
+  If the workflow has evaluators configured, they run automatically and the CLI
+  shows the real weighted score and per-evaluator pass/fail. If it has no
+  evaluators, the output is graded by a structural diff against the example's
+  stored expected output (every expected field must be present and equal; extra
+  output fields are ignored, arrays match by index). The summary reports two
+  separate signals: execution health ("ok"/"errored") and accuracy.
+
+Exit codes
+  1  a run errored (or, with --fail-on-mismatch, a graded example failed)
+  By default a grading failure is informational and does not change the exit code.
+`);
   addJsonFlag(withBaseUrl(program2.command("rerun <run-id>"))).description("Create a new run from a previous run's stored input snapshot.").option("--version <version>", "Version/source ref for the new run: latest, original, or an explicit ref", "latest").option("--wait", "Poll until the rerun reaches a terminal status").option("--interval <seconds>", "Polling interval in seconds", intArg, 2).option("--max-wait <seconds>", "Maximum wait before exit code 2", intArg, 1800).action(action(rerunTarget));
 }
 async function runTarget(target, opts) {
@@ -190820,7 +190787,8 @@ async function runTarget(target, opts) {
     }
     return runSavedWorkflowExamples(parsed.idOrSlug, [opts.example], {
       ...opts,
-      version: String(parsed.requestedVersion)
+      version: String(parsed.requestedVersion),
+      failOnMismatch: opts.failOnMismatch
     });
   }
   const client = buildClient(opts);
@@ -190865,73 +190833,73 @@ import {
   unlinkSync as unlinkSync2,
   writeFileSync as writeFileSync8
 } from "node:fs";
-import { sep as PATH_SEP, dirname as dirname5, join as join11, relative as relative3, resolve as resolve5 } from "node:path";
+import { sep as PATH_SEP, dirname as dirname5, join as join10, relative as relative3, resolve as resolve5 } from "node:path";
 import { fileURLToPath as fileURLToPath6 } from "node:url";
 
 // src/commands/skill/tools.ts
-import { join as join10 } from "node:path";
+import { join as join9 } from "node:path";
 var SKILL_NAME = "eigenpal";
 var TOOLS = [
   {
     id: "claude",
     label: "Claude Code",
-    relativePath: join10(".claude", "skills", SKILL_NAME),
+    relativePath: join9(".claude", "skills", SKILL_NAME),
     detectPaths: [".claude", "CLAUDE.md"]
   },
   {
     id: "cursor",
     label: "Cursor",
-    relativePath: join10(".cursor", "skills", SKILL_NAME),
+    relativePath: join9(".cursor", "skills", SKILL_NAME),
     detectPaths: [".cursor", ".cursorrules"]
   },
   {
     id: "codex",
     label: "Codex",
-    relativePath: join10(".codex", "skills", SKILL_NAME),
+    relativePath: join9(".codex", "skills", SKILL_NAME),
     detectPaths: [".codex"]
   },
   {
     id: "gemini",
     label: "Gemini CLI",
-    relativePath: join10(".gemini", "skills", SKILL_NAME),
+    relativePath: join9(".gemini", "skills", SKILL_NAME),
     detectPaths: [".gemini", "GEMINI.md"]
   },
   {
     id: "antigravity",
     label: "Antigravity",
-    relativePath: join10(".agent", "skills", SKILL_NAME),
+    relativePath: join9(".agent", "skills", SKILL_NAME),
     detectPaths: [".agent"]
   },
   {
     id: "opencode",
     label: "OpenCode",
-    relativePath: join10(".opencode", "skills", SKILL_NAME),
+    relativePath: join9(".opencode", "skills", SKILL_NAME),
     detectPaths: [".opencode"]
   },
   {
     id: "pi",
     label: "Pi",
-    relativePath: join10(".pi", "skills", SKILL_NAME),
+    relativePath: join9(".pi", "skills", SKILL_NAME),
     detectPaths: [".pi"]
   },
   {
     id: "windsurf",
     label: "Windsurf",
-    relativePath: join10(".windsurf", "skills", SKILL_NAME),
+    relativePath: join9(".windsurf", "skills", SKILL_NAME),
     detectPaths: [".windsurf", ".windsurfrules"]
   },
   {
     id: "github-copilot",
     label: "GitHub Copilot",
-    relativePath: join10(".github", "skills", SKILL_NAME),
+    relativePath: join9(".github", "skills", SKILL_NAME),
     detectPaths: [
-      join10(".github", "copilot-instructions.md"),
-      join10(".github", "instructions"),
-      join10(".github", "prompts"),
-      join10(".github", "agents"),
-      join10(".github", "skills"),
-      join10(".github", "workflows", "copilot-setup-steps.yml"),
-      join10(".github", ".mcp.json")
+      join9(".github", "copilot-instructions.md"),
+      join9(".github", "instructions"),
+      join9(".github", "prompts"),
+      join9(".github", "agents"),
+      join9(".github", "skills"),
+      join9(".github", "workflows", "copilot-setup-steps.yml"),
+      join9(".github", ".mcp.json")
     ]
   }
 ];
@@ -190951,7 +190919,7 @@ function resolveSkillSource() {
   ${distCandidate}`);
 }
 function defaultClaudeTarget() {
-  return join11(process.cwd(), TOOLS[0].relativePath);
+  return join10(process.cwd(), TOOLS[0].relativePath);
 }
 function resolveSkillTarget(opts) {
   return opts.target ?? defaultClaudeTarget();
@@ -190979,8 +190947,8 @@ async function installSkill(opts) {
   let skipped = 0;
   let prompted = 0;
   for (const relativePath of sourceFiles) {
-    const sourcePath = join11(source2, relativePath);
-    const targetPath = join11(target, relativePath);
+    const sourcePath = join10(source2, relativePath);
+    const targetPath = join10(target, relativePath);
     const sourceBytes = templateSourceFile(relativePath, readFileSync14(sourcePath), cliVersion);
     const sourceHash = sha2562(sourceBytes);
     if (!existsSync15(targetPath)) {
@@ -191017,7 +190985,7 @@ async function installSkill(opts) {
   }
   const manifestEntries = [];
   for (const relativePath of sourceFiles) {
-    const targetPath = join11(target, relativePath);
+    const targetPath = join10(target, relativePath);
     if (!existsSync15(targetPath))
       continue;
     manifestEntries.push({ path: relativePath, sha256: sha2562(readFileSync14(targetPath)) });
@@ -191027,7 +190995,7 @@ async function installSkill(opts) {
     installedAt: new Date().toISOString(),
     files: manifestEntries
   };
-  writeFileSync8(join11(target, MANIFEST_NAME), `${JSON.stringify(manifest, null, 2)}
+  writeFileSync8(join10(target, MANIFEST_NAME), `${JSON.stringify(manifest, null, 2)}
 `);
   const displayTarget = relative3(process.cwd(), target) || target;
   success2(`Installed skill at ${ui.bold(displayTarget)} — ${written} written, ${skipped} unchanged${prompted > 0 ? `, ${prompted} prompted` : ""}.`);
@@ -191036,7 +191004,7 @@ async function uninstallSkill(opts) {
   const target = resolveSkillTarget(opts);
   const manifest = readManifest(target);
   if (!manifest) {
-    error51(`No manifest found at ${join11(target, MANIFEST_NAME)}; nothing to uninstall.`);
+    error51(`No manifest found at ${join10(target, MANIFEST_NAME)}; nothing to uninstall.`);
     process.exit(1);
   }
   if (!opts.yes && process.stdin.isTTY) {
@@ -191069,7 +191037,7 @@ async function uninstallSkill(opts) {
     unlinkSync2(path15);
     removed += 1;
   }
-  rmSync2(join11(target, MANIFEST_NAME), { force: true });
+  rmSync2(join10(target, MANIFEST_NAME), { force: true });
   pruneEmptyDirs(target);
   const displayTarget = relative3(process.cwd(), target) || target;
   success2(`Uninstalled skill from ${ui.bold(displayTarget)} — ${removed} removed${kept > 0 ? `, ${kept} user-edited kept` : ""}.`);
@@ -191130,14 +191098,14 @@ async function installSkillTools(opts) {
   for (const tool of toInstall) {
     await installSkill({
       source: opts.source,
-      target: join11(cwd, tool.relativePath),
+      target: join10(cwd, tool.relativePath),
       force: opts.force,
       yes: opts.yes
     });
   }
   for (const tool of toUninstall) {
     await uninstallSkill({
-      target: join11(cwd, tool.relativePath),
+      target: join10(cwd, tool.relativePath),
       yes: true
     });
   }
@@ -191200,7 +191168,7 @@ async function uninstallSkillTools(opts) {
   }
   for (const tool of targets) {
     await uninstallSkill({
-      target: join11(cwd, tool.relativePath),
+      target: join10(cwd, tool.relativePath),
       yes: true
     });
   }
@@ -191208,7 +191176,7 @@ async function uninstallSkillTools(opts) {
 function listSkillTools(opts = {}) {
   const cwd = process.cwd();
   const rows = TOOLS.flatMap((tool) => {
-    const targetDir = join11(cwd, tool.relativePath);
+    const targetDir = join10(cwd, tool.relativePath);
     const manifest = readManifest(targetDir);
     if (!manifest)
       return [];
@@ -191261,10 +191229,10 @@ function hintFor(tool, installed, detected) {
   return tool.relativePath;
 }
 function isInstalled(cwd, tool) {
-  return existsSync15(join11(cwd, tool.relativePath, MANIFEST_NAME));
+  return existsSync15(join10(cwd, tool.relativePath, MANIFEST_NAME));
 }
 function isProjectUsing(cwd, tool) {
-  return tool.detectPaths.some((p4) => existsSync15(join11(cwd, p4)));
+  return tool.detectPaths.some((p4) => existsSync15(join10(cwd, p4)));
 }
 function listFiles(root2) {
   const out = [];
@@ -191272,7 +191240,7 @@ function listFiles(root2) {
   while (stack2.length > 0) {
     const dir = stack2.pop();
     for (const entry of readdirSync5(dir)) {
-      const full = join11(dir, entry);
+      const full = join10(dir, entry);
       const rel = relative3(root2, full);
       const stats = lstatSync(full);
       if (stats.isDirectory()) {
@@ -191287,7 +191255,7 @@ function listFiles(root2) {
   return out.sort();
 }
 function readManifest(target) {
-  const path15 = join11(target, MANIFEST_NAME);
+  const path15 = join10(target, MANIFEST_NAME);
   if (!existsSync15(path15))
     return null;
   try {
@@ -191322,7 +191290,7 @@ function pruneEmptyDirs(root2) {
   const stack2 = [];
   const collect = (dir) => {
     for (const entry of readdirSync5(dir)) {
-      const full = join11(dir, entry);
+      const full = join10(dir, entry);
       if (lstatSync(full).isDirectory())
         collect(full);
     }
@@ -191468,12 +191436,12 @@ import { createInterface as createInterface3 } from "node:readline/promises";
 
 // src/commands/workflow/clear.ts
 import { existsSync as existsSync16, readdirSync as readdirSync6, rmSync as rmSync3, statSync as statSync5 } from "fs";
-import { basename, join as join12 } from "path";
+import { basename, join as join11 } from "path";
 var EMPTY_REPORT = { removed: 0, kept: 0, touched: 0 };
 function getDirectoryEntries(path15) {
   if (!existsSync16(path15))
     return [];
-  return readdirSync6(path15).filter((entry) => statSync5(join12(path15, entry)).isDirectory()).sort();
+  return readdirSync6(path15).filter((entry) => statSync5(join11(path15, entry)).isDirectory()).sort();
 }
 function removeDirEntriesWithRetention(path15, removeAll) {
   const entries = getDirectoryEntries(path15);
@@ -191486,7 +191454,7 @@ function removeDirEntriesWithRetention(path15, removeAll) {
   for (const entry of entries) {
     if (keepSet.has(entry))
       continue;
-    rmSync3(join12(path15, entry), { recursive: true, force: true });
+    rmSync3(join11(path15, entry), { recursive: true, force: true });
     removed++;
   }
   return { removed, kept: keepSet.size, touched: entries.length };
@@ -191513,7 +191481,7 @@ function removeJudgeSummaryFilesWithRetention(exampleDir, removeAll) {
       kept++;
       continue;
     }
-    rmSync3(join12(exampleDir, fileName), { force: true });
+    rmSync3(join11(exampleDir, fileName), { force: true });
     removed++;
   }
   return { removed, kept, touched: stampByFile.size };
@@ -191521,17 +191489,17 @@ function removeJudgeSummaryFilesWithRetention(exampleDir, removeAll) {
 function getTargetExampleDirs(dir, exampleNames) {
   const baseDir = resolveEvalBaseDir(dir);
   if (!baseDir) {
-    error51(`No dataset/examples directory at ${join12(dir, "dataset", "examples")}`);
+    error51(`No dataset/examples directory at ${join11(dir, "dataset", "examples")}`);
     process.exit(1);
   }
-  const all = readdirSync6(baseDir).filter((entry) => statSync5(join12(baseDir, entry)).isDirectory());
+  const all = readdirSync6(baseDir).filter((entry) => statSync5(join11(baseDir, entry)).isDirectory());
   if (exampleNames.length === 0)
-    return all.map((name) => join12(baseDir, name));
+    return all.map((name) => join11(baseDir, name));
   const matched = all.filter((name) => exampleNames.includes(name));
   if (matched.length === 0) {
     warn(`None of [${exampleNames.join(", ")}] match an example under ./dataset/examples/. ` + `Heads up: the \`clear-local\` positional changed — it used to take a workflow slug, ` + `now it takes example names (or omit to clean every example).`);
   }
-  return matched.map((name) => join12(baseDir, name));
+  return matched.map((name) => join11(baseDir, name));
 }
 function runSection(args) {
   let removed = 0;
@@ -191563,8 +191531,8 @@ async function clearEvalOutputs(dir, exampleNames = [], options8 = {}) {
   const exampleDirs = getTargetExampleDirs(dir, exampleNames);
   runSection({
     exampleDirs,
-    clean: (exampleDir) => removeDirEntriesWithRetention(join12(exampleDir, "executions"), removeAll),
-    shouldCount: (exampleDir) => existsSync16(join12(exampleDir, "executions")),
+    clean: (exampleDir) => removeDirEntriesWithRetention(join11(exampleDir, "executions"), removeAll),
+    shouldCount: (exampleDir) => existsSync16(join11(exampleDir, "executions")),
     successLabel: "Execution cleanup complete",
     scannedLabel: "examples scanned:",
     seenLabel: "runs seen:       ",
@@ -192139,7 +192107,7 @@ function looksLikeFreeform(prop) {
 // src/commands/workflow/validation.ts
 var import_yaml8 = __toESM(require_dist4(), 1);
 import { existsSync as existsSync17, readdirSync as readdirSync7, readFileSync as readFileSync15, statSync as statSync6 } from "fs";
-import { join as join13, resolve as resolve6 } from "path";
+import { join as join12, resolve as resolve6 } from "path";
 var NAME_RE = /^[a-z0-9][a-z0-9-_]*$/;
 function isDirSafe(path15, issues, field) {
   try {
@@ -192259,19 +192227,19 @@ function validateDatasetFolder(root2) {
   if (!existsSync17(root2)) {
     return [{ field: root2, message: "Folder not found." }];
   }
-  if (existsSync17(join13(root2, "manifest.json"))) {
+  if (existsSync17(join12(root2, "manifest.json"))) {
     issues.push({
       field: "manifest.json",
       message: "Top-level manifest.json is the legacy format. Re-export the dataset using the folder convention."
     });
   }
-  const examplesDir = join13(root2, "examples");
+  const examplesDir = join12(root2, "examples");
   if (!existsSync17(examplesDir)) {
     issues.push({ field: "examples/", message: "No examples/ directory found at dataset root." });
     return issues;
   }
   for (const exampleName of readdirSync7(examplesDir)) {
-    const exampleFolder = join13(examplesDir, exampleName);
+    const exampleFolder = join12(examplesDir, exampleName);
     if (!isDirSafe(exampleFolder, issues, `examples/${exampleName}`))
       continue;
     if (!NAME_RE.test(exampleName)) {
@@ -192281,7 +192249,7 @@ function validateDatasetFolder(root2) {
       });
       continue;
     }
-    const inputJsonPath = join13(exampleFolder, "input.json");
+    const inputJsonPath = join12(exampleFolder, "input.json");
     const hasInputJson = existsSync17(inputJsonPath);
     if (!hasInputJson) {
       issues.push({
@@ -192308,8 +192276,8 @@ function validateDatasetFolder(root2) {
       });
       continue;
     }
-    const inputDir = join13(exampleFolder, "input");
-    if (existsSync17(join13(inputDir, "arguments.json"))) {
+    const inputDir = join12(exampleFolder, "input");
+    if (existsSync17(join12(inputDir, "arguments.json"))) {
       issues.push({
         field: `examples/${exampleName}/input/arguments.json`,
         message: "Legacy input/arguments.json is no longer accepted. Use input.json instead."
@@ -192324,8 +192292,8 @@ function validateDatasetFolder(root2) {
     })) {
       issues.push(issue2);
     }
-    const expectedDir = join13(exampleFolder, "expected");
-    const expectedJsonPath = join13(exampleFolder, "expected.json");
+    const expectedDir = join12(exampleFolder, "expected");
+    const expectedJsonPath = join12(exampleFolder, "expected.json");
     if (existsSync17(expectedDir)) {
       for (const entry of readdirSync7(expectedDir)) {
         if (entry === "output.json" || entry === "error.json") {
@@ -192335,7 +192303,7 @@ function validateDatasetFolder(root2) {
           });
           continue;
         }
-        const sub = join13(expectedDir, entry);
+        const sub = join12(expectedDir, entry);
         const field = `examples/${exampleName}/expected/${entry}`;
         let isDir;
         try {
@@ -192391,7 +192359,7 @@ function validateDatasetFolder(root2) {
       } else {
         for (const issue2 of validateFileRefs({
           refs: collectDatasetFileRefs(parsed),
-          filesRoot: join13(exampleFolder, "expected"),
+          filesRoot: join12(exampleFolder, "expected"),
           root: "expected",
           fieldPrefix: `examples/${exampleName}/expected.json`
         })) {
@@ -192444,7 +192412,7 @@ function validateFileRefs(args) {
       });
       continue;
     }
-    if (!existsSync17(join13(args.filesRoot, relative4))) {
+    if (!existsSync17(join12(args.filesRoot, relative4))) {
       issues.push({
         field: `${args.fieldPrefix}:${ref.jsonPath}`,
         message: `Referenced file does not exist: ${ref.path}.`
@@ -192498,7 +192466,7 @@ Note
       return;
     }
     const root2 = rootDir(opts);
-    const wfFile = join13(root2, "workflow.yaml");
+    const wfFile = join12(root2, "workflow.yaml");
     const wfResult = validateWorkflowYaml(wfFile);
     const wfPath = relPath(root2, wfFile);
     const wfOk = printIssues("workflow.yaml", wfPath, wfResult.issues);
@@ -192506,8 +192474,8 @@ Note
       printSchemaQualityWarnings(wfPath, wfResult.warnings);
       printInvokeOnlineHint(wfResult.invokeStepCount, opts.online);
     }
-    const evOk = printIssues("evaluators.yaml", relPath(root2, join13(root2, "evaluators.yaml")), validateEvaluatorsYaml(join13(root2, "evaluators.yaml")));
-    const dsOk = printIssues("dataset/", relPath(root2, join13(root2, "dataset")), validateDatasetFolder(join13(root2, "dataset")));
+    const evOk = printIssues("evaluators.yaml", relPath(root2, join12(root2, "evaluators.yaml")), validateEvaluatorsYaml(join12(root2, "evaluators.yaml")));
+    const dsOk = printIssues("dataset/", relPath(root2, join12(root2, "dataset")), validateDatasetFolder(join12(root2, "dataset")));
     let onlineOk = true;
     if (wfOk && opts.online && existsSync17(wfFile)) {
       onlineOk = printIssues("invoke refs (server)", wfPath, await onlineInvokeIssues(readFileSync15(wfFile, "utf-8"), opts));
@@ -192587,7 +192555,7 @@ async function writeOrPrint(out, content) {
 }
 async function readDirAsZip2(dir) {
   const tree = {};
-  async function walk2(current, relPrefix) {
+  async function walk3(current, relPrefix) {
     const entries = await fs17.readdir(current, { withFileTypes: true });
     for (const e8 of entries) {
       if (e8.name.startsWith(".") || e8.name === "node_modules")
@@ -192595,12 +192563,12 @@ async function readDirAsZip2(dir) {
       const full = path15.join(current, e8.name);
       const rel = relPrefix ? `${relPrefix}/${e8.name}` : e8.name;
       if (e8.isDirectory())
-        await walk2(full, rel);
+        await walk3(full, rel);
       else if (e8.isFile())
         tree[rel] = new Uint8Array(await fs17.readFile(full));
     }
   }
-  await walk2(dir, "");
+  await walk3(dir, "");
   if (Object.keys(tree).length === 0) {
     throw new Error(`directory is empty: ${dir}`);
   }

package/dist/skill/reference/cli/run.md

@@ -18,6 +18,7 @@ Start a workflow or agent run, e.g. workflows.extract-invoice.
 | `--input-file <field=path>` | no       | `[]`    | Input file to upload as multipart form-data. Repeat for multiple files; bare paths use field "file". |
 | `--example <name>`          | no       |         | Run one persisted example by name                                                                    |
 | `--dir <dir>`               | no       |         | Local eigenpal directory for workflow examples                                                       |
+| `--fail-on-mismatch`        | no       |         | For --example runs, exit non-zero when a graded example fails (evaluator fail, or output mismatch)   |
 | `--wait`                    | no       |         | Poll until the run reaches a terminal status                                                         |
 | `--interval <seconds>`      | no       | `2`     | Polling interval in seconds                                                                          |
 | `--max-wait <seconds>`      | no       | `1800`  | Maximum wait before exit code 2                                                                      |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eigenpal/cli",
-  "version": "0.10.15",
+  "version": "0.10.16",
   "description": "Eigenpal CLI — author and run document workflows from your terminal. Agent-ready.",
   "keywords": [
     "eigenpal",