@runtypelabs/cli 2.22.17 → 2.23.1

package/dist/index.js

@@ -179,8 +179,8 @@ var CallbackServer = class {
   expectedState;
   constructor() {
     this.app = express();
-    this.codePromise = new Promise((resolve11, reject) => {
-      this.codeResolve = resolve11;
+    this.codePromise = new Promise((resolve12, reject) => {
+      this.codeResolve = resolve12;
       this.codeReject = reject;
     });
     this.app.get("/callback", (req, res) => {
@@ -20404,6 +20404,41 @@ function validateUpsertRecordSourceShape(flowSteps, buckets) {
     }
   }
 }
+function validateStoreVectorSource(flowSteps, buckets, declaredFlowInputs) {
+  const declaredVariables = collectDeclaredFlowInputs(flowSteps, declaredFlowInputs);
+  for (const [stepIndex, step] of flowSteps.entries()) {
+    if (step.enabled === false) continue;
+    if (!isObjectRecord(step.config)) continue;
+    const config3 = step.config;
+    if (step.type === "store-vector") {
+      const rawSource = typeof config3.vectorsSource === "string" ? config3.vectorsSource.trim() : "";
+      if (rawSource) {
+        const templateMatch = rawSource.match(/^\s*\{\{\s*([^}]+?)\s*\}\}\s*$/);
+        const reference = (templateMatch?.[1] ?? rawSource).trim();
+        const classification = classifyVariableReference(reference);
+        if (classification.namespace === "plain" || classification.namespace === "flow") {
+          const baseName = classification.baseName;
+          const rootVariable = baseName.split(".")[0] || "";
+          if (rootVariable && !declaredVariables.has(rootVariable) && !declaredVariables.has(baseName)) {
+            addIssue(
+              "warning",
+              {
+                code: "STORE_VECTOR_SOURCE_UNRESOLVED",
+                message: `Vectors source "${rawSource}" references variable "${rootVariable}", but no earlier step declares an output variable "${rootVariable}" (and it is not a flow input). This will fail at runtime with "Could not resolve vectors". Set a prior step's outputVariable to "${rootVariable}" (typically a generate-embedding step), or declare "${rootVariable}" as a flow input.`,
+                path: `flowSteps[${stepIndex}].config.vectorsSource`,
+                step: { index: stepIndex, name: step.name, type: step.type },
+                details: { vectorsSource: rawSource, rootVariable }
+              },
+              buckets
+            );
+          }
+        }
+      }
+    }
+    const outputVar = getStepOutputVariable(step);
+    if (outputVar) declaredVariables.add(outputVar);
+  }
+}
 function checkConditionExpression(expr, path19, stepRef, buckets) {
   if (typeof expr !== "string" || !expr.includes("{{")) return;
   const match = UNQUOTED_TEMPLATE_BEFORE_OP.exec(expr) || UNQUOTED_TEMPLATE_AFTER_OP.exec(expr);
@@ -21061,6 +21096,7 @@ function collectFlowStructureIssues(flowData, deps, buckets) {
     deps.declaredFlowInputs
   );
   validateUpsertRecordSourceShape(flowData.flowSteps, buckets);
+  validateStoreVectorSource(flowData.flowSteps, buckets, deps.declaredFlowInputs);
   validateConditionExpressions(flowData.flowSteps, buckets, conditionalStepsExceedingDepth);
   return { pendingChecks };
 }
@@ -37587,7 +37623,43 @@ var checkGraderSchema = external_exports.discriminatedUnion("kind", [
   }),
   external_exports.object({ kind: external_exports.literal("latency"), maxMs: external_exports.number().int().positive() }),
   // Today's implicit "success" made explicit: the case produced output without erroring.
-  external_exports.object({ kind: external_exports.literal("no_error") })
+  external_exports.object({ kind: external_exports.literal("no_error") }),
+  // -------------------------------------------------------------------------
+  // Trace checks — deterministic, free, pure assertions over the run's
+  // EXECUTION TRACE (which tools/steps ran, in what order, whether it
+  // completed, what it cost) rather than its final output text. Scored by the
+  // same pure `runCheck` engine against `GradingTarget.trace`. These are the
+  // assertions a string/JSON check can't express (planning doc §3.1).
+  // -------------------------------------------------------------------------
+  // At least one tool call named `name` happened. Optional filters narrow the
+  // match: `input`/`output` deep-equal a call's resolved input/result,
+  // `isError` matches a call's error flag, and `times` asserts the matching
+  // count EXACTLY (omit `times` for "at least once").
+  external_exports.object({
+    kind: external_exports.literal("called_tool"),
+    name: external_exports.string().min(1),
+    input: external_exports.unknown().optional(),
+    output: external_exports.unknown().optional(),
+    isError: external_exports.boolean().optional(),
+    times: external_exports.number().int().positive().optional()
+  }),
+  // No tool named `name` was called.
+  external_exports.object({ kind: external_exports.literal("not_called_tool"), name: external_exports.string().min(1) }),
+  // The run made no tool calls at all.
+  external_exports.object({ kind: external_exports.literal("used_no_tools") }),
+  // The run made at most `max` tool calls.
+  external_exports.object({ kind: external_exports.literal("max_tool_calls"), max: external_exports.number().int().nonnegative() }),
+  // `tools` appears as an ordered SUBSEQUENCE of the tool-call names (other
+  // calls may interleave; relative order of the listed tools must hold).
+  external_exports.object({ kind: external_exports.literal("tool_order"), tools: external_exports.array(external_exports.string()).min(1) }),
+  // A step named (or typed) `name` ran.
+  external_exports.object({ kind: external_exports.literal("ran_step"), name: external_exports.string().min(1) }),
+  // `steps` appears as an ordered SUBSEQUENCE of the steps that ran.
+  external_exports.object({ kind: external_exports.literal("step_order"), steps: external_exports.array(external_exports.string()).min(1) }),
+  // The run completed (finished without erroring and was not left paused).
+  external_exports.object({ kind: external_exports.literal("completed") }),
+  // Total run cost was within `maxUsd` (US dollars).
+  external_exports.object({ kind: external_exports.literal("cost"), maxUsd: external_exports.number().positive() })
 ]);
 var aiGraderSchema = external_exports.object({
   kind: external_exports.literal("ai"),
@@ -38065,6 +38137,9 @@ var MODEL_FAMILY_PROVIDER_IDS = {
   "glm-5-2": {
     "vercel": "zai/glm-5.2"
   },
+  "glm-5-2-fast": {
+    "vercel": "zai/glm-5.2-fast"
+  },
   "glm-5-turbo": {
     "vercel": "zai/glm-5-turbo"
   },
@@ -38074,6 +38149,9 @@ var MODEL_FAMILY_PROVIDER_IDS = {
   "glm-5.2": {
     "vercel": "zai/glm-5.2"
   },
+  "glm-5.2-fast": {
+    "vercel": "zai/glm-5.2-fast"
+  },
   "glm-5v-turbo": {
     "vercel": "zai/glm-5v-turbo"
   },
@@ -38118,6 +38196,12 @@ var MODEL_FAMILY_PROVIDER_IDS = {
   "gpt-4o-mini-search-preview": {
     "vercel": "openai/gpt-4o-mini-search-preview"
   },
+  "gpt-4o-mini-transcribe": {
+    "vercel": "openai/gpt-4o-mini-transcribe"
+  },
+  "gpt-4o-transcribe": {
+    "vercel": "openai/gpt-4o-transcribe"
+  },
   "gpt-5": {
     "openai": "gpt-5",
     "vercel": "openai/gpt-5"
@@ -38405,6 +38489,18 @@ var MODEL_FAMILY_PROVIDER_IDS = {
   "grok-imagine-video-1.5-preview": {
     "vercel": "xai/grok-imagine-video-1.5-preview"
   },
+  "grok-stt": {
+    "vercel": "xai/grok-stt"
+  },
+  "grok-tts": {
+    "vercel": "xai/grok-tts"
+  },
+  "grok-voice-think-fast-1-0": {
+    "vercel": "xai/grok-voice-think-fast-1.0"
+  },
+  "grok-voice-think-fast-1.0": {
+    "vercel": "xai/grok-voice-think-fast-1.0"
+  },
   "imagen-4-0-fast-generate-001": {
     "vercel": "google/imagen-4.0-fast-generate-001"
   },
@@ -39138,6 +39234,12 @@ var MODEL_FAMILY_PROVIDER_IDS = {
   "trinity-mini": {
     "vercel": "arcee-ai/trinity-mini"
   },
+  "tts-1": {
+    "vercel": "openai/tts-1"
+  },
+  "tts-1-hd": {
+    "vercel": "openai/tts-1-hd"
+  },
   "veo-3-0-fast-generate-001": {
     "vercel": "google/veo-3.0-fast-generate-001"
   },
@@ -39234,6 +39336,9 @@ var MODEL_FAMILY_PROVIDER_IDS = {
   "wan-v2.6-t2v": {
     "vercel": "alibaba/wan-v2.6-t2v"
   },
+  "whisper-1": {
+    "vercel": "openai/whisper-1"
+  },
   "zai-org/GLM-5": {
     "togetherai": "togetherai/zai-org/GLM-5"
   },
@@ -42010,7 +42115,7 @@ var FLOW_STEP_TYPE_METADATA = {
     configHints: "provider, query, maxResults, outputVariable"
   },
   "generate-embedding": {
-    description: "Create a vector embedding from text using an embedding model.",
+    description: "Create a vector embedding from text using an embedding model. Writes { embedding, model, dimensions, textLength, metadata } to outputVariable; feed that variable into a store-vector step via vectorsSource.",
     category: "vector",
     isPrompt: false,
     configHints: "inputSource, text, embeddingModel, outputVariable"
@@ -42022,10 +42127,10 @@ var FLOW_STEP_TYPE_METADATA = {
     configHints: "query, limit, threshold, outputVariable"
   },
   "store-vector": {
-    description: "Store vector embeddings in a vector database.",
+    description: "Store vector embeddings in a vector database (pgvector, Weaviate, or Vectorize). vectorsSource accepts a bare variable name, a dot-path, or a {{var}} template, and must resolve to a number[] or an { embedding: number[] } object (the output of a prior generate-embedding step). The vector length must match the target index dimension.",
     category: "vector",
     isPrompt: false,
-    configHints: "vectorsSource, destination, outputVariable"
+    configHints: "vectorsSource, destination, idTemplate, outputVariable"
   },
   crawl: {
     description: "Crawl a website and extract content from pages.",
@@ -44831,14 +44936,14 @@ async function promptConfirm(message, options) {
     output: process.stdout,
     terminal: true
   });
-  return new Promise((resolve11) => {
+  return new Promise((resolve12) => {
     rl.question(chalk3.cyan(`${message} (${hint}): `), (answer) => {
       rl.close();
       const trimmed = answer.trim().toLowerCase();
       if (trimmed === "") {
-        resolve11(defaultYes);
+        resolve12(defaultYes);
       } else {
-        resolve11(trimmed === "y" || trimmed === "yes");
+        resolve12(trimmed === "y" || trimmed === "yes");
       }
     });
   });
@@ -46546,13 +46651,13 @@ promptsCommand.command("delete <id>").description("Delete a prompt").option("--t
     await waitUntilExit2();
     return;
   }
-  const confirmed = await new Promise((resolve11) => {
+  const confirmed = await new Promise((resolve12) => {
     const { unmount } = render4(
       React4.createElement(ConfirmPrompt, {
         message: `Delete prompt ${id}?`,
         defaultValue: false,
         onConfirm: (result) => {
-          resolve11(result);
+          resolve12(result);
           unmount();
         }
       })
@@ -47437,13 +47542,13 @@ secretsCommand.command("delete <id>").description("Delete a secret").option("--t
     await waitUntilExit2();
     return;
   }
-  const confirmed = await new Promise((resolve11) => {
+  const confirmed = await new Promise((resolve12) => {
     const { unmount } = render6(
       React6.createElement(ConfirmPrompt, {
         message: `Delete secret ${id}?`,
         defaultValue: false,
         onConfirm: (result) => {
-          resolve11(result);
+          resolve12(result);
           unmount();
         }
       })
@@ -47855,13 +47960,13 @@ toolsCommand.command("delete <id>").description("Delete a tool").option("--tty",
     await waitUntilExit2();
     return;
   }
-  const confirmed = await new Promise((resolve11) => {
+  const confirmed = await new Promise((resolve12) => {
     const { unmount } = render7(
       React7.createElement(ConfirmPrompt, {
         message: `Delete tool ${id}?`,
         defaultValue: false,
         onConfirm: (result) => {
-          resolve11(result);
+          resolve12(result);
           unmount();
         }
       })
@@ -50283,13 +50388,13 @@ conversationsCommand.command("delete <id>").description("Delete a conversation")
     await waitUntilExit2();
     return;
   }
-  const confirmed = await new Promise((resolve11) => {
+  const confirmed = await new Promise((resolve12) => {
     const { unmount } = render14(
       React14.createElement(ConfirmPrompt, {
         message: `Delete conversation ${id}?`,
         defaultValue: false,
         onConfirm: (result) => {
-          resolve11(result);
+          resolve12(result);
           unmount();
         }
       })
@@ -56154,8 +56259,8 @@ function MarathonApp({
         setIsCheckpointExploring(false);
         setIsTerminalCheckpoint(Boolean(isTerminal));
         isTerminalCheckpointRef.current = Boolean(isTerminal);
-        return new Promise((resolve11) => {
-          checkpointResolveRef.current = resolve11;
+        return new Promise((resolve12) => {
+          checkpointResolveRef.current = resolve12;
         });
       },
       updateMilestone: (milestone) => {
@@ -57565,16 +57670,16 @@ function MarathonStartupShell({
   latestAppPropsRef.current = marathonAppProps;
   useEffect26(() => {
     if (scene !== "app" || !appReadyResolverRef.current) return;
-    const resolve11 = appReadyResolverRef.current;
+    const resolve12 = appReadyResolverRef.current;
     appReadyResolverRef.current = null;
-    resolve11();
+    resolve12();
   }, [scene]);
   const beginTransition = (target) => {
     if (transitionPromiseRef.current) return transitionPromiseRef.current;
     if (target === "app" && !latestAppPropsRef.current) {
       throw new Error("Cannot complete startup before marathon app props are ready.");
     }
-    const promise2 = new Promise((resolve11) => {
+    const promise2 = new Promise((resolve12) => {
       globalThis.setTimeout(() => {
         setPrompt(null);
         setModelChoices(null);
@@ -57595,12 +57700,12 @@ function MarathonStartupShell({
         if (target === "app") {
           appReadyResolverRef.current = () => {
             transitionPromiseRef.current = null;
-            resolve11();
+            resolve12();
           };
         } else {
           dismissResolverRef.current = () => {
             transitionPromiseRef.current = null;
-            resolve11();
+            resolve12();
           };
         }
       }, Math.max(0, MIN_HOLD_MS - (Date.now() - mountedAtRef.current)));
@@ -57617,8 +57722,8 @@ function MarathonStartupShell({
         setModelChoices(null);
         setPrompt(nextPrompt);
         setSelectedPromptIndex(0);
-        return new Promise((resolve11) => {
-          promptResolverRef.current = resolve11;
+        return new Promise((resolve12) => {
+          promptResolverRef.current = resolve12;
         });
       },
       requestModelChoice: async (nextCurrentModel, models) => {
@@ -57626,8 +57731,8 @@ function MarathonStartupShell({
         setPlaybookConfirm(null);
         setCurrentModel(nextCurrentModel);
         setModelChoices(models);
-        return new Promise((resolve11) => {
-          modelResolverRef.current = resolve11;
+        return new Promise((resolve12) => {
+          modelResolverRef.current = resolve12;
         });
       },
       requestPlaybookModelConfirm: async (playbookName, milestoneModels) => {
@@ -57642,8 +57747,8 @@ function MarathonStartupShell({
           // Default selection is the "Confirm" action (first item after milestones)
           selectedIndex: names.length
         });
-        return new Promise((resolve11) => {
-          playbookConfirmResolverRef.current = resolve11;
+        return new Promise((resolve12) => {
+          playbookConfirmResolverRef.current = resolve12;
         });
       },
       completeStartup: () => beginTransition("app"),
@@ -58157,15 +58262,15 @@ async function retryOnNetworkError(fn, opts = {}) {
       }
       const delay = Math.min(baseDelay * 2 ** attempt, maxDelay);
       opts.onRetry?.(attempt + 1, delay, error51);
-      await new Promise((resolve11) => {
+      await new Promise((resolve12) => {
         const signal = opts.signal;
         const onAbort = () => {
           clearTimeout(timer);
-          resolve11();
+          resolve12();
         };
         const timer = setTimeout(() => {
           signal?.removeEventListener("abort", onAbort);
-          resolve11();
+          resolve12();
         }, delay);
         if (signal) {
           if (signal.aborted) onAbort();
@@ -58431,14 +58536,14 @@ async function promptNumericSelect(choices, promptLabel) {
     output: process.stdout,
     terminal: true
   });
-  return new Promise((resolve11) => {
+  return new Promise((resolve12) => {
     const ask = () => {
       rl.question(chalk21.cyan(`
 ${promptLabel} (1-${choices.length}): `), (answer) => {
         const value = parseInt(answer.trim(), 10);
         if (value >= 1 && value <= choices.length) {
           rl.close();
-          resolve11(choices[value - 1].value);
+          resolve12(choices[value - 1].value);
           return;
         }
         console.log(chalk21.red(`Please enter a number between 1 and ${choices.length}.`));
@@ -58466,7 +58571,7 @@ ${message}`));
   const previousRawMode = input.isRaw === true;
   let selectedIndex = 0;
   let renderedLineCount = 0;
-  return new Promise((resolve11) => {
+  return new Promise((resolve12) => {
     const renderMenu = () => {
       if (renderedLineCount > 0) {
         clearRenderedLines(output, renderedLineCount);
@@ -58488,7 +58593,7 @@ ${message}`));
     };
     const finish = (value) => {
       cleanup();
-      resolve11(value);
+      resolve12(value);
     };
     const onKeypress = (_2, key) => {
       if (key.ctrl && key.name === "c") {
@@ -61089,7 +61194,7 @@ async function taskAction(agent, options) {
     waitForUiExit = renderedShell.waitUntilExit;
     rerenderUi = renderedShell.rerender;
     unmountUi = renderedShell.unmount;
-    await new Promise((resolve11) => setTimeout(resolve11, 0));
+    await new Promise((resolve12) => setTimeout(resolve12, 0));
     if (!startupShellRef.current) {
       exitAltScreen();
       unmountUi?.();
@@ -61722,7 +61827,7 @@ Saving state... done. Session saved to ${filePath}`);
       waitForUiExit = renderedApp.waitUntilExit;
       unmountUi = renderedApp.unmount;
     }
-    await new Promise((resolve11) => setTimeout(resolve11, 0));
+    await new Promise((resolve12) => setTimeout(resolve12, 0));
     const streamActions = streamRef.current;
     if (!streamActions) {
       exitAltScreen();
@@ -61901,7 +62006,7 @@ Saving state... done. Session saved to ${filePath}`);
           };
           if (event.phase === "start") {
             currentActions.startContextCompaction(absoluteEvent);
-            await new Promise((resolve11) => setTimeout(resolve11, 0));
+            await new Promise((resolve12) => setTimeout(resolve12, 0));
             return;
           }
           currentActions.finishContextCompaction(absoluteEvent);
@@ -62333,8 +62438,8 @@ Saving state... done. Session saved to ${filePath}`);
       const INDEXING_SETTLE_GRACE_MS = 5e3;
       await Promise.race([
         Promise.allSettled(indexingRequests),
-        new Promise((resolve11) => {
-          const timer = setTimeout(resolve11, INDEXING_SETTLE_GRACE_MS);
+        new Promise((resolve12) => {
+          const timer = setTimeout(resolve12, INDEXING_SETTLE_GRACE_MS);
           timer.unref?.();
         })
       ]);
@@ -63768,7 +63873,122 @@ import React19 from "react";
 import { render as render19 } from "ink";
 import { useState as useState36, useEffect as useEffect30 } from "react";
 import { Text as Text34 } from "ink";
-import { readFileSync as readFileSync16 } from "fs";
+import { readFileSync as readFileSync16, writeFileSync as writeFileSync6, mkdirSync as mkdirSync8 } from "fs";
+import { dirname as dirname8, relative as relative6, resolve as resolve10 } from "path";
+
+// src/lib/eval-discovery.ts
+import { readdirSync as readdirSync6 } from "fs";
+import { join as join11, relative as relative5, sep as sep5 } from "path";
+import { createJiti as createJiti2 } from "jiti";
+var EVAL_FILE_SUFFIXES = [".eval.ts", ".eval.mts"];
+var IGNORED_DIRS = /* @__PURE__ */ new Set([
+  "node_modules",
+  ".git",
+  "dist",
+  "build",
+  ".next",
+  ".turbo",
+  "coverage",
+  ".cache"
+]);
+function isEvalFile(name) {
+  return EVAL_FILE_SUFFIXES.some((suffix) => name.endsWith(suffix));
+}
+function discoverEvalFiles(rootDir) {
+  const found = [];
+  const walk = (dir) => {
+    let entries;
+    try {
+      entries = readdirSync6(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const name = entry.name.toString();
+      if (entry.isDirectory()) {
+        if (!IGNORED_DIRS.has(name) && !name.startsWith(".")) {
+          walk(join11(dir, name));
+        }
+      } else if (entry.isFile() && isEvalFile(name)) {
+        found.push(join11(dir, name));
+      }
+    }
+  };
+  walk(rootDir);
+  return found.sort();
+}
+function filterEvalFilesByPrefix(files, rootDir, prefix) {
+  const normalized = prefix.replace(/\//g, sep5);
+  return files.filter((file2) => {
+    const rel = relative5(rootDir, file2);
+    const base = rel.split(sep5).pop() ?? rel;
+    return rel.startsWith(normalized) || base.includes(prefix);
+  });
+}
+var jitiLoader2;
+async function loadEvalDefinition(filePath) {
+  jitiLoader2 ??= createJiti2(import.meta.url, { interopDefault: false });
+  let mod;
+  try {
+    mod = await jitiLoader2.import(filePath);
+  } catch (error51) {
+    const message = error51 instanceof Error ? error51.message : String(error51);
+    throw new Error(`Failed to load eval file ${filePath}: ${message}`, { cause: error51 });
+  }
+  const exported = mod.default;
+  if (exported === void 0 || exported === null) {
+    throw new Error(
+      `Eval file ${filePath} must have a default export: the result of defineEval({...}) from @runtypelabs/sdk.`
+    );
+  }
+  if (typeof exported !== "object" || !("target" in exported) || !("cases" in exported) || !Array.isArray(exported.cases)) {
+    throw new Error(
+      `Eval file ${filePath} default export is not an eval definition. Export defineEval({ target, cases, ... }).`
+    );
+  }
+  return exported;
+}
+
+// src/lib/eval-junit.ts
+function escapeXml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+}
+function buildJUnitXml(suites) {
+  const totalTests = suites.reduce((sum, s) => sum + s.cases.length, 0);
+  const totalFailures = suites.reduce(
+    (sum, s) => sum + s.cases.filter((c2) => !c2.passed).length,
+    0
+  );
+  const lines = [];
+  lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+  lines.push(
+    `<testsuites name="runtype-eval" tests="${totalTests}" failures="${totalFailures}">`
+  );
+  for (const suite of suites) {
+    const suiteFailures = suite.cases.filter((c2) => !c2.passed).length;
+    lines.push(
+      `  <testsuite name="${escapeXml(suite.name)}" tests="${suite.cases.length}" failures="${suiteFailures}">`
+    );
+    for (const testCase of suite.cases) {
+      const classname = escapeXml(suite.name);
+      const caseName = escapeXml(testCase.name);
+      if (testCase.passed) {
+        lines.push(`    <testcase name="${caseName}" classname="${classname}" />`);
+      } else {
+        const reason = testCase.errored ? "Case errored while producing output" : "One or more graders failed";
+        const detail = testCase.failedGraders.length ? testCase.failedGraders.join("\n") : reason;
+        lines.push(`    <testcase name="${caseName}" classname="${classname}">`);
+        lines.push(`      <failure message="${escapeXml(reason)}">${escapeXml(detail)}</failure>`);
+        lines.push("    </testcase>");
+      }
+    }
+    lines.push("  </testsuite>");
+  }
+  lines.push("</testsuites>");
+  return lines.join("\n") + "\n";
+}
+
+// src/commands/eval.ts
 var evalCommand = new Command20("eval").description("Manage evaluations");
 evalCommand.command("submit").description("Submit an eval batch").requiredOption("-f, --flow <id>", "Flow ID to evaluate").requiredOption("-r, --records <file>", "JSON file with record IDs").option("-n, --name <name>", "Eval batch name").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (options) => {
   const apiKey = await ensureAuth();
@@ -64067,6 +64287,143 @@ evalCommand.command("compare <groupId>").description("Compare evals in a group")
   const { waitUntilExit } = render19(React19.createElement(App));
   await waitUntilExit();
 });
+var EVAL_RUN_EXIT = { pass: 0, fail: 1, config: 2 };
+function failConfig(message) {
+  console.error(chalk27.red(`Error: ${message}`));
+  process.exit(EVAL_RUN_EXIT.config);
+}
+function printSuiteResult(rootDir, outcome) {
+  const { definition, result } = outcome;
+  const rel = relative6(rootDir, outcome.file) || outcome.file;
+  const badge = result.passed ? chalk27.green("PASS") : chalk27.red("FAIL");
+  const score = `${result.passedCases}/${result.totalCases}`;
+  console.log(`${badge} ${chalk27.bold(definition.name)} ${chalk27.gray(`(${score} cases \xB7 ${rel})`)}`);
+  for (const testCase of result.cases) {
+    const caseBadge = testCase.passed ? chalk27.green("  \u2713") : chalk27.red("  \u2717");
+    console.log(`${caseBadge} ${testCase.name}`);
+    if (!testCase.passed) {
+      for (const outcomeItem of testCase.outcomes.filter((o) => !o.passed)) {
+        const reason = outcomeItem.reasoning ? `: ${outcomeItem.reasoning}` : "";
+        console.log(chalk27.red(`      \u2717 ${outcomeItem.kind}${reason}`));
+      }
+      if (testCase.errored) {
+        console.log(chalk27.red(`      \u2717 errored: ${testCase.outputExcerpt.slice(0, 200)}`));
+      }
+    }
+  }
+}
+function toJUnitSuite(outcome) {
+  return {
+    name: outcome.definition.name,
+    cases: outcome.result.cases.map((c2) => ({
+      name: c2.name,
+      passed: c2.passed,
+      errored: c2.errored,
+      failedGraders: c2.outcomes.filter((o) => !o.passed).map((o) => `${o.kind}${o.reasoning ? `: ${o.reasoning}` : ""}`)
+    }))
+  };
+}
+evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval suites (**/*.eval.ts) as a CI gate (exit 0 pass / 1 fail / 2 config)").option("--strict", "Fail on soft-threshold misses too (no-op until severity lands)").option("--virtual", "Run inline without persisting a suite/batch to the dashboard").option("--junit <path>", "Write JUnit XML results to <path>").option("--url <api>", "Override the API base URL (e.g. staging)").option("--cwd <dir>", "Directory to discover *.eval.ts under (default: current directory)").action(
+  async (idOrDirPrefix, options) => {
+    const apiKey = await ensureAuth();
+    if (!apiKey) {
+      failConfig("Not authenticated. Set RUNTYPE_API_KEY or run `runtype login`.");
+    }
+    const rootDir = resolve10(options.cwd ?? process.cwd());
+    const allFiles = discoverEvalFiles(rootDir);
+    if (allFiles.length === 0) {
+      failConfig(`No *.eval.ts files found under ${rootDir}.`);
+    }
+    const loadAll = async (files) => {
+      const loaded2 = [];
+      for (const file2 of files) {
+        try {
+          loaded2.push({ file: file2, def: await loadEvalDefinition(file2) });
+        } catch (error51) {
+          failConfig(error51 instanceof Error ? error51.message : String(error51));
+        }
+      }
+      return loaded2;
+    };
+    let loaded;
+    if (idOrDirPrefix) {
+      const byPath = filterEvalFilesByPrefix(allFiles, rootDir, idOrDirPrefix);
+      loaded = byPath.length > 0 ? await loadAll(byPath) : [];
+      if (loaded.length === 0) {
+        const everything = await loadAll(allFiles);
+        loaded = everything.filter(
+          ({ def }) => def.name === idOrDirPrefix || def.name.includes(idOrDirPrefix)
+        );
+      }
+      if (loaded.length === 0) {
+        failConfig(`No eval suites matched "${idOrDirPrefix}".`);
+      }
+    } else {
+      loaded = await loadAll(allFiles);
+    }
+    if (options.strict) {
+      console.log(
+        chalk27.gray("Note: --strict has no effect yet (grader severity lands in a later increment).")
+      );
+    }
+    const client = createCliClient(apiKey, options.url);
+    const outcomes = [];
+    for (const { file: file2, def } of loaded) {
+      const runVirtual = options.virtual || def.virtual;
+      try {
+        let result;
+        if (runVirtual) {
+          result = await client.post("/eval/run", { definition: def });
+        } else {
+          const ensured = await client.post("/eval/ensure", {
+            name: def.name,
+            definition: def
+          });
+          if (!ensured.suiteId) {
+            throw new Error(`ensure did not return a suiteId (result: ${ensured.result})`);
+          }
+          result = await client.post("/eval/run", { suiteId: ensured.suiteId });
+        }
+        outcomes.push({ file: file2, definition: def, result });
+        printSuiteResult(rootDir, { file: file2, definition: def, result });
+      } catch (error51) {
+        failConfig(
+          `Suite "${def.name}" (${relative6(rootDir, file2) || file2}): ` + (error51 instanceof Error ? error51.message : String(error51))
+        );
+      }
+    }
+    if (options.junit) {
+      try {
+        const xml = buildJUnitXml(outcomes.map(toJUnitSuite));
+        const outPath = resolve10(options.junit);
+        mkdirSync8(dirname8(outPath), { recursive: true });
+        writeFileSync6(outPath, xml, "utf-8");
+        console.log(chalk27.gray(`JUnit results written to ${options.junit}`));
+      } catch (error51) {
+        failConfig(`Failed to write JUnit report: ${error51 instanceof Error ? error51.message : String(error51)}`);
+      }
+    }
+    const failedSuites = outcomes.filter((o) => !o.result.passed);
+    const totalCases = outcomes.reduce((sum, o) => sum + o.result.totalCases, 0);
+    const passedCases = outcomes.reduce((sum, o) => sum + o.result.passedCases, 0);
+    console.log();
+    if (failedSuites.length === 0) {
+      console.log(
+        chalk27.green(
+          `All ${outcomes.length} suite(s) passed (${passedCases}/${totalCases} cases).`
+        )
+      );
+      process.exit(EVAL_RUN_EXIT.pass);
+    } else {
+      console.log(
+        chalk27.red(
+          `${failedSuites.length} of ${outcomes.length} suite(s) failed (${passedCases}/${totalCases} cases passed).`
+        )
+      );
+      process.exit(EVAL_RUN_EXIT.fail);
+    }
+  }
+);
 
 // src/commands/api-keys.ts
 import { Command as Command21 } from "commander";
@@ -64333,13 +64690,13 @@ apiKeysCommand.command("delete <id>").description("Delete an API key").option("-
     await waitUntilExit2();
     return;
   }
-  const confirmed = await new Promise((resolve11) => {
+  const confirmed = await new Promise((resolve12) => {
     const { unmount } = render20(
       React20.createElement(ConfirmPrompt, {
         message: `Delete API key ${id}?`,
         defaultValue: false,
         onConfirm: (result) => {
-          resolve11(result);
+          resolve12(result);
           unmount();
         }
       })
@@ -64805,13 +65162,13 @@ clientTokensCommand.command("delete <id>").description("Delete a client token").
     await waitUntilExit2();
     return;
   }
-  const confirmed = await new Promise((resolve11) => {
+  const confirmed = await new Promise((resolve12) => {
     const { unmount } = render21(
       React21.createElement(ConfirmPrompt, {
         message: `Delete client token ${id}?`,
         defaultValue: false,
         onConfirm: (result) => {
-          resolve11(result);
+          resolve12(result);
           unmount();
         }
       })
@@ -65433,7 +65790,7 @@ async function runPersonaInit(options) {
 // src/lib/persona-demo.ts
 import { createServer } from "http";
 import { createServer as createNetServer } from "net";
-import { existsSync as existsSync13, mkdirSync as mkdirSync8, readFileSync as readFileSync17, writeFileSync as writeFileSync6 } from "fs";
+import { existsSync as existsSync13, mkdirSync as mkdirSync9, readFileSync as readFileSync17, writeFileSync as writeFileSync7 } from "fs";
 import path16 from "path";
 var PERSONA_DEMO_DEFAULT_DIR = "persona-demo";
 var PERSONA_DEMO_DEFAULT_PORT = 43110;
@@ -65469,12 +65826,12 @@ function ensurePersonaDemoCanWrite(directory, force) {
 }
 function writePersonaDemoPage(options) {
   ensurePersonaDemoCanWrite(options.directory, options.force);
-  mkdirSync8(options.directory, { recursive: true });
+  mkdirSync9(options.directory, { recursive: true });
   const filePath = personaDemoIndexPath(options.directory);
   const readmePath = personaDemoReadmePath(options.directory);
-  writeFileSync6(filePath, `${options.html.trim()}
+  writeFileSync7(filePath, `${options.html.trim()}
 `, "utf8");
-  writeFileSync6(readmePath, `${options.readme.trim()}
+  writeFileSync7(readmePath, `${options.readme.trim()}
 `, "utf8");
   return { directory: options.directory, filePath, readmePath };
 }
@@ -65488,13 +65845,13 @@ async function findAvailablePersonaDemoPort(preferredPort = PERSONA_DEMO_DEFAULT
   throw new Error(`Could not find an available local port starting at ${preferredPort}`);
 }
 function canListenOnPort(port) {
-  return new Promise((resolve11) => {
+  return new Promise((resolve12) => {
     const server = createNetServer();
     server.once("error", () => {
-      resolve11(false);
+      resolve12(false);
     });
     server.once("listening", () => {
-      server.close(() => resolve11(true));
+      server.close(() => resolve12(true));
     });
     server.listen(port, "127.0.0.1");
   });
@@ -65518,10 +65875,10 @@ async function startPersonaDemoServer(options) {
   return {
     url: `http://127.0.0.1:${options.port}/`,
     port: options.port,
-    close: () => new Promise((resolve11, reject) => {
+    close: () => new Promise((resolve12, reject) => {
       server.close((error51) => {
         if (error51) reject(error51);
-        else resolve11();
+        else resolve12();
       });
     })
   };
@@ -65530,11 +65887,11 @@ function readIndexHtml(filePath) {
   return readFileSync17(filePath, "utf8");
 }
 function listen(server, port) {
-  return new Promise((resolve11, reject) => {
+  return new Promise((resolve12, reject) => {
     server.once("error", reject);
     server.listen(port, "127.0.0.1", () => {
       server.off("error", reject);
-      resolve11();
+      resolve12();
     });
   });
 }
@@ -66139,8 +66496,8 @@ function parsePort(raw, fallback) {
 }
 async function promptLine(rl, question, defaultValue) {
   const hint = defaultValue ? chalk30.dim(` (${defaultValue})`) : "";
-  const answer = await new Promise((resolve11) => {
-    rl.question(`${question}${hint}: `, resolve11);
+  const answer = await new Promise((resolve12) => {
+    rl.question(`${question}${hint}: `, resolve12);
   });
   const trimmed = answer.trim();
   if (!trimmed && defaultValue !== void 0) {
@@ -66150,8 +66507,8 @@ async function promptLine(rl, question, defaultValue) {
 }
 async function promptConfirm2(rl, message, defaultYes = false) {
   const hint = defaultYes ? chalk30.dim(" (Y/n)") : chalk30.dim(" (y/N)");
-  const answer = await new Promise((resolve11) => {
-    rl.question(`${message}${hint}: `, resolve11);
+  const answer = await new Promise((resolve12) => {
+    rl.question(`${message}${hint}: `, resolve12);
   });
   const t = answer.trim().toLowerCase();
   if (t === "") return defaultYes;
@@ -66239,13 +66596,13 @@ Dashboard: ${initial.dashboardUrl}`));
     output: process.stdout,
     terminal: true
   });
-  await new Promise((resolve11) => {
+  await new Promise((resolve12) => {
     let finished = false;
     const finish = () => {
       if (finished) return;
       finished = true;
       rl.close();
-      resolve11();
+      resolve12();
     };
     const runAction = async (name) => {
       if (name === "q") {
@@ -67835,11 +68192,11 @@ async function runTail(options) {
       process.stderr.write(
         (useColor ? chalk35.gray(`Retrying in ${delay / 1e3}s (attempt ${attempt}/${MAX_ATTEMPTS})...`) : `Retrying in ${delay / 1e3}s (attempt ${attempt}/${MAX_ATTEMPTS})...`) + "\n"
       );
-      await new Promise((resolve11) => {
-        const timer = setTimeout(resolve11, delay);
+      await new Promise((resolve12) => {
+        const timer = setTimeout(resolve12, delay);
         const onAbort = () => {
           clearTimeout(timer);
-          resolve11();
+          resolve12();
         };
         controller.signal.addEventListener("abort", onAbort, { once: true });
       });
@@ -69181,7 +69538,7 @@ import { Command as Command31 } from "commander";
 import chalk39 from "chalk";
 
 // src/lib/skills-install.ts
-import { mkdirSync as mkdirSync10, readFileSync as readFileSync19, writeFileSync as writeFileSync8 } from "fs";
+import { mkdirSync as mkdirSync11, readFileSync as readFileSync19, writeFileSync as writeFileSync9 } from "fs";
 import path18 from "path";
 import readline4 from "readline";
 import chalk38 from "chalk";
@@ -69205,8 +69562,8 @@ function readSkillsInstallMetadata() {
 }
 function writeSkillsInstallMetadata(metadata) {
   try {
-    mkdirSync10(path18.dirname(metadataPath()), { recursive: true });
-    writeFileSync8(metadataPath(), JSON.stringify(metadata, null, 2));
+    mkdirSync11(path18.dirname(metadataPath()), { recursive: true });
+    writeFileSync9(metadataPath(), JSON.stringify(metadata, null, 2));
   } catch {
   }
 }
@@ -69256,8 +69613,8 @@ async function promptConfirm3(message, defaultYes = true) {
   const rl = readline4.createInterface({ input: process.stdin, output: process.stdout });
   const hint = defaultYes ? chalk38.dim(" (Y/n)") : chalk38.dim(" (y/N)");
   try {
-    const answer = await new Promise((resolve11) => {
-      rl.question(`${message}${hint}: `, resolve11);
+    const answer = await new Promise((resolve12) => {
+      rl.question(`${message}${hint}: `, resolve12);
     });
     const t = answer.trim().toLowerCase();
     if (t === "") return defaultYes;
@@ -69462,8 +69819,8 @@ skillsCommand.command("install").description(`Install Runtype skills (${SKILLS_R
 );
 
 // src/commands/apps.ts
-import { readdirSync as readdirSync6, readFileSync as readFileSync20, lstatSync, statSync as statSync7, existsSync as existsSync15 } from "fs";
-import { join as join12, relative as relative5 } from "path";
+import { readdirSync as readdirSync7, readFileSync as readFileSync20, lstatSync, statSync as statSync7, existsSync as existsSync15 } from "fs";
+import { join as join13, relative as relative7 } from "path";
 import { Command as Command32 } from "commander";
 import chalk40 from "chalk";
 import { zipSync } from "fflate";
@@ -69474,16 +69831,16 @@ var MANIFEST_FILENAME = "runtype.app.json";
 function collectBundleFiles(dir) {
   const zippable = {};
   const walk = (current) => {
-    for (const entry of readdirSync6(current)) {
+    for (const entry of readdirSync7(current)) {
       if (entry.startsWith(".")) continue;
-      const full = join12(current, entry);
+      const full = join13(current, entry);
       const stat = lstatSync(full);
       if (stat.isSymbolicLink()) continue;
       if (stat.isDirectory()) {
         if (entry === "node_modules") continue;
         walk(full);
       } else {
-        zippable[relative5(dir, full).split("\\").join("/")] = readFileSync20(full);
+        zippable[relative7(dir, full).split("\\").join("/")] = readFileSync20(full);
       }
     }
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runtypelabs/cli",
-  "version": "2.22.17",
+  "version": "2.23.1",
   "description": "Command-line interface for Runtype AI platform",
   "type": "module",
   "main": "dist/index.js",
@@ -24,7 +24,7 @@
     "rosie-skills": "0.8.1",
     "yaml": "^2.9.0",
     "@runtypelabs/ink-components": "0.3.4",
-    "@runtypelabs/sdk": "5.3.1",
+    "@runtypelabs/sdk": "5.5.0",
     "@runtypelabs/terminal-animations": "0.2.1"
   },
   "devDependencies": {
@@ -39,7 +39,7 @@
     "tsx": "^4.7.1",
     "typescript": "^6.0.3",
     "vitest": "^4.1.0",
-    "@runtypelabs/shared": "1.42.3"
+    "@runtypelabs/shared": "1.42.5"
   },
   "engines": {
     "node": ">=22.0.0"