@runtypelabs/cli 2.23.1 → 2.24.1

package/dist/index.js

@@ -549,7 +549,7 @@ import { RuntypeApiError } from "@runtypelabs/sdk";
 // src/lib/sdk-client.ts
 import { createClient } from "@runtypelabs/sdk";
 
-// ../shared/dist/chunk-MYXAZOFG.mjs
+// ../shared/dist/chunk-6FWWIIGE.mjs
 var travrseConfig = {
   name: "Travrse",
   nameLower: "travrse",
@@ -573,7 +573,7 @@ var runtypeConfig = {
   name: "Runtype",
   nameLower: "runtype",
   tagline: "Your AI Product Platform",
-  description: "Build and deploy AI experiences faster than ever. Powerful API for developers, intuitive dashboard for teams.",
+  description: "The fastest way to ship agentic products. Prototype in minutes, prove it with evals, and ship to every surface on our cloud or yours.",
   url: "https://www.runtype.com",
   website: "https://runtype.com",
   domain: "runtype.com",
@@ -15566,7 +15566,7 @@ function resolveFallback(parsed, context, tracking) {
 }
 var templateEngine = new SimpleTemplateEngine();
 
-// ../shared/dist/chunk-MMDRPM6Q.mjs
+// ../shared/dist/chunk-ONVX3P3J.mjs
 var tokensSchema = external_exports.object({ input: external_exports.number(), output: external_exports.number() });
 var wireStopReasonSchema = external_exports.enum([
   "end_turn",
@@ -15904,7 +15904,17 @@ var unifiedAwaitEventSchema = external_exports.object({
   parameters: external_exports.record(external_exports.string(), external_exports.unknown()).optional(),
   awaitedAt: external_exports.string().optional(),
   origin: external_exports.enum(["webmcp", "sdk"]).optional(),
-  pageOrigin: external_exports.string().optional()
+  pageOrigin: external_exports.string().optional(),
+  // Discriminates an AUTO-RESUMING durable pause (`crawl_pending` /
+  // `durable_poll`, resumed server-side by the crawl / wait-until
+  // Workflow) from a CLIENT-RESOLVABLE local-tool / WebMCP await (absent), so a
+  // streaming consumer does not render a "resume" prompt for a pause that
+  // resumes itself. Mirrors the JSON `pausedReason` discriminator
+  // (execution-pipeline.ts); the server-resumed pauses carry `crawlId` /
+  // `stepId` instead of the tool fields. UX context; never a control signal.
+  awaitReason: external_exports.string().optional(),
+  crawlId: external_exports.string().optional(),
+  stepId: external_exports.string().optional()
 });
 var unifiedErrorEventSchema = external_exports.object({
   ...unifiedBase,
@@ -16400,7 +16410,17 @@ var flowAwaitEventSchema = external_exports.object({
   parameters: external_exports.record(external_exports.string(), external_exports.unknown()).optional(),
   awaitedAt: external_exports.string(),
   origin: external_exports.enum(["webmcp", "sdk"]).optional(),
-  pageOrigin: external_exports.string().optional()
+  pageOrigin: external_exports.string().optional(),
+  // Discriminates an AUTO-RESUMING durable pause (`crawl_pending` /
+  // `durable_poll`, resumed server-side by the crawl / wait-until
+  // Workflow) from a CLIENT-RESOLVABLE local-tool / WebMCP await (absent). The
+  // server-resumed pauses carry `crawlId` / `stepId` instead of the tool fields.
+  // Open `z.string()` (not an enum) for forward-compat — kept in lockstep with
+  // the JSON `pausedReason` discriminator in execution-pipeline.ts. UX context;
+  // never a control signal.
+  awaitReason: external_exports.string().optional(),
+  crawlId: external_exports.string().optional(),
+  stepId: external_exports.string().optional()
 });
 var stepStartEventSchema = external_exports.object({
   ...baseFlowEvent,
@@ -33327,7 +33347,7 @@ var CORE_BUILTIN_TOOLS_REGISTRY = [
   {
     id: "exa",
     name: "Exa",
-    description: "Search the web using neural and keyword algorithms. Returns relevant content with optional summaries, highlights, and full text.",
+    description: "Search the web using neural and keyword algorithms. Returns relevant content with optional summaries, highlights, and full text. Tuned for retrieval and not blocked by anti-bot defenses, so prefer this over the raw fetch-url/crawl flow steps for research against sites that block datacenter traffic.",
     category: BuiltInToolCategory.WEB_SEARCH,
     providers: [BuiltInToolProvider.MULTI],
     parametersSchema: {
@@ -33402,7 +33422,7 @@ var CORE_BUILTIN_TOOLS_REGISTRY = [
   {
     id: "firecrawl",
     name: "Firecrawl",
-    description: "Scrape and extract content from a URL as markdown. Provide a url and optional formats array.",
+    description: "Scrape and extract content from a URL as markdown. Provide a url and optional formats array. Handles rendering and anti-bot measures, so prefer this over a raw HTTP fetch when a site blocks, rate-limits, or challenges datacenter (non-human) traffic.",
     category: BuiltInToolCategory.WEB_SCRAPING,
     providers: [BuiltInToolProvider.MULTI],
     parametersSchema: {
@@ -37591,39 +37611,46 @@ var BUILT_IN_GRADER_IDS = [
   "rightTone",
   "safeToSend"
 ];
+var graderSeveritySchema = external_exports.enum(["gate", "soft"]);
+var severityFields = { severity: graderSeveritySchema.optional() };
 var checkGraderSchema = external_exports.discriminatedUnion("kind", [
   external_exports.object({
     kind: external_exports.literal("contains"),
     value: external_exports.string(),
-    caseSensitive: external_exports.boolean().optional()
+    caseSensitive: external_exports.boolean().optional(),
+    ...severityFields
   }),
   external_exports.object({
     kind: external_exports.literal("not_contains"),
     value: external_exports.string(),
-    caseSensitive: external_exports.boolean().optional()
+    caseSensitive: external_exports.boolean().optional(),
+    ...severityFields
   }),
   // Exact/normalized match against `case.expected.text`.
-  external_exports.object({ kind: external_exports.literal("matches_expected") }),
+  external_exports.object({ kind: external_exports.literal("matches_expected"), ...severityFields }),
   external_exports.object({
     kind: external_exports.literal("regex"),
     pattern: external_exports.string(),
-    flags: external_exports.string().optional()
+    flags: external_exports.string().optional(),
+    ...severityFields
   }),
-  external_exports.object({ kind: external_exports.literal("valid_json") }),
+  external_exports.object({ kind: external_exports.literal("valid_json"), ...severityFields }),
   external_exports.object({
     kind: external_exports.literal("json_field"),
     path: external_exports.string(),
     equals: external_exports.unknown().optional(),
-    exists: external_exports.boolean().optional()
+    exists: external_exports.boolean().optional(),
+    ...severityFields
   }),
   external_exports.object({
     kind: external_exports.literal("length"),
     minChars: external_exports.number().int().nonnegative().optional(),
-    maxChars: external_exports.number().int().nonnegative().optional()
+    maxChars: external_exports.number().int().nonnegative().optional(),
+    ...severityFields
   }),
-  external_exports.object({ kind: external_exports.literal("latency"), maxMs: external_exports.number().int().positive() }),
+  external_exports.object({ kind: external_exports.literal("latency"), maxMs: external_exports.number().int().positive(), ...severityFields }),
   // Today's implicit "success" made explicit: the case produced output without erroring.
-  external_exports.object({ kind: external_exports.literal("no_error") }),
+  external_exports.object({ kind: external_exports.literal("no_error"), ...severityFields }),
   // -------------------------------------------------------------------------
   // Trace checks — deterministic, free, pure assertions over the run's
   // EXECUTION TRACE (which tools/steps ran, in what order, whether it
@@ -37641,25 +37668,30 @@ var checkGraderSchema = external_exports.discriminatedUnion("kind", [
     input: external_exports.unknown().optional(),
     output: external_exports.unknown().optional(),
     isError: external_exports.boolean().optional(),
-    times: external_exports.number().int().positive().optional()
+    times: external_exports.number().int().positive().optional(),
+    ...severityFields
   }),
   // No tool named `name` was called.
-  external_exports.object({ kind: external_exports.literal("not_called_tool"), name: external_exports.string().min(1) }),
+  external_exports.object({ kind: external_exports.literal("not_called_tool"), name: external_exports.string().min(1), ...severityFields }),
   // The run made no tool calls at all.
-  external_exports.object({ kind: external_exports.literal("used_no_tools") }),
+  external_exports.object({ kind: external_exports.literal("used_no_tools"), ...severityFields }),
   // The run made at most `max` tool calls.
-  external_exports.object({ kind: external_exports.literal("max_tool_calls"), max: external_exports.number().int().nonnegative() }),
+  external_exports.object({
+    kind: external_exports.literal("max_tool_calls"),
+    max: external_exports.number().int().nonnegative(),
+    ...severityFields
+  }),
   // `tools` appears as an ordered SUBSEQUENCE of the tool-call names (other
   // calls may interleave; relative order of the listed tools must hold).
-  external_exports.object({ kind: external_exports.literal("tool_order"), tools: external_exports.array(external_exports.string()).min(1) }),
+  external_exports.object({ kind: external_exports.literal("tool_order"), tools: external_exports.array(external_exports.string()).min(1), ...severityFields }),
   // A step named (or typed) `name` ran.
-  external_exports.object({ kind: external_exports.literal("ran_step"), name: external_exports.string().min(1) }),
+  external_exports.object({ kind: external_exports.literal("ran_step"), name: external_exports.string().min(1), ...severityFields }),
   // `steps` appears as an ordered SUBSEQUENCE of the steps that ran.
-  external_exports.object({ kind: external_exports.literal("step_order"), steps: external_exports.array(external_exports.string()).min(1) }),
+  external_exports.object({ kind: external_exports.literal("step_order"), steps: external_exports.array(external_exports.string()).min(1), ...severityFields }),
   // The run completed (finished without erroring and was not left paused).
-  external_exports.object({ kind: external_exports.literal("completed") }),
+  external_exports.object({ kind: external_exports.literal("completed"), ...severityFields }),
   // Total run cost was within `maxUsd` (US dollars).
-  external_exports.object({ kind: external_exports.literal("cost"), maxUsd: external_exports.number().positive() })
+  external_exports.object({ kind: external_exports.literal("cost"), maxUsd: external_exports.number().positive(), ...severityFields })
 ]);
 var aiGraderSchema = external_exports.object({
   kind: external_exports.literal("ai"),
@@ -37671,7 +37703,8 @@ var aiGraderSchema = external_exports.object({
   /** Defaults to a cheap routed model (e.g. claude-haiku-4-5) at execution time. */
   model: external_exports.string().optional(),
   /** Pass cutoff for the 1-5 scale. */
-  threshold: external_exports.number().min(1).max(5).optional()
+  threshold: external_exports.number().min(1).max(5).optional(),
+  ...severityFields
 });
 var graderConfigSchema = external_exports.union([checkGraderSchema, aiGraderSchema]);
 var gradersSchema = external_exports.array(graderConfigSchema);
@@ -42133,10 +42166,10 @@ var FLOW_STEP_TYPE_METADATA = {
     configHints: "vectorsSource, destination, idTemplate, outputVariable"
   },
   crawl: {
-    description: "Crawl a website and extract content from pages.",
+    description: "Crawl a website with Cloudflare Browser Rendering and extract page content as markdown. Renders the SEED page fully; at depth >= 1 it enumerates child links WITHOUT rendering them (1 rendered seed record + N empty-markdown child records) and follows only ABSOLUTE link hrefs (relative links like item?id= are never followed). Writes { crawlId, status, result, resultPages[] } to outputVariable \u2014 flatten resultPages[].records[] (each { url, status, markdown?, metadata? }) to read every page. GOTCHA: the URL query string is DROPPED (https://site/item?id=42 crawls https://site/item) \u2014 for query-string URLs, JSON APIs, SPAs, or single detail/comment pages use fetch-url or the builtin:firecrawl scrape tool instead (firecrawl preserves query strings). ASYNC by default: with no streaming consumer (MCP run_flow/dispatch) the flow durably PAUSES while a Workflow polls; if the crawl exceeds completionTimeoutMs (default 120000 = 2 min) the step SILENTLY resumes with output = null. For real crawls raise completionTimeoutMs (e.g. 600000) and set pollIntervalMs (e.g. 5000), or set asyncCrawl:false to poll inline within the request (good for smoke tests).",
     category: "integration",
     isPrompt: false,
-    configHints: "url, limit, depth, outputVariable"
+    configHints: 'url (required; query string is dropped), outputVariable (required), depth (0 = seed-only render, recommended), limit, formats (e.g. ["markdown"]), render, options ({ includePatterns, excludePatterns, allowSubdomains, allowExternalLinks, ignoreSitemap, sitemapOnly }), waitForSelector, completionTimeoutMs (default 120000; raise for slow crawls), pollIntervalMs, asyncCrawl (false = synchronous inline polling), errorHandling ({ onError: "continue"|"fail" } OBJECT, not a bare string), defaultValue. Pattern for "index + detail/comment pages": crawl the index with depth 0, transform-data to extract the links/ids from result markdown, then an agent prompt step (mode: "agent") with tools.toolIds ["builtin:firecrawl"] to fetch each query-string detail URL.'
   },
   "paginate-api": {
     description: "Fetch paginated data from an API endpoint across multiple pages.",
@@ -42854,6 +42887,7 @@ ${orthogonalLines.join("\n")}
 - **Web search (provider-native, inline citations)** \u2192 see Provider-Native Search section above
 - **Web search (standalone, AI-powered)** \u2192 \`builtin:exa\`
 - **Simple web scraping to markdown** \u2192 \`builtin:firecrawl\`
+- **Research/scraping a site that blocks bots** \u2192 prefer \`builtin:firecrawl\` or \`builtin:exa\` over the raw \`fetch-url\`/\`crawl\` flow steps. Those steps request from datacenter IPs flagged as non-human traffic, so sites behind a WAF/anti-bot service often return a 403, CAPTCHA, or empty body. A mixed approach (fetch first, fall back to Firecrawl/Exa on a block) is the most reliable.
 - **Rendered web pages, screenshots, structured extraction, or link harvesting** \u2192 \`builtin:browser:markdown\`, \`builtin:browser:snapshot\`, \`builtin:browser:json\`, \`builtin:browser:scrape\`, \`builtin:browser:links\` (stateless, one-shot \u2014 cheap, no session overhead)
 - **Autonomous browsing (just want the outcome \u2014 extract data, complete a form, find info on a page)** \u2192 \`builtin:browser:agent\` with a natural-language \`goal\` plus \`startUrl\` (fresh browser) or \`sessionId\` (continue an existing session). Stagehand handles observe/act/extract; you get back text or, with an \`outputSchema\`, structured JSON. Try this BEFORE reaching for the manual session tools below.
 - **Interactive browsing (precise control over individual actions)** \u2192 \`builtin:browser:open\` to start a session, then \`builtin:browser:click\`, \`builtin:browser:type\`, \`builtin:browser:scroll\`, \`builtin:browser:session-screenshot\`, \`builtin:browser:evaluate\`, \`builtin:browser:navigate\`, and \`builtin:browser:close\` when done. Session tools maintain cookies and page state across calls but cost more than stateless tools \u2014 prefer stateless tools when a single page read suffices.
@@ -42919,6 +42953,7 @@ Tools are added to prompt steps via \`tools: { toolIds: ["<id>"] }\`. The AI mod
 - **Web search (provider-native, inline citations)** \u2192 see the Provider-Native Search section
 - **Web search (standalone, AI-powered)** \u2192 \`builtin:exa\`
 - **Simple web scraping to markdown** \u2192 \`builtin:firecrawl\`
+- **Research/scraping a site that blocks bots** \u2192 prefer \`builtin:firecrawl\` or \`builtin:exa\` over the raw \`fetch-url\`/\`crawl\` flow steps. Those steps request from datacenter IPs flagged as non-human traffic, so sites behind a WAF/anti-bot service often return a 403, CAPTCHA, or empty body. A mixed approach (fetch first, fall back to Firecrawl/Exa on a block) is the most reliable.
 - **Rendered web pages, screenshots, structured extraction, or link harvesting** \u2192 \`builtin:browser:markdown\`, \`builtin:browser:snapshot\`, \`builtin:browser:json\`, \`builtin:browser:scrape\`, \`builtin:browser:links\` (stateless, one-shot)
 - **Autonomous browsing (just want the outcome)** \u2192 \`builtin:browser:agent\` with a natural-language \`goal\` plus \`startUrl\` or \`sessionId\` \u2014 try this BEFORE the manual session tools
 - **Interactive browsing (precise control)** \u2192 \`builtin:browser:open\`, then \`builtin:browser:click\` / \`type\` / \`scroll\` / \`session-screenshot\` / \`evaluate\` / \`navigate\`, and \`builtin:browser:close\` when done
@@ -43939,6 +43974,15 @@ var RuntimeProxyToolConfigSchema = external_exports.object({
   toolCallStrategy: external_exports.enum(["auto", "required", "none"]).optional(),
   onToolError: external_exports.enum(["fail", "continue", "retry"]).optional()
 });
+var RuntimeProxyProviderTransportSchema = external_exports.object({
+  mixlayer: external_exports.object({
+    modelApi: external_exports.enum(["chat", "responses"]).optional(),
+    responsesTransport: external_exports.enum(["http", "websocket"]).optional()
+  }).optional(),
+  openai: external_exports.object({
+    responsesTransport: external_exports.enum(["http", "websocket"]).optional()
+  }).optional()
+});
 var RuntimeProxyRequestSchema = external_exports.object({
   model: external_exports.string().min(1),
   prompt: external_exports.string().default(""),
@@ -43955,6 +43999,7 @@ var RuntimeProxyRequestSchema = external_exports.object({
     seed: external_exports.number().int().optional(),
     tools: external_exports.array(RuntimeProxyToolSchema).optional(),
     providerTools: external_exports.array(RuntimeProxyProviderToolSchema).optional(),
+    providerTransport: RuntimeProxyProviderTransportSchema.optional(),
     toolConfiguration: RuntimeProxyToolConfigSchema.optional()
   }).optional().default({})
 });
@@ -63990,185 +64035,191 @@ function buildJUnitXml(suites) {
 
 // src/commands/eval.ts
 var evalCommand = new Command20("eval").description("Manage evaluations");
-evalCommand.command("submit").description("Submit an eval batch").requiredOption("-f, --flow <id>", "Flow ID to evaluate").requiredOption("-r, --records <file>", "JSON file with record IDs").option("-n, --name <name>", "Eval batch name").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (options) => {
-  const apiKey = await ensureAuth();
-  if (!apiKey) return;
-  let recordIds;
-  try {
-    const content = readFileSync16(options.records, "utf-8");
-    const parsed = JSON.parse(content);
-    recordIds = Array.isArray(parsed) ? parsed : parsed.recordIds || parsed.records || [];
-  } catch (error51) {
-    const message = error51 instanceof Error ? error51.message : "Unknown error";
-    console.error(chalk27.red(`Failed to read records file: ${message}`));
-    process.exit(1);
-    return;
-  }
-  const client = createCliClient(apiKey);
-  if (!isTTY(options) || options.json) {
+evalCommand.command("submit").description("Submit an eval batch").requiredOption("-f, --flow <id>", "Flow ID to evaluate").requiredOption("-r, --records <file>", "JSON file with record IDs").option("-n, --name <name>", "Eval batch name").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(
+  async (options) => {
+    const apiKey = await ensureAuth();
+    if (!apiKey) return;
+    let recordIds;
     try {
-      const data = await client.post("/eval/submit", {
-        flowId: options.flow,
-        recordIds,
-        name: options.name
-      });
-      if (options.json) {
-        printJson(data);
-      } else {
-        console.log(chalk27.green("Eval submitted"));
-        console.log(`  Batch ID: ${chalk27.green(data.id)}`);
-        if (data.name) console.log(`  Name: ${data.name}`);
-        console.log(`  Status: ${data.status}`);
-        console.log(`  Records: ${data.totalRecords}`);
-        if (data.groupId) console.log(`  Group: ${data.groupId}`);
-      }
+      const content = readFileSync16(options.records, "utf-8");
+      const parsed = JSON.parse(content);
+      recordIds = Array.isArray(parsed) ? parsed : parsed.recordIds || parsed.records || [];
     } catch (error51) {
       const message = error51 instanceof Error ? error51.message : "Unknown error";
-      console.error(chalk27.red("Failed to submit eval"));
-      console.error(chalk27.red(message));
+      console.error(chalk27.red(`Failed to read records file: ${message}`));
       process.exit(1);
+      return;
     }
-    return;
-  }
-  const App = () => {
-    const [loading, setLoading] = useState36(true);
-    const [success2, setSuccess] = useState36(null);
-    const [error51, setError] = useState36(null);
-    const [resultNode, setResultNode] = useState36(void 0);
-    useEffect30(() => {
-      const run2 = async () => {
-        try {
-          const data = await client.post("/eval/submit", {
-            flowId: options.flow,
-            recordIds,
-            name: options.name
-          });
-          const fields = [
-            { label: "Batch ID", value: data.id, color: "green" }
-          ];
-          if (data.name) fields.push({ label: "Name", value: data.name });
-          fields.push({ label: "Status", value: data.status });
-          fields.push({ label: "Records", value: data.totalRecords });
-          if (data.groupId) fields.push({ label: "Group", value: data.groupId });
-          setResultNode(React19.createElement(EntityCard, { fields }));
-          setSuccess(true);
-          setLoading(false);
-        } catch (err) {
-          setError(err instanceof Error ? err : new Error(String(err)));
-          setSuccess(false);
-          setLoading(false);
-        }
-      };
-      run2();
-    }, []);
-    return React19.createElement(MutationResult, {
-      loading,
-      loadingLabel: `Submitting eval with ${recordIds.length} records...`,
-      success: success2,
-      successMessage: "Eval submitted",
-      error: error51,
-      result: resultNode
-    });
-  };
-  const { waitUntilExit } = render19(React19.createElement(App));
-  await waitUntilExit();
-});
-evalCommand.command("list").description("List eval batches").option("--flow <id>", "Filter by flow ID").option("--limit <n>", "Limit results", "20").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (options) => {
-  const apiKey = await ensureAuth();
-  if (!apiKey) return;
-  const client = createCliClient(apiKey);
-  const params = { limit: options.limit };
-  if (options.flow) params.flowId = options.flow;
-  if (!isTTY(options) || options.json) {
-    try {
-      const data = await client.get("/eval/batches", params);
-      if (options.json) {
-        printJson(data);
-      } else {
-        const batches = data.data ?? [];
-        if (batches.length === 0) {
-          console.log(chalk27.gray("No eval batches found"));
-          return;
-        }
-        console.log(chalk27.cyan("Eval Batches:"));
-        for (const batch of batches) {
-          const name = batch.name || batch.id;
-          const progress = batch.totalRecords ? `${batch.completedRecords ?? 0}/${batch.totalRecords}` : "";
-          const statusColor = batch.status === "completed" ? "green" : "yellow";
-          console.log(
-            `  ${chalk27.green(batch.id)} ${name} ${chalk27[statusColor](`[${batch.status}]`)} ${chalk27.gray(progress)}`
-          );
-        }
-        const total = getTotalCount(data.pagination);
-        if (total !== void 0) {
-          console.log(chalk27.dim(`
-  Total: ${total} batches`));
+    const client = createCliClient(apiKey);
+    if (!isTTY(options) || options.json) {
+      try {
+        const data = await client.post("/eval/submit", {
+          flowId: options.flow,
+          recordIds,
+          name: options.name
+        });
+        if (options.json) {
+          printJson(data);
+        } else {
+          console.log(chalk27.green("Eval submitted"));
+          console.log(`  Batch ID: ${chalk27.green(data.id)}`);
+          if (data.name) console.log(`  Name: ${data.name}`);
+          console.log(`  Status: ${data.status}`);
+          console.log(`  Records: ${data.totalRecords}`);
+          if (data.groupId) console.log(`  Group: ${data.groupId}`);
         }
+      } catch (error51) {
+        const message = error51 instanceof Error ? error51.message : "Unknown error";
+        console.error(chalk27.red("Failed to submit eval"));
+        console.error(chalk27.red(message));
+        process.exit(1);
       }
-    } catch (error51) {
-      const message = error51 instanceof Error ? error51.message : "Unknown error";
-      console.error(chalk27.red("Failed to fetch eval batches"));
-      console.error(chalk27.red(message));
-      process.exit(1);
+      return;
     }
-    return;
+    const App = () => {
+      const [loading, setLoading] = useState36(true);
+      const [success2, setSuccess] = useState36(null);
+      const [error51, setError] = useState36(null);
+      const [resultNode, setResultNode] = useState36(void 0);
+      useEffect30(() => {
+        const run2 = async () => {
+          try {
+            const data = await client.post("/eval/submit", {
+              flowId: options.flow,
+              recordIds,
+              name: options.name
+            });
+            const fields = [{ label: "Batch ID", value: data.id, color: "green" }];
+            if (data.name) fields.push({ label: "Name", value: data.name });
+            fields.push({ label: "Status", value: data.status });
+            fields.push({ label: "Records", value: data.totalRecords });
+            if (data.groupId) fields.push({ label: "Group", value: data.groupId });
+            setResultNode(React19.createElement(EntityCard, { fields }));
+            setSuccess(true);
+            setLoading(false);
+          } catch (err) {
+            setError(err instanceof Error ? err : new Error(String(err)));
+            setSuccess(false);
+            setLoading(false);
+          }
+        };
+        run2();
+      }, []);
+      return React19.createElement(MutationResult, {
+        loading,
+        loadingLabel: `Submitting eval with ${recordIds.length} records...`,
+        success: success2,
+        successMessage: "Eval submitted",
+        error: error51,
+        result: resultNode
+      });
+    };
+    const { waitUntilExit } = render19(React19.createElement(App));
+    await waitUntilExit();
   }
-  const App = () => {
-    const [loading, setLoading] = useState36(true);
-    const [items, setItems] = useState36(null);
-    const [total, setTotal] = useState36(void 0);
-    const [error51, setError] = useState36(null);
-    useEffect30(() => {
-      const run2 = async () => {
-        try {
-          const data = await client.get("/eval/batches", params);
-          setItems(data.data ?? []);
-          setTotal(getTotalCount(data.pagination));
-          setLoading(false);
-        } catch (err) {
-          setError(err instanceof Error ? err : new Error(String(err)));
-          setLoading(false);
+);
+evalCommand.command("list").description("List eval batches").option("--flow <id>", "Filter by flow ID").option("--limit <n>", "Limit results", "20").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(
+  async (options) => {
+    const apiKey = await ensureAuth();
+    if (!apiKey) return;
+    const client = createCliClient(apiKey);
+    const params = { limit: options.limit };
+    if (options.flow) params.flowId = options.flow;
+    if (!isTTY(options) || options.json) {
+      try {
+        const data = await client.get("/eval/batches", params);
+        if (options.json) {
+          printJson(data);
+        } else {
+          const batches = data.data ?? [];
+          if (batches.length === 0) {
+            console.log(chalk27.gray("No eval batches found"));
+            return;
+          }
+          console.log(chalk27.cyan("Eval Batches:"));
+          for (const batch of batches) {
+            const name = batch.name || batch.id;
+            const progress = batch.totalRecords ? `${batch.completedRecords ?? 0}/${batch.totalRecords}` : "";
+            const statusColor = batch.status === "completed" ? "green" : "yellow";
+            console.log(
+              `  ${chalk27.green(batch.id)} ${name} ${chalk27[statusColor](`[${batch.status}]`)} ${chalk27.gray(progress)}`
+            );
+          }
+          const total = getTotalCount(data.pagination);
+          if (total !== void 0) {
+            console.log(chalk27.dim(`
+  Total: ${total} batches`));
+          }
         }
-      };
-      run2();
-    }, []);
-    return React19.createElement(DataList, {
-      title: "Eval Batches",
-      items,
-      error: error51,
-      loading,
-      total,
-      emptyMessage: "No eval batches found",
-      renderCard: (item) => {
-        const b = item;
-        const name = b.name || b.id;
-        const progress = b.totalRecords ? `${b.completedRecords ?? 0}/${b.totalRecords}` : "";
-        const statusColor = b.status === "completed" ? "green" : "yellow";
-        return React19.createElement(
-          Text34,
-          { color: statusColor },
-          `  ${b.id} ${name} [${b.status}] ${progress}`
-        );
+      } catch (error51) {
+        const message = error51 instanceof Error ? error51.message : "Unknown error";
+        console.error(chalk27.red("Failed to fetch eval batches"));
+        console.error(chalk27.red(message));
+        process.exit(1);
       }
-    });
-  };
-  const { waitUntilExit } = render19(React19.createElement(App));
-  await waitUntilExit();
-});
+      return;
+    }
+    const App = () => {
+      const [loading, setLoading] = useState36(true);
+      const [items, setItems] = useState36(null);
+      const [total, setTotal] = useState36(void 0);
+      const [error51, setError] = useState36(null);
+      useEffect30(() => {
+        const run2 = async () => {
+          try {
+            const data = await client.get("/eval/batches", params);
+            setItems(data.data ?? []);
+            setTotal(getTotalCount(data.pagination));
+            setLoading(false);
+          } catch (err) {
+            setError(err instanceof Error ? err : new Error(String(err)));
+            setLoading(false);
+          }
+        };
+        run2();
+      }, []);
+      return React19.createElement(DataList, {
+        title: "Eval Batches",
+        items,
+        error: error51,
+        loading,
+        total,
+        emptyMessage: "No eval batches found",
+        renderCard: (item) => {
+          const b = item;
+          const name = b.name || b.id;
+          const progress = b.totalRecords ? `${b.completedRecords ?? 0}/${b.totalRecords}` : "";
+          const statusColor = b.status === "completed" ? "green" : "yellow";
+          return React19.createElement(
+            Text34,
+            { color: statusColor },
+            `  ${b.id} ${name} [${b.status}] ${progress}`
+          );
+        }
+      });
+    };
+    const { waitUntilExit } = render19(React19.createElement(App));
+    await waitUntilExit();
+  }
+);
 evalCommand.command("results <id>").description("Get eval batch results").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (id, options) => {
   const apiKey = await ensureAuth();
   if (!apiKey) return;
   const client = createCliClient(apiKey);
   if (!isTTY(options) || options.json) {
     try {
-      const data = await client.get(`/eval/${id}/results`);
+      const data = await client.get(
+        `/eval/${id}/results`
+      );
       if (options.json) {
         printJson(data);
       } else {
         if (data.batch) {
           console.log(chalk27.cyan(`Eval: ${data.batch.name || data.batch.id}`));
           console.log(`  Status: ${data.batch.status}`);
-          console.log(`  Progress: ${data.batch.completedRecords ?? 0}/${data.batch.totalRecords ?? 0}`);
+          console.log(
+            `  Progress: ${data.batch.completedRecords ?? 0}/${data.batch.totalRecords ?? 0}`
+          );
           console.log();
         }
         const results = data.data ?? [];
@@ -64201,19 +64252,28 @@ evalCommand.command("results <id>").description("Get eval batch results").option
     useEffect30(() => {
       const run2 = async () => {
         try {
-          const data = await client.get(`/eval/${id}/results`);
+          const data = await client.get(
+            `/eval/${id}/results`
+          );
           const results = data.data ?? [];
           const fields = [];
           if (data.batch) {
             fields.push({ label: "Eval", value: data.batch.name || data.batch.id });
             fields.push({ label: "Status", value: data.batch.status });
-            fields.push({ label: "Progress", value: `${data.batch.completedRecords ?? 0}/${data.batch.totalRecords ?? 0}` });
+            fields.push({
+              label: "Progress",
+              value: `${data.batch.completedRecords ?? 0}/${data.batch.totalRecords ?? 0}`
+            });
           }
           fields.push({ label: "Results", value: results.length });
           if (results.length > 0) {
             const completed = results.filter((r) => r.status === "completed").length;
             const avgScore = results.filter((r) => r.score !== void 0).reduce((sum, r) => sum + (r.score ?? 0), 0) / (results.filter((r) => r.score !== void 0).length || 1);
-            fields.push({ label: "Completed", value: `${completed}/${results.length}`, color: "green" });
+            fields.push({
+              label: "Completed",
+              value: `${completed}/${results.length}`,
+              color: "green"
+            });
             if (results.some((r) => r.score !== void 0)) {
               fields.push({ label: "Avg Score", value: avgScore.toFixed(2) });
             }
@@ -64309,6 +64369,13 @@ function printSuiteResult(rootDir, outcome) {
       if (testCase.errored) {
         console.log(chalk27.red(`      \u2717 errored: ${testCase.outputExcerpt.slice(0, 200)}`));
       }
+    } else {
+      for (const outcomeItem of testCase.outcomes.filter(
+        (o) => !o.passed && o.severity === "soft"
+      )) {
+        const reason = outcomeItem.reasoning ? `: ${outcomeItem.reasoning}` : "";
+        console.log(chalk27.yellow(`      \u26A0 soft ${outcomeItem.kind}${reason}`));
+      }
     }
   }
 }
@@ -64323,7 +64390,12 @@ function toJUnitSuite(outcome) {
     }))
   };
 }
-evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval suites (**/*.eval.ts) as a CI gate (exit 0 pass / 1 fail / 2 config)").option("--strict", "Fail on soft-threshold misses too (no-op until severity lands)").option("--virtual", "Run inline without persisting a suite/batch to the dashboard").option("--junit <path>", "Write JUnit XML results to <path>").option("--url <api>", "Override the API base URL (e.g. staging)").option("--cwd <dir>", "Directory to discover *.eval.ts under (default: current directory)").action(
+evalCommand.command("run [idOrDirPrefix]").description(
+  "Run code-colocated eval suites (**/*.eval.ts) as a CI gate (exit 0 pass / 1 fail / 2 config)"
+).option(
+  "--strict",
+  "Fail the exit code on soft grader misses too (default: soft misses are reported but do not fail)"
+).option("--virtual", "Run inline without persisting a suite/batch to the dashboard").option("--junit <path>", "Write JUnit XML results to <path>").option("--url <api>", "Override the API base URL (e.g. staging)").option("--cwd <dir>", "Directory to discover *.eval.ts under (default: current directory)").action(
   async (idOrDirPrefix, options) => {
     const apiKey = await ensureAuth();
     if (!apiKey) {
@@ -64361,11 +64433,6 @@ evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval
     } else {
       loaded = await loadAll(allFiles);
     }
-    if (options.strict) {
-      console.log(
-        chalk27.gray("Note: --strict has no effect yet (grader severity lands in a later increment).")
-      );
-    }
     const client = createCliClient(apiKey, options.url);
     const outcomes = [];
     for (const { file: file2, def } of loaded) {
@@ -64373,16 +64440,25 @@ evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval
       try {
         let result;
         if (runVirtual) {
-          result = await client.post("/eval/run", { definition: def });
-        } else {
-          const ensured = await client.post("/eval/ensure", {
-            name: def.name,
-            definition: def
+          result = await client.post("/eval/run", {
+            definition: def,
+            strict: options.strict ?? false
           });
+        } else {
+          const ensured = await client.post(
+            "/eval/ensure",
+            {
+              name: def.name,
+              definition: def
+            }
+          );
           if (!ensured.suiteId) {
             throw new Error(`ensure did not return a suiteId (result: ${ensured.result})`);
           }
-          result = await client.post("/eval/run", { suiteId: ensured.suiteId });
+          result = await client.post("/eval/run", {
+            suiteId: ensured.suiteId,
+            strict: options.strict ?? false
+          });
         }
         outcomes.push({ file: file2, definition: def, result });
         printSuiteResult(rootDir, { file: file2, definition: def, result });
@@ -64400,7 +64476,9 @@ evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval
         writeFileSync6(outPath, xml, "utf-8");
         console.log(chalk27.gray(`JUnit results written to ${options.junit}`));
       } catch (error51) {
-        failConfig(`Failed to write JUnit report: ${error51 instanceof Error ? error51.message : String(error51)}`);
+        failConfig(
+          `Failed to write JUnit report: ${error51 instanceof Error ? error51.message : String(error51)}`
+        );
       }
     }
     const failedSuites = outcomes.filter((o) => !o.result.passed);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runtypelabs/cli",
-  "version": "2.23.1",
+  "version": "2.24.1",
   "description": "Command-line interface for Runtype AI platform",
   "type": "module",
   "main": "dist/index.js",
@@ -24,7 +24,7 @@
     "rosie-skills": "0.8.1",
     "yaml": "^2.9.0",
     "@runtypelabs/ink-components": "0.3.4",
-    "@runtypelabs/sdk": "5.5.0",
+    "@runtypelabs/sdk": "5.7.0",
     "@runtypelabs/terminal-animations": "0.2.1"
   },
   "devDependencies": {
@@ -39,7 +39,7 @@
     "tsx": "^4.7.1",
     "typescript": "^6.0.3",
     "vitest": "^4.1.0",
-    "@runtypelabs/shared": "1.42.5"
+    "@runtypelabs/shared": "1.43.1"
   },
   "engines": {
     "node": ">=22.0.0"