@runtypelabs/cli 2.22.17 → 2.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +358 -73
  2. package/package.json +3 -3
package/dist/index.js CHANGED
@@ -179,8 +179,8 @@ var CallbackServer = class {
179
179
  expectedState;
180
180
  constructor() {
181
181
  this.app = express();
182
- this.codePromise = new Promise((resolve11, reject) => {
183
- this.codeResolve = resolve11;
182
+ this.codePromise = new Promise((resolve12, reject) => {
183
+ this.codeResolve = resolve12;
184
184
  this.codeReject = reject;
185
185
  });
186
186
  this.app.get("/callback", (req, res) => {
@@ -38065,6 +38065,9 @@ var MODEL_FAMILY_PROVIDER_IDS = {
38065
38065
  "glm-5-2": {
38066
38066
  "vercel": "zai/glm-5.2"
38067
38067
  },
38068
+ "glm-5-2-fast": {
38069
+ "vercel": "zai/glm-5.2-fast"
38070
+ },
38068
38071
  "glm-5-turbo": {
38069
38072
  "vercel": "zai/glm-5-turbo"
38070
38073
  },
@@ -38074,6 +38077,9 @@ var MODEL_FAMILY_PROVIDER_IDS = {
38074
38077
  "glm-5.2": {
38075
38078
  "vercel": "zai/glm-5.2"
38076
38079
  },
38080
+ "glm-5.2-fast": {
38081
+ "vercel": "zai/glm-5.2-fast"
38082
+ },
38077
38083
  "glm-5v-turbo": {
38078
38084
  "vercel": "zai/glm-5v-turbo"
38079
38085
  },
@@ -38118,6 +38124,12 @@ var MODEL_FAMILY_PROVIDER_IDS = {
38118
38124
  "gpt-4o-mini-search-preview": {
38119
38125
  "vercel": "openai/gpt-4o-mini-search-preview"
38120
38126
  },
38127
+ "gpt-4o-mini-transcribe": {
38128
+ "vercel": "openai/gpt-4o-mini-transcribe"
38129
+ },
38130
+ "gpt-4o-transcribe": {
38131
+ "vercel": "openai/gpt-4o-transcribe"
38132
+ },
38121
38133
  "gpt-5": {
38122
38134
  "openai": "gpt-5",
38123
38135
  "vercel": "openai/gpt-5"
@@ -38405,6 +38417,18 @@ var MODEL_FAMILY_PROVIDER_IDS = {
38405
38417
  "grok-imagine-video-1.5-preview": {
38406
38418
  "vercel": "xai/grok-imagine-video-1.5-preview"
38407
38419
  },
38420
+ "grok-stt": {
38421
+ "vercel": "xai/grok-stt"
38422
+ },
38423
+ "grok-tts": {
38424
+ "vercel": "xai/grok-tts"
38425
+ },
38426
+ "grok-voice-think-fast-1-0": {
38427
+ "vercel": "xai/grok-voice-think-fast-1.0"
38428
+ },
38429
+ "grok-voice-think-fast-1.0": {
38430
+ "vercel": "xai/grok-voice-think-fast-1.0"
38431
+ },
38408
38432
  "imagen-4-0-fast-generate-001": {
38409
38433
  "vercel": "google/imagen-4.0-fast-generate-001"
38410
38434
  },
@@ -39138,6 +39162,12 @@ var MODEL_FAMILY_PROVIDER_IDS = {
39138
39162
  "trinity-mini": {
39139
39163
  "vercel": "arcee-ai/trinity-mini"
39140
39164
  },
39165
+ "tts-1": {
39166
+ "vercel": "openai/tts-1"
39167
+ },
39168
+ "tts-1-hd": {
39169
+ "vercel": "openai/tts-1-hd"
39170
+ },
39141
39171
  "veo-3-0-fast-generate-001": {
39142
39172
  "vercel": "google/veo-3.0-fast-generate-001"
39143
39173
  },
@@ -39234,6 +39264,9 @@ var MODEL_FAMILY_PROVIDER_IDS = {
39234
39264
  "wan-v2.6-t2v": {
39235
39265
  "vercel": "alibaba/wan-v2.6-t2v"
39236
39266
  },
39267
+ "whisper-1": {
39268
+ "vercel": "openai/whisper-1"
39269
+ },
39237
39270
  "zai-org/GLM-5": {
39238
39271
  "togetherai": "togetherai/zai-org/GLM-5"
39239
39272
  },
@@ -44831,14 +44864,14 @@ async function promptConfirm(message, options) {
44831
44864
  output: process.stdout,
44832
44865
  terminal: true
44833
44866
  });
44834
- return new Promise((resolve11) => {
44867
+ return new Promise((resolve12) => {
44835
44868
  rl.question(chalk3.cyan(`${message} (${hint}): `), (answer) => {
44836
44869
  rl.close();
44837
44870
  const trimmed = answer.trim().toLowerCase();
44838
44871
  if (trimmed === "") {
44839
- resolve11(defaultYes);
44872
+ resolve12(defaultYes);
44840
44873
  } else {
44841
- resolve11(trimmed === "y" || trimmed === "yes");
44874
+ resolve12(trimmed === "y" || trimmed === "yes");
44842
44875
  }
44843
44876
  });
44844
44877
  });
@@ -46546,13 +46579,13 @@ promptsCommand.command("delete <id>").description("Delete a prompt").option("--t
46546
46579
  await waitUntilExit2();
46547
46580
  return;
46548
46581
  }
46549
- const confirmed = await new Promise((resolve11) => {
46582
+ const confirmed = await new Promise((resolve12) => {
46550
46583
  const { unmount } = render4(
46551
46584
  React4.createElement(ConfirmPrompt, {
46552
46585
  message: `Delete prompt ${id}?`,
46553
46586
  defaultValue: false,
46554
46587
  onConfirm: (result) => {
46555
- resolve11(result);
46588
+ resolve12(result);
46556
46589
  unmount();
46557
46590
  }
46558
46591
  })
@@ -47437,13 +47470,13 @@ secretsCommand.command("delete <id>").description("Delete a secret").option("--t
47437
47470
  await waitUntilExit2();
47438
47471
  return;
47439
47472
  }
47440
- const confirmed = await new Promise((resolve11) => {
47473
+ const confirmed = await new Promise((resolve12) => {
47441
47474
  const { unmount } = render6(
47442
47475
  React6.createElement(ConfirmPrompt, {
47443
47476
  message: `Delete secret ${id}?`,
47444
47477
  defaultValue: false,
47445
47478
  onConfirm: (result) => {
47446
- resolve11(result);
47479
+ resolve12(result);
47447
47480
  unmount();
47448
47481
  }
47449
47482
  })
@@ -47855,13 +47888,13 @@ toolsCommand.command("delete <id>").description("Delete a tool").option("--tty",
47855
47888
  await waitUntilExit2();
47856
47889
  return;
47857
47890
  }
47858
- const confirmed = await new Promise((resolve11) => {
47891
+ const confirmed = await new Promise((resolve12) => {
47859
47892
  const { unmount } = render7(
47860
47893
  React7.createElement(ConfirmPrompt, {
47861
47894
  message: `Delete tool ${id}?`,
47862
47895
  defaultValue: false,
47863
47896
  onConfirm: (result) => {
47864
- resolve11(result);
47897
+ resolve12(result);
47865
47898
  unmount();
47866
47899
  }
47867
47900
  })
@@ -50283,13 +50316,13 @@ conversationsCommand.command("delete <id>").description("Delete a conversation")
50283
50316
  await waitUntilExit2();
50284
50317
  return;
50285
50318
  }
50286
- const confirmed = await new Promise((resolve11) => {
50319
+ const confirmed = await new Promise((resolve12) => {
50287
50320
  const { unmount } = render14(
50288
50321
  React14.createElement(ConfirmPrompt, {
50289
50322
  message: `Delete conversation ${id}?`,
50290
50323
  defaultValue: false,
50291
50324
  onConfirm: (result) => {
50292
- resolve11(result);
50325
+ resolve12(result);
50293
50326
  unmount();
50294
50327
  }
50295
50328
  })
@@ -56154,8 +56187,8 @@ function MarathonApp({
56154
56187
  setIsCheckpointExploring(false);
56155
56188
  setIsTerminalCheckpoint(Boolean(isTerminal));
56156
56189
  isTerminalCheckpointRef.current = Boolean(isTerminal);
56157
- return new Promise((resolve11) => {
56158
- checkpointResolveRef.current = resolve11;
56190
+ return new Promise((resolve12) => {
56191
+ checkpointResolveRef.current = resolve12;
56159
56192
  });
56160
56193
  },
56161
56194
  updateMilestone: (milestone) => {
@@ -57565,16 +57598,16 @@ function MarathonStartupShell({
57565
57598
  latestAppPropsRef.current = marathonAppProps;
57566
57599
  useEffect26(() => {
57567
57600
  if (scene !== "app" || !appReadyResolverRef.current) return;
57568
- const resolve11 = appReadyResolverRef.current;
57601
+ const resolve12 = appReadyResolverRef.current;
57569
57602
  appReadyResolverRef.current = null;
57570
- resolve11();
57603
+ resolve12();
57571
57604
  }, [scene]);
57572
57605
  const beginTransition = (target) => {
57573
57606
  if (transitionPromiseRef.current) return transitionPromiseRef.current;
57574
57607
  if (target === "app" && !latestAppPropsRef.current) {
57575
57608
  throw new Error("Cannot complete startup before marathon app props are ready.");
57576
57609
  }
57577
- const promise2 = new Promise((resolve11) => {
57610
+ const promise2 = new Promise((resolve12) => {
57578
57611
  globalThis.setTimeout(() => {
57579
57612
  setPrompt(null);
57580
57613
  setModelChoices(null);
@@ -57595,12 +57628,12 @@ function MarathonStartupShell({
57595
57628
  if (target === "app") {
57596
57629
  appReadyResolverRef.current = () => {
57597
57630
  transitionPromiseRef.current = null;
57598
- resolve11();
57631
+ resolve12();
57599
57632
  };
57600
57633
  } else {
57601
57634
  dismissResolverRef.current = () => {
57602
57635
  transitionPromiseRef.current = null;
57603
- resolve11();
57636
+ resolve12();
57604
57637
  };
57605
57638
  }
57606
57639
  }, Math.max(0, MIN_HOLD_MS - (Date.now() - mountedAtRef.current)));
@@ -57617,8 +57650,8 @@ function MarathonStartupShell({
57617
57650
  setModelChoices(null);
57618
57651
  setPrompt(nextPrompt);
57619
57652
  setSelectedPromptIndex(0);
57620
- return new Promise((resolve11) => {
57621
- promptResolverRef.current = resolve11;
57653
+ return new Promise((resolve12) => {
57654
+ promptResolverRef.current = resolve12;
57622
57655
  });
57623
57656
  },
57624
57657
  requestModelChoice: async (nextCurrentModel, models) => {
@@ -57626,8 +57659,8 @@ function MarathonStartupShell({
57626
57659
  setPlaybookConfirm(null);
57627
57660
  setCurrentModel(nextCurrentModel);
57628
57661
  setModelChoices(models);
57629
- return new Promise((resolve11) => {
57630
- modelResolverRef.current = resolve11;
57662
+ return new Promise((resolve12) => {
57663
+ modelResolverRef.current = resolve12;
57631
57664
  });
57632
57665
  },
57633
57666
  requestPlaybookModelConfirm: async (playbookName, milestoneModels) => {
@@ -57642,8 +57675,8 @@ function MarathonStartupShell({
57642
57675
  // Default selection is the "Confirm" action (first item after milestones)
57643
57676
  selectedIndex: names.length
57644
57677
  });
57645
- return new Promise((resolve11) => {
57646
- playbookConfirmResolverRef.current = resolve11;
57678
+ return new Promise((resolve12) => {
57679
+ playbookConfirmResolverRef.current = resolve12;
57647
57680
  });
57648
57681
  },
57649
57682
  completeStartup: () => beginTransition("app"),
@@ -58157,15 +58190,15 @@ async function retryOnNetworkError(fn, opts = {}) {
58157
58190
  }
58158
58191
  const delay = Math.min(baseDelay * 2 ** attempt, maxDelay);
58159
58192
  opts.onRetry?.(attempt + 1, delay, error51);
58160
- await new Promise((resolve11) => {
58193
+ await new Promise((resolve12) => {
58161
58194
  const signal = opts.signal;
58162
58195
  const onAbort = () => {
58163
58196
  clearTimeout(timer);
58164
- resolve11();
58197
+ resolve12();
58165
58198
  };
58166
58199
  const timer = setTimeout(() => {
58167
58200
  signal?.removeEventListener("abort", onAbort);
58168
- resolve11();
58201
+ resolve12();
58169
58202
  }, delay);
58170
58203
  if (signal) {
58171
58204
  if (signal.aborted) onAbort();
@@ -58431,14 +58464,14 @@ async function promptNumericSelect(choices, promptLabel) {
58431
58464
  output: process.stdout,
58432
58465
  terminal: true
58433
58466
  });
58434
- return new Promise((resolve11) => {
58467
+ return new Promise((resolve12) => {
58435
58468
  const ask = () => {
58436
58469
  rl.question(chalk21.cyan(`
58437
58470
  ${promptLabel} (1-${choices.length}): `), (answer) => {
58438
58471
  const value = parseInt(answer.trim(), 10);
58439
58472
  if (value >= 1 && value <= choices.length) {
58440
58473
  rl.close();
58441
- resolve11(choices[value - 1].value);
58474
+ resolve12(choices[value - 1].value);
58442
58475
  return;
58443
58476
  }
58444
58477
  console.log(chalk21.red(`Please enter a number between 1 and ${choices.length}.`));
@@ -58466,7 +58499,7 @@ ${message}`));
58466
58499
  const previousRawMode = input.isRaw === true;
58467
58500
  let selectedIndex = 0;
58468
58501
  let renderedLineCount = 0;
58469
- return new Promise((resolve11) => {
58502
+ return new Promise((resolve12) => {
58470
58503
  const renderMenu = () => {
58471
58504
  if (renderedLineCount > 0) {
58472
58505
  clearRenderedLines(output, renderedLineCount);
@@ -58488,7 +58521,7 @@ ${message}`));
58488
58521
  };
58489
58522
  const finish = (value) => {
58490
58523
  cleanup();
58491
- resolve11(value);
58524
+ resolve12(value);
58492
58525
  };
58493
58526
  const onKeypress = (_2, key) => {
58494
58527
  if (key.ctrl && key.name === "c") {
@@ -61089,7 +61122,7 @@ async function taskAction(agent, options) {
61089
61122
  waitForUiExit = renderedShell.waitUntilExit;
61090
61123
  rerenderUi = renderedShell.rerender;
61091
61124
  unmountUi = renderedShell.unmount;
61092
- await new Promise((resolve11) => setTimeout(resolve11, 0));
61125
+ await new Promise((resolve12) => setTimeout(resolve12, 0));
61093
61126
  if (!startupShellRef.current) {
61094
61127
  exitAltScreen();
61095
61128
  unmountUi?.();
@@ -61722,7 +61755,7 @@ Saving state... done. Session saved to ${filePath}`);
61722
61755
  waitForUiExit = renderedApp.waitUntilExit;
61723
61756
  unmountUi = renderedApp.unmount;
61724
61757
  }
61725
- await new Promise((resolve11) => setTimeout(resolve11, 0));
61758
+ await new Promise((resolve12) => setTimeout(resolve12, 0));
61726
61759
  const streamActions = streamRef.current;
61727
61760
  if (!streamActions) {
61728
61761
  exitAltScreen();
@@ -61901,7 +61934,7 @@ Saving state... done. Session saved to ${filePath}`);
61901
61934
  };
61902
61935
  if (event.phase === "start") {
61903
61936
  currentActions.startContextCompaction(absoluteEvent);
61904
- await new Promise((resolve11) => setTimeout(resolve11, 0));
61937
+ await new Promise((resolve12) => setTimeout(resolve12, 0));
61905
61938
  return;
61906
61939
  }
61907
61940
  currentActions.finishContextCompaction(absoluteEvent);
@@ -62333,8 +62366,8 @@ Saving state... done. Session saved to ${filePath}`);
62333
62366
  const INDEXING_SETTLE_GRACE_MS = 5e3;
62334
62367
  await Promise.race([
62335
62368
  Promise.allSettled(indexingRequests),
62336
- new Promise((resolve11) => {
62337
- const timer = setTimeout(resolve11, INDEXING_SETTLE_GRACE_MS);
62369
+ new Promise((resolve12) => {
62370
+ const timer = setTimeout(resolve12, INDEXING_SETTLE_GRACE_MS);
62338
62371
  timer.unref?.();
62339
62372
  })
62340
62373
  ]);
@@ -63768,7 +63801,122 @@ import React19 from "react";
63768
63801
  import { render as render19 } from "ink";
63769
63802
  import { useState as useState36, useEffect as useEffect30 } from "react";
63770
63803
  import { Text as Text34 } from "ink";
63771
- import { readFileSync as readFileSync16 } from "fs";
63804
+ import { readFileSync as readFileSync16, writeFileSync as writeFileSync6, mkdirSync as mkdirSync8 } from "fs";
63805
+ import { dirname as dirname8, relative as relative6, resolve as resolve10 } from "path";
63806
+
63807
+ // src/lib/eval-discovery.ts
63808
+ import { readdirSync as readdirSync6 } from "fs";
63809
+ import { join as join11, relative as relative5, sep as sep5 } from "path";
63810
+ import { createJiti as createJiti2 } from "jiti";
63811
+ var EVAL_FILE_SUFFIXES = [".eval.ts", ".eval.mts"];
63812
+ var IGNORED_DIRS = /* @__PURE__ */ new Set([
63813
+ "node_modules",
63814
+ ".git",
63815
+ "dist",
63816
+ "build",
63817
+ ".next",
63818
+ ".turbo",
63819
+ "coverage",
63820
+ ".cache"
63821
+ ]);
63822
+ function isEvalFile(name) {
63823
+ return EVAL_FILE_SUFFIXES.some((suffix) => name.endsWith(suffix));
63824
+ }
63825
+ function discoverEvalFiles(rootDir) {
63826
+ const found = [];
63827
+ const walk = (dir) => {
63828
+ let entries;
63829
+ try {
63830
+ entries = readdirSync6(dir, { withFileTypes: true });
63831
+ } catch {
63832
+ return;
63833
+ }
63834
+ for (const entry of entries) {
63835
+ const name = entry.name.toString();
63836
+ if (entry.isDirectory()) {
63837
+ if (!IGNORED_DIRS.has(name) && !name.startsWith(".")) {
63838
+ walk(join11(dir, name));
63839
+ }
63840
+ } else if (entry.isFile() && isEvalFile(name)) {
63841
+ found.push(join11(dir, name));
63842
+ }
63843
+ }
63844
+ };
63845
+ walk(rootDir);
63846
+ return found.sort();
63847
+ }
63848
+ function filterEvalFilesByPrefix(files, rootDir, prefix) {
63849
+ const normalized = prefix.replace(/\//g, sep5);
63850
+ return files.filter((file2) => {
63851
+ const rel = relative5(rootDir, file2);
63852
+ const base = rel.split(sep5).pop() ?? rel;
63853
+ return rel.startsWith(normalized) || base.includes(prefix);
63854
+ });
63855
+ }
63856
+ var jitiLoader2;
63857
+ async function loadEvalDefinition(filePath) {
63858
+ jitiLoader2 ??= createJiti2(import.meta.url, { interopDefault: false });
63859
+ let mod;
63860
+ try {
63861
+ mod = await jitiLoader2.import(filePath);
63862
+ } catch (error51) {
63863
+ const message = error51 instanceof Error ? error51.message : String(error51);
63864
+ throw new Error(`Failed to load eval file ${filePath}: ${message}`, { cause: error51 });
63865
+ }
63866
+ const exported = mod.default;
63867
+ if (exported === void 0 || exported === null) {
63868
+ throw new Error(
63869
+ `Eval file ${filePath} must have a default export: the result of defineEval({...}) from @runtypelabs/sdk.`
63870
+ );
63871
+ }
63872
+ if (typeof exported !== "object" || !("target" in exported) || !("cases" in exported) || !Array.isArray(exported.cases)) {
63873
+ throw new Error(
63874
+ `Eval file ${filePath} default export is not an eval definition. Export defineEval({ target, cases, ... }).`
63875
+ );
63876
+ }
63877
+ return exported;
63878
+ }
63879
+
63880
+ // src/lib/eval-junit.ts
63881
+ function escapeXml(value) {
63882
+ return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
63883
+ }
63884
+ function buildJUnitXml(suites) {
63885
+ const totalTests = suites.reduce((sum, s) => sum + s.cases.length, 0);
63886
+ const totalFailures = suites.reduce(
63887
+ (sum, s) => sum + s.cases.filter((c2) => !c2.passed).length,
63888
+ 0
63889
+ );
63890
+ const lines = [];
63891
+ lines.push('<?xml version="1.0" encoding="UTF-8"?>');
63892
+ lines.push(
63893
+ `<testsuites name="runtype-eval" tests="${totalTests}" failures="${totalFailures}">`
63894
+ );
63895
+ for (const suite of suites) {
63896
+ const suiteFailures = suite.cases.filter((c2) => !c2.passed).length;
63897
+ lines.push(
63898
+ ` <testsuite name="${escapeXml(suite.name)}" tests="${suite.cases.length}" failures="${suiteFailures}">`
63899
+ );
63900
+ for (const testCase of suite.cases) {
63901
+ const classname = escapeXml(suite.name);
63902
+ const caseName = escapeXml(testCase.name);
63903
+ if (testCase.passed) {
63904
+ lines.push(` <testcase name="${caseName}" classname="${classname}" />`);
63905
+ } else {
63906
+ const reason = testCase.errored ? "Case errored while producing output" : "One or more graders failed";
63907
+ const detail = testCase.failedGraders.length ? testCase.failedGraders.join("\n") : reason;
63908
+ lines.push(` <testcase name="${caseName}" classname="${classname}">`);
63909
+ lines.push(` <failure message="${escapeXml(reason)}">${escapeXml(detail)}</failure>`);
63910
+ lines.push(" </testcase>");
63911
+ }
63912
+ }
63913
+ lines.push(" </testsuite>");
63914
+ }
63915
+ lines.push("</testsuites>");
63916
+ return lines.join("\n") + "\n";
63917
+ }
63918
+
63919
+ // src/commands/eval.ts
63772
63920
  var evalCommand = new Command20("eval").description("Manage evaluations");
63773
63921
  evalCommand.command("submit").description("Submit an eval batch").requiredOption("-f, --flow <id>", "Flow ID to evaluate").requiredOption("-r, --records <file>", "JSON file with record IDs").option("-n, --name <name>", "Eval batch name").option("--json", "Output as JSON").option("--tty", "Force TTY mode").option("--no-tty", "Force non-TTY mode").action(async (options) => {
63774
63922
  const apiKey = await ensureAuth();
@@ -64067,6 +64215,143 @@ evalCommand.command("compare <groupId>").description("Compare evals in a group")
64067
64215
  const { waitUntilExit } = render19(React19.createElement(App));
64068
64216
  await waitUntilExit();
64069
64217
  });
64218
+ var EVAL_RUN_EXIT = { pass: 0, fail: 1, config: 2 };
64219
+ function failConfig(message) {
64220
+ console.error(chalk27.red(`Error: ${message}`));
64221
+ process.exit(EVAL_RUN_EXIT.config);
64222
+ }
64223
+ function printSuiteResult(rootDir, outcome) {
64224
+ const { definition, result } = outcome;
64225
+ const rel = relative6(rootDir, outcome.file) || outcome.file;
64226
+ const badge = result.passed ? chalk27.green("PASS") : chalk27.red("FAIL");
64227
+ const score = `${result.passedCases}/${result.totalCases}`;
64228
+ console.log(`${badge} ${chalk27.bold(definition.name)} ${chalk27.gray(`(${score} cases \xB7 ${rel})`)}`);
64229
+ for (const testCase of result.cases) {
64230
+ const caseBadge = testCase.passed ? chalk27.green(" \u2713") : chalk27.red(" \u2717");
64231
+ console.log(`${caseBadge} ${testCase.name}`);
64232
+ if (!testCase.passed) {
64233
+ for (const outcomeItem of testCase.outcomes.filter((o) => !o.passed)) {
64234
+ const reason = outcomeItem.reasoning ? `: ${outcomeItem.reasoning}` : "";
64235
+ console.log(chalk27.red(` \u2717 ${outcomeItem.kind}${reason}`));
64236
+ }
64237
+ if (testCase.errored) {
64238
+ console.log(chalk27.red(` \u2717 errored: ${testCase.outputExcerpt.slice(0, 200)}`));
64239
+ }
64240
+ }
64241
+ }
64242
+ }
64243
+ function toJUnitSuite(outcome) {
64244
+ return {
64245
+ name: outcome.definition.name,
64246
+ cases: outcome.result.cases.map((c2) => ({
64247
+ name: c2.name,
64248
+ passed: c2.passed,
64249
+ errored: c2.errored,
64250
+ failedGraders: c2.outcomes.filter((o) => !o.passed).map((o) => `${o.kind}${o.reasoning ? `: ${o.reasoning}` : ""}`)
64251
+ }))
64252
+ };
64253
+ }
64254
+ evalCommand.command("run [idOrDirPrefix]").description("Run code-colocated eval suites (**/*.eval.ts) as a CI gate (exit 0 pass / 1 fail / 2 config)").option("--strict", "Fail on soft-threshold misses too (no-op until severity lands)").option("--virtual", "Run inline without persisting a suite/batch to the dashboard").option("--junit <path>", "Write JUnit XML results to <path>").option("--url <api>", "Override the API base URL (e.g. staging)").option("--cwd <dir>", "Directory to discover *.eval.ts under (default: current directory)").action(
64255
+ async (idOrDirPrefix, options) => {
64256
+ const apiKey = await ensureAuth();
64257
+ if (!apiKey) {
64258
+ failConfig("Not authenticated. Set RUNTYPE_API_KEY or run `runtype login`.");
64259
+ }
64260
+ const rootDir = resolve10(options.cwd ?? process.cwd());
64261
+ const allFiles = discoverEvalFiles(rootDir);
64262
+ if (allFiles.length === 0) {
64263
+ failConfig(`No *.eval.ts files found under ${rootDir}.`);
64264
+ }
64265
+ const loadAll = async (files) => {
64266
+ const loaded2 = [];
64267
+ for (const file2 of files) {
64268
+ try {
64269
+ loaded2.push({ file: file2, def: await loadEvalDefinition(file2) });
64270
+ } catch (error51) {
64271
+ failConfig(error51 instanceof Error ? error51.message : String(error51));
64272
+ }
64273
+ }
64274
+ return loaded2;
64275
+ };
64276
+ let loaded;
64277
+ if (idOrDirPrefix) {
64278
+ const byPath = filterEvalFilesByPrefix(allFiles, rootDir, idOrDirPrefix);
64279
+ loaded = byPath.length > 0 ? await loadAll(byPath) : [];
64280
+ if (loaded.length === 0) {
64281
+ const everything = await loadAll(allFiles);
64282
+ loaded = everything.filter(
64283
+ ({ def }) => def.name === idOrDirPrefix || def.name.includes(idOrDirPrefix)
64284
+ );
64285
+ }
64286
+ if (loaded.length === 0) {
64287
+ failConfig(`No eval suites matched "${idOrDirPrefix}".`);
64288
+ }
64289
+ } else {
64290
+ loaded = await loadAll(allFiles);
64291
+ }
64292
+ if (options.strict) {
64293
+ console.log(
64294
+ chalk27.gray("Note: --strict has no effect yet (grader severity lands in a later increment).")
64295
+ );
64296
+ }
64297
+ const client = createCliClient(apiKey, options.url);
64298
+ const outcomes = [];
64299
+ for (const { file: file2, def } of loaded) {
64300
+ const runVirtual = options.virtual || def.virtual;
64301
+ try {
64302
+ let result;
64303
+ if (runVirtual) {
64304
+ result = await client.post("/eval/run", { definition: def });
64305
+ } else {
64306
+ const ensured = await client.post("/eval/ensure", {
64307
+ name: def.name,
64308
+ definition: def
64309
+ });
64310
+ if (!ensured.suiteId) {
64311
+ throw new Error(`ensure did not return a suiteId (result: ${ensured.result})`);
64312
+ }
64313
+ result = await client.post("/eval/run", { suiteId: ensured.suiteId });
64314
+ }
64315
+ outcomes.push({ file: file2, definition: def, result });
64316
+ printSuiteResult(rootDir, { file: file2, definition: def, result });
64317
+ } catch (error51) {
64318
+ failConfig(
64319
+ `Suite "${def.name}" (${relative6(rootDir, file2) || file2}): ` + (error51 instanceof Error ? error51.message : String(error51))
64320
+ );
64321
+ }
64322
+ }
64323
+ if (options.junit) {
64324
+ try {
64325
+ const xml = buildJUnitXml(outcomes.map(toJUnitSuite));
64326
+ const outPath = resolve10(options.junit);
64327
+ mkdirSync8(dirname8(outPath), { recursive: true });
64328
+ writeFileSync6(outPath, xml, "utf-8");
64329
+ console.log(chalk27.gray(`JUnit results written to ${options.junit}`));
64330
+ } catch (error51) {
64331
+ failConfig(`Failed to write JUnit report: ${error51 instanceof Error ? error51.message : String(error51)}`);
64332
+ }
64333
+ }
64334
+ const failedSuites = outcomes.filter((o) => !o.result.passed);
64335
+ const totalCases = outcomes.reduce((sum, o) => sum + o.result.totalCases, 0);
64336
+ const passedCases = outcomes.reduce((sum, o) => sum + o.result.passedCases, 0);
64337
+ console.log();
64338
+ if (failedSuites.length === 0) {
64339
+ console.log(
64340
+ chalk27.green(
64341
+ `All ${outcomes.length} suite(s) passed (${passedCases}/${totalCases} cases).`
64342
+ )
64343
+ );
64344
+ process.exit(EVAL_RUN_EXIT.pass);
64345
+ } else {
64346
+ console.log(
64347
+ chalk27.red(
64348
+ `${failedSuites.length} of ${outcomes.length} suite(s) failed (${passedCases}/${totalCases} cases passed).`
64349
+ )
64350
+ );
64351
+ process.exit(EVAL_RUN_EXIT.fail);
64352
+ }
64353
+ }
64354
+ );
64070
64355
 
64071
64356
  // src/commands/api-keys.ts
64072
64357
  import { Command as Command21 } from "commander";
@@ -64333,13 +64618,13 @@ apiKeysCommand.command("delete <id>").description("Delete an API key").option("-
64333
64618
  await waitUntilExit2();
64334
64619
  return;
64335
64620
  }
64336
- const confirmed = await new Promise((resolve11) => {
64621
+ const confirmed = await new Promise((resolve12) => {
64337
64622
  const { unmount } = render20(
64338
64623
  React20.createElement(ConfirmPrompt, {
64339
64624
  message: `Delete API key ${id}?`,
64340
64625
  defaultValue: false,
64341
64626
  onConfirm: (result) => {
64342
- resolve11(result);
64627
+ resolve12(result);
64343
64628
  unmount();
64344
64629
  }
64345
64630
  })
@@ -64805,13 +65090,13 @@ clientTokensCommand.command("delete <id>").description("Delete a client token").
64805
65090
  await waitUntilExit2();
64806
65091
  return;
64807
65092
  }
64808
- const confirmed = await new Promise((resolve11) => {
65093
+ const confirmed = await new Promise((resolve12) => {
64809
65094
  const { unmount } = render21(
64810
65095
  React21.createElement(ConfirmPrompt, {
64811
65096
  message: `Delete client token ${id}?`,
64812
65097
  defaultValue: false,
64813
65098
  onConfirm: (result) => {
64814
- resolve11(result);
65099
+ resolve12(result);
64815
65100
  unmount();
64816
65101
  }
64817
65102
  })
@@ -65433,7 +65718,7 @@ async function runPersonaInit(options) {
65433
65718
  // src/lib/persona-demo.ts
65434
65719
  import { createServer } from "http";
65435
65720
  import { createServer as createNetServer } from "net";
65436
- import { existsSync as existsSync13, mkdirSync as mkdirSync8, readFileSync as readFileSync17, writeFileSync as writeFileSync6 } from "fs";
65721
+ import { existsSync as existsSync13, mkdirSync as mkdirSync9, readFileSync as readFileSync17, writeFileSync as writeFileSync7 } from "fs";
65437
65722
  import path16 from "path";
65438
65723
  var PERSONA_DEMO_DEFAULT_DIR = "persona-demo";
65439
65724
  var PERSONA_DEMO_DEFAULT_PORT = 43110;
@@ -65469,12 +65754,12 @@ function ensurePersonaDemoCanWrite(directory, force) {
65469
65754
  }
65470
65755
  function writePersonaDemoPage(options) {
65471
65756
  ensurePersonaDemoCanWrite(options.directory, options.force);
65472
- mkdirSync8(options.directory, { recursive: true });
65757
+ mkdirSync9(options.directory, { recursive: true });
65473
65758
  const filePath = personaDemoIndexPath(options.directory);
65474
65759
  const readmePath = personaDemoReadmePath(options.directory);
65475
- writeFileSync6(filePath, `${options.html.trim()}
65760
+ writeFileSync7(filePath, `${options.html.trim()}
65476
65761
  `, "utf8");
65477
- writeFileSync6(readmePath, `${options.readme.trim()}
65762
+ writeFileSync7(readmePath, `${options.readme.trim()}
65478
65763
  `, "utf8");
65479
65764
  return { directory: options.directory, filePath, readmePath };
65480
65765
  }
@@ -65488,13 +65773,13 @@ async function findAvailablePersonaDemoPort(preferredPort = PERSONA_DEMO_DEFAULT
65488
65773
  throw new Error(`Could not find an available local port starting at ${preferredPort}`);
65489
65774
  }
65490
65775
  function canListenOnPort(port) {
65491
- return new Promise((resolve11) => {
65776
+ return new Promise((resolve12) => {
65492
65777
  const server = createNetServer();
65493
65778
  server.once("error", () => {
65494
- resolve11(false);
65779
+ resolve12(false);
65495
65780
  });
65496
65781
  server.once("listening", () => {
65497
- server.close(() => resolve11(true));
65782
+ server.close(() => resolve12(true));
65498
65783
  });
65499
65784
  server.listen(port, "127.0.0.1");
65500
65785
  });
@@ -65518,10 +65803,10 @@ async function startPersonaDemoServer(options) {
65518
65803
  return {
65519
65804
  url: `http://127.0.0.1:${options.port}/`,
65520
65805
  port: options.port,
65521
- close: () => new Promise((resolve11, reject) => {
65806
+ close: () => new Promise((resolve12, reject) => {
65522
65807
  server.close((error51) => {
65523
65808
  if (error51) reject(error51);
65524
- else resolve11();
65809
+ else resolve12();
65525
65810
  });
65526
65811
  })
65527
65812
  };
@@ -65530,11 +65815,11 @@ function readIndexHtml(filePath) {
65530
65815
  return readFileSync17(filePath, "utf8");
65531
65816
  }
65532
65817
  function listen(server, port) {
65533
- return new Promise((resolve11, reject) => {
65818
+ return new Promise((resolve12, reject) => {
65534
65819
  server.once("error", reject);
65535
65820
  server.listen(port, "127.0.0.1", () => {
65536
65821
  server.off("error", reject);
65537
- resolve11();
65822
+ resolve12();
65538
65823
  });
65539
65824
  });
65540
65825
  }
@@ -66139,8 +66424,8 @@ function parsePort(raw, fallback) {
66139
66424
  }
66140
66425
  async function promptLine(rl, question, defaultValue) {
66141
66426
  const hint = defaultValue ? chalk30.dim(` (${defaultValue})`) : "";
66142
- const answer = await new Promise((resolve11) => {
66143
- rl.question(`${question}${hint}: `, resolve11);
66427
+ const answer = await new Promise((resolve12) => {
66428
+ rl.question(`${question}${hint}: `, resolve12);
66144
66429
  });
66145
66430
  const trimmed = answer.trim();
66146
66431
  if (!trimmed && defaultValue !== void 0) {
@@ -66150,8 +66435,8 @@ async function promptLine(rl, question, defaultValue) {
66150
66435
  }
66151
66436
  async function promptConfirm2(rl, message, defaultYes = false) {
66152
66437
  const hint = defaultYes ? chalk30.dim(" (Y/n)") : chalk30.dim(" (y/N)");
66153
- const answer = await new Promise((resolve11) => {
66154
- rl.question(`${message}${hint}: `, resolve11);
66438
+ const answer = await new Promise((resolve12) => {
66439
+ rl.question(`${message}${hint}: `, resolve12);
66155
66440
  });
66156
66441
  const t = answer.trim().toLowerCase();
66157
66442
  if (t === "") return defaultYes;
@@ -66239,13 +66524,13 @@ Dashboard: ${initial.dashboardUrl}`));
66239
66524
  output: process.stdout,
66240
66525
  terminal: true
66241
66526
  });
66242
- await new Promise((resolve11) => {
66527
+ await new Promise((resolve12) => {
66243
66528
  let finished = false;
66244
66529
  const finish = () => {
66245
66530
  if (finished) return;
66246
66531
  finished = true;
66247
66532
  rl.close();
66248
- resolve11();
66533
+ resolve12();
66249
66534
  };
66250
66535
  const runAction = async (name) => {
66251
66536
  if (name === "q") {
@@ -67835,11 +68120,11 @@ async function runTail(options) {
67835
68120
  process.stderr.write(
67836
68121
  (useColor ? chalk35.gray(`Retrying in ${delay / 1e3}s (attempt ${attempt}/${MAX_ATTEMPTS})...`) : `Retrying in ${delay / 1e3}s (attempt ${attempt}/${MAX_ATTEMPTS})...`) + "\n"
67837
68122
  );
67838
- await new Promise((resolve11) => {
67839
- const timer = setTimeout(resolve11, delay);
68123
+ await new Promise((resolve12) => {
68124
+ const timer = setTimeout(resolve12, delay);
67840
68125
  const onAbort = () => {
67841
68126
  clearTimeout(timer);
67842
- resolve11();
68127
+ resolve12();
67843
68128
  };
67844
68129
  controller.signal.addEventListener("abort", onAbort, { once: true });
67845
68130
  });
@@ -69181,7 +69466,7 @@ import { Command as Command31 } from "commander";
69181
69466
  import chalk39 from "chalk";
69182
69467
 
69183
69468
  // src/lib/skills-install.ts
69184
- import { mkdirSync as mkdirSync10, readFileSync as readFileSync19, writeFileSync as writeFileSync8 } from "fs";
69469
+ import { mkdirSync as mkdirSync11, readFileSync as readFileSync19, writeFileSync as writeFileSync9 } from "fs";
69185
69470
  import path18 from "path";
69186
69471
  import readline4 from "readline";
69187
69472
  import chalk38 from "chalk";
@@ -69205,8 +69490,8 @@ function readSkillsInstallMetadata() {
69205
69490
  }
69206
69491
  function writeSkillsInstallMetadata(metadata) {
69207
69492
  try {
69208
- mkdirSync10(path18.dirname(metadataPath()), { recursive: true });
69209
- writeFileSync8(metadataPath(), JSON.stringify(metadata, null, 2));
69493
+ mkdirSync11(path18.dirname(metadataPath()), { recursive: true });
69494
+ writeFileSync9(metadataPath(), JSON.stringify(metadata, null, 2));
69210
69495
  } catch {
69211
69496
  }
69212
69497
  }
@@ -69256,8 +69541,8 @@ async function promptConfirm3(message, defaultYes = true) {
69256
69541
  const rl = readline4.createInterface({ input: process.stdin, output: process.stdout });
69257
69542
  const hint = defaultYes ? chalk38.dim(" (Y/n)") : chalk38.dim(" (y/N)");
69258
69543
  try {
69259
- const answer = await new Promise((resolve11) => {
69260
- rl.question(`${message}${hint}: `, resolve11);
69544
+ const answer = await new Promise((resolve12) => {
69545
+ rl.question(`${message}${hint}: `, resolve12);
69261
69546
  });
69262
69547
  const t = answer.trim().toLowerCase();
69263
69548
  if (t === "") return defaultYes;
@@ -69462,8 +69747,8 @@ skillsCommand.command("install").description(`Install Runtype skills (${SKILLS_R
69462
69747
  );
69463
69748
 
69464
69749
  // src/commands/apps.ts
69465
- import { readdirSync as readdirSync6, readFileSync as readFileSync20, lstatSync, statSync as statSync7, existsSync as existsSync15 } from "fs";
69466
- import { join as join12, relative as relative5 } from "path";
69750
+ import { readdirSync as readdirSync7, readFileSync as readFileSync20, lstatSync, statSync as statSync7, existsSync as existsSync15 } from "fs";
69751
+ import { join as join13, relative as relative7 } from "path";
69467
69752
  import { Command as Command32 } from "commander";
69468
69753
  import chalk40 from "chalk";
69469
69754
  import { zipSync } from "fflate";
@@ -69474,16 +69759,16 @@ var MANIFEST_FILENAME = "runtype.app.json";
69474
69759
  function collectBundleFiles(dir) {
69475
69760
  const zippable = {};
69476
69761
  const walk = (current) => {
69477
- for (const entry of readdirSync6(current)) {
69762
+ for (const entry of readdirSync7(current)) {
69478
69763
  if (entry.startsWith(".")) continue;
69479
- const full = join12(current, entry);
69764
+ const full = join13(current, entry);
69480
69765
  const stat = lstatSync(full);
69481
69766
  if (stat.isSymbolicLink()) continue;
69482
69767
  if (stat.isDirectory()) {
69483
69768
  if (entry === "node_modules") continue;
69484
69769
  walk(full);
69485
69770
  } else {
69486
- zippable[relative5(dir, full).split("\\").join("/")] = readFileSync20(full);
69771
+ zippable[relative7(dir, full).split("\\").join("/")] = readFileSync20(full);
69487
69772
  }
69488
69773
  }
69489
69774
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runtypelabs/cli",
3
- "version": "2.22.17",
3
+ "version": "2.23.0",
4
4
  "description": "Command-line interface for Runtype AI platform",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -24,7 +24,7 @@
24
24
  "rosie-skills": "0.8.1",
25
25
  "yaml": "^2.9.0",
26
26
  "@runtypelabs/ink-components": "0.3.4",
27
- "@runtypelabs/sdk": "5.3.1",
27
+ "@runtypelabs/sdk": "5.4.0",
28
28
  "@runtypelabs/terminal-animations": "0.2.1"
29
29
  },
30
30
  "devDependencies": {
@@ -39,7 +39,7 @@
39
39
  "tsx": "^4.7.1",
40
40
  "typescript": "^6.0.3",
41
41
  "vitest": "^4.1.0",
42
- "@runtypelabs/shared": "1.42.3"
42
+ "@runtypelabs/shared": "1.42.4"
43
43
  },
44
44
  "engines": {
45
45
  "node": ">=22.0.0"