@vatio-ai/cli 0.80.0 → 0.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,7 @@ login | logout Authorize this machine in a browser, or revoke and f
54
54
  doctor Node, config, workspace and token status
55
55
 
56
56
  push Validate the workspace, then update preview
57
- publish Take preview live, once `vatio eval` has tested it
57
+ publish Take preview live, once `vatio test` has tested it
58
58
  diff [--env NAME] What this directory would change
59
59
  status Preview and live deployment state
60
60
  rollback Restore the previous live deployment
@@ -65,7 +65,8 @@ chat show [CHAT_ID] [--json]
65
65
  The chat you are in, or any chat in the workspace, with its tool calls
66
66
 
67
67
  flags Every open report from the inbox, for your coding agent to fix
68
- eval Test every report against preview; exits 1 if one fails
68
+ test Test every report against preview, in the background
69
+ test --wait Wait for the report; exits 1 if a case fails
69
70
  tools [--env NAME] How each tool's backend has been answering
70
71
 
71
72
  kb list|show|create|rm Knowledge bases
package/bin/vatio.mjs CHANGED
@@ -16,12 +16,12 @@ import { keys, secrets, tokens } from "../lib/commands/workspace-admin.mjs";
16
16
  import { templates, webhooks } from "../lib/commands/webhooks.mjs";
17
17
  import { routines } from "../lib/commands/routines.mjs";
18
18
  import { auth } from "../lib/commands/identity-key.mjs";
19
- import { chat } from "../lib/commands/chat.mjs";
19
+ import { chat, chatHelp } from "../lib/commands/chat.mjs";
20
20
  import { kb } from "../lib/commands/knowledge.mjs";
21
21
  import { business } from "../lib/commands/business.mjs";
22
22
  import { instagram, whatsapp } from "../lib/commands/channels.mjs";
23
23
  import { docs, feedback, shareSession } from "../lib/commands/support-commands.mjs";
24
- import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
24
+ import { evalCommand, flags, test, tools } from "../lib/commands/improve.mjs";
25
25
  import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
26
26
  import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
27
27
 
@@ -38,7 +38,7 @@ const RETIRED = {
38
38
  mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
39
39
  update: "is npm's job now: `npx @vatio-ai/cli@latest` always runs the current release, and\n" +
40
40
  "`npm install -g @vatio-ai/cli@latest` updates a global install.",
41
- propose: "was removed in 0.71.0. Test with `vatio eval` and publish with `vatio publish`."
41
+ propose: "was removed in 0.71.0. Test with `vatio test` and publish with `vatio publish`."
42
42
  };
43
43
 
44
44
  const config = new Config({ startDir: process.cwd() });
@@ -54,6 +54,12 @@ async function main(argv) {
54
54
  }
55
55
 
56
56
  async function run(command, args) {
57
+ // `--help` after a command asks about it. Left to the command, it reads as a
58
+ // message to send, an argument, or nothing at all: `vatio chat --help` said
59
+ // "--help" to the agent, and `vatio eval --help` started a run.
60
+ if (command !== undefined && (args.includes("--help") || args.includes("-h"))) {
61
+ return console.log(command === "chat" ? chatHelp() : helpText());
62
+ }
57
63
 
58
64
  switch (command) {
59
65
  case "version":
@@ -108,6 +114,8 @@ async function run(command, args) {
108
114
  return await instagram(config, args);
109
115
  case "flags":
110
116
  return await flags(config, args);
117
+ case "test":
118
+ return await test(config, args);
111
119
  case "eval":
112
120
  return await evalCommand(config, args);
113
121
  case "tools":
@@ -59,7 +59,7 @@ export async function chat(config, args) {
59
59
  return await converse(config, options, message);
60
60
  }
61
61
 
62
- function chatHelp() {
62
+ export function chatHelp() {
63
63
  return `Usage:
64
64
  vatio chat "message" [--env NAME] [--new] [--timeout N]
65
65
  vatio chat show [CHAT_ID] [--env NAME] [--last N] [--json]
@@ -183,8 +183,8 @@ function regressionsMessage(error) {
183
183
  lines.push(
184
184
  "",
185
185
  broken.length > 0
186
- ? "Fix them, then `vatio push` and `vatio eval` again before publishing."
187
- : "Run `vatio eval` again, then publish.",
186
+ ? "Fix them, then `vatio push` and `vatio test --wait` again before publishing."
187
+ : "Run `vatio test --wait` again, then publish.",
188
188
  "`vatio publish --force` publishes anyway: only if you know what you're doing."
189
189
  );
190
190
  return lines.join("\n");
@@ -1,8 +1,8 @@
1
- // flags / eval / tools -- what supervisors reported, proving a fix for it, and
1
+ // flags / test / tools -- what supervisors reported, proving a fix for it, and
2
2
  // how the tools' backend has been answering.
3
3
  //
4
4
  // All print markdown, because the reader is usually a coding agent: `vatio
5
- // flags` is the document it fixes the agent from, `vatio eval` is the
5
+ // flags` is the document it fixes the agent from, `vatio test` is the
6
6
  // evidence that the fix worked on preview before `vatio publish` takes it
7
7
  // live, and `vatio tools` is the inbox's Tools page for the failures nobody
8
8
  // reported.
@@ -88,16 +88,32 @@ export async function tools(config, args) {
88
88
  process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
89
89
  }
90
90
 
91
- // Replays every report with a note against an environment and waits for the
92
- // judge. Exits 1 when a case fails, so it can gate CI.
91
+ // Replays every report with a note against an environment. Starts the runs
92
+ // and returns: a run takes minutes, and a command that sat waiting for it was
93
+ // one a coding agent had to remember to put in the background. `--wait` waits
94
+ // for the judge and exits 1 when a case fails, so it can gate CI.
95
+ export async function test(config, args) {
96
+ return await runTests(config, args, { name: "test", waitByDefault: false });
97
+ }
98
+
99
+ // What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
100
+ // for the CI jobs and older skills that still call it; the name also reads as
101
+ // the shell's eval to command guards, which refused it.
93
102
  export async function evalCommand(config, args) {
103
+ console.error("vatio: `vatio eval` is deprecated; use `vatio test --wait` (or `vatio test` to start the run and return).");
104
+ return await runTests(config, args, { name: "eval", waitByDefault: true });
105
+ }
106
+
107
+ async function runTests(config, args, { name, waitByDefault }) {
94
108
  const show = takeValue(args, "--show");
95
- if (show) return await evalShow(config, show);
109
+ if (show) return await testShow(config, show);
96
110
 
97
111
  const environment = takeEnv(args, { fallback: "preview" });
98
112
  const agent = takeValue(args, "--agent");
113
+ const wait = takeFlag(args, "--wait");
99
114
  const noWait = takeFlag(args, "--no-wait");
100
115
  const timeoutS = Number(takeValue(args, "--timeout") ?? DEFAULT_TIMEOUT_S);
116
+ const waiting = wait || (waitByDefault && !noWait);
101
117
 
102
118
  const { url, token } = workspaceBase(config);
103
119
  const client = new HttpClient({ baseUrl: url, token });
@@ -115,17 +131,26 @@ export async function evalCommand(config, args) {
115
131
  const total = runs.reduce((sum, run) => sum + run.cases, 0);
116
132
  console.error(`vatio: replaying ${total} case${total === 1 ? "" : "s"} on ${environment}…`);
117
133
  await printSkillsHint();
118
- if (noWait) {
119
- for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio eval --show ${run.id}`);
134
+ if (!waiting) {
135
+ for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
136
+ if (name === "test") {
137
+ console.log(
138
+ "The runs keep going on Vatio's servers, five cases at a time. `vatio test --show RUN_ID` prints the\n" +
139
+ "report so far; `vatio test --wait` waits for this version's runs and exits 1 if a case fails."
140
+ );
141
+ }
120
142
  return;
121
143
  }
144
+ // Said before waiting, so the id is there to read the run again later
145
+ // however this command ends.
146
+ for (const run of runs) console.error(`vatio: run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
122
147
 
123
148
  const deadline = Date.now() + timeoutS * 1000;
124
149
  const finished = [];
125
150
  for (const run of runs) {
126
151
  let row = run;
127
152
  while (!row.finished) {
128
- if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio eval --show ${run.id}`);
153
+ if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio ${name} --show ${run.id}`);
129
154
  await sleep(POLL_MS);
130
155
  row = await client.get(`/evals/runs/${run.id}`);
131
156
  }
@@ -133,12 +158,11 @@ export async function evalCommand(config, args) {
133
158
  }
134
159
 
135
160
  for (const run of finished) {
136
- process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `eval run ${run.id}`));
161
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `test run ${run.id}`));
137
162
  }
138
163
  if (finished.some((run) => run.failed > 0)) process.exitCode = 1;
139
164
  }
140
165
 
141
- export async function evalShow(config, id) {
142
- if (!id) fail("Usage: vatio eval --show RUN_ID");
143
- process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `eval run ${id}`));
166
+ async function testShow(config, id) {
167
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `test run ${id}`));
144
168
  }
@@ -139,7 +139,7 @@ const AGENTS_NOTE = [
139
139
  "",
140
140
  "To fix what supervisors reported in the inbox, take every open report in",
141
141
  "one session: `vatio flags` (follow the instructions at the top of its",
142
- "output), fix, `vatio push`, `vatio eval` until the reports pass, then",
142
+ "output), fix, `vatio push`, `vatio test` until the reports pass, then",
143
143
  "`vatio publish`. The `vatio-improve` skill walks through it.",
144
144
  "`vatio tools --env live` shows how each tool's backend has been answering:",
145
145
  "error rate, timeouts, latency and the calls that failed.",
package/lib/help.mjs CHANGED
@@ -39,7 +39,7 @@ Commands default as listed.
39
39
  --force replaces an unpublished push from
40
40
  another git branch
41
41
  publish [--force] Take preview live, with its knowledge changes.
42
- Once published, refused until \`vatio eval\` has
42
+ Once published, refused until \`vatio test\` has
43
43
  tested preview's version, and while it fails a
44
44
  report live passes; --force publishes anyway
45
45
  rollback Put live back to the previous snapshot: config and knowledge
@@ -61,7 +61,7 @@ Commands default as listed.
61
61
  (web, WhatsApp, ...): messages, tool calls, channel
62
62
 
63
63
  Improve it from what supervisors reported and how its tools answered
64
- (flags, fix, push, eval, publish):
64
+ (flags, fix, push, test, publish):
65
65
  flags [--save [PATH]|--json]
66
66
  Every open report: the conversation, what each tool
67
67
  answered, and what should have happened
@@ -70,12 +70,16 @@ Commands default as listed.
70
70
  rate, timeouts and p50/p95, then the calls that
71
71
  failed with what your backend answered. Default:
72
72
  live, 7d
73
- eval [--env NAME] [--agent SLUG] [--no-wait] [--timeout N]
73
+ test [--env NAME] [--agent SLUG]
74
74
  Test every report with a note against preview
75
- (default) and judge it; exits 1 if one fails.
76
- Can take minutes: in Claude Code, run it in the
77
- background
78
- eval --show RUN_ID One run's report
75
+ (default): starts the runs on Vatio's servers,
76
+ prints their ids and returns.
77
+ --env live also resolves the reports it passes
78
+ test --wait [--timeout N] Wait for the judge and print the report; exits 1
79
+ if a case fails. Joins this version's running run
80
+ test --show RUN_ID One run's report so far, with the replay chat of
81
+ each case that did not pass (vatio chat show ID)
82
+ eval Deprecated: \`vatio test --wait\` under its old name
79
83
 
80
84
  Knowledge:
81
85
  kb [list] Bases, and whether a deployed agent reads each
@@ -105,7 +109,7 @@ Commands default as listed.
105
109
  Without VALUE, set asks for it with the input hidden.
106
110
  With --env, the value only that environment uses;
107
111
  otherwise the value everywhere. With --tests, the
108
- value preview and every eval use, like a staging URL
112
+ value preview and every test run use, like a staging URL
109
113
  tokens list|create [--env NAME] [--label NAME] [--origin URL]...|revoke PREFIX
110
114
  tokens origins PREFIX URL...
111
115
  The origins a token is accepted from (replaces the list)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.80.0",
3
+ "version": "0.81.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",