@vatio-ai/cli 0.80.0 → 0.82.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,7 @@ login | logout Authorize this machine in a browser, or revoke and f
54
54
  doctor Node, config, workspace and token status
55
55
 
56
56
  push Validate the workspace, then update preview
57
- publish Take preview live, once `vatio eval` has tested it
57
+ publish Take preview live, once `vatio test` has tested it
58
58
  diff [--env NAME] What this directory would change
59
59
  status Preview and live deployment state
60
60
  rollback Restore the previous live deployment
@@ -65,7 +65,8 @@ chat show [CHAT_ID] [--json]
65
65
  The chat you are in, or any chat in the workspace, with its tool calls
66
66
 
67
67
  flags Every open report from the inbox, for your coding agent to fix
68
- eval Test every report against preview; exits 1 if one fails
68
+ test Test every report against preview, in the background
69
+ test --wait Wait for the report; exits 1 if a case fails
69
70
  tools [--env NAME] How each tool's backend has been answering
70
71
 
71
72
  kb list|show|create|rm Knowledge bases
package/bin/vatio.mjs CHANGED
@@ -5,7 +5,7 @@
5
5
  // terminal; the credential lives in ~/.vatio/config.json.
6
6
 
7
7
  import { Config, ConfigError } from "../lib/config.mjs";
8
- import { HttpError, UnauthorizedError, ForbiddenError, NotFoundError } from "../lib/http.mjs";
8
+ import { CliOutdatedError, HttpError, UnauthorizedError, ForbiddenError, NotFoundError } from "../lib/http.mjs";
9
9
  import { UserError, fail } from "../lib/support.mjs";
10
10
  import { helpText } from "../lib/help.mjs";
11
11
  import { login, logout } from "../lib/commands/auth.mjs";
@@ -16,13 +16,14 @@ import { keys, secrets, tokens } from "../lib/commands/workspace-admin.mjs";
16
16
  import { templates, webhooks } from "../lib/commands/webhooks.mjs";
17
17
  import { routines } from "../lib/commands/routines.mjs";
18
18
  import { auth } from "../lib/commands/identity-key.mjs";
19
- import { chat } from "../lib/commands/chat.mjs";
19
+ import { chat, chatHelp } from "../lib/commands/chat.mjs";
20
20
  import { kb } from "../lib/commands/knowledge.mjs";
21
21
  import { business } from "../lib/commands/business.mjs";
22
22
  import { instagram, whatsapp } from "../lib/commands/channels.mjs";
23
23
  import { docs, feedback, shareSession } from "../lib/commands/support-commands.mjs";
24
- import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
24
+ import { evalCommand, flags, test, tools } from "../lib/commands/improve.mjs";
25
25
  import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
26
+ import { runLatest } from "../lib/run-latest.mjs";
26
27
  import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
27
28
 
28
29
  // Commands that existed and are gone, answered by name rather than as
@@ -38,7 +39,7 @@ const RETIRED = {
38
39
  mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
39
40
  update: "is npm's job now: `npx @vatio-ai/cli@latest` always runs the current release, and\n" +
40
41
  "`npm install -g @vatio-ai/cli@latest` updates a global install.",
41
- propose: "was removed in 0.71.0. Test with `vatio eval` and publish with `vatio publish`."
42
+ propose: "was removed in 0.71.0. Test with `vatio test` and publish with `vatio publish`."
42
43
  };
43
44
 
44
45
  const config = new Config({ startDir: process.cwd() });
@@ -54,6 +55,12 @@ async function main(argv) {
54
55
  }
55
56
 
56
57
  async function run(command, args) {
58
+ // `--help` after a command asks about it. Left to the command, it reads as a
59
+ // message to send, an argument, or nothing at all: `vatio chat --help` said
60
+ // "--help" to the agent, and `vatio eval --help` started a run.
61
+ if (command !== undefined && (args.includes("--help") || args.includes("-h"))) {
62
+ return console.log(command === "chat" ? chatHelp() : helpText());
63
+ }
57
64
 
58
65
  switch (command) {
59
66
  case "version":
@@ -108,6 +115,8 @@ async function run(command, args) {
108
115
  return await instagram(config, args);
109
116
  case "flags":
110
117
  return await flags(config, args);
118
+ case "test":
119
+ return await test(config, args);
111
120
  case "eval":
112
121
  return await evalCommand(config, args);
113
122
  case "tools":
@@ -131,6 +140,11 @@ async function run(command, args) {
131
140
 
132
141
 
133
142
  main(process.argv).catch(async (error) => {
143
+ if (error instanceof CliOutdatedError) {
144
+ // The run it starts reports itself; this one would count the command twice.
145
+ const code = runLatest(error, process.argv);
146
+ if (code !== null) process.exit(code);
147
+ }
134
148
  printError(error);
135
149
  await reportRun(config, { exitCode: 1, error });
136
150
  process.exit(1);
@@ -59,7 +59,7 @@ export async function chat(config, args) {
59
59
  return await converse(config, options, message);
60
60
  }
61
61
 
62
- function chatHelp() {
62
+ export function chatHelp() {
63
63
  return `Usage:
64
64
  vatio chat "message" [--env NAME] [--new] [--timeout N]
65
65
  vatio chat show [CHAT_ID] [--env NAME] [--last N] [--json]
@@ -183,8 +183,8 @@ function regressionsMessage(error) {
183
183
  lines.push(
184
184
  "",
185
185
  broken.length > 0
186
- ? "Fix them, then `vatio push` and `vatio eval` again before publishing."
187
- : "Run `vatio eval` again, then publish.",
186
+ ? "Fix them, then `vatio push` and `vatio test --wait` again before publishing."
187
+ : "Run `vatio test --wait` again, then publish.",
188
188
  "`vatio publish --force` publishes anyway: only if you know what you're doing."
189
189
  );
190
190
  return lines.join("\n");
@@ -1,8 +1,8 @@
1
- // flags / eval / tools -- what supervisors reported, proving a fix for it, and
1
+ // flags / test / tools -- what supervisors reported, proving a fix for it, and
2
2
  // how the tools' backend has been answering.
3
3
  //
4
4
  // All print markdown, because the reader is usually a coding agent: `vatio
5
- // flags` is the document it fixes the agent from, `vatio eval` is the
5
+ // flags` is the document it fixes the agent from, `vatio test` is the
6
6
  // evidence that the fix worked on preview before `vatio publish` takes it
7
7
  // live, and `vatio tools` is the inbox's Tools page for the failures nobody
8
8
  // reported.
@@ -10,7 +10,7 @@
10
10
  import { mkdirSync, writeFileSync } from "node:fs";
11
11
  import { dirname, resolve } from "node:path";
12
12
 
13
- import { HttpClient } from "../http.mjs";
13
+ import { HttpClient, NotFoundError, throwIfOutdated } from "../http.mjs";
14
14
  import { clientHeaders } from "../telemetry.mjs";
15
15
  import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
16
16
  import { printSkillsHint } from "../skills-hint.mjs";
@@ -31,6 +31,7 @@ async function fetchMarkdown(config, path, what) {
31
31
  headers: { ...clientHeaders(), Accept: "text/markdown, text/plain", Authorization: `Bearer ${token}` },
32
32
  signal: AbortSignal.timeout(30_000)
33
33
  });
34
+ await throwIfOutdated(response);
34
35
  if (!response.ok) {
35
36
  const body = await response.json().catch(() => null);
36
37
  const reason = body?.error?.message ?? `HTTP ${response.status}`;
@@ -43,6 +44,8 @@ async function fetchMarkdown(config, path, what) {
43
44
  // before it, every tool call of that turn with what your backend answered,
44
45
  // and what the supervisor says should have happened.
45
46
  export async function flags(config, args) {
47
+ if (["rm", "remove", "delete"].includes(args[0])) return await removeFlags(config, args.slice(1));
48
+
46
49
  const json = takeFlag(args, "--json");
47
50
  const save = takeFlag(args, "--save");
48
51
  const savePath = save ? args.shift() ?? null : null;
@@ -68,6 +71,28 @@ export async function flags(config, args) {
68
71
  await printSkillsHint();
69
72
  }
70
73
 
74
+ // What the inbox's Quitar does, by the ids `vatio flags` prints: the flag and
75
+ // its test case go, for everyone in the workspace. The developer's call, not
76
+ // the coding agent's: the vatio-improve skill says to run it only when asked.
77
+ async function removeFlags(config, ids) {
78
+ if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio flags rm ID [ID...]");
79
+
80
+ const { url, token } = workspaceBase(config);
81
+ const client = new HttpClient({ baseUrl: url, token });
82
+ for (const id of ids) {
83
+ let removed;
84
+ try {
85
+ removed = await client.delete(`/flags/${encodeURIComponent(id)}`);
86
+ } catch (error) {
87
+ // The platform's own words name the id and where to find the right one.
88
+ if (error instanceof NotFoundError) fail(error.message);
89
+ throw error;
90
+ }
91
+ const was = removed.state === "resolved" ? "a resolved flag, so no longer a regression case" : "an open flag";
92
+ console.log(`Removed ${removed.id} (${was}) and its test case.`);
93
+ }
94
+ }
95
+
71
96
  // Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
72
97
  // then the calls that failed with what your backend answered: the inbox's
73
98
  // Tools page.
@@ -88,16 +113,32 @@ export async function tools(config, args) {
88
113
  process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
89
114
  }
90
115
 
91
- // Replays every report with a note against an environment and waits for the
92
- // judge. Exits 1 when a case fails, so it can gate CI.
116
+ // Replays every report with a note against an environment. Starts the runs
117
+ // and returns: a run takes minutes, and a command that sat waiting for it was
118
+ // one a coding agent had to remember to put in the background. `--wait` waits
119
+ // for the judge and exits 1 when a case fails, so it can gate CI.
120
+ export async function test(config, args) {
121
+ return await runTests(config, args, { name: "test", waitByDefault: false });
122
+ }
123
+
124
+ // What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
125
+ // for the CI jobs and older skills that still call it; the name also reads as
126
+ // the shell's eval to command guards, which refused it.
93
127
  export async function evalCommand(config, args) {
128
+ console.error("vatio: `vatio eval` is deprecated; use `vatio test --wait` (or `vatio test` to start the run and return).");
129
+ return await runTests(config, args, { name: "eval", waitByDefault: true });
130
+ }
131
+
132
+ async function runTests(config, args, { name, waitByDefault }) {
94
133
  const show = takeValue(args, "--show");
95
- if (show) return await evalShow(config, show);
134
+ if (show) return await testShow(config, show);
96
135
 
97
136
  const environment = takeEnv(args, { fallback: "preview" });
98
137
  const agent = takeValue(args, "--agent");
138
+ const wait = takeFlag(args, "--wait");
99
139
  const noWait = takeFlag(args, "--no-wait");
100
140
  const timeoutS = Number(takeValue(args, "--timeout") ?? DEFAULT_TIMEOUT_S);
141
+ const waiting = wait || (waitByDefault && !noWait);
101
142
 
102
143
  const { url, token } = workspaceBase(config);
103
144
  const client = new HttpClient({ baseUrl: url, token });
@@ -115,17 +156,26 @@ export async function evalCommand(config, args) {
115
156
  const total = runs.reduce((sum, run) => sum + run.cases, 0);
116
157
  console.error(`vatio: replaying ${total} case${total === 1 ? "" : "s"} on ${environment}…`);
117
158
  await printSkillsHint();
118
- if (noWait) {
119
- for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio eval --show ${run.id}`);
159
+ if (!waiting) {
160
+ for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
161
+ if (name === "test") {
162
+ console.log(
163
+ "The runs keep going on Vatio's servers, five cases at a time. `vatio test --show RUN_ID` prints the\n" +
164
+ "report so far; `vatio test --wait` waits for this version's runs and exits 1 if a case fails."
165
+ );
166
+ }
120
167
  return;
121
168
  }
169
+ // Said before waiting, so the id is there to read the run again later
170
+ // however this command ends.
171
+ for (const run of runs) console.error(`vatio: run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
122
172
 
123
173
  const deadline = Date.now() + timeoutS * 1000;
124
174
  const finished = [];
125
175
  for (const run of runs) {
126
176
  let row = run;
127
177
  while (!row.finished) {
128
- if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio eval --show ${run.id}`);
178
+ if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio ${name} --show ${run.id}`);
129
179
  await sleep(POLL_MS);
130
180
  row = await client.get(`/evals/runs/${run.id}`);
131
181
  }
@@ -133,12 +183,11 @@ export async function evalCommand(config, args) {
133
183
  }
134
184
 
135
185
  for (const run of finished) {
136
- process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `eval run ${run.id}`));
186
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `test run ${run.id}`));
137
187
  }
138
188
  if (finished.some((run) => run.failed > 0)) process.exitCode = 1;
139
189
  }
140
190
 
141
- export async function evalShow(config, id) {
142
- if (!id) fail("Usage: vatio eval --show RUN_ID");
143
- process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `eval run ${id}`));
191
+ async function testShow(config, id) {
192
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `test run ${id}`));
144
193
  }
@@ -16,7 +16,7 @@
16
16
  import { readFileSync, statSync } from "node:fs";
17
17
 
18
18
  import { deployClient } from "./deploy.mjs";
19
- import { fail, takeEnv, takeValue } from "../support.mjs";
19
+ import { fail, rememberStdin, takeEnv, takeValue } from "../support.mjs";
20
20
 
21
21
  // Only what reads or writes entries takes an environment; bases belong to the
22
22
  // workspace.
@@ -272,7 +272,7 @@ function readStdin() {
272
272
  if (process.stdin.isTTY) return "";
273
273
 
274
274
  try {
275
- return readFileSync(0, "utf8");
275
+ return rememberStdin(readFileSync(0, "utf8"));
276
276
  } catch {
277
277
  return "";
278
278
  }
@@ -139,7 +139,7 @@ const AGENTS_NOTE = [
139
139
  "",
140
140
  "To fix what supervisors reported in the inbox, take every open report in",
141
141
  "one session: `vatio flags` (follow the instructions at the top of its",
142
- "output), fix, `vatio push`, `vatio eval` until the reports pass, then",
142
+ "output), fix, `vatio push`, `vatio test` until the reports pass, then",
143
143
  "`vatio publish`. The `vatio-improve` skill walks through it.",
144
144
  "`vatio tools --env live` shows how each tool's backend has been answering:",
145
145
  "error rate, timeouts, latency and the calls that failed.",
@@ -5,6 +5,7 @@
5
5
  // Unauthenticated by definition -- this is what mints the token -- so it does
6
6
  // not go through HttpClient, which requires one.
7
7
 
8
+ import { throwIfOutdated } from "./http.mjs";
8
9
  import { clientHeaders } from "./telemetry.mjs";
9
10
 
10
11
  export class DeviceAuthError extends Error {}
@@ -43,6 +44,7 @@ export class DeviceAuthClient {
43
44
  const reason = error?.cause?.code ?? error?.name ?? "request failed";
44
45
  throw new DeviceAuthError(`${reason}: ${url}`);
45
46
  }
47
+ await throwIfOutdated(response);
46
48
 
47
49
  const raw = await response.text();
48
50
  let parsed;
package/lib/help.mjs CHANGED
@@ -39,7 +39,7 @@ Commands default as listed.
39
39
  --force replaces an unpublished push from
40
40
  another git branch
41
41
  publish [--force] Take preview live, with its knowledge changes.
42
- Once published, refused until \`vatio eval\` has
42
+ Once published, refused until \`vatio test\` has
43
43
  tested preview's version, and while it fails a
44
44
  report live passes; --force publishes anyway
45
45
  rollback Put live back to the previous snapshot: config and knowledge
@@ -61,21 +61,27 @@ Commands default as listed.
61
61
  (web, WhatsApp, ...): messages, tool calls, channel
62
62
 
63
63
  Improve it from what supervisors reported and how its tools answered
64
- (flags, fix, push, eval, publish):
64
+ (flags, fix, push, test, publish):
65
65
  flags [--save [PATH]|--json]
66
66
  Every open report: the conversation, what each tool
67
67
  answered, and what should have happened
68
+ flags rm ID [ID...] Remove a report and its test case, open or
69
+ resolved, as the inbox's Remove does
68
70
  tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
69
71
  The inbox's Tools page: each tool's calls, error
70
72
  rate, timeouts and p50/p95, then the calls that
71
73
  failed with what your backend answered. Default:
72
74
  live, 7d
73
- eval [--env NAME] [--agent SLUG] [--no-wait] [--timeout N]
75
+ test [--env NAME] [--agent SLUG]
74
76
  Test every report with a note against preview
75
- (default) and judge it; exits 1 if one fails.
76
- Can take minutes: in Claude Code, run it in the
77
- background
78
- eval --show RUN_ID One run's report
77
+ (default): starts the runs on Vatio's servers,
78
+ prints their ids and returns.
79
+ --env live also resolves the reports it passes
80
+ test --wait [--timeout N] Wait for the judge and print the report; exits 1
81
+ if a case fails. Joins this version's running run
82
+ test --show RUN_ID One run's report so far, with the replay chat of
83
+ each case that did not pass (vatio chat show ID)
84
+ eval Deprecated: \`vatio test --wait\` under its old name
79
85
 
80
86
  Knowledge:
81
87
  kb [list] Bases, and whether a deployed agent reads each
@@ -105,7 +111,7 @@ Commands default as listed.
105
111
  Without VALUE, set asks for it with the input hidden.
106
112
  With --env, the value only that environment uses;
107
113
  otherwise the value everywhere. With --tests, the
108
- value preview and every eval use, like a staging URL
114
+ value preview and every test run use, like a staging URL
109
115
  tokens list|create [--env NAME] [--label NAME] [--origin URL]...|revoke PREFIX
110
116
  tokens origins PREFIX URL...
111
117
  The origins a token is accepted from (replaces the list)
package/lib/http.mjs CHANGED
@@ -21,6 +21,27 @@ export class UnauthorizedError extends HttpError {}
21
21
  export class ForbiddenError extends HttpError {}
22
22
  export class NotFoundError extends HttpError {}
23
23
 
24
+ // The platform takes calls only from the CLI's current release
25
+ // (RequiresLatestCli on the backend): `latest` is the version to run instead.
26
+ // The refused call did nothing, so bin/vatio.mjs runs the whole command again
27
+ // as that version (run-latest.mjs).
28
+ export class CliOutdatedError extends HttpError {
29
+ constructor(message, options) {
30
+ super(message, options);
31
+ this.latest = typeof options.body?.latest === "string" ? options.body.latest : null;
32
+ }
33
+ }
34
+
35
+ // For the calls made with fetch() directly rather than through HttpClient.
36
+ export async function throwIfOutdated(response) {
37
+ if (response.status !== 426) return;
38
+
39
+ const raw = await response.clone().text();
40
+ const body = parseJson(raw) ?? {};
41
+ if (body.error_key !== "cli_outdated") return;
42
+ throw new CliOutdatedError(responseMessage(body, raw, response), { status: 426, requestId: body.request_id ?? null, body });
43
+ }
44
+
24
45
  export class UnprocessableError extends HttpError {
25
46
  constructor(body) {
26
47
  const hash = body && typeof body === "object" ? body : {};
@@ -125,6 +146,7 @@ export class HttpClient {
125
146
  }
126
147
 
127
148
  const options = { status, requestId, body: body ?? {} };
149
+ if (status === 426 && body?.error_key === "cli_outdated") throw new CliOutdatedError(message, options);
128
150
  if (status === 401) throw new UnauthorizedError(message, options);
129
151
  if (status === 403) throw new ForbiddenError(message, options);
130
152
  if (status === 404) throw new NotFoundError(message, options);
@@ -0,0 +1,36 @@
1
+ // What the CLI does when the platform refuses a call because this is not its
2
+ // current release (426 cli_outdated, see CliOutdatedError): it runs the
3
+ // release it was told to through npx, with the same arguments, and exits with
4
+ // that run's code. The refused call did nothing, so the whole command can run
5
+ // again; stdin a command already read is handed on, so
6
+ // `op read … | vatio secrets set KEY` still works.
7
+ //
8
+ // Returns null when it cannot, and the caller prints the refusal, which says
9
+ // how to update: never twice in a row (a release published in between), and
10
+ // not on Windows, where npx is a .cmd that Node starts only through a shell
11
+ // that would split the arguments again.
12
+
13
+ import { spawnSync } from "node:child_process";
14
+
15
+ import { consumedStdin } from "./support.mjs";
16
+ import { VERSION } from "./version.mjs";
17
+
18
+ const RAN_LATEST = "VATIO_CLI_RAN_LATEST";
19
+
20
+ export function runLatest(error, argv) {
21
+ const latest = error.latest;
22
+ if (!/^\d+\.\d+\.\d+$/.test(latest ?? "") || process.env[RAN_LATEST] || process.platform === "win32") return null;
23
+
24
+ console.error(
25
+ `vatio: ${VERSION} is not the current release, so this runs ${latest} through npx. ` +
26
+ "`npm install -g @vatio-ai/cli@latest` installs it."
27
+ );
28
+ const input = consumedStdin();
29
+ const child = spawnSync("npx", ["--yes", `@vatio-ai/cli@${latest}`, ...argv.slice(2)], {
30
+ stdio: input === null ? "inherit" : ["pipe", "inherit", "inherit"],
31
+ input: input ?? undefined,
32
+ env: { ...process.env, [RAN_LATEST]: latest }
33
+ });
34
+ if (child.error) return null;
35
+ return child.status ?? 1;
36
+ }
package/lib/support.mjs CHANGED
@@ -178,8 +178,21 @@ export async function askSecret(question) {
178
178
  });
179
179
  }
180
180
 
181
+ // What a command read from stdin, kept for the run of the current release
182
+ // that may follow a refused call (run-latest.mjs): stdin can only be read once.
183
+ let stdinRead = null;
184
+
185
+ export function rememberStdin(text) {
186
+ stdinRead = text;
187
+ return text;
188
+ }
189
+
190
+ export function consumedStdin() {
191
+ return stdinRead;
192
+ }
193
+
181
194
  async function readStdin() {
182
195
  const chunks = [];
183
196
  for await (const chunk of process.stdin) chunks.push(chunk);
184
- return Buffer.concat(chunks).toString("utf8");
197
+ return rememberStdin(Buffer.concat(chunks).toString("utf8"));
185
198
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.80.0",
3
+ "version": "0.82.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",