@vatio-ai/cli 0.79.0 → 0.81.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,7 @@ login | logout Authorize this machine in a browser, or revoke and f
54
54
  doctor Node, config, workspace and token status
55
55
 
56
56
  push Validate the workspace, then update preview
57
- publish Take preview live, once `vatio eval` has tested it
57
+ publish Take preview live, once `vatio test` has tested it
58
58
  diff [--env NAME] What this directory would change
59
59
  status Preview and live deployment state
60
60
  rollback Restore the previous live deployment
@@ -65,7 +65,8 @@ chat show [CHAT_ID] [--json]
65
65
  The chat you are in, or any chat in the workspace, with its tool calls
66
66
 
67
67
  flags Every open report from the inbox, for your coding agent to fix
68
- eval Test every report against preview; exits 1 if one fails
68
+ test Test every report against preview, in the background
69
+ test --wait Wait for the report; exits 1 if a case fails
69
70
  tools [--env NAME] How each tool's backend has been answering
70
71
 
71
72
  kb list|show|create|rm Knowledge bases
package/bin/vatio.mjs CHANGED
@@ -16,12 +16,12 @@ import { keys, secrets, tokens } from "../lib/commands/workspace-admin.mjs";
16
16
  import { templates, webhooks } from "../lib/commands/webhooks.mjs";
17
17
  import { routines } from "../lib/commands/routines.mjs";
18
18
  import { auth } from "../lib/commands/identity-key.mjs";
19
- import { chat } from "../lib/commands/chat.mjs";
19
+ import { chat, chatHelp } from "../lib/commands/chat.mjs";
20
20
  import { kb } from "../lib/commands/knowledge.mjs";
21
21
  import { business } from "../lib/commands/business.mjs";
22
22
  import { instagram, whatsapp } from "../lib/commands/channels.mjs";
23
23
  import { docs, feedback, shareSession } from "../lib/commands/support-commands.mjs";
24
- import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
24
+ import { evalCommand, flags, test, tools } from "../lib/commands/improve.mjs";
25
25
  import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
26
26
  import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
27
27
 
@@ -38,7 +38,7 @@ const RETIRED = {
38
38
  mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
39
39
  update: "is npm's job now: `npx @vatio-ai/cli@latest` always runs the current release, and\n" +
40
40
  "`npm install -g @vatio-ai/cli@latest` updates a global install.",
41
- propose: "was removed in 0.71.0. Test with `vatio eval` and publish with `vatio publish`."
41
+ propose: "was removed in 0.71.0. Test with `vatio test` and publish with `vatio publish`."
42
42
  };
43
43
 
44
44
  const config = new Config({ startDir: process.cwd() });
@@ -54,6 +54,12 @@ async function main(argv) {
54
54
  }
55
55
 
56
56
  async function run(command, args) {
57
+ // `--help` after a command asks about it. Left to the command, it reads as a
58
+ // message to send, an argument, or nothing at all: `vatio chat --help` said
59
+ // "--help" to the agent, and `vatio eval --help` started a run.
60
+ if (command !== undefined && (args.includes("--help") || args.includes("-h"))) {
61
+ return console.log(command === "chat" ? chatHelp() : helpText());
62
+ }
57
63
 
58
64
  switch (command) {
59
65
  case "version":
@@ -108,6 +114,8 @@ async function run(command, args) {
108
114
  return await instagram(config, args);
109
115
  case "flags":
110
116
  return await flags(config, args);
117
+ case "test":
118
+ return await test(config, args);
111
119
  case "eval":
112
120
  return await evalCommand(config, args);
113
121
  case "tools":
@@ -59,7 +59,7 @@ export async function chat(config, args) {
59
59
  return await converse(config, options, message);
60
60
  }
61
61
 
62
- function chatHelp() {
62
+ export function chatHelp() {
63
63
  return `Usage:
64
64
  vatio chat "message" [--env NAME] [--new] [--timeout N]
65
65
  vatio chat show [CHAT_ID] [--env NAME] [--last N] [--json]
@@ -71,6 +71,10 @@ your inbox.
71
71
  \`show\` prints the chat you are in, with every tool call the agent made. Give
72
72
  it an id to read any chat in the workspace -- one a visitor had on the web,
73
73
  WhatsApp or Instagram -- with the environment and deployment that answered it.
74
+ A [your backend] line under a tool call is how your backend answered it.
75
+ A [safeguard] line says which of Vatio's checks discarded or rewrote a draft,
76
+ and a [vatio] line marks a reply Vatio wrote because the agent's turn produced
77
+ none: why, and whether anything in your workspace caused it.
74
78
 
75
79
  The open chat per deployment is remembered in ${STATE_FILE}, so a second
76
80
  \`vatio chat\` continues the first. \`--new\` starts over as a visitor the
@@ -243,15 +247,64 @@ async function showChat(config, options, args) {
243
247
  if (limit) console.log(` [tool limit] ${limit}`);
244
248
  continue;
245
249
  }
246
- const flags = message.discarded ? " (discarded)" : "";
250
+ const safeguard = message.safeguard ?? null;
251
+ const flags = message.discarded
252
+ ? safeguard?.policy
253
+ ? ` (discarded by the ${safeguard.policy} safeguard)`
254
+ : " (discarded)"
255
+ : "";
247
256
  if (content !== "" || flags) console.log(`[${message.role}]${flags} ${content}`);
248
257
  for (const call of Array.isArray(message.tool_calls) ? message.tool_calls : []) {
249
258
  console.log(` -> ${call.name}(${truncate(JSON.stringify(call.arguments ?? {}), 400)})`);
259
+ if (call.backend) console.log(` [your backend] ${backendLine(call.backend)}`);
250
260
  }
261
+ if (safeguard) console.log(` [safeguard] ${safeguardLine(safeguard)}`);
262
+ if (message.fallback) console.log(` [vatio] ${fallbackLine(message.fallback)}`);
251
263
  }
252
264
  if (messages.length === 0) console.log("(no messages)");
253
265
  }
254
266
 
267
+ // How the developer's backend answered a tool call, as Vatio measured it.
268
+ // Anything but `ok` is the backend's to fix, not the agent's.
269
+ function backendLine(backend) {
270
+ const request = [backend.method, backend.url].filter(Boolean).join(" ");
271
+ const status = [backend.http_status && `HTTP ${backend.http_status}`, backend.outcome].filter(Boolean).join(" · ");
272
+ const failed = backend.outcome && backend.outcome !== "ok" ? " — failed in your backend, not in the agent" : "";
273
+ return `${request} → ${status} in ${backend.duration_ms ?? "?"} ms${failed}`;
274
+ }
275
+
276
+ // What a safeguard did to a draft, in words: Vatio::Safeguards' policies,
277
+ // which run on every reply and cannot be turned off.
278
+ function safeguardLine(safeguard) {
279
+ const detail = safeguard.detail ?? {};
280
+ const what =
281
+ safeguard.policy === "no_links"
282
+ ? `a link that is not in links:, instructions or a tool result (${asList(detail.forbidden_urls)})`
283
+ : safeguard.policy === "no_generated_media"
284
+ ? `embedded media (${asList(detail.markers)})`
285
+ : safeguard.policy === "no_empty_content"
286
+ ? "an empty draft"
287
+ : "a draft that broke a rule";
288
+ const action =
289
+ safeguard.action === "discarded"
290
+ ? "discarded it and asked the model again"
291
+ : "rewrote it before it went out";
292
+ return `${safeguard.policy}: ${what}; Vatio ${action}`;
293
+ }
294
+
295
+ // Why Vatio wrote this reply instead of the agent, and whose it is to fix.
296
+ function fallbackLine(fallback) {
297
+ const why = `Vatio wrote this reply because ${fallback.reason}.`;
298
+ if (fallback.owner === "vatio") return `${why} Not your agent: nothing in your workspace caused it.`;
299
+ if (fallback.cause === "tool_loop") return `${why} See which tool the agent repeated above, and what it answered.`;
300
+ return `${why} The fix is in your agent.`;
301
+ }
302
+
303
+ function asList(value) {
304
+ const items = Array.isArray(value) ? value : [];
305
+ return items.length > 0 ? truncate(items.join(", "), 200) : "not recorded";
306
+ }
307
+
255
308
  // Vatio::Runtime::ToolBudget's marks on a tool result, said on a line of their
256
309
  // own: the note rides at the end of the result and truncate() can cut it off,
257
310
  // which left a turn that ran out of calls looking like one that just stopped.
@@ -173,11 +173,18 @@ export async function publish(config, args) {
173
173
  }
174
174
 
175
175
  function regressionsMessage(error) {
176
+ const rows = asArray(error.body.regressions);
176
177
  const lines = [error.message];
177
- for (const row of asArray(error.body.regressions)) lines.push(` ${row.id} — ${row.title}`);
178
+ for (const row of rows) {
179
+ const untested = row.vatio_error ? " (not tested: an error on Vatio's side, not your agent)" : "";
180
+ lines.push(` ${row.id} — ${row.title}${untested}`);
181
+ }
182
+ const broken = rows.filter((row) => !row.vatio_error);
178
183
  lines.push(
179
184
  "",
180
- "Fix them, then `vatio push` and `vatio eval` again before publishing.",
185
+ broken.length > 0
186
+ ? "Fix them, then `vatio push` and `vatio test --wait` again before publishing."
187
+ : "Run `vatio test --wait` again, then publish.",
181
188
  "`vatio publish --force` publishes anyway: only if you know what you're doing."
182
189
  );
183
190
  return lines.join("\n");
@@ -204,6 +211,35 @@ export async function status(config) {
204
211
  `reads it — add \`knowledge: [${base.name}]\` to vatio.yml and push.`
205
212
  );
206
213
  }
214
+ printVatioSide(payload.vatio);
215
+ }
216
+
217
+ // Is it Vatio or is it me? The replies Vatio wrote here because the agent's
218
+ // turn produced none, split by whose they are to fix. Also on stderr, for the
219
+ // same reason as the warning above, and only when there is something to say:
220
+ // a line on every call would be noise to anything reading both.
221
+ function printVatioSide(vatio) {
222
+ if (!vatio) return;
223
+ const latest = asArray(vatio.fallback_replies?.latest);
224
+ const byCause = vatio.fallback_replies?.by_cause ?? {};
225
+ const total = Object.values(byCause).reduce((sum, count) => sum + Number(count), 0);
226
+ if (total === 0) return;
227
+
228
+ const counts = Object.entries(byCause)
229
+ .map(([cause, count]) => `${cause} ${count}`)
230
+ .join(", ");
231
+ console.error(`Replies Vatio wrote because the agent's turn produced none, last 7 days: ${total} (${counts}).`);
232
+ for (const reply of latest) {
233
+ const whose = reply.owner === "vatio" ? "Vatio's side, not your agent" : "your agent";
234
+ console.error(
235
+ ` ${reply.at} ${reply.environment} — ${reply.reason} (${whose}): vatio chat show ${reply.chat_id}`
236
+ );
237
+ }
238
+ if (latest.some((reply) => reply.owner === "vatio")) {
239
+ console.error(
240
+ ' Nothing to fix in your agent for those. If they keep happening, tell us: vatio feedback "…"'
241
+ );
242
+ }
207
243
  }
208
244
 
209
245
  export async function diff(config, args) {
@@ -1,8 +1,8 @@
1
- // flags / eval / tools -- what supervisors reported, proving a fix for it, and
1
+ // flags / test / tools -- what supervisors reported, proving a fix for it, and
2
2
  // how the tools' backend has been answering.
3
3
  //
4
4
  // All print markdown, because the reader is usually a coding agent: `vatio
5
- // flags` is the document it fixes the agent from, `vatio eval` is the
5
+ // flags` is the document it fixes the agent from, `vatio test` is the
6
6
  // evidence that the fix worked on preview before `vatio publish` takes it
7
7
  // live, and `vatio tools` is the inbox's Tools page for the failures nobody
8
8
  // reported.
@@ -88,16 +88,32 @@ export async function tools(config, args) {
88
88
  process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
89
89
  }
90
90
 
91
- // Replays every report with a note against an environment and waits for the
92
- // judge. Exits 1 when a case fails, so it can gate CI.
91
+ // Replays every report with a note against an environment. Starts the runs
92
+ // and returns: a run takes minutes, and a command that sat waiting for it was
93
+ // one a coding agent had to remember to put in the background. `--wait` waits
94
+ // for the judge and exits 1 when a case fails, so it can gate CI.
95
+ export async function test(config, args) {
96
+ return await runTests(config, args, { name: "test", waitByDefault: false });
97
+ }
98
+
99
+ // What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
100
+ // for the CI jobs and older skills that still call it; the name also reads as
101
+ // the shell's eval to command guards, which refused it.
93
102
  export async function evalCommand(config, args) {
103
+ console.error("vatio: `vatio eval` is deprecated; use `vatio test --wait` (or `vatio test` to start the run and return).");
104
+ return await runTests(config, args, { name: "eval", waitByDefault: true });
105
+ }
106
+
107
+ async function runTests(config, args, { name, waitByDefault }) {
94
108
  const show = takeValue(args, "--show");
95
- if (show) return await evalShow(config, show);
109
+ if (show) return await testShow(config, show);
96
110
 
97
111
  const environment = takeEnv(args, { fallback: "preview" });
98
112
  const agent = takeValue(args, "--agent");
113
+ const wait = takeFlag(args, "--wait");
99
114
  const noWait = takeFlag(args, "--no-wait");
100
115
  const timeoutS = Number(takeValue(args, "--timeout") ?? DEFAULT_TIMEOUT_S);
116
+ const waiting = wait || (waitByDefault && !noWait);
101
117
 
102
118
  const { url, token } = workspaceBase(config);
103
119
  const client = new HttpClient({ baseUrl: url, token });
@@ -115,17 +131,26 @@ export async function evalCommand(config, args) {
115
131
  const total = runs.reduce((sum, run) => sum + run.cases, 0);
116
132
  console.error(`vatio: replaying ${total} case${total === 1 ? "" : "s"} on ${environment}…`);
117
133
  await printSkillsHint();
118
- if (noWait) {
119
- for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio eval --show ${run.id}`);
134
+ if (!waiting) {
135
+ for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
136
+ if (name === "test") {
137
+ console.log(
138
+ "The runs keep going on Vatio's servers, five cases at a time. `vatio test --show RUN_ID` prints the\n" +
139
+ "report so far; `vatio test --wait` waits for this version's runs and exits 1 if a case fails."
140
+ );
141
+ }
120
142
  return;
121
143
  }
144
+ // Said before waiting, so the id is there to read the run again later
145
+ // however this command ends.
146
+ for (const run of runs) console.error(`vatio: run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
122
147
 
123
148
  const deadline = Date.now() + timeoutS * 1000;
124
149
  const finished = [];
125
150
  for (const run of runs) {
126
151
  let row = run;
127
152
  while (!row.finished) {
128
- if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio eval --show ${run.id}`);
153
+ if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio ${name} --show ${run.id}`);
129
154
  await sleep(POLL_MS);
130
155
  row = await client.get(`/evals/runs/${run.id}`);
131
156
  }
@@ -133,12 +158,11 @@ export async function evalCommand(config, args) {
133
158
  }
134
159
 
135
160
  for (const run of finished) {
136
- process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `eval run ${run.id}`));
161
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `test run ${run.id}`));
137
162
  }
138
163
  if (finished.some((run) => run.failed > 0)) process.exitCode = 1;
139
164
  }
140
165
 
141
- export async function evalShow(config, id) {
142
- if (!id) fail("Usage: vatio eval --show RUN_ID");
143
- process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `eval run ${id}`));
166
+ async function testShow(config, id) {
167
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `test run ${id}`));
144
168
  }
@@ -139,7 +139,7 @@ const AGENTS_NOTE = [
139
139
  "",
140
140
  "To fix what supervisors reported in the inbox, take every open report in",
141
141
  "one session: `vatio flags` (follow the instructions at the top of its",
142
- "output), fix, `vatio push`, `vatio eval` until the reports pass, then",
142
+ "output), fix, `vatio push`, `vatio test` until the reports pass, then",
143
143
  "`vatio publish`. The `vatio-improve` skill walks through it.",
144
144
  "`vatio tools --env live` shows how each tool's backend has been answering:",
145
145
  "error rate, timeouts, latency and the calls that failed.",
package/lib/help.mjs CHANGED
@@ -39,11 +39,12 @@ Commands default as listed.
39
39
  --force replaces an unpublished push from
40
40
  another git branch
41
41
  publish [--force] Take preview live, with its knowledge changes.
42
- Once published, refused until \`vatio eval\` has
42
+ Once published, refused until \`vatio test\` has
43
43
  tested preview's version, and while it fails a
44
44
  report live passes; --force publishes anyway
45
45
  rollback Put live back to the previous snapshot: config and knowledge
46
- status Preview and live deployment state
46
+ status Preview and live deployment state, and the replies
47
+ Vatio wrote for the agent, and why
47
48
  env [preview|live] An environment: token, widget snippet, link,
48
49
  knowledge changes and conflicts (default: preview)
49
50
  env list Both environments and what each runs
@@ -60,7 +61,7 @@ Commands default as listed.
60
61
  (web, WhatsApp, ...): messages, tool calls, channel
61
62
 
62
63
  Improve it from what supervisors reported and how its tools answered
63
- (flags, fix, push, eval, publish):
64
+ (flags, fix, push, test, publish):
64
65
  flags [--save [PATH]|--json]
65
66
  Every open report: the conversation, what each tool
66
67
  answered, and what should have happened
@@ -69,12 +70,16 @@ Commands default as listed.
69
70
  rate, timeouts and p50/p95, then the calls that
70
71
  failed with what your backend answered. Default:
71
72
  live, 7d
72
- eval [--env NAME] [--agent SLUG] [--no-wait] [--timeout N]
73
+ test [--env NAME] [--agent SLUG]
73
74
  Test every report with a note against preview
74
- (default) and judge it; exits 1 if one fails.
75
- Can take minutes: in Claude Code, run it in the
76
- background
77
- eval --show RUN_ID One run's report
75
+ (default): starts the runs on Vatio's servers,
76
+ prints their ids and returns.
77
+ --env live also resolves the reports it passes
78
+ test --wait [--timeout N] Wait for the judge and print the report; exits 1
79
+ if a case fails. Joins this version's running run
80
+ test --show RUN_ID One run's report so far, with the replay chat of
81
+ each case that did not pass (vatio chat show ID)
82
+ eval Deprecated: \`vatio test --wait\` under its old name
78
83
 
79
84
  Knowledge:
80
85
  kb [list] Bases, and whether a deployed agent reads each
@@ -104,7 +109,7 @@ Commands default as listed.
104
109
  Without VALUE, set asks for it with the input hidden.
105
110
  With --env, the value only that environment uses;
106
111
  otherwise the value everywhere. With --tests, the
107
- value preview and every eval use, like a staging URL
112
+ value preview and every test run use, like a staging URL
108
113
  tokens list|create [--env NAME] [--label NAME] [--origin URL]...|revoke PREFIX
109
114
  tokens origins PREFIX URL...
110
115
  The origins a token is accepted from (replaces the list)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.79.0",
3
+ "version": "0.81.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",