@vatio-ai/cli 0.118.0 → 0.120.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -101,7 +101,7 @@ your inbox.
101
101
  \`show\` prints the chat you are in, with every tool call the agent made. Give
102
102
  it an id to read any chat in the workspace -- one a visitor had on the web,
103
103
  WhatsApp or Instagram -- with the environment and deployment that answered it.
104
- Each message carries its id (#4821), the one \`vatio flags add --message\` takes.
104
+ Each message carries its id (#4821), the one \`vatio test add --message\` takes.
105
105
  A [your backend] line under a tool call is how your backend answered it.
106
106
  A [safeguard] line says which of Vatio's checks discarded or rewrote a draft,
107
107
  and a [vatio] line marks a reply Vatio wrote because the agent's turn produced
@@ -215,7 +215,7 @@ function chatsFor(config, apiUrl) {
215
215
  return state.chats && typeof state.chats === "object" ? state.chats : {};
216
216
  }
217
217
 
218
- // The chat `vatio chat` is in on this environment, for `vatio flags add`.
218
+ // The chat `vatio chat` is in on this environment, for `vatio test add`.
219
219
  export function rememberedChatId(config, environment) {
220
220
  return chatsFor(config, config.resolveApiUrl())[environment] ?? null;
221
221
  }
@@ -473,6 +473,22 @@ function truncate(text, max) {
473
473
  return text.length > max ? `${text.slice(0, max)}…` : text;
474
474
  }
475
475
 
476
+ // One question in a new chat, for `vatio test add "QUESTION" "WHAT IT SHOULD
477
+ // DO"`: the chat becomes the one `vatio chat` continues, and the reply is the
478
+ // one the test is made on.
479
+ export async function askInNewChat(config, { environment, asPhone = null, identity = null, timeout = 120 }, question) {
480
+ const options = { environment, asPhone, identity, timeout, apiUrl: null, token: null };
481
+ const apiUrl = config.resolveApiUrl({ explicit: null });
482
+ const payload = await openChat(config, options, { apiUrl, chatId: chatsFor(config, apiUrl)[environment] ?? null });
483
+ const chatId = Number(payload.chat_id);
484
+ const api = client(config, options);
485
+ const sent = await api.sendChatMessage({ chatId, content: question });
486
+ const reply = await waitForAssistant(api, { chatId, after: sent.user_message_id, timeout });
487
+ console.log(`[you] ${question}`);
488
+ console.log(formatVisitor(reply));
489
+ return reply;
490
+ }
491
+
476
492
  async function startOver(config, options) {
477
493
  const apiUrl = config.resolveApiUrl({ explicit: options.apiUrl });
478
494
  const payload = await openChat(config, options, {
@@ -420,11 +420,14 @@ function printImpact(impact) {
420
420
  if (saved.total > 0) {
421
421
  parts.push(
422
422
  `Vatio saved ${saved.total} ${saved.total === 1 ? "reply" : "replies"} on live ` +
423
- `(${saved.corrected?.total ?? 0} corrected before sending, ${saved.tool_errors?.total ?? 0} after a tool failed)`
423
+ `(${saved.corrected?.total ?? 0} corrected before sending, ${saved.recovered?.total ?? 0} recovered after an error, ` +
424
+ `${saved.stopped?.total ?? 0} with a tool call stopped before your backend)`
424
425
  );
425
426
  }
426
427
  const hours = impact.after_hours ?? {};
427
428
  if (hours.after_hours > 0) parts.push(`${hours.after_hours} of ${hours.conversations} conversations came in after hours`);
429
+ const asked = impact.asked_team ?? {};
430
+ if (asked.questions > 0) parts.push(`the agent asked your team ${asked.questions} ${asked.questions === 1 ? "time" : "times"} instead of guessing`);
428
431
  if (impact.test_replays > 0) parts.push(`Vatio replayed ${impact.test_replays} test conversations`);
429
432
  const held = impact.regressions_held ?? {};
430
433
  if (held.count > 0) parts.push(`the publish gate kept ${held.count} ${held.count === 1 ? "regression" : "regressions"} off live`);
@@ -1,8 +1,8 @@
1
- // flags / test / tools -- what supervisors reported, proving a fix for it, and
2
- // how the tools' backend has been answering.
1
+ // test / tools -- what the agent should do and what supervisors reported,
2
+ // proving a fix for it, and how the tools' backend has been answering.
3
3
  //
4
4
  // All print markdown, because the reader is usually a coding agent: `vatio
5
- // flags` is the document it fixes the agent from, `vatio test` is the
5
+ // test list` is the document it fixes the agent from, `vatio test` is the
6
6
  // evidence that the fix worked on preview before `vatio publish` takes it
7
7
  // live, and `vatio tools` is how each tool's backend has been answering, for
8
8
  // the failures nobody reported.
@@ -11,7 +11,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
11
11
  import { dirname, resolve } from "node:path";
12
12
 
13
13
  import { HttpClient, NotFoundError, throwIfOutdated } from "../http.mjs";
14
- import { rememberedChatId } from "./chat.mjs";
14
+ import { askInNewChat, rememberedChatId } from "./chat.mjs";
15
15
  import { clientHeaders } from "../telemetry.mjs";
16
16
  import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
17
17
  import { printSkillsHint } from "../skills-hint.mjs";
@@ -63,116 +63,16 @@ async function fetchMarkdown(config, path, what) {
63
63
  }
64
64
  }
65
65
 
66
- // The open flags as one document: each flagged reply with the conversation
67
- // before it, every tool call of that turn with what your backend answered,
68
- // and what the supervisor says should have happened.
66
+ // `vatio flags` until 0.120.0, when reports and tests became one list: kept
67
+ // for the skills, scripts and muscle memory that still call it.
69
68
  export async function flags(config, args) {
70
- if (args[0] === "add") return await addFlag(config, args.slice(1));
71
- if (["rm", "remove", "delete"].includes(args[0])) return await removeFlags(config, args.slice(1));
72
-
73
- const json = takeFlag(args, "--json");
74
- const save = takeFlag(args, "--save");
75
- const savePath = save ? args.shift() ?? null : null;
76
-
77
- if (json) {
78
- const { url, token } = workspaceBase(config);
79
- const body = await new HttpClient({ baseUrl: url, token }).get("/flags");
80
- process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
81
- return;
82
- }
83
-
84
- const markdown = await fetchMarkdown(config, "/flags.md", "the flags");
85
- if (!save) {
86
- process.stdout.write(markdown);
87
- await printSkillsHint();
88
- return;
89
- }
90
-
91
- const target = resolve(savePath ?? "vatio-flags.md");
92
- mkdirSync(dirname(target), { recursive: true });
93
- writeFileSync(target, markdown);
94
- console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
95
- await printSkillsHint();
96
- }
97
-
98
- const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
99
-
100
- // The reply `vatio flags add` reports and `vatio test add` keeps: the agent's
101
- // last one in the chat `vatio chat` is in, or --chat's, or --message.
102
- function replyTarget(config, args, usage) {
103
- const environment = takeEnv(args, { fallback: "preview" });
104
- const chatId = takeValue(args, "--chat");
105
- const messageId = takeValue(args, "--message");
106
- const note = args.join(" ").trim();
107
- if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${usage}`);
108
- if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${usage}`);
109
-
110
- const target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
111
- if (!target.message_id && !target.chat_id) {
112
- fail(`No ${environment} chat yet: run \`vatio chat "hello"\` first, or pass --chat ID or --message ID.`);
113
- }
114
- return { ...target, note };
115
- }
116
-
117
- // What the inbox's Reportar does, from the terminal: a reply that was wrong,
118
- // and what it should have done. On live the note makes it a report and a
119
- // test case; on preview it is a note for the coding agent, closed by the
120
- // next push. A reply that is already right is not a flag: `vatio test add`
121
- // keeps it.
122
- async function addFlag(config, args) {
123
- if (args.includes("--passes")) {
124
- fail('A reply that is already right is a test, not a report: `vatio test add "WHAT IT SHOULD DO"` keeps it.');
125
- }
126
- const body = replyTarget(config, args, ADD_USAGE);
127
-
128
- const { url, token } = workspaceBase(config);
129
- let flag;
130
- try {
131
- flag = await new HttpClient({ baseUrl: url, token }).post("/flags", body);
132
- } catch (error) {
133
- // The platform's own words name the chat or message and where to find the right one.
134
- if (error instanceof NotFoundError) fail(error.message);
135
- throw error;
136
- }
137
-
138
- const reply = String(flag.reply ?? "").trim().replace(/\s+/g, " ");
139
- const note = flag.environment === "preview";
140
- console.log(
141
- `${flag.rewritten ? "Rewrote" : note ? "Noted" : "Reported"} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`
142
- );
143
- console.log(` [agent #${flag.message_id}] ${reply.length > 200 ? `${reply.slice(0, 200)}…` : reply}`);
144
- if (note) {
145
- console.log(
146
- "A note on preview, not a test case: `vatio flags` lists it for your coding agent until your next `vatio push` closes it. " +
147
- 'Once the agent gets it right, `vatio test add "WHAT IT SHOULD DO"` keeps that reply as a test.'
148
- );
149
- } else if (flag.test_case) {
150
- console.log("It is a test case now: `vatio test` replays it, `vatio flags` lists it, `vatio flags rm` removes it.");
151
- } else {
152
- console.log("No visitor message comes before that reply, so there is nothing to replay: it is not a test case.");
153
- }
154
- }
155
-
156
- // What the inbox's Quitar does, by the ids `vatio flags` prints: the flag and
157
- // its test case go, for everyone in the workspace. The developer's call, not
158
- // the coding agent's: the vatio-improve skill says to run it only when asked.
159
- async function removeFlags(config, ids) {
160
- if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio flags rm ID [ID...]");
161
-
162
- const { url, token } = workspaceBase(config);
163
- const client = new HttpClient({ baseUrl: url, token });
164
- for (const id of ids) {
165
- let removed;
166
- try {
167
- removed = await client.delete(`/flags/${encodeURIComponent(id)}`);
168
- } catch (error) {
169
- // The platform's own words name the id and where to find the right one.
170
- if (error instanceof NotFoundError) fail(error.message);
171
- throw error;
172
- }
173
- const was = removed.state === "resolved" ? "a resolved flag, so no longer a regression case" : "an open flag";
174
- console.log(`Removed ${removed.id} (${was}) and its test case.`);
175
- }
69
+ const sub = args[0];
70
+ const removing = ["rm", "remove", "delete"].includes(sub);
71
+ const now = sub === "add" ? "vatio test add" : removing ? "vatio test rm" : "vatio test list";
72
+ console.error(`vatio: \`vatio flags${sub === "add" || removing ? ` ${sub}` : ""}\` is \`${now}\` now.`);
73
+ if (sub === "add") return await addTest(config, args.slice(1));
74
+ if (removing) return await removeTests(config, args.slice(1));
75
+ return await listTests(config, args);
176
76
  }
177
77
 
178
78
  // Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
@@ -205,63 +105,117 @@ export async function test(config, args) {
205
105
  return await runTests(config, args, { name: "test", waitByDefault: false });
206
106
  }
207
107
 
208
- const TEST_ADD_USAGE = 'vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
108
+ const TEST_ADD_USAGE =
109
+ 'vatio test add "QUESTION" "WHAT IT SHOULD DO" [--as-phone me|PHONE] [--as-subject ID] [--env NAME]\n' +
110
+ ' vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
209
111
 
210
- // A reply kept as a test of what it should do, right already or not yet: a
211
- // photo of its turn, like a report's case, with no report for Vatio to
212
- // triage or fix. A set of `vatio chat --new` + `vatio test add` is a test
213
- // suite, and on preview the only way to make one: a flag there is a note.
112
+ // What the agent should do when asked something, checked on every version.
113
+ // With a question, Vatio asks it in a new chat and the test is made on that
114
+ // turn; with only what it should do, on the agent's last reply in the chat
115
+ // `vatio chat` is in, or --chat's, or --message's (a reply a visitor got, or a
116
+ // report's, whose note it rewrites). Either way the reply is judged at once,
117
+ // so you know whether the test is a guard or work still to do.
214
118
  async function addTest(config, args) {
215
- const body = replyTarget(config, args, TEST_ADD_USAGE);
119
+ const environment = takeEnv(args, { fallback: "preview" });
120
+ const chatId = takeValue(args, "--chat");
121
+ const messageId = takeValue(args, "--message");
122
+ const asPhone = takeValue(args, "--as-phone");
123
+ const timeout = Number(takeValue(args, "--timeout") ?? 120);
124
+ const identity = takeIdentity(args, config);
125
+ if (args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${TEST_ADD_USAGE}`);
126
+ if (args.length > 2) {
127
+ fail(`Put the question and what the agent should do in quotes, one argument each. Usage: ${TEST_ADD_USAGE}`);
128
+ }
129
+ const [first, second] = args.map((arg) => arg.trim());
130
+ if (!first || (args.length === 2 && !second)) fail(`Usage: ${TEST_ADD_USAGE}`);
131
+
132
+ let target;
133
+ let note;
134
+ if (args.length === 2) {
135
+ if (chatId || messageId) fail("A question opens its own chat: drop --chat and --message, or the question.");
136
+ const reply = await askInNewChat(config, { environment, asPhone, identity, timeout }, first);
137
+ target = { message_id: reply.id };
138
+ note = second;
139
+ } else {
140
+ if (asPhone || identity) fail("--as-phone and --as-subject go with a question: `vatio test add \"QUESTION\" \"WHAT IT SHOULD DO\"`.");
141
+ if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${TEST_ADD_USAGE}`);
142
+ target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
143
+ if (!target.message_id && !target.chat_id) {
144
+ fail(`No ${environment} chat yet: \`vatio test add "QUESTION" "WHAT IT SHOULD DO"\` asks it, or pass --chat ID or --message ID.`);
145
+ }
146
+ note = first;
147
+ }
148
+
216
149
  const { url, token } = workspaceBase(config);
217
150
  let added;
218
151
  try {
219
- added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", body);
152
+ added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", { ...target, note, judge: true });
220
153
  } catch (error) {
221
154
  if (error instanceof NotFoundError) fail(error.message);
222
155
  throw error;
223
156
  }
224
- console.log(`${added.rewritten ? "Rewrote" : "Added"} test ${added.id} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
157
+ const report = String(added.id).startsWith("F-");
158
+ const what = report ? `report ${added.id}'s note` : `test ${added.id}`;
159
+ console.log(`${added.rewritten ? "Rewrote" : "Added"} ${what} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
225
160
  console.log(` Visitor: ${String(added.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 200)}`);
226
- console.log(` Should: ${added.note}`);
227
- console.log(
228
- "`vatio test` replays its turn on every version; once live passes it, `vatio publish` refuses a version that fails it."
229
- );
161
+ console.log(` Should: ${added.note}`);
162
+ const judged = added.judged ?? {};
163
+ if (judged.verdict === "pass") {
164
+ console.log(` Passes now: ${judged.reason}`);
165
+ console.log("`vatio test` replays it on every version; once live passes it, `vatio publish` refuses a version that fails it.");
166
+ } else if (judged.verdict === "fail") {
167
+ console.log(` Fails now: ${judged.reason}`);
168
+ console.log("Fix the agent, `vatio push`, then `vatio test`: `vatio test list` shows it with its replay once it has run.");
169
+ } else {
170
+ console.log(` Not judged now${judged.error ? `: ${judged.error}` : ""}. \`vatio test\` judges it with the rest.`);
171
+ }
230
172
  }
231
173
 
174
+ // What there is to fix, then the whole suite: the reports still open, each
175
+ // with its conversation and what every tool answered; the tests that fail on
176
+ // preview or that the agent answers differently each time; every test.
232
177
  async function listTests(config, args) {
233
178
  const json = takeFlag(args, "--json");
234
- if (args.length > 0) fail("Usage: vatio test list [--json]");
235
- const { url, token } = workspaceBase(config);
236
- const body = await new HttpClient({ baseUrl: url, token }).get("/evals/cases");
179
+ const save = takeFlag(args, "--save");
180
+ const savePath = save ? args.shift() ?? null : null;
181
+ if (args.length > 0) fail("Usage: vatio test list [--save [PATH] | --json]");
182
+
237
183
  if (json) {
238
- process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
184
+ const { url, token } = workspaceBase(config);
185
+ const client = new HttpClient({ baseUrl: url, token });
186
+ const [tests, flags] = await Promise.all([client.get("/evals/cases"), client.get("/flags")]);
187
+ process.stdout.write(`${JSON.stringify({ tests: tests.tests ?? [], flags: flags.flags ?? [] }, null, 2)}\n`);
239
188
  return;
240
189
  }
241
- const tests = body.tests ?? [];
242
- if (tests.length === 0) {
243
- console.log('No tests yet. `vatio chat --new "..."`, then `vatio test add "WHAT IT SHOULD DO"` keeps a reply that is right.');
190
+
191
+ const markdown = await fetchMarkdown(config, "/flags.md", "the tests");
192
+ if (!save) {
193
+ process.stdout.write(markdown);
194
+ await printSkillsHint();
244
195
  return;
245
196
  }
246
- for (const row of tests) {
247
- console.log(`${row.id} ${row.agent} ${String(row.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 80)}`);
248
- console.log(` should: ${row.note}`);
249
- }
250
- console.log("Reports are tests too: `vatio flags` lists the open ones.");
197
+
198
+ const target = resolve(savePath ?? "vatio-tests.md");
199
+ mkdirSync(dirname(target), { recursive: true });
200
+ writeFileSync(target, markdown);
201
+ console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio test list --save\` to refresh.`);
202
+ await printSkillsHint();
251
203
  }
252
204
 
205
+ // A report (F-) goes with its test, for everyone, as the inbox's Remove does.
253
206
  async function removeTests(config, ids) {
254
207
  if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio test rm ID [ID...]");
255
208
  const { url, token } = workspaceBase(config);
256
209
  const client = new HttpClient({ baseUrl: url, token });
257
210
  for (const id of ids) {
211
+ let removed;
258
212
  try {
259
- await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
213
+ removed = await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
260
214
  } catch (error) {
261
215
  if (error instanceof NotFoundError) fail(error.message);
262
216
  throw error;
263
217
  }
264
- console.log(`Removed test ${id.toUpperCase()}.`);
218
+ console.log(removed.report ? `Removed report ${removed.id} and its test.` : `Removed test ${removed.id}.`);
265
219
  }
266
220
  }
267
221
 
@@ -312,23 +266,17 @@ async function runTests(config, args, { name, waitByDefault }) {
312
266
  );
313
267
  return;
314
268
  }
315
- // What this said until now was "report a reply in the inbox", which is
316
- // the door for a reply that went *wrong*. On a workspace where nothing
317
- // has gone wrong yet there is nothing to report, so the answer read as
318
- // "there is nothing to do here" and the suite never got started: of the
319
- // twelve workspaces, six have never run a case, including the three
320
- // newest. A suite is built out of replies that are already right, and
321
- // `vatio test add` is how one is kept.
269
+ // "Report a reply in the inbox" read as "nothing to do here" on a
270
+ // workspace where nothing had gone wrong yet, and the suite never got
271
+ // started. A suite is what the agent should do when asked, written down.
322
272
  console.log(
323
273
  "No tests yet, so there is nothing to replay.\n" +
324
274
  "\n" +
325
- "A test is a reply you have seen be right, kept so later versions keep it right:\n" +
326
- ' vatio chat --new "what a visitor would ask"\n' +
327
- ' vatio test add "what the answer has to say"\n' +
275
+ "A test is what the agent should do when someone asks something:\n" +
276
+ ' vatio test add "what a visitor would ask" "what the answer has to do"\n' +
328
277
  "\n" +
329
278
  "Then `vatio test` replays them all on every version, and `vatio publish` refuses\n" +
330
- "one that breaks them. A reply that was wrong is a report instead:\n" +
331
- '`vatio flags add "what it should have done"`, or Reportar in the inbox.'
279
+ "one that breaks them. Replies the team reports in the inbox become tests too."
332
280
  );
333
281
  return;
334
282
  }
@@ -167,7 +167,7 @@ export async function init(config, args) {
167
167
 
168
168
  // A pointer, not a copy of the docs: whatever coding agent works in this
169
169
  // directory reads AGENTS.md, and these lines are how it learns that `vatio
170
- // docs` is the contract and `vatio flags` is where fixing the agent starts.
170
+ // docs` is the contract and `vatio test list` is where fixing the agent starts.
171
171
  // The instructions themselves come from the platform at run time, so nothing
172
172
  // written here goes stale. An AGENTS.md that already mentions Vatio is left
173
173
  // alone; any other gets the section appended.
@@ -178,10 +178,11 @@ const AGENTS_NOTE = [
178
178
  "contract for the manifest, tools, knowledge and the CLI; read it before",
179
179
  "changing anything here.",
180
180
  "",
181
- "To fix what supervisors reported in the inbox, take every open report in",
182
- "one session: `vatio flags` (follow the instructions at the top of its",
183
- "output), fix, `vatio push`, `vatio test` until the reports pass, then",
181
+ "To fix what supervisors reported and the tests that fail, take them all in",
182
+ "one session: `vatio test list` (follow the instructions at the top of its",
183
+ "output), fix, `vatio push`, `vatio test` until they pass, then",
184
184
  "`vatio publish`. The `vatio-improve` skill walks through it.",
185
+ "`vatio test add \"QUESTION\" \"WHAT IT SHOULD DO\"` adds a test.",
185
186
  "`vatio tools --env live` shows how each tool's backend has been answering:",
186
187
  "error rate, timeouts, latency and the calls that failed.",
187
188
  ""
package/lib/help.mjs CHANGED
@@ -86,19 +86,8 @@ the two you are in. Commands default as listed.
86
86
  line as JSON. Default: live. Any team member,
87
87
  viewers included; every call is in the activity
88
88
 
89
- Improve it from what supervisors reported and how its tools answered
90
- (flags, fix, push, test, publish):
91
- flags [--save [PATH]|--json]
92
- Every open report: the conversation, what each tool
93
- answered, and what should have happened
94
- flags add "NOTE" [--chat ID|--message ID] [--env NAME]
95
- Flag a reply and what should have happened:
96
- the agent's last one in the chat \`vatio chat\`
97
- is in (default: preview). On live it is a report
98
- and a test case; on preview, a note for your
99
- coding agent that the next push closes
100
- flags rm ID [ID...] Remove a report and its test case, open or
101
- resolved, as the inbox's Remove does
89
+ Test and improve it: what it should do, what supervisors reported, and how
90
+ its tools answered (test add, test list, fix, push, test, publish):
102
91
  tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
103
92
  Each tool's calls, error rate, timeouts and
104
93
  p50/p95, then the calls that failed with what
@@ -114,12 +103,21 @@ the two you are in. Commands default as listed.
114
103
  if a case fails. Joins this version's running run
115
104
  test --show RUN_ID One run's report so far, with the replay chat of
116
105
  each case that did not pass (vatio chat show ID)
117
- test add "NOTE" [--chat ID|--message ID] [--env NAME]
118
- Keep a reply as a test of what it should do: the
119
- agent's last one in the chat \`vatio chat\` is in
120
- test list [--json] | test rm ID [ID...]
121
- The tests \`test add\` made; reports are \`flags\`
106
+ test add "QUESTION" "WHAT IT SHOULD DO" [--as-phone me|PHONE] [--as-subject ID]
107
+ Ask it in a new chat (default: preview) and keep
108
+ the turn as a test, judged at once
109
+ test add "WHAT IT SHOULD DO" [--chat ID|--message ID]
110
+ The same on a reply you already have: the agent's
111
+ last one in the chat \`vatio chat\` is in, or a
112
+ report's, whose note it rewrites
113
+ test list [--save [PATH]|--json]
114
+ What to fix: open reports with their conversation
115
+ and what each tool answered, tests that fail or
116
+ answer differently each time; then every test
117
+ test rm ID [ID...] Remove a test, or a report (F-) and its test
122
118
  eval Deprecated: \`vatio test --wait\` under its old name
119
+ flags [add|rm] Deprecated: \`vatio test list\`, \`test add\` and
120
+ \`test rm\` under their old name
123
121
 
124
122
  Knowledge:
125
123
  kb [list] Bases, and whether a deployed agent reads each
package/lib/suggest.mjs CHANGED
@@ -72,7 +72,7 @@ export const COMMANDS = [
72
72
  // listing them here would be dead. scripts/check-docs.mjs checks the command
73
73
  // names on the right against the dispatch.
74
74
  const SUBCOMMAND_OF = {
75
- add: ["flags", "test"],
75
+ add: ["test"],
76
76
  cat: ["kb"],
77
77
  create: ["kb", "keys", "templates", "webhooks"],
78
78
  delete: ["contacts"],
@@ -88,9 +88,9 @@ const SUBCOMMAND_OF = {
88
88
  off: ["quiet-hours"],
89
89
  origins: ["keys"],
90
90
  refresh: ["kb"],
91
- remove: ["secrets", "webhooks", "flags", "test"],
91
+ remove: ["secrets", "webhooks", "test"],
92
92
  revoke: ["keys"],
93
- rm: ["env", "kb", "secrets", "webhooks", "flags", "test"],
93
+ rm: ["env", "kb", "secrets", "webhooks", "test"],
94
94
  "rm-entry": ["kb"],
95
95
  set: ["business", "quiet-hours", "secrets", "config"],
96
96
  show: ["chat", "kb", "business", "quiet-hours", "config"],
package/lib/telemetry.mjs CHANGED
@@ -38,7 +38,9 @@ const SUBCOMMANDS = {
38
38
  contacts: ["export", "delete"],
39
39
  whatsapp: ["only", "everyone"],
40
40
  config: ["show", "get", "set", "unset"],
41
- "share-session": ["--should-ask", "--never"]
41
+ "share-session": ["--should-ask", "--never"],
42
+ test: ["add", "list", "rm"],
43
+ flags: ["add", "rm"]
42
44
  };
43
45
 
44
46
  export const USER_AGENT = `vatio-cli/${VERSION} (node ${process.version}; ${platform()} ${arch()})`;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.118.0",
3
+ "version": "0.120.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",