@vatio-ai/cli 0.90.0 → 0.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -76,6 +76,7 @@ your inbox.
76
76
  \`show\` prints the chat you are in, with every tool call the agent made. Give
77
77
  it an id to read any chat in the workspace -- one a visitor had on the web,
78
78
  WhatsApp or Instagram -- with the environment and deployment that answered it.
79
+ Each message carries its id (#4821), the one \`vatio flags add --message\` takes.
79
80
  A [your backend] line under a tool call is how your backend answered it.
80
81
  A [safeguard] line says which of Vatio's checks discarded or rewrote a draft,
81
82
  and a [vatio] line marks a reply Vatio wrote because the agent's turn produced
@@ -175,6 +176,11 @@ function chatsFor(config, apiUrl) {
175
176
  return state.chats && typeof state.chats === "object" ? state.chats : {};
176
177
  }
177
178
 
179
+ // The chat `vatio chat` is in on this environment, for `vatio flags add`.
180
+ export function rememberedChatId(config, environment) {
181
+ return chatsFor(config, config.resolveApiUrl())[environment] ?? null;
182
+ }
183
+
178
184
  function rememberChat(config, { apiUrl, environment, chatId }) {
179
185
  const chats = { ...chatsFor(config, apiUrl), [environment]: chatId };
180
186
  writeFileSync(statePath(config), `${JSON.stringify({ api_url: apiUrl, chats }, null, 2)}\n`);
@@ -310,7 +316,7 @@ async function showChat(config, options, args) {
310
316
  ? ` (discarded by the ${safeguard.policy} safeguard)`
311
317
  : " (discarded)"
312
318
  : "";
313
- if (content !== "" || flags) console.log(`[${message.role}]${flags} ${content}`);
319
+ if (content !== "" || flags) console.log(`[${message.role} #${message.id}]${flags} ${content}`);
314
320
  for (const call of Array.isArray(message.tool_calls) ? message.tool_calls : []) {
315
321
  console.log(` -> ${call.name}(${truncate(JSON.stringify(call.arguments ?? {}), 400)})`);
316
322
  if (call.backend) console.log(` [your backend] ${backendLine(call.backend)}`);
@@ -11,6 +11,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
11
11
  import { dirname, resolve } from "node:path";
12
12
 
13
13
  import { HttpClient, NotFoundError, throwIfOutdated } from "../http.mjs";
14
+ import { rememberedChatId } from "./chat.mjs";
14
15
  import { clientHeaders } from "../telemetry.mjs";
15
16
  import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
16
17
  import { printSkillsHint } from "../skills-hint.mjs";
@@ -44,6 +45,7 @@ async function fetchMarkdown(config, path, what) {
44
45
  // before it, every tool call of that turn with what your backend answered,
45
46
  // and what the supervisor says should have happened.
46
47
  export async function flags(config, args) {
48
+ if (args[0] === "add") return await addFlag(config, args.slice(1));
47
49
  if (["rm", "remove", "delete"].includes(args[0])) return await removeFlags(config, args.slice(1));
48
50
 
49
51
  const json = takeFlag(args, "--json");
@@ -71,6 +73,54 @@ export async function flags(config, args) {
71
73
  await printSkillsHint();
72
74
  }
73
75
 
76
+ const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
77
+
78
+ // The reply `vatio flags add` reports and `vatio test add` keeps: the agent's
79
+ // last one in the chat `vatio chat` is in, or --chat's, or --message.
80
+ function replyTarget(config, args, usage) {
81
+ const environment = takeEnv(args, { fallback: "preview" });
82
+ const chatId = takeValue(args, "--chat");
83
+ const messageId = takeValue(args, "--message");
84
+ const note = args.join(" ").trim();
85
+ if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${usage}`);
86
+ if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${usage}`);
87
+
88
+ const target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
89
+ if (!target.message_id && !target.chat_id) {
90
+ fail(`No ${environment} chat yet: run \`vatio chat "hello"\` first, or pass --chat ID or --message ID.`);
91
+ }
92
+ return { ...target, note };
93
+ }
94
+
95
+ // What the inbox's Reportar does, from the terminal: a reply that was wrong,
96
+ // and what it should have done. The note makes it a test case. A reply that
97
+ // is already right is not a report: `vatio test add` keeps it.
98
+ async function addFlag(config, args) {
99
+ if (args.includes("--passes")) {
100
+ fail('A reply that is already right is a test, not a report: `vatio test add "WHAT IT SHOULD DO"` keeps it.');
101
+ }
102
+ const body = replyTarget(config, args, ADD_USAGE);
103
+
104
+ const { url, token } = workspaceBase(config);
105
+ let flag;
106
+ try {
107
+ flag = await new HttpClient({ baseUrl: url, token }).post("/flags", body);
108
+ } catch (error) {
109
+ // The platform's own words name the chat or message and where to find the right one.
110
+ if (error instanceof NotFoundError) fail(error.message);
111
+ throw error;
112
+ }
113
+
114
+ const reply = String(flag.reply ?? "").trim().replace(/\s+/g, " ");
115
+ console.log(`${flag.rewritten ? "Rewrote" : "Reported"} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`);
116
+ console.log(` [agent #${flag.message_id}] ${reply.length > 200 ? `${reply.slice(0, 200)}…` : reply}`);
117
+ if (flag.test_case) {
118
+ console.log("It is a test case now: `vatio test` replays it, `vatio flags` lists it, `vatio flags rm` removes it.");
119
+ } else {
120
+ console.log("No visitor message comes before that reply, so there is nothing to replay: it is not a test case.");
121
+ }
122
+ }
123
+
74
124
  // What the inbox's Quitar does, by the ids `vatio flags` prints: the flag and
75
125
  // its test case go, for everyone in the workspace. The developer's call, not
76
126
  // the coding agent's: the vatio-improve skill says to run it only when asked.
@@ -118,9 +168,71 @@ export async function tools(config, args) {
118
168
  // one a coding agent had to remember to put in the background. `--wait` waits
119
169
  // for the judge and exits 1 when a case fails, so it can gate CI.
120
170
  export async function test(config, args) {
171
+ if (args[0] === "add") return await addTest(config, args.slice(1));
172
+ if (args[0] === "list") return await listTests(config, args.slice(1));
173
+ if (["rm", "remove", "delete"].includes(args[0])) return await removeTests(config, args.slice(1));
121
174
  return await runTests(config, args, { name: "test", waitByDefault: false });
122
175
  }
123
176
 
177
+ const TEST_ADD_USAGE = 'vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
178
+
179
+ // A reply that is right, kept as a test so it stays right: a photo of its
180
+ // turn, like a report's case, with no report for Vatio to triage or fix. A
181
+ // set of `vatio chat --new` + `vatio test add` is a test suite.
182
+ async function addTest(config, args) {
183
+ const body = replyTarget(config, args, TEST_ADD_USAGE);
184
+ const { url, token } = workspaceBase(config);
185
+ let added;
186
+ try {
187
+ added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", body);
188
+ } catch (error) {
189
+ if (error instanceof NotFoundError) fail(error.message);
190
+ throw error;
191
+ }
192
+ console.log(`${added.rewritten ? "Rewrote" : "Added"} test ${added.id} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
193
+ console.log(` Visitor: ${String(added.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 200)}`);
194
+ console.log(` Should: ${added.note}`);
195
+ console.log(
196
+ "`vatio test` replays its turn on every version; once live passes it, `vatio publish` refuses a version that fails it."
197
+ );
198
+ }
199
+
200
+ async function listTests(config, args) {
201
+ const json = takeFlag(args, "--json");
202
+ if (args.length > 0) fail("Usage: vatio test list [--json]");
203
+ const { url, token } = workspaceBase(config);
204
+ const body = await new HttpClient({ baseUrl: url, token }).get("/evals/cases");
205
+ if (json) {
206
+ process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
207
+ return;
208
+ }
209
+ const tests = body.tests ?? [];
210
+ if (tests.length === 0) {
211
+ console.log('No tests yet. `vatio chat --new "..."`, then `vatio test add "WHAT IT SHOULD DO"` keeps a reply that is right.');
212
+ return;
213
+ }
214
+ for (const row of tests) {
215
+ console.log(`${row.id} ${row.agent} ${String(row.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 80)}`);
216
+ console.log(` should: ${row.note}`);
217
+ }
218
+ console.log("Reports are tests too: `vatio flags` lists the open ones.");
219
+ }
220
+
221
+ async function removeTests(config, ids) {
222
+ if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio test rm ID [ID...]");
223
+ const { url, token } = workspaceBase(config);
224
+ const client = new HttpClient({ baseUrl: url, token });
225
+ for (const id of ids) {
226
+ try {
227
+ await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
228
+ } catch (error) {
229
+ if (error instanceof NotFoundError) fail(error.message);
230
+ throw error;
231
+ }
232
+ console.log(`Removed test ${id.toUpperCase()}.`);
233
+ }
234
+ }
235
+
124
236
  // What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
125
237
  // for the CI jobs and older skills that still call it; the name also reads as
126
238
  // the shell's eval to command guards, which refused it.
@@ -149,7 +261,10 @@ async function runTests(config, args, { name, waitByDefault }) {
149
261
  }
150
262
  const runs = started.runs ?? [];
151
263
  if (runs.length === 0) {
152
- console.log("No reports to test yet. Report a reply in the inbox and write what it should have said.");
264
+ console.log(
265
+ "No reports to test yet. Report a reply in the inbox, or after a `vatio chat` with\n" +
266
+ '`vatio flags add "what it should have done"`.'
267
+ );
153
268
  return;
154
269
  }
155
270
 
package/lib/help.mjs CHANGED
@@ -78,6 +78,10 @@ Commands default as listed.
78
78
  flags [--save [PATH]|--json]
79
79
  Every open report: the conversation, what each tool
80
80
  answered, and what should have happened
81
+ flags add "NOTE" [--chat ID|--message ID] [--env NAME]
82
+ Report a reply and what should have happened:
83
+ the agent's last one in the chat \`vatio chat\`
84
+ is in (default: preview). It is a test case now
81
85
  flags rm ID [ID...] Remove a report and its test case, open or
82
86
  resolved, as the inbox's Remove does
83
87
  tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
@@ -86,7 +90,7 @@ Commands default as listed.
86
90
  failed with what your backend answered. Default:
87
91
  live, 7d
88
92
  test [--env NAME] [--agent SLUG]
89
- Test every report with a note against preview
93
+ Test every report and test against preview
90
94
  (default): starts the runs on Vatio's servers,
91
95
  prints their ids and returns.
92
96
  --env live also resolves the reports it passes
@@ -94,6 +98,11 @@ Commands default as listed.
94
98
  if a case fails. Joins this version's running run
95
99
  test --show RUN_ID One run's report so far, with the replay chat of
96
100
  each case that did not pass (vatio chat show ID)
101
+ test add "NOTE" [--chat ID|--message ID] [--env NAME]
102
+ Keep a reply that is right as a test: the agent's
103
+ last one in the chat \`vatio chat\` is in
104
+ test list [--json] | test rm ID [ID...]
105
+ The tests \`test add\` made; reports are \`flags\`
97
106
  eval Deprecated: \`vatio test --wait\` under its old name
98
107
 
99
108
  Knowledge:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.90.0",
3
+ "version": "0.92.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",