@vatio-ai/cli 0.91.0 → 0.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -73,32 +73,38 @@ export async function flags(config, args) {
73
73
  await printSkillsHint();
74
74
  }
75
75
 
76
- const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--passes] [--chat ID | --message ID] [--env NAME]';
77
-
78
- // What the inbox's Reportar does, from the terminal: the agent's last reply in
79
- // the chat `vatio chat` is in (or --chat's, or --message), and what it should
80
- // have done. The note makes it a test case, so a set of `vatio chat --new` +
81
- // `vatio flags add` is a test suite `vatio test` replays on every version.
82
- // --passes is for a reply that is already right: the case keeps it so, filed
83
- // resolved, with nothing for Vatio to triage or fix.
84
- async function addFlag(config, args) {
76
+ const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
77
+
78
+ // The reply `vatio flags add` reports and `vatio test add` keeps: the agent's
79
+ // last one in the chat `vatio chat` is in, or --chat's, or --message.
80
+ function replyTarget(config, args, usage) {
85
81
  const environment = takeEnv(args, { fallback: "preview" });
86
82
  const chatId = takeValue(args, "--chat");
87
83
  const messageId = takeValue(args, "--message");
88
- const passing = takeFlag(args, "--passes");
89
84
  const note = args.join(" ").trim();
90
- if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${ADD_USAGE}`);
91
- if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${ADD_USAGE}`);
85
+ if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${usage}`);
86
+ if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${usage}`);
92
87
 
93
88
  const target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
94
89
  if (!target.message_id && !target.chat_id) {
95
90
  fail(`No ${environment} chat yet: run \`vatio chat "hello"\` first, or pass --chat ID or --message ID.`);
96
91
  }
92
+ return { ...target, note };
93
+ }
94
+
95
+ // What the inbox's Reportar does, from the terminal: a reply that was wrong,
96
+ // and what it should have done. The note makes it a test case. A reply that
97
+ // is already right is not a report: `vatio test add` keeps it.
98
+ async function addFlag(config, args) {
99
+ if (args.includes("--passes")) {
100
+ fail('A reply that is already right is a test, not a report: `vatio test add "WHAT IT SHOULD DO"` keeps it.');
101
+ }
102
+ const body = replyTarget(config, args, ADD_USAGE);
97
103
 
98
104
  const { url, token } = workspaceBase(config);
99
105
  let flag;
100
106
  try {
101
- flag = await new HttpClient({ baseUrl: url, token }).post("/flags", { ...target, note, passing });
107
+ flag = await new HttpClient({ baseUrl: url, token }).post("/flags", body);
102
108
  } catch (error) {
103
109
  // The platform's own words name the chat or message and where to find the right one.
104
110
  if (error instanceof NotFoundError) fail(error.message);
@@ -106,15 +112,9 @@ async function addFlag(config, args) {
106
112
  }
107
113
 
108
114
  const reply = String(flag.reply ?? "").trim().replace(/\s+/g, " ");
109
- const verb = flag.passing ? "Kept as right" : flag.rewritten ? "Rewrote" : "Reported";
110
- console.log(`${verb} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`);
115
+ console.log(`${flag.rewritten ? "Rewrote" : "Reported"} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`);
111
116
  console.log(` [agent #${flag.message_id}] ${reply.length > 200 ? `${reply.slice(0, 200)}…` : reply}`);
112
- if (flag.test_case && flag.passing) {
113
- console.log(
114
- `It is a test case, resolved${flag.resolved_in ? ` in ${flag.resolved_in}` : ""}: nothing for Vatio to fix, nothing pending\n` +
115
- "in the inbox. `vatio test` replays it; once live passes it, `vatio publish` refuses a version that fails it."
116
- );
117
- } else if (flag.test_case) {
117
+ if (flag.test_case) {
118
118
  console.log("It is a test case now: `vatio test` replays it, `vatio flags` lists it, `vatio flags rm` removes it.");
119
119
  } else {
120
120
  console.log("No visitor message comes before that reply, so there is nothing to replay: it is not a test case.");
@@ -168,9 +168,71 @@ export async function tools(config, args) {
168
168
  // one a coding agent had to remember to put in the background. `--wait` waits
169
169
  // for the judge and exits 1 when a case fails, so it can gate CI.
170
170
  export async function test(config, args) {
171
+ if (args[0] === "add") return await addTest(config, args.slice(1));
172
+ if (args[0] === "list") return await listTests(config, args.slice(1));
173
+ if (["rm", "remove", "delete"].includes(args[0])) return await removeTests(config, args.slice(1));
171
174
  return await runTests(config, args, { name: "test", waitByDefault: false });
172
175
  }
173
176
 
177
+ const TEST_ADD_USAGE = 'vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
178
+
179
+ // A reply that is right, kept as a test so it stays right: a photo of its
180
+ // turn, like a report's case, with no report for Vatio to triage or fix. A
181
+ // set of `vatio chat --new` + `vatio test add` is a test suite.
182
+ async function addTest(config, args) {
183
+ const body = replyTarget(config, args, TEST_ADD_USAGE);
184
+ const { url, token } = workspaceBase(config);
185
+ let added;
186
+ try {
187
+ added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", body);
188
+ } catch (error) {
189
+ if (error instanceof NotFoundError) fail(error.message);
190
+ throw error;
191
+ }
192
+ console.log(`${added.rewritten ? "Rewrote" : "Added"} test ${added.id} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
193
+ console.log(` Visitor: ${String(added.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 200)}`);
194
+ console.log(` Should: ${added.note}`);
195
+ console.log(
196
+ "`vatio test` replays its turn on every version; once live passes it, `vatio publish` refuses a version that fails it."
197
+ );
198
+ }
199
+
200
+ async function listTests(config, args) {
201
+ const json = takeFlag(args, "--json");
202
+ if (args.length > 0) fail("Usage: vatio test list [--json]");
203
+ const { url, token } = workspaceBase(config);
204
+ const body = await new HttpClient({ baseUrl: url, token }).get("/evals/cases");
205
+ if (json) {
206
+ process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
207
+ return;
208
+ }
209
+ const tests = body.tests ?? [];
210
+ if (tests.length === 0) {
211
+ console.log('No tests yet. `vatio chat --new "..."`, then `vatio test add "WHAT IT SHOULD DO"` keeps a reply that is right.');
212
+ return;
213
+ }
214
+ for (const row of tests) {
215
+ console.log(`${row.id} ${row.agent} ${String(row.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 80)}`);
216
+ console.log(` should: ${row.note}`);
217
+ }
218
+ console.log("Reports are tests too: `vatio flags` lists the open ones.");
219
+ }
220
+
221
+ async function removeTests(config, ids) {
222
+ if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio test rm ID [ID...]");
223
+ const { url, token } = workspaceBase(config);
224
+ const client = new HttpClient({ baseUrl: url, token });
225
+ for (const id of ids) {
226
+ try {
227
+ await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
228
+ } catch (error) {
229
+ if (error instanceof NotFoundError) fail(error.message);
230
+ throw error;
231
+ }
232
+ console.log(`Removed test ${id.toUpperCase()}.`);
233
+ }
234
+ }
235
+
174
236
  // What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
175
237
  // for the CI jobs and older skills that still call it; the name also reads as
176
238
  // the shell's eval to command guards, which refused it.
package/lib/help.mjs CHANGED
@@ -78,11 +78,10 @@ Commands default as listed.
78
78
  flags [--save [PATH]|--json]
79
79
  Every open report: the conversation, what each tool
80
80
  answered, and what should have happened
81
- flags add "NOTE" [--passes] [--chat ID|--message ID] [--env NAME]
81
+ flags add "NOTE" [--chat ID|--message ID] [--env NAME]
82
82
  Report a reply and what should have happened:
83
83
  the agent's last one in the chat \`vatio chat\`
84
- is in (default: preview). It is a test case now.
85
- --passes: the reply is right; the case keeps it so
84
+ is in (default: preview). It is a test case now
86
85
  flags rm ID [ID...] Remove a report and its test case, open or
87
86
  resolved, as the inbox's Remove does
88
87
  tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
@@ -91,7 +90,7 @@ Commands default as listed.
91
90
  failed with what your backend answered. Default:
92
91
  live, 7d
93
92
  test [--env NAME] [--agent SLUG]
94
- Test every report with a note against preview
93
+ Test every report and test against preview
95
94
  (default): starts the runs on Vatio's servers,
96
95
  prints their ids and returns.
97
96
  --env live also resolves the reports it passes
@@ -99,6 +98,11 @@ Commands default as listed.
99
98
  if a case fails. Joins this version's running run
100
99
  test --show RUN_ID One run's report so far, with the replay chat of
101
100
  each case that did not pass (vatio chat show ID)
101
+ test add "NOTE" [--chat ID|--message ID] [--env NAME]
102
+ Keep a reply that is right as a test: the agent's
103
+ last one in the chat \`vatio chat\` is in
104
+ test list [--json] | test rm ID [ID...]
105
+ The tests \`test add\` made; reports are \`flags\`
102
106
  eval Deprecated: \`vatio test --wait\` under its old name
103
107
 
104
108
  Knowledge:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.91.0",
3
+ "version": "0.92.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",