@vatio-ai/cli 0.90.0 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/chat.mjs +7 -1
- package/lib/commands/improve.mjs +116 -1
- package/lib/help.mjs +10 -1
- package/package.json +1 -1
package/lib/commands/chat.mjs
CHANGED
|
@@ -76,6 +76,7 @@ your inbox.
|
|
|
76
76
|
\`show\` prints the chat you are in, with every tool call the agent made. Give
|
|
77
77
|
it an id to read any chat in the workspace -- one a visitor had on the web,
|
|
78
78
|
WhatsApp or Instagram -- with the environment and deployment that answered it.
|
|
79
|
+
Each message carries its id (#4821), the one \`vatio flags add --message\` takes.
|
|
79
80
|
A [your backend] line under a tool call is how your backend answered it.
|
|
80
81
|
A [safeguard] line says which of Vatio's checks discarded or rewrote a draft,
|
|
81
82
|
and a [vatio] line marks a reply Vatio wrote because the agent's turn produced
|
|
@@ -175,6 +176,11 @@ function chatsFor(config, apiUrl) {
|
|
|
175
176
|
return state.chats && typeof state.chats === "object" ? state.chats : {};
|
|
176
177
|
}
|
|
177
178
|
|
|
179
|
+
// The chat `vatio chat` is in on this environment, for `vatio flags add`.
|
|
180
|
+
export function rememberedChatId(config, environment) {
|
|
181
|
+
return chatsFor(config, config.resolveApiUrl())[environment] ?? null;
|
|
182
|
+
}
|
|
183
|
+
|
|
178
184
|
function rememberChat(config, { apiUrl, environment, chatId }) {
|
|
179
185
|
const chats = { ...chatsFor(config, apiUrl), [environment]: chatId };
|
|
180
186
|
writeFileSync(statePath(config), `${JSON.stringify({ api_url: apiUrl, chats }, null, 2)}\n`);
|
|
@@ -310,7 +316,7 @@ async function showChat(config, options, args) {
|
|
|
310
316
|
? ` (discarded by the ${safeguard.policy} safeguard)`
|
|
311
317
|
: " (discarded)"
|
|
312
318
|
: "";
|
|
313
|
-
if (content !== "" || flags) console.log(`[${message.role}]${flags} ${content}`);
|
|
319
|
+
if (content !== "" || flags) console.log(`[${message.role} #${message.id}]${flags} ${content}`);
|
|
314
320
|
for (const call of Array.isArray(message.tool_calls) ? message.tool_calls : []) {
|
|
315
321
|
console.log(` -> ${call.name}(${truncate(JSON.stringify(call.arguments ?? {}), 400)})`);
|
|
316
322
|
if (call.backend) console.log(` [your backend] ${backendLine(call.backend)}`);
|
package/lib/commands/improve.mjs
CHANGED
|
@@ -11,6 +11,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
11
11
|
import { dirname, resolve } from "node:path";
|
|
12
12
|
|
|
13
13
|
import { HttpClient, NotFoundError, throwIfOutdated } from "../http.mjs";
|
|
14
|
+
import { rememberedChatId } from "./chat.mjs";
|
|
14
15
|
import { clientHeaders } from "../telemetry.mjs";
|
|
15
16
|
import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
|
|
16
17
|
import { printSkillsHint } from "../skills-hint.mjs";
|
|
@@ -44,6 +45,7 @@ async function fetchMarkdown(config, path, what) {
|
|
|
44
45
|
// before it, every tool call of that turn with what your backend answered,
|
|
45
46
|
// and what the supervisor says should have happened.
|
|
46
47
|
export async function flags(config, args) {
|
|
48
|
+
if (args[0] === "add") return await addFlag(config, args.slice(1));
|
|
47
49
|
if (["rm", "remove", "delete"].includes(args[0])) return await removeFlags(config, args.slice(1));
|
|
48
50
|
|
|
49
51
|
const json = takeFlag(args, "--json");
|
|
@@ -71,6 +73,54 @@ export async function flags(config, args) {
|
|
|
71
73
|
await printSkillsHint();
|
|
72
74
|
}
|
|
73
75
|
|
|
76
|
+
const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
|
|
77
|
+
|
|
78
|
+
// The reply `vatio flags add` reports and `vatio test add` keeps: the agent's
|
|
79
|
+
// last one in the chat `vatio chat` is in, or --chat's, or --message.
|
|
80
|
+
function replyTarget(config, args, usage) {
|
|
81
|
+
const environment = takeEnv(args, { fallback: "preview" });
|
|
82
|
+
const chatId = takeValue(args, "--chat");
|
|
83
|
+
const messageId = takeValue(args, "--message");
|
|
84
|
+
const note = args.join(" ").trim();
|
|
85
|
+
if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${usage}`);
|
|
86
|
+
if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${usage}`);
|
|
87
|
+
|
|
88
|
+
const target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
|
|
89
|
+
if (!target.message_id && !target.chat_id) {
|
|
90
|
+
fail(`No ${environment} chat yet: run \`vatio chat "hello"\` first, or pass --chat ID or --message ID.`);
|
|
91
|
+
}
|
|
92
|
+
return { ...target, note };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// What the inbox's Reportar does, from the terminal: a reply that was wrong,
|
|
96
|
+
// and what it should have done. The note makes it a test case. A reply that
|
|
97
|
+
// is already right is not a report: `vatio test add` keeps it.
|
|
98
|
+
async function addFlag(config, args) {
|
|
99
|
+
if (args.includes("--passes")) {
|
|
100
|
+
fail('A reply that is already right is a test, not a report: `vatio test add "WHAT IT SHOULD DO"` keeps it.');
|
|
101
|
+
}
|
|
102
|
+
const body = replyTarget(config, args, ADD_USAGE);
|
|
103
|
+
|
|
104
|
+
const { url, token } = workspaceBase(config);
|
|
105
|
+
let flag;
|
|
106
|
+
try {
|
|
107
|
+
flag = await new HttpClient({ baseUrl: url, token }).post("/flags", body);
|
|
108
|
+
} catch (error) {
|
|
109
|
+
// The platform's own words name the chat or message and where to find the right one.
|
|
110
|
+
if (error instanceof NotFoundError) fail(error.message);
|
|
111
|
+
throw error;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const reply = String(flag.reply ?? "").trim().replace(/\s+/g, " ");
|
|
115
|
+
console.log(`${flag.rewritten ? "Rewrote" : "Reported"} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`);
|
|
116
|
+
console.log(` [agent #${flag.message_id}] ${reply.length > 200 ? `${reply.slice(0, 200)}…` : reply}`);
|
|
117
|
+
if (flag.test_case) {
|
|
118
|
+
console.log("It is a test case now: `vatio test` replays it, `vatio flags` lists it, `vatio flags rm` removes it.");
|
|
119
|
+
} else {
|
|
120
|
+
console.log("No visitor message comes before that reply, so there is nothing to replay: it is not a test case.");
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
74
124
|
// What the inbox's Quitar does, by the ids `vatio flags` prints: the flag and
|
|
75
125
|
// its test case go, for everyone in the workspace. The developer's call, not
|
|
76
126
|
// the coding agent's: the vatio-improve skill says to run it only when asked.
|
|
@@ -118,9 +168,71 @@ export async function tools(config, args) {
|
|
|
118
168
|
// one a coding agent had to remember to put in the background. `--wait` waits
|
|
119
169
|
// for the judge and exits 1 when a case fails, so it can gate CI.
|
|
120
170
|
export async function test(config, args) {
|
|
171
|
+
if (args[0] === "add") return await addTest(config, args.slice(1));
|
|
172
|
+
if (args[0] === "list") return await listTests(config, args.slice(1));
|
|
173
|
+
if (["rm", "remove", "delete"].includes(args[0])) return await removeTests(config, args.slice(1));
|
|
121
174
|
return await runTests(config, args, { name: "test", waitByDefault: false });
|
|
122
175
|
}
|
|
123
176
|
|
|
177
|
+
const TEST_ADD_USAGE = 'vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
|
|
178
|
+
|
|
179
|
+
// A reply that is right, kept as a test so it stays right: a photo of its
|
|
180
|
+
// turn, like a report's case, with no report for Vatio to triage or fix. A
|
|
181
|
+
// set of `vatio chat --new` + `vatio test add` is a test suite.
|
|
182
|
+
async function addTest(config, args) {
|
|
183
|
+
const body = replyTarget(config, args, TEST_ADD_USAGE);
|
|
184
|
+
const { url, token } = workspaceBase(config);
|
|
185
|
+
let added;
|
|
186
|
+
try {
|
|
187
|
+
added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", body);
|
|
188
|
+
} catch (error) {
|
|
189
|
+
if (error instanceof NotFoundError) fail(error.message);
|
|
190
|
+
throw error;
|
|
191
|
+
}
|
|
192
|
+
console.log(`${added.rewritten ? "Rewrote" : "Added"} test ${added.id} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
|
|
193
|
+
console.log(` Visitor: ${String(added.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 200)}`);
|
|
194
|
+
console.log(` Should: ${added.note}`);
|
|
195
|
+
console.log(
|
|
196
|
+
"`vatio test` replays its turn on every version; once live passes it, `vatio publish` refuses a version that fails it."
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
async function listTests(config, args) {
|
|
201
|
+
const json = takeFlag(args, "--json");
|
|
202
|
+
if (args.length > 0) fail("Usage: vatio test list [--json]");
|
|
203
|
+
const { url, token } = workspaceBase(config);
|
|
204
|
+
const body = await new HttpClient({ baseUrl: url, token }).get("/evals/cases");
|
|
205
|
+
if (json) {
|
|
206
|
+
process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
const tests = body.tests ?? [];
|
|
210
|
+
if (tests.length === 0) {
|
|
211
|
+
console.log('No tests yet. `vatio chat --new "..."`, then `vatio test add "WHAT IT SHOULD DO"` keeps a reply that is right.');
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
for (const row of tests) {
|
|
215
|
+
console.log(`${row.id} ${row.agent} ${String(row.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 80)}`);
|
|
216
|
+
console.log(` should: ${row.note}`);
|
|
217
|
+
}
|
|
218
|
+
console.log("Reports are tests too: `vatio flags` lists the open ones.");
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
async function removeTests(config, ids) {
|
|
222
|
+
if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio test rm ID [ID...]");
|
|
223
|
+
const { url, token } = workspaceBase(config);
|
|
224
|
+
const client = new HttpClient({ baseUrl: url, token });
|
|
225
|
+
for (const id of ids) {
|
|
226
|
+
try {
|
|
227
|
+
await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
|
|
228
|
+
} catch (error) {
|
|
229
|
+
if (error instanceof NotFoundError) fail(error.message);
|
|
230
|
+
throw error;
|
|
231
|
+
}
|
|
232
|
+
console.log(`Removed test ${id.toUpperCase()}.`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
124
236
|
// What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
|
|
125
237
|
// for the CI jobs and older skills that still call it; the name also reads as
|
|
126
238
|
// the shell's eval to command guards, which refused it.
|
|
@@ -149,7 +261,10 @@ async function runTests(config, args, { name, waitByDefault }) {
|
|
|
149
261
|
}
|
|
150
262
|
const runs = started.runs ?? [];
|
|
151
263
|
if (runs.length === 0) {
|
|
152
|
-
console.log(
|
|
264
|
+
console.log(
|
|
265
|
+
"No reports to test yet. Report a reply in the inbox, or after a `vatio chat` with\n" +
|
|
266
|
+
'`vatio flags add "what it should have done"`.'
|
|
267
|
+
);
|
|
153
268
|
return;
|
|
154
269
|
}
|
|
155
270
|
|
package/lib/help.mjs
CHANGED
|
@@ -78,6 +78,10 @@ Commands default as listed.
|
|
|
78
78
|
flags [--save [PATH]|--json]
|
|
79
79
|
Every open report: the conversation, what each tool
|
|
80
80
|
answered, and what should have happened
|
|
81
|
+
flags add "NOTE" [--chat ID|--message ID] [--env NAME]
|
|
82
|
+
Report a reply and what should have happened:
|
|
83
|
+
the agent's last one in the chat \`vatio chat\`
|
|
84
|
+
is in (default: preview). It is a test case now
|
|
81
85
|
flags rm ID [ID...] Remove a report and its test case, open or
|
|
82
86
|
resolved, as the inbox's Remove does
|
|
83
87
|
tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
|
|
@@ -86,7 +90,7 @@ Commands default as listed.
|
|
|
86
90
|
failed with what your backend answered. Default:
|
|
87
91
|
live, 7d
|
|
88
92
|
test [--env NAME] [--agent SLUG]
|
|
89
|
-
Test every report
|
|
93
|
+
Test every report and test against preview
|
|
90
94
|
(default): starts the runs on Vatio's servers,
|
|
91
95
|
prints their ids and returns.
|
|
92
96
|
--env live also resolves the reports it passes
|
|
@@ -94,6 +98,11 @@ Commands default as listed.
|
|
|
94
98
|
if a case fails. Joins this version's running run
|
|
95
99
|
test --show RUN_ID One run's report so far, with the replay chat of
|
|
96
100
|
each case that did not pass (vatio chat show ID)
|
|
101
|
+
test add "NOTE" [--chat ID|--message ID] [--env NAME]
|
|
102
|
+
Keep a reply that is right as a test: the agent's
|
|
103
|
+
last one in the chat \`vatio chat\` is in
|
|
104
|
+
test list [--json] | test rm ID [ID...]
|
|
105
|
+
The tests \`test add\` made; reports are \`flags\`
|
|
97
106
|
eval Deprecated: \`vatio test --wait\` under its old name
|
|
98
107
|
|
|
99
108
|
Knowledge:
|