@vatio-ai/cli 0.91.0 → 0.92.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/improve.mjs +85 -24
- package/lib/help.mjs +11 -8
- package/package.json +1 -1
package/lib/commands/improve.mjs
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
// All print markdown, because the reader is usually a coding agent: `vatio
|
|
5
5
|
// flags` is the document it fixes the agent from, `vatio test` is the
|
|
6
6
|
// evidence that the fix worked on preview before `vatio publish` takes it
|
|
7
|
-
// live, and `vatio tools` is
|
|
8
|
-
// reported.
|
|
7
|
+
// live, and `vatio tools` is how each tool's backend has been answering, for
|
|
8
|
+
// the failures nobody reported.
|
|
9
9
|
|
|
10
10
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
11
11
|
import { dirname, resolve } from "node:path";
|
|
@@ -73,32 +73,38 @@ export async function flags(config, args) {
|
|
|
73
73
|
await printSkillsHint();
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
-
const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--
|
|
76
|
+
const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
|
|
77
77
|
|
|
78
|
-
//
|
|
79
|
-
// the chat `vatio chat` is in
|
|
80
|
-
|
|
81
|
-
// `vatio flags add` is a test suite `vatio test` replays on every version.
|
|
82
|
-
// --passes is for a reply that is already right: the case keeps it so, filed
|
|
83
|
-
// resolved, with nothing for Vatio to triage or fix.
|
|
84
|
-
async function addFlag(config, args) {
|
|
78
|
+
// The reply `vatio flags add` reports and `vatio test add` keeps: the agent's
|
|
79
|
+
// last one in the chat `vatio chat` is in, or --chat's, or --message.
|
|
80
|
+
function replyTarget(config, args, usage) {
|
|
85
81
|
const environment = takeEnv(args, { fallback: "preview" });
|
|
86
82
|
const chatId = takeValue(args, "--chat");
|
|
87
83
|
const messageId = takeValue(args, "--message");
|
|
88
|
-
const passing = takeFlag(args, "--passes");
|
|
89
84
|
const note = args.join(" ").trim();
|
|
90
|
-
if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${
|
|
91
|
-
if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${
|
|
85
|
+
if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${usage}`);
|
|
86
|
+
if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${usage}`);
|
|
92
87
|
|
|
93
88
|
const target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
|
|
94
89
|
if (!target.message_id && !target.chat_id) {
|
|
95
90
|
fail(`No ${environment} chat yet: run \`vatio chat "hello"\` first, or pass --chat ID or --message ID.`);
|
|
96
91
|
}
|
|
92
|
+
return { ...target, note };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// What the inbox's Reportar does, from the terminal: a reply that was wrong,
|
|
96
|
+
// and what it should have done. The note makes it a test case. A reply that
|
|
97
|
+
// is already right is not a report: `vatio test add` keeps it.
|
|
98
|
+
async function addFlag(config, args) {
|
|
99
|
+
if (args.includes("--passes")) {
|
|
100
|
+
fail('A reply that is already right is a test, not a report: `vatio test add "WHAT IT SHOULD DO"` keeps it.');
|
|
101
|
+
}
|
|
102
|
+
const body = replyTarget(config, args, ADD_USAGE);
|
|
97
103
|
|
|
98
104
|
const { url, token } = workspaceBase(config);
|
|
99
105
|
let flag;
|
|
100
106
|
try {
|
|
101
|
-
flag = await new HttpClient({ baseUrl: url, token }).post("/flags",
|
|
107
|
+
flag = await new HttpClient({ baseUrl: url, token }).post("/flags", body);
|
|
102
108
|
} catch (error) {
|
|
103
109
|
// The platform's own words name the chat or message and where to find the right one.
|
|
104
110
|
if (error instanceof NotFoundError) fail(error.message);
|
|
@@ -106,15 +112,9 @@ async function addFlag(config, args) {
|
|
|
106
112
|
}
|
|
107
113
|
|
|
108
114
|
const reply = String(flag.reply ?? "").trim().replace(/\s+/g, " ");
|
|
109
|
-
|
|
110
|
-
console.log(`${verb} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`);
|
|
115
|
+
console.log(`${flag.rewritten ? "Rewrote" : "Reported"} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`);
|
|
111
116
|
console.log(` [agent #${flag.message_id}] ${reply.length > 200 ? `${reply.slice(0, 200)}…` : reply}`);
|
|
112
|
-
if (flag.test_case
|
|
113
|
-
console.log(
|
|
114
|
-
`It is a test case, resolved${flag.resolved_in ? ` in ${flag.resolved_in}` : ""}: nothing for Vatio to fix, nothing pending\n` +
|
|
115
|
-
"in the inbox. `vatio test` replays it; once live passes it, `vatio publish` refuses a version that fails it."
|
|
116
|
-
);
|
|
117
|
-
} else if (flag.test_case) {
|
|
117
|
+
if (flag.test_case) {
|
|
118
118
|
console.log("It is a test case now: `vatio test` replays it, `vatio flags` lists it, `vatio flags rm` removes it.");
|
|
119
119
|
} else {
|
|
120
120
|
console.log("No visitor message comes before that reply, so there is nothing to replay: it is not a test case.");
|
|
@@ -144,8 +144,7 @@ async function removeFlags(config, ids) {
|
|
|
144
144
|
}
|
|
145
145
|
|
|
146
146
|
// Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
|
|
147
|
-
// then the calls that failed with what your backend answered
|
|
148
|
-
// Tools page.
|
|
147
|
+
// then the calls that failed with what your backend answered.
|
|
149
148
|
export async function tools(config, args) {
|
|
150
149
|
const environment = takeEnv(args, { fallback: "live" });
|
|
151
150
|
const period = takeValue(args, "--period");
|
|
@@ -168,9 +167,71 @@ export async function tools(config, args) {
|
|
|
168
167
|
// one a coding agent had to remember to put in the background. `--wait` waits
|
|
169
168
|
// for the judge and exits 1 when a case fails, so it can gate CI.
|
|
170
169
|
export async function test(config, args) {
|
|
170
|
+
if (args[0] === "add") return await addTest(config, args.slice(1));
|
|
171
|
+
if (args[0] === "list") return await listTests(config, args.slice(1));
|
|
172
|
+
if (["rm", "remove", "delete"].includes(args[0])) return await removeTests(config, args.slice(1));
|
|
171
173
|
return await runTests(config, args, { name: "test", waitByDefault: false });
|
|
172
174
|
}
|
|
173
175
|
|
|
176
|
+
const TEST_ADD_USAGE = 'vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
|
|
177
|
+
|
|
178
|
+
// A reply that is right, kept as a test so it stays right: a photo of its
|
|
179
|
+
// turn, like a report's case, with no report for Vatio to triage or fix. A
|
|
180
|
+
// set of `vatio chat --new` + `vatio test add` is a test suite.
|
|
181
|
+
async function addTest(config, args) {
|
|
182
|
+
const body = replyTarget(config, args, TEST_ADD_USAGE);
|
|
183
|
+
const { url, token } = workspaceBase(config);
|
|
184
|
+
let added;
|
|
185
|
+
try {
|
|
186
|
+
added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", body);
|
|
187
|
+
} catch (error) {
|
|
188
|
+
if (error instanceof NotFoundError) fail(error.message);
|
|
189
|
+
throw error;
|
|
190
|
+
}
|
|
191
|
+
console.log(`${added.rewritten ? "Rewrote" : "Added"} test ${added.id} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
|
|
192
|
+
console.log(` Visitor: ${String(added.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 200)}`);
|
|
193
|
+
console.log(` Should: ${added.note}`);
|
|
194
|
+
console.log(
|
|
195
|
+
"`vatio test` replays its turn on every version; once live passes it, `vatio publish` refuses a version that fails it."
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
async function listTests(config, args) {
|
|
200
|
+
const json = takeFlag(args, "--json");
|
|
201
|
+
if (args.length > 0) fail("Usage: vatio test list [--json]");
|
|
202
|
+
const { url, token } = workspaceBase(config);
|
|
203
|
+
const body = await new HttpClient({ baseUrl: url, token }).get("/evals/cases");
|
|
204
|
+
if (json) {
|
|
205
|
+
process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
const tests = body.tests ?? [];
|
|
209
|
+
if (tests.length === 0) {
|
|
210
|
+
console.log('No tests yet. `vatio chat --new "..."`, then `vatio test add "WHAT IT SHOULD DO"` keeps a reply that is right.');
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
for (const row of tests) {
|
|
214
|
+
console.log(`${row.id} ${row.agent} ${String(row.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 80)}`);
|
|
215
|
+
console.log(` should: ${row.note}`);
|
|
216
|
+
}
|
|
217
|
+
console.log("Reports are tests too: `vatio flags` lists the open ones.");
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function removeTests(config, ids) {
|
|
221
|
+
if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio test rm ID [ID...]");
|
|
222
|
+
const { url, token } = workspaceBase(config);
|
|
223
|
+
const client = new HttpClient({ baseUrl: url, token });
|
|
224
|
+
for (const id of ids) {
|
|
225
|
+
try {
|
|
226
|
+
await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
|
|
227
|
+
} catch (error) {
|
|
228
|
+
if (error instanceof NotFoundError) fail(error.message);
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
console.log(`Removed test ${id.toUpperCase()}.`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
174
235
|
// What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
|
|
175
236
|
// for the CI jobs and older skills that still call it; the name also reads as
|
|
176
237
|
// the shell's eval to command guards, which refused it.
|
package/lib/help.mjs
CHANGED
|
@@ -78,20 +78,18 @@ Commands default as listed.
|
|
|
78
78
|
flags [--save [PATH]|--json]
|
|
79
79
|
Every open report: the conversation, what each tool
|
|
80
80
|
answered, and what should have happened
|
|
81
|
-
flags add "NOTE" [--
|
|
81
|
+
flags add "NOTE" [--chat ID|--message ID] [--env NAME]
|
|
82
82
|
Report a reply and what should have happened:
|
|
83
83
|
the agent's last one in the chat \`vatio chat\`
|
|
84
|
-
is in (default: preview). It is a test case now
|
|
85
|
-
--passes: the reply is right; the case keeps it so
|
|
84
|
+
is in (default: preview). It is a test case now
|
|
86
85
|
flags rm ID [ID...] Remove a report and its test case, open or
|
|
87
86
|
resolved, as the inbox's Remove does
|
|
88
87
|
tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
live, 7d
|
|
88
|
+
Each tool's calls, error rate, timeouts and
|
|
89
|
+
p50/p95, then the calls that failed with what
|
|
90
|
+
your backend answered. Default: live, 7d
|
|
93
91
|
test [--env NAME] [--agent SLUG]
|
|
94
|
-
Test every report
|
|
92
|
+
Test every report and test against preview
|
|
95
93
|
(default): starts the runs on Vatio's servers,
|
|
96
94
|
prints their ids and returns.
|
|
97
95
|
--env live also resolves the reports it passes
|
|
@@ -99,6 +97,11 @@ Commands default as listed.
|
|
|
99
97
|
if a case fails. Joins this version's running run
|
|
100
98
|
test --show RUN_ID One run's report so far, with the replay chat of
|
|
101
99
|
each case that did not pass (vatio chat show ID)
|
|
100
|
+
test add "NOTE" [--chat ID|--message ID] [--env NAME]
|
|
101
|
+
Keep a reply that is right as a test: the agent's
|
|
102
|
+
last one in the chat \`vatio chat\` is in
|
|
103
|
+
test list [--json] | test rm ID [ID...]
|
|
104
|
+
The tests \`test add\` made; reports are \`flags\`
|
|
102
105
|
eval Deprecated: \`vatio test --wait\` under its old name
|
|
103
106
|
|
|
104
107
|
Knowledge:
|