@vatio-ai/cli 0.80.0 → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/bin/vatio.mjs +11 -3
- package/lib/commands/chat.mjs +1 -1
- package/lib/commands/deploy.mjs +2 -2
- package/lib/commands/improve.mjs +36 -12
- package/lib/commands/misc.mjs +1 -1
- package/lib/help.mjs +12 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -54,7 +54,7 @@ login | logout Authorize this machine in a browser, or revoke and f
|
|
|
54
54
|
doctor Node, config, workspace and token status
|
|
55
55
|
|
|
56
56
|
push Validate the workspace, then update preview
|
|
57
|
-
publish Take preview live, once `vatio
|
|
57
|
+
publish Take preview live, once `vatio test` has tested it
|
|
58
58
|
diff [--env NAME] What this directory would change
|
|
59
59
|
status Preview and live deployment state
|
|
60
60
|
rollback Restore the previous live deployment
|
|
@@ -65,7 +65,8 @@ chat show [CHAT_ID] [--json]
|
|
|
65
65
|
The chat you are in, or any chat in the workspace, with its tool calls
|
|
66
66
|
|
|
67
67
|
flags Every open report from the inbox, for your coding agent to fix
|
|
68
|
-
|
|
68
|
+
test Test every report against preview, in the background
|
|
69
|
+
test --wait Wait for the report; exits 1 if a case fails
|
|
69
70
|
tools [--env NAME] How each tool's backend has been answering
|
|
70
71
|
|
|
71
72
|
kb list|show|create|rm Knowledge bases
|
package/bin/vatio.mjs
CHANGED
|
@@ -16,12 +16,12 @@ import { keys, secrets, tokens } from "../lib/commands/workspace-admin.mjs";
|
|
|
16
16
|
import { templates, webhooks } from "../lib/commands/webhooks.mjs";
|
|
17
17
|
import { routines } from "../lib/commands/routines.mjs";
|
|
18
18
|
import { auth } from "../lib/commands/identity-key.mjs";
|
|
19
|
-
import { chat } from "../lib/commands/chat.mjs";
|
|
19
|
+
import { chat, chatHelp } from "../lib/commands/chat.mjs";
|
|
20
20
|
import { kb } from "../lib/commands/knowledge.mjs";
|
|
21
21
|
import { business } from "../lib/commands/business.mjs";
|
|
22
22
|
import { instagram, whatsapp } from "../lib/commands/channels.mjs";
|
|
23
23
|
import { docs, feedback, shareSession } from "../lib/commands/support-commands.mjs";
|
|
24
|
-
import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
|
|
24
|
+
import { evalCommand, flags, test, tools } from "../lib/commands/improve.mjs";
|
|
25
25
|
import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
|
|
26
26
|
import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
|
|
27
27
|
|
|
@@ -38,7 +38,7 @@ const RETIRED = {
|
|
|
38
38
|
mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
|
|
39
39
|
update: "is npm's job now: `npx @vatio-ai/cli@latest` always runs the current release, and\n" +
|
|
40
40
|
"`npm install -g @vatio-ai/cli@latest` updates a global install.",
|
|
41
|
-
propose: "was removed in 0.71.0. Test with `vatio
|
|
41
|
+
propose: "was removed in 0.71.0. Test with `vatio test` and publish with `vatio publish`."
|
|
42
42
|
};
|
|
43
43
|
|
|
44
44
|
const config = new Config({ startDir: process.cwd() });
|
|
@@ -54,6 +54,12 @@ async function main(argv) {
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
async function run(command, args) {
|
|
57
|
+
// `--help` after a command asks about it. Left to the command, it reads as a
|
|
58
|
+
// message to send, an argument, or nothing at all: `vatio chat --help` said
|
|
59
|
+
// "--help" to the agent, and `vatio eval --help` started a run.
|
|
60
|
+
if (command !== undefined && (args.includes("--help") || args.includes("-h"))) {
|
|
61
|
+
return console.log(command === "chat" ? chatHelp() : helpText());
|
|
62
|
+
}
|
|
57
63
|
|
|
58
64
|
switch (command) {
|
|
59
65
|
case "version":
|
|
@@ -108,6 +114,8 @@ async function run(command, args) {
|
|
|
108
114
|
return await instagram(config, args);
|
|
109
115
|
case "flags":
|
|
110
116
|
return await flags(config, args);
|
|
117
|
+
case "test":
|
|
118
|
+
return await test(config, args);
|
|
111
119
|
case "eval":
|
|
112
120
|
return await evalCommand(config, args);
|
|
113
121
|
case "tools":
|
package/lib/commands/chat.mjs
CHANGED
|
@@ -59,7 +59,7 @@ export async function chat(config, args) {
|
|
|
59
59
|
return await converse(config, options, message);
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
function chatHelp() {
|
|
62
|
+
export function chatHelp() {
|
|
63
63
|
return `Usage:
|
|
64
64
|
vatio chat "message" [--env NAME] [--new] [--timeout N]
|
|
65
65
|
vatio chat show [CHAT_ID] [--env NAME] [--last N] [--json]
|
package/lib/commands/deploy.mjs
CHANGED
|
@@ -183,8 +183,8 @@ function regressionsMessage(error) {
|
|
|
183
183
|
lines.push(
|
|
184
184
|
"",
|
|
185
185
|
broken.length > 0
|
|
186
|
-
? "Fix them, then `vatio push` and `vatio
|
|
187
|
-
: "Run `vatio
|
|
186
|
+
? "Fix them, then `vatio push` and `vatio test --wait` again before publishing."
|
|
187
|
+
: "Run `vatio test --wait` again, then publish.",
|
|
188
188
|
"`vatio publish --force` publishes anyway: only if you know what you're doing."
|
|
189
189
|
);
|
|
190
190
|
return lines.join("\n");
|
package/lib/commands/improve.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
// flags /
|
|
1
|
+
// flags / test / tools -- what supervisors reported, proving a fix for it, and
|
|
2
2
|
// how the tools' backend has been answering.
|
|
3
3
|
//
|
|
4
4
|
// All print markdown, because the reader is usually a coding agent: `vatio
|
|
5
|
-
// flags` is the document it fixes the agent from, `vatio
|
|
5
|
+
// flags` is the document it fixes the agent from, `vatio test` is the
|
|
6
6
|
// evidence that the fix worked on preview before `vatio publish` takes it
|
|
7
7
|
// live, and `vatio tools` is the inbox's Tools page for the failures nobody
|
|
8
8
|
// reported.
|
|
@@ -88,16 +88,32 @@ export async function tools(config, args) {
|
|
|
88
88
|
process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
-
// Replays every report with a note against an environment
|
|
92
|
-
//
|
|
91
|
+
// Replays every report with a note against an environment. Starts the runs
|
|
92
|
+
// and returns: a run takes minutes, and a command that sat waiting for it was
|
|
93
|
+
// one a coding agent had to remember to put in the background. `--wait` waits
|
|
94
|
+
// for the judge and exits 1 when a case fails, so it can gate CI.
|
|
95
|
+
export async function test(config, args) {
|
|
96
|
+
return await runTests(config, args, { name: "test", waitByDefault: false });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// What `vatio test` was called until 0.81.0. Kept, waiting as it always did,
|
|
100
|
+
// for the CI jobs and older skills that still call it; the name also reads as
|
|
101
|
+
// the shell's eval to command guards, which refused it.
|
|
93
102
|
export async function evalCommand(config, args) {
|
|
103
|
+
console.error("vatio: `vatio eval` is deprecated; use `vatio test --wait` (or `vatio test` to start the run and return).");
|
|
104
|
+
return await runTests(config, args, { name: "eval", waitByDefault: true });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function runTests(config, args, { name, waitByDefault }) {
|
|
94
108
|
const show = takeValue(args, "--show");
|
|
95
|
-
if (show) return await
|
|
109
|
+
if (show) return await testShow(config, show);
|
|
96
110
|
|
|
97
111
|
const environment = takeEnv(args, { fallback: "preview" });
|
|
98
112
|
const agent = takeValue(args, "--agent");
|
|
113
|
+
const wait = takeFlag(args, "--wait");
|
|
99
114
|
const noWait = takeFlag(args, "--no-wait");
|
|
100
115
|
const timeoutS = Number(takeValue(args, "--timeout") ?? DEFAULT_TIMEOUT_S);
|
|
116
|
+
const waiting = wait || (waitByDefault && !noWait);
|
|
101
117
|
|
|
102
118
|
const { url, token } = workspaceBase(config);
|
|
103
119
|
const client = new HttpClient({ baseUrl: url, token });
|
|
@@ -115,17 +131,26 @@ export async function evalCommand(config, args) {
|
|
|
115
131
|
const total = runs.reduce((sum, run) => sum + run.cases, 0);
|
|
116
132
|
console.error(`vatio: replaying ${total} case${total === 1 ? "" : "s"} on ${environment}…`);
|
|
117
133
|
await printSkillsHint();
|
|
118
|
-
if (
|
|
119
|
-
for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio
|
|
134
|
+
if (!waiting) {
|
|
135
|
+
for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
|
|
136
|
+
if (name === "test") {
|
|
137
|
+
console.log(
|
|
138
|
+
"The runs keep going on Vatio's servers, five cases at a time. `vatio test --show RUN_ID` prints the\n" +
|
|
139
|
+
"report so far; `vatio test --wait` waits for this version's runs and exits 1 if a case fails."
|
|
140
|
+
);
|
|
141
|
+
}
|
|
120
142
|
return;
|
|
121
143
|
}
|
|
144
|
+
// Said before waiting, so the id is there to read the run again later
|
|
145
|
+
// however this command ends.
|
|
146
|
+
for (const run of runs) console.error(`vatio: run ${run.id} (${run.agent}): vatio ${name} --show ${run.id}`);
|
|
122
147
|
|
|
123
148
|
const deadline = Date.now() + timeoutS * 1000;
|
|
124
149
|
const finished = [];
|
|
125
150
|
for (const run of runs) {
|
|
126
151
|
let row = run;
|
|
127
152
|
while (!row.finished) {
|
|
128
|
-
if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio
|
|
153
|
+
if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio ${name} --show ${run.id}`);
|
|
129
154
|
await sleep(POLL_MS);
|
|
130
155
|
row = await client.get(`/evals/runs/${run.id}`);
|
|
131
156
|
}
|
|
@@ -133,12 +158,11 @@ export async function evalCommand(config, args) {
|
|
|
133
158
|
}
|
|
134
159
|
|
|
135
160
|
for (const run of finished) {
|
|
136
|
-
process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `
|
|
161
|
+
process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `test run ${run.id}`));
|
|
137
162
|
}
|
|
138
163
|
if (finished.some((run) => run.failed > 0)) process.exitCode = 1;
|
|
139
164
|
}
|
|
140
165
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `eval run ${id}`));
|
|
166
|
+
async function testShow(config, id) {
|
|
167
|
+
process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `test run ${id}`));
|
|
144
168
|
}
|
package/lib/commands/misc.mjs
CHANGED
|
@@ -139,7 +139,7 @@ const AGENTS_NOTE = [
|
|
|
139
139
|
"",
|
|
140
140
|
"To fix what supervisors reported in the inbox, take every open report in",
|
|
141
141
|
"one session: `vatio flags` (follow the instructions at the top of its",
|
|
142
|
-
"output), fix, `vatio push`, `vatio
|
|
142
|
+
"output), fix, `vatio push`, `vatio test` until the reports pass, then",
|
|
143
143
|
"`vatio publish`. The `vatio-improve` skill walks through it.",
|
|
144
144
|
"`vatio tools --env live` shows how each tool's backend has been answering:",
|
|
145
145
|
"error rate, timeouts, latency and the calls that failed.",
|
package/lib/help.mjs
CHANGED
|
@@ -39,7 +39,7 @@ Commands default as listed.
|
|
|
39
39
|
--force replaces an unpublished push from
|
|
40
40
|
another git branch
|
|
41
41
|
publish [--force] Take preview live, with its knowledge changes.
|
|
42
|
-
Once published, refused until \`vatio
|
|
42
|
+
Once published, refused until \`vatio test\` has
|
|
43
43
|
tested preview's version, and while it fails a
|
|
44
44
|
report live passes; --force publishes anyway
|
|
45
45
|
rollback Put live back to the previous snapshot: config and knowledge
|
|
@@ -61,7 +61,7 @@ Commands default as listed.
|
|
|
61
61
|
(web, WhatsApp, ...): messages, tool calls, channel
|
|
62
62
|
|
|
63
63
|
Improve it from what supervisors reported and how its tools answered
|
|
64
|
-
(flags, fix, push,
|
|
64
|
+
(flags, fix, push, test, publish):
|
|
65
65
|
flags [--save [PATH]|--json]
|
|
66
66
|
Every open report: the conversation, what each tool
|
|
67
67
|
answered, and what should have happened
|
|
@@ -70,12 +70,16 @@ Commands default as listed.
|
|
|
70
70
|
rate, timeouts and p50/p95, then the calls that
|
|
71
71
|
failed with what your backend answered. Default:
|
|
72
72
|
live, 7d
|
|
73
|
-
|
|
73
|
+
test [--env NAME] [--agent SLUG]
|
|
74
74
|
Test every report with a note against preview
|
|
75
|
-
(default)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
75
|
+
(default): starts the runs on Vatio's servers,
|
|
76
|
+
prints their ids and returns.
|
|
77
|
+
--env live also resolves the reports it passes
|
|
78
|
+
test --wait [--timeout N] Wait for the judge and print the report; exits 1
|
|
79
|
+
if a case fails. Joins this version's running run
|
|
80
|
+
test --show RUN_ID One run's report so far, with the replay chat of
|
|
81
|
+
each case that did not pass (vatio chat show ID)
|
|
82
|
+
eval Deprecated: \`vatio test --wait\` under its old name
|
|
79
83
|
|
|
80
84
|
Knowledge:
|
|
81
85
|
kb [list] Bases, and whether a deployed agent reads each
|
|
@@ -105,7 +109,7 @@ Commands default as listed.
|
|
|
105
109
|
Without VALUE, set asks for it with the input hidden.
|
|
106
110
|
With --env, the value only that environment uses;
|
|
107
111
|
otherwise the value everywhere. With --tests, the
|
|
108
|
-
value preview and every
|
|
112
|
+
value preview and every test run use, like a staging URL
|
|
109
113
|
tokens list|create [--env NAME] [--label NAME] [--origin URL]...|revoke PREFIX
|
|
110
114
|
tokens origins PREFIX URL...
|
|
111
115
|
The origins a token is accepted from (replaces the list)
|