@vatio-ai/cli 0.119.0 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/claude-session.mjs +4 -2
- package/lib/commands/chat.mjs +18 -2
- package/lib/commands/deploy.mjs +37 -5
- package/lib/commands/improve.mjs +99 -151
- package/lib/commands/misc.mjs +5 -4
- package/lib/commands/support-commands.mjs +1 -1
- package/lib/help.mjs +16 -18
- package/lib/suggest.mjs +25 -3
- package/lib/telemetry.mjs +3 -1
- package/package.json +1 -1
package/lib/claude-session.mjs
CHANGED
|
@@ -59,10 +59,12 @@ const SECRET_PATTERNS = [
|
|
|
59
59
|
// The people a session is about rather than the one who sends it: customers
|
|
60
60
|
// that came up in a query or a log. The team never needs who they were. A
|
|
61
61
|
// phone counts only in international form; a bare run of digits is as likely
|
|
62
|
-
// an id.
|
|
62
|
+
// an id. A Chilean RUT has its check digit after a dash, the same pattern the
|
|
63
|
+
// Admin API redacts chats with (AdminApi::ChatTranscriptRedactor).
|
|
63
64
|
const PERSONAL_PATTERNS = [
|
|
64
65
|
[/(?<![\w.%+/-])(?!git@|no-?reply@)[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}\b/g, "[EMAIL]"],
|
|
65
|
-
[/(?<![\w+])\+\d(?:[ .-]?\d){7,14}\b/g, "[PHONE]"]
|
|
66
|
+
[/(?<![\w+])\+\d(?:[ .-]?\d){7,14}\b/g, "[PHONE]"],
|
|
67
|
+
[/(?<![\p{L}\p{N}])\d{1,2}\.?\d{3}\.?\d{3}-[\dkK](?![\p{L}\p{N}])/gu, "[RUT]"]
|
|
66
68
|
];
|
|
67
69
|
|
|
68
70
|
export function claudeHome() {
|
package/lib/commands/chat.mjs
CHANGED
|
@@ -101,7 +101,7 @@ your inbox.
|
|
|
101
101
|
\`show\` prints the chat you are in, with every tool call the agent made. Give
|
|
102
102
|
it an id to read any chat in the workspace -- one a visitor had on the web,
|
|
103
103
|
WhatsApp or Instagram -- with the environment and deployment that answered it.
|
|
104
|
-
Each message carries its id (#4821), the one \`vatio
|
|
104
|
+
Each message carries its id (#4821), the one \`vatio test add --message\` takes.
|
|
105
105
|
A [your backend] line under a tool call is how your backend answered it.
|
|
106
106
|
A [safeguard] line says which of Vatio's checks discarded or rewrote a draft,
|
|
107
107
|
and a [vatio] line marks a reply Vatio wrote because the agent's turn produced
|
|
@@ -215,7 +215,7 @@ function chatsFor(config, apiUrl) {
|
|
|
215
215
|
return state.chats && typeof state.chats === "object" ? state.chats : {};
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
// The chat `vatio chat` is in on this environment, for `vatio
|
|
218
|
+
// The chat `vatio chat` is in on this environment, for `vatio test add`.
|
|
219
219
|
export function rememberedChatId(config, environment) {
|
|
220
220
|
return chatsFor(config, config.resolveApiUrl())[environment] ?? null;
|
|
221
221
|
}
|
|
@@ -473,6 +473,22 @@ function truncate(text, max) {
|
|
|
473
473
|
return text.length > max ? `${text.slice(0, max)}…` : text;
|
|
474
474
|
}
|
|
475
475
|
|
|
476
|
+
// One question in a new chat, for `vatio test add "QUESTION" "WHAT IT SHOULD
|
|
477
|
+
// DO"`: the chat becomes the one `vatio chat` continues, and the reply is the
|
|
478
|
+
// one the test is made on.
|
|
479
|
+
export async function askInNewChat(config, { environment, asPhone = null, identity = null, timeout = 120 }, question) {
|
|
480
|
+
const options = { environment, asPhone, identity, timeout, apiUrl: null, token: null };
|
|
481
|
+
const apiUrl = config.resolveApiUrl({ explicit: null });
|
|
482
|
+
const payload = await openChat(config, options, { apiUrl, chatId: chatsFor(config, apiUrl)[environment] ?? null });
|
|
483
|
+
const chatId = Number(payload.chat_id);
|
|
484
|
+
const api = client(config, options);
|
|
485
|
+
const sent = await api.sendChatMessage({ chatId, content: question });
|
|
486
|
+
const reply = await waitForAssistant(api, { chatId, after: sent.user_message_id, timeout });
|
|
487
|
+
console.log(`[you] ${question}`);
|
|
488
|
+
console.log(formatVisitor(reply));
|
|
489
|
+
return reply;
|
|
490
|
+
}
|
|
491
|
+
|
|
476
492
|
async function startOver(config, options) {
|
|
477
493
|
const apiUrl = config.resolveApiUrl({ explicit: options.apiUrl });
|
|
478
494
|
const payload = await openChat(config, options, {
|
package/lib/commands/deploy.mjs
CHANGED
|
@@ -75,9 +75,34 @@ export async function checkWorkspace(config, workspace, { environment = null } =
|
|
|
75
75
|
return deployClient(config, workspace).check({ files, environment });
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
// A block vatio.yml still declares and the platform no longer reads. Among the
|
|
79
|
+
// other warnings it read as one more, and a developer took `allowed_origins:`
|
|
80
|
+
// under `widget:` as applied: said apart, as what it is.
|
|
81
|
+
const IGNORED_BLOCK = /^vatio\.yml: `[^`]+` is ignored/;
|
|
82
|
+
|
|
83
|
+
// Returns the ignored-block lines it held back, for a caller that prints them
|
|
84
|
+
// last (`vatio push`); otherwise they follow the other warnings.
|
|
85
|
+
export function printDiagnostics(payload, { holdIgnored = false } = {}) {
|
|
79
86
|
for (const row of asArray(payload.errors)) console.error(`error: ${diagnosticLine(row)}`);
|
|
80
|
-
|
|
87
|
+
const ignored = [];
|
|
88
|
+
for (const row of asArray(payload.warnings)) {
|
|
89
|
+
const line = diagnosticLine(row);
|
|
90
|
+
if (IGNORED_BLOCK.test(line)) ignored.push(line);
|
|
91
|
+
else console.error(`warning: ${line}`);
|
|
92
|
+
}
|
|
93
|
+
if (!holdIgnored) printIgnored(ignored);
|
|
94
|
+
return ignored;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function printIgnored(lines) {
|
|
98
|
+
if (lines.length === 0) return;
|
|
99
|
+
const one = lines.length === 1;
|
|
100
|
+
console.error("");
|
|
101
|
+
console.error(
|
|
102
|
+
`Not applied: vatio.yml declares ${one ? "a block" : `${lines.length} blocks`} Vatio does not read, ` +
|
|
103
|
+
`and nothing in ${one ? "it" : "them"} takes effect:`
|
|
104
|
+
);
|
|
105
|
+
for (const line of lines) console.error(` ${line}`);
|
|
81
106
|
}
|
|
82
107
|
|
|
83
108
|
// A diagnostic is { message, path } -- `path` is the file it is about, or null
|
|
@@ -118,8 +143,11 @@ export async function push(config, args) {
|
|
|
118
143
|
}
|
|
119
144
|
|
|
120
145
|
const check = await checkWorkspace(config, workspace, { environment });
|
|
121
|
-
printDiagnostics(check);
|
|
122
|
-
if (!check.ok)
|
|
146
|
+
const ignored = printDiagnostics(check, { holdIgnored: true });
|
|
147
|
+
if (!check.ok) {
|
|
148
|
+
printIgnored(ignored);
|
|
149
|
+
fail(`push aborted: check failed (${asArray(check.errors).length} error(s))`);
|
|
150
|
+
}
|
|
123
151
|
|
|
124
152
|
const client = deployClient(config, workspace);
|
|
125
153
|
const baseUrl = config.resolveBaseUrl();
|
|
@@ -156,7 +184,10 @@ export async function push(config, args) {
|
|
|
156
184
|
for (const line of consequenceLines(payload)) console.log(line);
|
|
157
185
|
// Where to try it: the inbox's Try the agent panel for that environment.
|
|
158
186
|
if (payload.share_url) console.log(`Try it: ${payload.share_url}`);
|
|
159
|
-
for (const warning of asArray(payload.warnings))
|
|
187
|
+
for (const warning of asArray(payload.warnings)) {
|
|
188
|
+
if (!IGNORED_BLOCK.test(String(warning))) console.log(` warning: ${warning}`);
|
|
189
|
+
else if (!ignored.includes(String(warning))) ignored.push(String(warning));
|
|
190
|
+
}
|
|
160
191
|
if (payload.preview_url) console.log(`Preview: ${payload.preview_url}`);
|
|
161
192
|
|
|
162
193
|
const hint = payload.first_push_hint;
|
|
@@ -166,6 +197,7 @@ export async function push(config, args) {
|
|
|
166
197
|
console.log(` ${hint.url}`);
|
|
167
198
|
}
|
|
168
199
|
printNotices(payload);
|
|
200
|
+
printIgnored(ignored);
|
|
169
201
|
}
|
|
170
202
|
|
|
171
203
|
// The two or three lines that say who sees this push, read off the platform's
|
package/lib/commands/improve.mjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
//
|
|
2
|
-
// how the tools' backend has been answering.
|
|
1
|
+
// test / tools -- what the agent should do and what supervisors reported,
|
|
2
|
+
// proving a fix for it, and how the tools' backend has been answering.
|
|
3
3
|
//
|
|
4
4
|
// All print markdown, because the reader is usually a coding agent: `vatio
|
|
5
|
-
//
|
|
5
|
+
// test list` is the document it fixes the agent from, `vatio test` is the
|
|
6
6
|
// evidence that the fix worked on preview before `vatio publish` takes it
|
|
7
7
|
// live, and `vatio tools` is how each tool's backend has been answering, for
|
|
8
8
|
// the failures nobody reported.
|
|
@@ -11,7 +11,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
11
11
|
import { dirname, resolve } from "node:path";
|
|
12
12
|
|
|
13
13
|
import { HttpClient, NotFoundError, throwIfOutdated } from "../http.mjs";
|
|
14
|
-
import { rememberedChatId } from "./chat.mjs";
|
|
14
|
+
import { askInNewChat, rememberedChatId } from "./chat.mjs";
|
|
15
15
|
import { clientHeaders } from "../telemetry.mjs";
|
|
16
16
|
import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
|
|
17
17
|
import { printSkillsHint } from "../skills-hint.mjs";
|
|
@@ -63,116 +63,16 @@ async function fetchMarkdown(config, path, what) {
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
// and what the supervisor says should have happened.
|
|
66
|
+
// `vatio flags` until 0.120.0, when reports and tests became one list: kept
|
|
67
|
+
// for the skills, scripts and muscle memory that still call it.
|
|
69
68
|
export async function flags(config, args) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (json) {
|
|
78
|
-
const { url, token } = workspaceBase(config);
|
|
79
|
-
const body = await new HttpClient({ baseUrl: url, token }).get("/flags");
|
|
80
|
-
process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
|
|
81
|
-
return;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
const markdown = await fetchMarkdown(config, "/flags.md", "the flags");
|
|
85
|
-
if (!save) {
|
|
86
|
-
process.stdout.write(markdown);
|
|
87
|
-
await printSkillsHint();
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
const target = resolve(savePath ?? "vatio-flags.md");
|
|
92
|
-
mkdirSync(dirname(target), { recursive: true });
|
|
93
|
-
writeFileSync(target, markdown);
|
|
94
|
-
console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
|
|
95
|
-
await printSkillsHint();
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
const ADD_USAGE = 'vatio flags add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
|
|
99
|
-
|
|
100
|
-
// The reply `vatio flags add` reports and `vatio test add` keeps: the agent's
|
|
101
|
-
// last one in the chat `vatio chat` is in, or --chat's, or --message.
|
|
102
|
-
function replyTarget(config, args, usage) {
|
|
103
|
-
const environment = takeEnv(args, { fallback: "preview" });
|
|
104
|
-
const chatId = takeValue(args, "--chat");
|
|
105
|
-
const messageId = takeValue(args, "--message");
|
|
106
|
-
const note = args.join(" ").trim();
|
|
107
|
-
if (note === "" || args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${usage}`);
|
|
108
|
-
if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${usage}`);
|
|
109
|
-
|
|
110
|
-
const target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
|
|
111
|
-
if (!target.message_id && !target.chat_id) {
|
|
112
|
-
fail(`No ${environment} chat yet: run \`vatio chat "hello"\` first, or pass --chat ID or --message ID.`);
|
|
113
|
-
}
|
|
114
|
-
return { ...target, note };
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
// What the inbox's Reportar does, from the terminal: a reply that was wrong,
|
|
118
|
-
// and what it should have done. On live the note makes it a report and a
|
|
119
|
-
// test case; on preview it is a note for the coding agent, closed by the
|
|
120
|
-
// next push. A reply that is already right is not a flag: `vatio test add`
|
|
121
|
-
// keeps it.
|
|
122
|
-
async function addFlag(config, args) {
|
|
123
|
-
if (args.includes("--passes")) {
|
|
124
|
-
fail('A reply that is already right is a test, not a report: `vatio test add "WHAT IT SHOULD DO"` keeps it.');
|
|
125
|
-
}
|
|
126
|
-
const body = replyTarget(config, args, ADD_USAGE);
|
|
127
|
-
|
|
128
|
-
const { url, token } = workspaceBase(config);
|
|
129
|
-
let flag;
|
|
130
|
-
try {
|
|
131
|
-
flag = await new HttpClient({ baseUrl: url, token }).post("/flags", body);
|
|
132
|
-
} catch (error) {
|
|
133
|
-
// The platform's own words name the chat or message and where to find the right one.
|
|
134
|
-
if (error instanceof NotFoundError) fail(error.message);
|
|
135
|
-
throw error;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
const reply = String(flag.reply ?? "").trim().replace(/\s+/g, " ");
|
|
139
|
-
const note = flag.environment === "preview";
|
|
140
|
-
console.log(
|
|
141
|
-
`${flag.rewritten ? "Rewrote" : note ? "Noted" : "Reported"} ${flag.public_id}, chat ${flag.chat_id} on ${flag.environment}:`
|
|
142
|
-
);
|
|
143
|
-
console.log(` [agent #${flag.message_id}] ${reply.length > 200 ? `${reply.slice(0, 200)}…` : reply}`);
|
|
144
|
-
if (note) {
|
|
145
|
-
console.log(
|
|
146
|
-
"A note on preview, not a test case: `vatio flags` lists it for your coding agent until your next `vatio push` closes it. " +
|
|
147
|
-
'Once the agent gets it right, `vatio test add "WHAT IT SHOULD DO"` keeps that reply as a test.'
|
|
148
|
-
);
|
|
149
|
-
} else if (flag.test_case) {
|
|
150
|
-
console.log("It is a test case now: `vatio test` replays it, `vatio flags` lists it, `vatio flags rm` removes it.");
|
|
151
|
-
} else {
|
|
152
|
-
console.log("No visitor message comes before that reply, so there is nothing to replay: it is not a test case.");
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
// What the inbox's Quitar does, by the ids `vatio flags` prints: the flag and
|
|
157
|
-
// its test case go, for everyone in the workspace. The developer's call, not
|
|
158
|
-
// the coding agent's: the vatio-improve skill says to run it only when asked.
|
|
159
|
-
async function removeFlags(config, ids) {
|
|
160
|
-
if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio flags rm ID [ID...]");
|
|
161
|
-
|
|
162
|
-
const { url, token } = workspaceBase(config);
|
|
163
|
-
const client = new HttpClient({ baseUrl: url, token });
|
|
164
|
-
for (const id of ids) {
|
|
165
|
-
let removed;
|
|
166
|
-
try {
|
|
167
|
-
removed = await client.delete(`/flags/${encodeURIComponent(id)}`);
|
|
168
|
-
} catch (error) {
|
|
169
|
-
// The platform's own words name the id and where to find the right one.
|
|
170
|
-
if (error instanceof NotFoundError) fail(error.message);
|
|
171
|
-
throw error;
|
|
172
|
-
}
|
|
173
|
-
const was = removed.state === "resolved" ? "a resolved flag, so no longer a regression case" : "an open flag";
|
|
174
|
-
console.log(`Removed ${removed.id} (${was}) and its test case.`);
|
|
175
|
-
}
|
|
69
|
+
const sub = args[0];
|
|
70
|
+
const removing = ["rm", "remove", "delete"].includes(sub);
|
|
71
|
+
const now = sub === "add" ? "vatio test add" : removing ? "vatio test rm" : "vatio test list";
|
|
72
|
+
console.error(`vatio: \`vatio flags${sub === "add" || removing ? ` ${sub}` : ""}\` is \`${now}\` now.`);
|
|
73
|
+
if (sub === "add") return await addTest(config, args.slice(1));
|
|
74
|
+
if (removing) return await removeTests(config, args.slice(1));
|
|
75
|
+
return await listTests(config, args);
|
|
176
76
|
}
|
|
177
77
|
|
|
178
78
|
// Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
|
|
@@ -205,63 +105,117 @@ export async function test(config, args) {
|
|
|
205
105
|
return await runTests(config, args, { name: "test", waitByDefault: false });
|
|
206
106
|
}
|
|
207
107
|
|
|
208
|
-
const TEST_ADD_USAGE =
|
|
108
|
+
const TEST_ADD_USAGE =
|
|
109
|
+
'vatio test add "QUESTION" "WHAT IT SHOULD DO" [--as-phone me|PHONE] [--as-subject ID] [--env NAME]\n' +
|
|
110
|
+
' vatio test add "WHAT IT SHOULD DO" [--chat ID | --message ID] [--env NAME]';
|
|
209
111
|
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
//
|
|
112
|
+
// What the agent should do when asked something, checked on every version.
|
|
113
|
+
// With a question, Vatio asks it in a new chat and the test is made on that
|
|
114
|
+
// turn; with only what it should do, on the agent's last reply in the chat
|
|
115
|
+
// `vatio chat` is in, or --chat's, or --message's (a reply a visitor got, or a
|
|
116
|
+
// report's, whose note it rewrites). Either way the reply is judged at once,
|
|
117
|
+
// so you know whether the test is a guard or work still to do.
|
|
214
118
|
async function addTest(config, args) {
|
|
215
|
-
const
|
|
119
|
+
const environment = takeEnv(args, { fallback: "preview" });
|
|
120
|
+
const chatId = takeValue(args, "--chat");
|
|
121
|
+
const messageId = takeValue(args, "--message");
|
|
122
|
+
const asPhone = takeValue(args, "--as-phone");
|
|
123
|
+
const timeout = Number(takeValue(args, "--timeout") ?? 120);
|
|
124
|
+
const identity = takeIdentity(args, config);
|
|
125
|
+
if (args.some((arg) => arg.startsWith("--"))) fail(`Usage: ${TEST_ADD_USAGE}`);
|
|
126
|
+
if (args.length > 2) {
|
|
127
|
+
fail(`Put the question and what the agent should do in quotes, one argument each. Usage: ${TEST_ADD_USAGE}`);
|
|
128
|
+
}
|
|
129
|
+
const [first, second] = args.map((arg) => arg.trim());
|
|
130
|
+
if (!first || (args.length === 2 && !second)) fail(`Usage: ${TEST_ADD_USAGE}`);
|
|
131
|
+
|
|
132
|
+
let target;
|
|
133
|
+
let note;
|
|
134
|
+
if (args.length === 2) {
|
|
135
|
+
if (chatId || messageId) fail("A question opens its own chat: drop --chat and --message, or the question.");
|
|
136
|
+
const reply = await askInNewChat(config, { environment, asPhone, identity, timeout }, first);
|
|
137
|
+
target = { message_id: reply.id };
|
|
138
|
+
note = second;
|
|
139
|
+
} else {
|
|
140
|
+
if (asPhone || identity) fail("--as-phone and --as-subject go with a question: `vatio test add \"QUESTION\" \"WHAT IT SHOULD DO\"`.");
|
|
141
|
+
if (chatId && messageId) fail(`--chat or --message, not both. Usage: ${TEST_ADD_USAGE}`);
|
|
142
|
+
target = messageId ? { message_id: messageId } : { chat_id: chatId ?? rememberedChatId(config, environment) };
|
|
143
|
+
if (!target.message_id && !target.chat_id) {
|
|
144
|
+
fail(`No ${environment} chat yet: \`vatio test add "QUESTION" "WHAT IT SHOULD DO"\` asks it, or pass --chat ID or --message ID.`);
|
|
145
|
+
}
|
|
146
|
+
note = first;
|
|
147
|
+
}
|
|
148
|
+
|
|
216
149
|
const { url, token } = workspaceBase(config);
|
|
217
150
|
let added;
|
|
218
151
|
try {
|
|
219
|
-
added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases",
|
|
152
|
+
added = await new HttpClient({ baseUrl: url, token }).post("/evals/cases", { ...target, note, judge: true });
|
|
220
153
|
} catch (error) {
|
|
221
154
|
if (error instanceof NotFoundError) fail(error.message);
|
|
222
155
|
throw error;
|
|
223
156
|
}
|
|
224
|
-
|
|
157
|
+
const report = String(added.id).startsWith("F-");
|
|
158
|
+
const what = report ? `report ${added.id}'s note` : `test ${added.id}`;
|
|
159
|
+
console.log(`${added.rewritten ? "Rewrote" : "Added"} ${what} (agent \`${added.agent}\`, chat ${added.chat_id}):`);
|
|
225
160
|
console.log(` Visitor: ${String(added.asked ?? "").trim().replace(/\s+/g, " ").slice(0, 200)}`);
|
|
226
|
-
console.log(` Should:
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
);
|
|
161
|
+
console.log(` Should: ${added.note}`);
|
|
162
|
+
const judged = added.judged ?? {};
|
|
163
|
+
if (judged.verdict === "pass") {
|
|
164
|
+
console.log(` Passes now: ${judged.reason}`);
|
|
165
|
+
console.log("`vatio test` replays it on every version; once live passes it, `vatio publish` refuses a version that fails it.");
|
|
166
|
+
} else if (judged.verdict === "fail") {
|
|
167
|
+
console.log(` Fails now: ${judged.reason}`);
|
|
168
|
+
console.log("Fix the agent, `vatio push`, then `vatio test`: `vatio test list` shows it with its replay once it has run.");
|
|
169
|
+
} else {
|
|
170
|
+
console.log(` Not judged now${judged.error ? `: ${judged.error}` : ""}. \`vatio test\` judges it with the rest.`);
|
|
171
|
+
}
|
|
230
172
|
}
|
|
231
173
|
|
|
174
|
+
// What there is to fix, then the whole suite: the reports still open, each
|
|
175
|
+
// with its conversation and what every tool answered; the tests that fail on
|
|
176
|
+
// preview or that the agent answers differently each time; every test.
|
|
232
177
|
async function listTests(config, args) {
|
|
233
178
|
const json = takeFlag(args, "--json");
|
|
234
|
-
|
|
235
|
-
const
|
|
236
|
-
|
|
179
|
+
const save = takeFlag(args, "--save");
|
|
180
|
+
const savePath = save ? args.shift() ?? null : null;
|
|
181
|
+
if (args.length > 0) fail("Usage: vatio test list [--save [PATH] | --json]");
|
|
182
|
+
|
|
237
183
|
if (json) {
|
|
238
|
-
|
|
184
|
+
const { url, token } = workspaceBase(config);
|
|
185
|
+
const client = new HttpClient({ baseUrl: url, token });
|
|
186
|
+
const [tests, flags] = await Promise.all([client.get("/evals/cases"), client.get("/flags")]);
|
|
187
|
+
process.stdout.write(`${JSON.stringify({ tests: tests.tests ?? [], flags: flags.flags ?? [] }, null, 2)}\n`);
|
|
239
188
|
return;
|
|
240
189
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
190
|
+
|
|
191
|
+
const markdown = await fetchMarkdown(config, "/flags.md", "the tests");
|
|
192
|
+
if (!save) {
|
|
193
|
+
process.stdout.write(markdown);
|
|
194
|
+
await printSkillsHint();
|
|
244
195
|
return;
|
|
245
196
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
console.log(
|
|
197
|
+
|
|
198
|
+
const target = resolve(savePath ?? "vatio-tests.md");
|
|
199
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
200
|
+
writeFileSync(target, markdown);
|
|
201
|
+
console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio test list --save\` to refresh.`);
|
|
202
|
+
await printSkillsHint();
|
|
251
203
|
}
|
|
252
204
|
|
|
205
|
+
// A report (F-) goes with its test, for everyone, as the inbox's Remove does.
|
|
253
206
|
async function removeTests(config, ids) {
|
|
254
207
|
if (ids.length === 0 || ids.some((id) => id.startsWith("-"))) fail("Usage: vatio test rm ID [ID...]");
|
|
255
208
|
const { url, token } = workspaceBase(config);
|
|
256
209
|
const client = new HttpClient({ baseUrl: url, token });
|
|
257
210
|
for (const id of ids) {
|
|
211
|
+
let removed;
|
|
258
212
|
try {
|
|
259
|
-
await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
|
|
213
|
+
removed = await client.delete(`/evals/cases/${encodeURIComponent(id)}`);
|
|
260
214
|
} catch (error) {
|
|
261
215
|
if (error instanceof NotFoundError) fail(error.message);
|
|
262
216
|
throw error;
|
|
263
217
|
}
|
|
264
|
-
console.log(`Removed test ${id
|
|
218
|
+
console.log(removed.report ? `Removed report ${removed.id} and its test.` : `Removed test ${removed.id}.`);
|
|
265
219
|
}
|
|
266
220
|
}
|
|
267
221
|
|
|
@@ -312,23 +266,17 @@ async function runTests(config, args, { name, waitByDefault }) {
|
|
|
312
266
|
);
|
|
313
267
|
return;
|
|
314
268
|
}
|
|
315
|
-
//
|
|
316
|
-
//
|
|
317
|
-
//
|
|
318
|
-
// "there is nothing to do here" and the suite never got started: of the
|
|
319
|
-
// twelve workspaces, six have never run a case, including the three
|
|
320
|
-
// newest. A suite is built out of replies that are already right, and
|
|
321
|
-
// `vatio test add` is how one is kept.
|
|
269
|
+
// "Report a reply in the inbox" read as "nothing to do here" on a
|
|
270
|
+
// workspace where nothing had gone wrong yet, and the suite never got
|
|
271
|
+
// started. A suite is what the agent should do when asked, written down.
|
|
322
272
|
console.log(
|
|
323
273
|
"No tests yet, so there is nothing to replay.\n" +
|
|
324
274
|
"\n" +
|
|
325
|
-
"A test is
|
|
326
|
-
' vatio
|
|
327
|
-
' vatio test add "what the answer has to say"\n' +
|
|
275
|
+
"A test is what the agent should do when someone asks something:\n" +
|
|
276
|
+
' vatio test add "what a visitor would ask" "what the answer has to do"\n' +
|
|
328
277
|
"\n" +
|
|
329
278
|
"Then `vatio test` replays them all on every version, and `vatio publish` refuses\n" +
|
|
330
|
-
"one that breaks them.
|
|
331
|
-
'`vatio flags add "what it should have done"`, or Reportar in the inbox.'
|
|
279
|
+
"one that breaks them. Replies the team reports in the inbox become tests too."
|
|
332
280
|
);
|
|
333
281
|
return;
|
|
334
282
|
}
|
package/lib/commands/misc.mjs
CHANGED
|
@@ -167,7 +167,7 @@ export async function init(config, args) {
|
|
|
167
167
|
|
|
168
168
|
// A pointer, not a copy of the docs: whatever coding agent works in this
|
|
169
169
|
// directory reads AGENTS.md, and these lines are how it learns that `vatio
|
|
170
|
-
// docs` is the contract and `vatio
|
|
170
|
+
// docs` is the contract and `vatio test list` is where fixing the agent starts.
|
|
171
171
|
// The instructions themselves come from the platform at run time, so nothing
|
|
172
172
|
// written here goes stale. An AGENTS.md that already mentions Vatio is left
|
|
173
173
|
// alone; any other gets the section appended.
|
|
@@ -178,10 +178,11 @@ const AGENTS_NOTE = [
|
|
|
178
178
|
"contract for the manifest, tools, knowledge and the CLI; read it before",
|
|
179
179
|
"changing anything here.",
|
|
180
180
|
"",
|
|
181
|
-
"To fix what supervisors reported
|
|
182
|
-
"one session: `vatio
|
|
183
|
-
"output), fix, `vatio push`, `vatio test` until
|
|
181
|
+
"To fix what supervisors reported and the tests that fail, take them all in",
|
|
182
|
+
"one session: `vatio test list` (follow the instructions at the top of its",
|
|
183
|
+
"output), fix, `vatio push`, `vatio test` until they pass, then",
|
|
184
184
|
"`vatio publish`. The `vatio-improve` skill walks through it.",
|
|
185
|
+
"`vatio test add \"QUESTION\" \"WHAT IT SHOULD DO\"` adds a test.",
|
|
185
186
|
"`vatio tools --env live` shows how each tool's backend has been answering:",
|
|
186
187
|
"error rate, timeouts, latency and the calls that failed.",
|
|
187
188
|
""
|
|
@@ -100,7 +100,7 @@ export async function shareSession(config, args) {
|
|
|
100
100
|
console.log(
|
|
101
101
|
` Left out: ${transcript.images > 0 ? `${transcript.images} image(s) and ` : ""}${label}'s own bookkeeping.` +
|
|
102
102
|
(transcript.redactions > 0 ? ` ${transcript.redactions} thing(s) that looked like a key, token or password are [REDACTED].` : "") +
|
|
103
|
-
(transcript.personal > 0 ? ` ${transcript.personal} email address(es)
|
|
103
|
+
(transcript.personal > 0 ? ` ${transcript.personal} email address(es), phone number(s) and RUT(s) are [EMAIL], [PHONE] and [RUT].` : "")
|
|
104
104
|
);
|
|
105
105
|
|
|
106
106
|
if (savePath) {
|
package/lib/help.mjs
CHANGED
|
@@ -86,19 +86,8 @@ the two you are in. Commands default as listed.
|
|
|
86
86
|
line as JSON. Default: live. Any team member,
|
|
87
87
|
viewers included; every call is in the activity
|
|
88
88
|
|
|
89
|
-
|
|
90
|
-
(
|
|
91
|
-
flags [--save [PATH]|--json]
|
|
92
|
-
Every open report: the conversation, what each tool
|
|
93
|
-
answered, and what should have happened
|
|
94
|
-
flags add "NOTE" [--chat ID|--message ID] [--env NAME]
|
|
95
|
-
Flag a reply and what should have happened:
|
|
96
|
-
the agent's last one in the chat \`vatio chat\`
|
|
97
|
-
is in (default: preview). On live it is a report
|
|
98
|
-
and a test case; on preview, a note for your
|
|
99
|
-
coding agent that the next push closes
|
|
100
|
-
flags rm ID [ID...] Remove a report and its test case, open or
|
|
101
|
-
resolved, as the inbox's Remove does
|
|
89
|
+
Test and improve it: what it should do, what supervisors reported, and how
|
|
90
|
+
its tools answered (test add, test list, fix, push, test, publish):
|
|
102
91
|
tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
|
|
103
92
|
Each tool's calls, error rate, timeouts and
|
|
104
93
|
p50/p95, then the calls that failed with what
|
|
@@ -114,12 +103,21 @@ the two you are in. Commands default as listed.
|
|
|
114
103
|
if a case fails. Joins this version's running run
|
|
115
104
|
test --show RUN_ID One run's report so far, with the replay chat of
|
|
116
105
|
each case that did not pass (vatio chat show ID)
|
|
117
|
-
test add "
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
test
|
|
121
|
-
The
|
|
106
|
+
test add "QUESTION" "WHAT IT SHOULD DO" [--as-phone me|PHONE] [--as-subject ID]
|
|
107
|
+
Ask it in a new chat (default: preview) and keep
|
|
108
|
+
the turn as a test, judged at once
|
|
109
|
+
test add "WHAT IT SHOULD DO" [--chat ID|--message ID]
|
|
110
|
+
The same on a reply you already have: the agent's
|
|
111
|
+
last one in the chat \`vatio chat\` is in, or a
|
|
112
|
+
report's, whose note it rewrites
|
|
113
|
+
test list [--save [PATH]|--json]
|
|
114
|
+
What to fix: open reports with their conversation
|
|
115
|
+
and what each tool answered, tests that fail or
|
|
116
|
+
answer differently each time; then every test
|
|
117
|
+
test rm ID [ID...] Remove a test, or a report (F-) and its test
|
|
122
118
|
eval Deprecated: \`vatio test --wait\` under its old name
|
|
119
|
+
flags [add|rm] Deprecated: \`vatio test list\`, \`test add\` and
|
|
120
|
+
\`test rm\` under their old name
|
|
123
121
|
|
|
124
122
|
Knowledge:
|
|
125
123
|
kb [list] Bases, and whether a deployed agent reads each
|
package/lib/suggest.mjs
CHANGED
|
@@ -72,7 +72,7 @@ export const COMMANDS = [
|
|
|
72
72
|
// listing them here would be dead. scripts/check-docs.mjs checks the command
|
|
73
73
|
// names on the right against the dispatch.
|
|
74
74
|
const SUBCOMMAND_OF = {
|
|
75
|
-
add: ["
|
|
75
|
+
add: ["test"],
|
|
76
76
|
cat: ["kb"],
|
|
77
77
|
create: ["kb", "keys", "templates", "webhooks"],
|
|
78
78
|
delete: ["contacts"],
|
|
@@ -88,9 +88,9 @@ const SUBCOMMAND_OF = {
|
|
|
88
88
|
off: ["quiet-hours"],
|
|
89
89
|
origins: ["keys"],
|
|
90
90
|
refresh: ["kb"],
|
|
91
|
-
remove: ["secrets", "webhooks", "
|
|
91
|
+
remove: ["secrets", "webhooks", "test"],
|
|
92
92
|
revoke: ["keys"],
|
|
93
|
-
rm: ["env", "kb", "secrets", "webhooks", "
|
|
93
|
+
rm: ["env", "kb", "secrets", "webhooks", "test"],
|
|
94
94
|
"rm-entry": ["kb"],
|
|
95
95
|
set: ["business", "quiet-hours", "secrets", "config"],
|
|
96
96
|
show: ["chat", "kb", "business", "quiet-hours", "config"],
|
|
@@ -100,6 +100,16 @@ const SUBCOMMAND_OF = {
|
|
|
100
100
|
write: ["kb"]
|
|
101
101
|
};
|
|
102
102
|
|
|
103
|
+
// Commands that were renamed. A token that is one of these is a habit (or an
|
|
104
|
+
// older doc) and the reader needs the new name, not an edit-distance guess:
|
|
105
|
+
// `tokens` is nowhere near `keys`, so it used to get the bare list.
|
|
106
|
+
const RENAMED = {
|
|
107
|
+
// `tokens create` made publishable keys; `keys create` names the kind.
|
|
108
|
+
tokens: (rest) => (rest[0] === "create" && !["publishable", "secret"].includes(rest[1])
|
|
109
|
+
? ["keys", "create", "publishable", ...rest.slice(1)]
|
|
110
|
+
: ["keys", ...rest])
|
|
111
|
+
};
|
|
112
|
+
|
|
103
113
|
// Exported so scripts/check-docs.mjs can check every command named above is
|
|
104
114
|
// one the CLI actually dispatches.
|
|
105
115
|
export const SUBCOMMAND_OWNERS = [...new Set(Object.values(SUBCOMMAND_OF).flat())];
|
|
@@ -215,6 +225,18 @@ export function unknownCommandMessage(command, args = []) {
|
|
|
215
225
|
].join("\n");
|
|
216
226
|
}
|
|
217
227
|
|
|
228
|
+
const rename = RENAMED[token.toLowerCase()];
|
|
229
|
+
if (rename) {
|
|
230
|
+
const [renamed, ...renamedArgs] = rename(rest);
|
|
231
|
+
return [
|
|
232
|
+
`\`vatio ${token}\` is now \`vatio ${renamed}\`.`,
|
|
233
|
+
"",
|
|
234
|
+
`Run: vatio ${[renamed, ...renamedArgs].join(" ")}`,
|
|
235
|
+
"",
|
|
236
|
+
"`vatio help` lists every command."
|
|
237
|
+
].join("\n");
|
|
238
|
+
}
|
|
239
|
+
|
|
218
240
|
const belongsTo = SUBCOMMAND_OF[token.toLowerCase()];
|
|
219
241
|
if (belongsTo) {
|
|
220
242
|
const options = belongsTo.map((owner) => `vatio ${owner} ${token}${trailing}`);
|
package/lib/telemetry.mjs
CHANGED
|
@@ -38,7 +38,9 @@ const SUBCOMMANDS = {
|
|
|
38
38
|
contacts: ["export", "delete"],
|
|
39
39
|
whatsapp: ["only", "everyone"],
|
|
40
40
|
config: ["show", "get", "set", "unset"],
|
|
41
|
-
"share-session": ["--should-ask", "--never"]
|
|
41
|
+
"share-session": ["--should-ask", "--never"],
|
|
42
|
+
test: ["add", "list", "rm"],
|
|
43
|
+
flags: ["add", "rm"]
|
|
42
44
|
};
|
|
43
45
|
|
|
44
46
|
export const USER_AGENT = `vatio-cli/${VERSION} (node ${process.version}; ${platform()} ${arch()})`;
|