@vatio-ai/cli 0.65.0 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/vatio.mjs +3 -3
- package/lib/api-client.mjs +6 -3
- package/lib/commands/improve.mjs +31 -5
- package/lib/commands/misc.mjs +2 -0
- package/lib/commands/workspace-admin.mjs +20 -10
- package/lib/help.mjs +10 -3
- package/package.json +1 -1
package/bin/vatio.mjs
CHANGED
|
@@ -20,7 +20,7 @@ import { kb } from "../lib/commands/knowledge.mjs";
|
|
|
20
20
|
import { business } from "../lib/commands/business.mjs";
|
|
21
21
|
import { instagram, whatsapp } from "../lib/commands/channels.mjs";
|
|
22
22
|
import { docs, feedback } from "../lib/commands/support-commands.mjs";
|
|
23
|
-
import { evalCommand, flags } from "../lib/commands/improve.mjs";
|
|
23
|
+
import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
|
|
24
24
|
import { propose } from "../lib/commands/propose.mjs";
|
|
25
25
|
import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
|
|
26
26
|
import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
|
|
@@ -33,8 +33,6 @@ const RETIRED = {
|
|
|
33
33
|
"or read the manifest at GET /api/developer/v1/:slug/manifest.",
|
|
34
34
|
issue: "was removed: report a problem on the public repository it belongs to --\n" +
|
|
35
35
|
"https://github.com/vatio-ai/cli/issues (or sdk, ios, ruby under github.com/vatio-ai).",
|
|
36
|
-
tools: "was removed: `vatio push` validates before it deploys, and `vatio diff` validates\n" +
|
|
37
|
-
"without deploying.",
|
|
38
36
|
widget: "was removed: `vatio env` prints the snippet and its token, and the look is edited on\n" +
|
|
39
37
|
"the console's Widget page.",
|
|
40
38
|
mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
|
|
@@ -109,6 +107,8 @@ async function run(command, args) {
|
|
|
109
107
|
return await flags(config, args);
|
|
110
108
|
case "eval":
|
|
111
109
|
return await evalCommand(config, args);
|
|
110
|
+
case "tools":
|
|
111
|
+
return await tools(config, args);
|
|
112
112
|
case "propose":
|
|
113
113
|
return await propose(config, args);
|
|
114
114
|
case "docs":
|
package/lib/api-client.mjs
CHANGED
|
@@ -91,14 +91,17 @@ export class ApiClient extends HttpClient {
|
|
|
91
91
|
return this.get("/secrets", { environment });
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
-
|
|
94
|
+
// `tests`: the value every environment but live, and every eval replay,
|
|
95
|
+
// uses instead.
|
|
96
|
+
upsertSecret({ key, value, environment = null, tests = false }) {
|
|
95
97
|
const body = { value };
|
|
96
98
|
if (environment) body.environment = environment;
|
|
99
|
+
if (tests) body.tests = true;
|
|
97
100
|
return this.put(`/secrets/${encode(key)}`, body);
|
|
98
101
|
}
|
|
99
102
|
|
|
100
|
-
deleteSecret({ key, environment = null }) {
|
|
101
|
-
return this.delete(`/secrets/${encode(key)}${query({ environment })}`);
|
|
103
|
+
deleteSecret({ key, environment = null, tests = false }) {
|
|
104
|
+
return this.delete(`/secrets/${encode(key)}${query({ environment, tests: tests ? 1 : null })}`);
|
|
102
105
|
}
|
|
103
106
|
|
|
104
107
|
// --- knowledge bases ------------------------------------------------
|
package/lib/commands/improve.mjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
// flags / eval -- what supervisors flagged,
|
|
1
|
+
// flags / eval / tools -- what supervisors flagged, proving a fix for it, and
|
|
2
|
+
// how the tools' backend has been answering.
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
// flags` is the document it fixes the agent from,
|
|
5
|
-
// evidence that the fix worked on a branch before anyone merges it
|
|
4
|
+
// All print markdown, because the reader is usually a coding agent: `vatio
|
|
5
|
+
// flags` is the document it fixes the agent from, `vatio eval` is the
|
|
6
|
+
// evidence that the fix worked on a branch before anyone merges it, and
|
|
7
|
+
// `vatio tools` is the console's Tools page for the failures nobody flagged.
|
|
6
8
|
|
|
7
9
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
8
10
|
import { dirname, resolve } from "node:path";
|
|
@@ -27,7 +29,11 @@ async function fetchMarkdown(config, path, what) {
|
|
|
27
29
|
headers: { ...clientHeaders(), Accept: "text/markdown, text/plain", Authorization: `Bearer ${token}` },
|
|
28
30
|
signal: AbortSignal.timeout(30_000)
|
|
29
31
|
});
|
|
30
|
-
if (!response.ok)
|
|
32
|
+
if (!response.ok) {
|
|
33
|
+
const body = await response.json().catch(() => null);
|
|
34
|
+
const reason = body?.error?.message ?? `HTTP ${response.status}`;
|
|
35
|
+
fail(`Could not read ${what} (${reason})`);
|
|
36
|
+
}
|
|
31
37
|
return await response.text();
|
|
32
38
|
}
|
|
33
39
|
|
|
@@ -58,6 +64,26 @@ export async function flags(config, args) {
|
|
|
58
64
|
console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
|
|
59
65
|
}
|
|
60
66
|
|
|
67
|
+
// Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
|
|
68
|
+
// then the calls that failed with what your backend answered: the console's
|
|
69
|
+
// Tools page.
|
|
70
|
+
export async function tools(config, args) {
|
|
71
|
+
const environment = takeEnv(args, { fallback: "live", cwd: config.workspaceRoot });
|
|
72
|
+
const period = takeValue(args, "--period");
|
|
73
|
+
const json = takeFlag(args, "--json");
|
|
74
|
+
if (args.length > 0) fail("Usage: vatio tools [--env NAME] [--period 1h|1d|7d|30d] [--json]");
|
|
75
|
+
|
|
76
|
+
if (json) {
|
|
77
|
+
const { url, token } = workspaceBase(config);
|
|
78
|
+
const body = await new HttpClient({ baseUrl: url, token }).get("/tools", { environment, period });
|
|
79
|
+
process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const query = new URLSearchParams({ environment, ...(period ? { period } : {}) });
|
|
84
|
+
process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
|
|
85
|
+
}
|
|
86
|
+
|
|
61
87
|
// Replays every eval case -- each flag is one -- against an environment and
|
|
62
88
|
// waits for the judge. Exits 1 when a case fails, so it can gate CI.
|
|
63
89
|
export async function evalCommand(config, args) {
|
package/lib/commands/misc.mjs
CHANGED
|
@@ -140,6 +140,8 @@ const AGENTS_NOTE = [
|
|
|
140
140
|
"To improve the agent from what supervisors flagged in the inbox, run",
|
|
141
141
|
"`vatio flags` and follow the instructions at the top of its output. Prove",
|
|
142
142
|
"each fix with `vatio eval` on a branch before opening a pull request.",
|
|
143
|
+
"`vatio tools --env live` shows how each tool's backend has been answering:",
|
|
144
|
+
"error rate, timeouts, latency and the calls that failed.",
|
|
143
145
|
""
|
|
144
146
|
].join("\n");
|
|
145
147
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// them is in vatio.yml.
|
|
5
5
|
|
|
6
6
|
import { deployClient } from "./deploy.mjs";
|
|
7
|
-
import { askSecret, branchDefault, fail, paint, takeValue } from "../support.mjs";
|
|
7
|
+
import { askSecret, branchDefault, fail, paint, takeFlag, takeValue } from "../support.mjs";
|
|
8
8
|
|
|
9
9
|
// live, preview, or a branch's own like fix-pagos.
|
|
10
10
|
const ENVIRONMENT_NAME = /^[a-z0-9]+(?:[-_][a-z0-9]+)*$/;
|
|
@@ -23,12 +23,22 @@ function environmentOption(args, fallback, cwd) {
|
|
|
23
23
|
|
|
24
24
|
// Without an environment a secret is the value everywhere. On a branch, or
|
|
25
25
|
// with --env, it is the value only that environment uses -- a preview pointing
|
|
26
|
-
// its tools at staging while live keeps production.
|
|
26
|
+
// its tools at staging while live keeps production. With --tests, the value
|
|
27
|
+
// every environment but live and every eval replay uses, branches included,
|
|
28
|
+
// so it never takes the branch's environment.
|
|
27
29
|
export async function secrets(config, args) {
|
|
28
30
|
const workspace = config.resolveWorkspaceRequired();
|
|
29
31
|
const client = deployClient(config, workspace);
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
+
const tests = takeFlag(args, "--tests");
|
|
33
|
+
if (tests && (takeValue(args, "--env") ?? takeValue(args, "--environment") ?? takeValue(args, "-e"))) {
|
|
34
|
+
fail("--tests is the value for every environment but live; drop --env");
|
|
35
|
+
}
|
|
36
|
+
const environment = tests ? null : environmentOption(args, null, config.workspaceRoot);
|
|
37
|
+
const where = tests
|
|
38
|
+
? `workspace ${workspace}, for tests`
|
|
39
|
+
: environment
|
|
40
|
+
? `${environment} on workspace ${workspace}`
|
|
41
|
+
: `workspace ${workspace}`;
|
|
32
42
|
const sub = args.shift() ?? "list";
|
|
33
43
|
|
|
34
44
|
if (sub === "list") {
|
|
@@ -44,31 +54,31 @@ export async function secrets(config, args) {
|
|
|
44
54
|
// into a terminal scrollback.
|
|
45
55
|
console.log(`Secrets on ${where} (${keys.length}):`);
|
|
46
56
|
for (const secret of keys) {
|
|
47
|
-
const scope = secret.environment ? ` (only ${secret.environment})` : "";
|
|
57
|
+
const scope = secret.tests ? " (tests)" : secret.environment ? ` (only ${secret.environment})` : "";
|
|
48
58
|
console.log(` ${secret.key ?? secret}${scope}`);
|
|
49
59
|
}
|
|
50
60
|
return;
|
|
51
61
|
}
|
|
52
62
|
if (sub === "set") {
|
|
53
63
|
const key = args.shift();
|
|
54
|
-
if (!key) fail("Usage: vatio secrets set KEY [VALUE] [--env NAME]");
|
|
64
|
+
if (!key) fail("Usage: vatio secrets set KEY [VALUE] [--env NAME | --tests]");
|
|
55
65
|
// Asked for rather than typed on the command line, where it would stay in
|
|
56
66
|
// shell history.
|
|
57
67
|
const value = args.shift() ?? (await askSecret(`${paint("green", "?")} Paste your secret: `));
|
|
58
68
|
if (value === "") fail(`No value given for ${key}`);
|
|
59
|
-
await client.upsertSecret({ key, value, environment });
|
|
69
|
+
await client.upsertSecret({ key, value, environment, tests });
|
|
60
70
|
console.log(`${paint("green", "✓")} Set secret ${key} on ${where}`);
|
|
61
71
|
return;
|
|
62
72
|
}
|
|
63
73
|
if (sub === "rm" || sub === "remove" || sub === "delete") {
|
|
64
74
|
const key = args.shift();
|
|
65
|
-
if (!key) fail("Usage: vatio secrets rm KEY [--env NAME]");
|
|
66
|
-
await client.deleteSecret({ key, environment });
|
|
75
|
+
if (!key) fail("Usage: vatio secrets rm KEY [--env NAME | --tests]");
|
|
76
|
+
await client.deleteSecret({ key, environment, tests });
|
|
67
77
|
console.log(`Removed ${key} from ${where}`);
|
|
68
78
|
return;
|
|
69
79
|
}
|
|
70
80
|
|
|
71
|
-
fail("Usage: vatio secrets list|set KEY [VALUE]|rm KEY [--env NAME]");
|
|
81
|
+
fail("Usage: vatio secrets list|set KEY [VALUE]|rm KEY [--env NAME | --tests]");
|
|
72
82
|
}
|
|
73
83
|
|
|
74
84
|
// Repeatable: `--origin https://a.com --origin https://b.com`.
|
package/lib/help.mjs
CHANGED
|
@@ -51,10 +51,15 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
|
|
|
51
51
|
The chat you are in, or any chat in the workspace
|
|
52
52
|
(web, WhatsApp, ...): messages, tool calls, channel
|
|
53
53
|
|
|
54
|
-
Improve it from what supervisors flagged:
|
|
54
|
+
Improve it from what supervisors flagged and how its tools answered:
|
|
55
55
|
flags [--save [PATH]|--json]
|
|
56
56
|
Every open flag: the conversation, what each tool
|
|
57
57
|
answered, and what should have happened
|
|
58
|
+
tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
|
|
59
|
+
The console's Tools page: each tool's calls, error
|
|
60
|
+
rate, timeouts and p50/p95, then the calls that
|
|
61
|
+
failed with what your backend answered. Default:
|
|
62
|
+
the branch's environment, else live; 7d
|
|
58
63
|
eval [--env NAME] [--agent SLUG] [--no-wait]
|
|
59
64
|
Replay every eval case (each flag is one) and
|
|
60
65
|
judge it; exits 1 if one fails. Default: the
|
|
@@ -91,10 +96,12 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
|
|
|
91
96
|
business hours --clear No hours: someone is always available
|
|
92
97
|
|
|
93
98
|
Credentials:
|
|
94
|
-
secrets list|set KEY [VALUE]|rm KEY [--env NAME]
|
|
99
|
+
secrets list|set KEY [VALUE]|rm KEY [--env NAME | --tests]
|
|
95
100
|
Without VALUE, set asks for it with the input hidden.
|
|
96
101
|
On a branch or with --env, the value only that
|
|
97
|
-
environment uses; otherwise the value everywhere
|
|
102
|
+
environment uses; otherwise the value everywhere.
|
|
103
|
+
With --tests, the value every environment but live
|
|
104
|
+
and every eval uses, like a staging URL
|
|
98
105
|
tokens list|create [--env NAME] [--label NAME] [--origin URL]...|revoke PREFIX
|
|
99
106
|
tokens origins PREFIX URL...
|
|
100
107
|
The origins a token is accepted from (replaces the list)
|