@vatio-ai/cli 0.66.0 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/vatio.mjs +3 -3
- package/lib/commands/improve.mjs +31 -5
- package/lib/commands/misc.mjs +2 -0
- package/lib/help.mjs +6 -1
- package/package.json +1 -1
package/bin/vatio.mjs
CHANGED
|
@@ -20,7 +20,7 @@ import { kb } from "../lib/commands/knowledge.mjs";
|
|
|
20
20
|
import { business } from "../lib/commands/business.mjs";
|
|
21
21
|
import { instagram, whatsapp } from "../lib/commands/channels.mjs";
|
|
22
22
|
import { docs, feedback } from "../lib/commands/support-commands.mjs";
|
|
23
|
-
import { evalCommand, flags } from "../lib/commands/improve.mjs";
|
|
23
|
+
import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
|
|
24
24
|
import { propose } from "../lib/commands/propose.mjs";
|
|
25
25
|
import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
|
|
26
26
|
import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
|
|
@@ -33,8 +33,6 @@ const RETIRED = {
|
|
|
33
33
|
"or read the manifest at GET /api/developer/v1/:slug/manifest.",
|
|
34
34
|
issue: "was removed: report a problem on the public repository it belongs to --\n" +
|
|
35
35
|
"https://github.com/vatio-ai/cli/issues (or sdk, ios, ruby under github.com/vatio-ai).",
|
|
36
|
-
tools: "was removed: `vatio push` validates before it deploys, and `vatio diff` validates\n" +
|
|
37
|
-
"without deploying.",
|
|
38
36
|
widget: "was removed: `vatio env` prints the snippet and its token, and the look is edited on\n" +
|
|
39
37
|
"the console's Widget page.",
|
|
40
38
|
mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
|
|
@@ -109,6 +107,8 @@ async function run(command, args) {
|
|
|
109
107
|
return await flags(config, args);
|
|
110
108
|
case "eval":
|
|
111
109
|
return await evalCommand(config, args);
|
|
110
|
+
case "tools":
|
|
111
|
+
return await tools(config, args);
|
|
112
112
|
case "propose":
|
|
113
113
|
return await propose(config, args);
|
|
114
114
|
case "docs":
|
package/lib/commands/improve.mjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
// flags / eval -- what supervisors flagged,
|
|
1
|
+
// flags / eval / tools -- what supervisors flagged, proving a fix for it, and
|
|
2
|
+
// how the tools' backend has been answering.
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
// flags` is the document it fixes the agent from,
|
|
5
|
-
// evidence that the fix worked on a branch before anyone merges it
|
|
4
|
+
// All print markdown, because the reader is usually a coding agent: `vatio
|
|
5
|
+
// flags` is the document it fixes the agent from, `vatio eval` is the
|
|
6
|
+
// evidence that the fix worked on a branch before anyone merges it, and
|
|
7
|
+
// `vatio tools` is the console's Tools page for the failures nobody flagged.
|
|
6
8
|
|
|
7
9
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
8
10
|
import { dirname, resolve } from "node:path";
|
|
@@ -27,7 +29,11 @@ async function fetchMarkdown(config, path, what) {
|
|
|
27
29
|
headers: { ...clientHeaders(), Accept: "text/markdown, text/plain", Authorization: `Bearer ${token}` },
|
|
28
30
|
signal: AbortSignal.timeout(30_000)
|
|
29
31
|
});
|
|
30
|
-
if (!response.ok)
|
|
32
|
+
if (!response.ok) {
|
|
33
|
+
const body = await response.json().catch(() => null);
|
|
34
|
+
const reason = body?.error?.message ?? `HTTP ${response.status}`;
|
|
35
|
+
fail(`Could not read ${what} (${reason})`);
|
|
36
|
+
}
|
|
31
37
|
return await response.text();
|
|
32
38
|
}
|
|
33
39
|
|
|
@@ -58,6 +64,26 @@ export async function flags(config, args) {
|
|
|
58
64
|
console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
|
|
59
65
|
}
|
|
60
66
|
|
|
67
|
+
// Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
|
|
68
|
+
// then the calls that failed with what your backend answered: the console's
|
|
69
|
+
// Tools page.
|
|
70
|
+
export async function tools(config, args) {
|
|
71
|
+
const environment = takeEnv(args, { fallback: "live", cwd: config.workspaceRoot });
|
|
72
|
+
const period = takeValue(args, "--period");
|
|
73
|
+
const json = takeFlag(args, "--json");
|
|
74
|
+
if (args.length > 0) fail("Usage: vatio tools [--env NAME] [--period 1h|1d|7d|30d] [--json]");
|
|
75
|
+
|
|
76
|
+
if (json) {
|
|
77
|
+
const { url, token } = workspaceBase(config);
|
|
78
|
+
const body = await new HttpClient({ baseUrl: url, token }).get("/tools", { environment, period });
|
|
79
|
+
process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const query = new URLSearchParams({ environment, ...(period ? { period } : {}) });
|
|
84
|
+
process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
|
|
85
|
+
}
|
|
86
|
+
|
|
61
87
|
// Replays every eval case -- each flag is one -- against an environment and
|
|
62
88
|
// waits for the judge. Exits 1 when a case fails, so it can gate CI.
|
|
63
89
|
export async function evalCommand(config, args) {
|
package/lib/commands/misc.mjs
CHANGED
|
@@ -140,6 +140,8 @@ const AGENTS_NOTE = [
|
|
|
140
140
|
"To improve the agent from what supervisors flagged in the inbox, run",
|
|
141
141
|
"`vatio flags` and follow the instructions at the top of its output. Prove",
|
|
142
142
|
"each fix with `vatio eval` on a branch before opening a pull request.",
|
|
143
|
+
"`vatio tools --env live` shows how each tool's backend has been answering:",
|
|
144
|
+
"error rate, timeouts, latency and the calls that failed.",
|
|
143
145
|
""
|
|
144
146
|
].join("\n");
|
|
145
147
|
|
package/lib/help.mjs
CHANGED
|
@@ -51,10 +51,15 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
|
|
|
51
51
|
The chat you are in, or any chat in the workspace
|
|
52
52
|
(web, WhatsApp, ...): messages, tool calls, channel
|
|
53
53
|
|
|
54
|
-
Improve it from what supervisors flagged:
|
|
54
|
+
Improve it from what supervisors flagged and how its tools answered:
|
|
55
55
|
flags [--save [PATH]|--json]
|
|
56
56
|
Every open flag: the conversation, what each tool
|
|
57
57
|
answered, and what should have happened
|
|
58
|
+
tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
|
|
59
|
+
The console's Tools page: each tool's calls, error
|
|
60
|
+
rate, timeouts and p50/p95, then the calls that
|
|
61
|
+
failed with what your backend answered. Default:
|
|
62
|
+
the branch's environment, else live; 7d
|
|
58
63
|
eval [--env NAME] [--agent SLUG] [--no-wait]
|
|
59
64
|
Replay every eval case (each flag is one) and
|
|
60
65
|
judge it; exits 1 if one fails. Default: the
|