@vatio-ai/cli 0.65.0 → 0.67.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/vatio.mjs CHANGED
@@ -20,7 +20,7 @@ import { kb } from "../lib/commands/knowledge.mjs";
20
20
  import { business } from "../lib/commands/business.mjs";
21
21
  import { instagram, whatsapp } from "../lib/commands/channels.mjs";
22
22
  import { docs, feedback } from "../lib/commands/support-commands.mjs";
23
- import { evalCommand, flags } from "../lib/commands/improve.mjs";
23
+ import { evalCommand, flags, tools } from "../lib/commands/improve.mjs";
24
24
  import { propose } from "../lib/commands/propose.mjs";
25
25
  import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
26
26
  import { beginRun, isCodingAgent, reportRun, telemetryEnabled } from "../lib/telemetry.mjs";
@@ -33,8 +33,6 @@ const RETIRED = {
33
33
  "or read the manifest at GET /api/developer/v1/:slug/manifest.",
34
34
  issue: "was removed: report a problem on the public repository it belongs to --\n" +
35
35
  "https://github.com/vatio-ai/cli/issues (or sdk, ios, ruby under github.com/vatio-ai).",
36
- tools: "was removed: `vatio push` validates before it deploys, and `vatio diff` validates\n" +
37
- "without deploying.",
38
36
  widget: "was removed: `vatio env` prints the snippet and its token, and the look is edited on\n" +
39
37
  "the console's Widget page.",
40
38
  mcp: "was removed. Point your coding agent at `vatio docs --save` and let it run the CLI.",
@@ -109,6 +107,8 @@ async function run(command, args) {
109
107
  return await flags(config, args);
110
108
  case "eval":
111
109
  return await evalCommand(config, args);
110
+ case "tools":
111
+ return await tools(config, args);
112
112
  case "propose":
113
113
  return await propose(config, args);
114
114
  case "docs":
@@ -91,14 +91,17 @@ export class ApiClient extends HttpClient {
91
91
  return this.get("/secrets", { environment });
92
92
  }
93
93
 
94
- upsertSecret({ key, value, environment = null }) {
94
+ // `tests`: the value every environment but live, and every eval replay,
95
+ // uses instead.
96
+ upsertSecret({ key, value, environment = null, tests = false }) {
95
97
  const body = { value };
96
98
  if (environment) body.environment = environment;
99
+ if (tests) body.tests = true;
97
100
  return this.put(`/secrets/${encode(key)}`, body);
98
101
  }
99
102
 
100
- deleteSecret({ key, environment = null }) {
101
- return this.delete(`/secrets/${encode(key)}${query({ environment })}`);
103
+ deleteSecret({ key, environment = null, tests = false }) {
104
+ return this.delete(`/secrets/${encode(key)}${query({ environment, tests: tests ? 1 : null })}`);
102
105
  }
103
106
 
104
107
  // --- knowledge bases ------------------------------------------------
@@ -1,8 +1,10 @@
1
- // flags / eval -- what supervisors flagged, and proving a fix for it.
1
+ // flags / eval / tools -- what supervisors flagged, proving a fix for it, and
2
+ // how the tools' backend has been answering.
2
3
  //
3
- // Both print markdown, because the reader is usually a coding agent: `vatio
4
- // flags` is the document it fixes the agent from, and `vatio eval` is the
5
- // evidence that the fix worked on a branch before anyone merges it.
4
+ // All print markdown, because the reader is usually a coding agent: `vatio
5
+ // flags` is the document it fixes the agent from, `vatio eval` is the
6
+ // evidence that the fix worked on a branch before anyone merges it, and
7
+ // `vatio tools` is the console's Tools page for the failures nobody flagged.
6
8
 
7
9
  import { mkdirSync, writeFileSync } from "node:fs";
8
10
  import { dirname, resolve } from "node:path";
@@ -27,7 +29,11 @@ async function fetchMarkdown(config, path, what) {
27
29
  headers: { ...clientHeaders(), Accept: "text/markdown, text/plain", Authorization: `Bearer ${token}` },
28
30
  signal: AbortSignal.timeout(30_000)
29
31
  });
30
- if (!response.ok) fail(`Could not read ${what} (HTTP ${response.status})`);
32
+ if (!response.ok) {
33
+ const body = await response.json().catch(() => null);
34
+ const reason = body?.error?.message ?? `HTTP ${response.status}`;
35
+ fail(`Could not read ${what} (${reason})`);
36
+ }
31
37
  return await response.text();
32
38
  }
33
39
 
@@ -58,6 +64,26 @@ export async function flags(config, args) {
58
64
  console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
59
65
  }
60
66
 
67
+ // Each tool's calls, error rate, timeouts and p50/p95 latency over a period,
68
+ // then the calls that failed with what your backend answered: the console's
69
+ // Tools page.
70
+ export async function tools(config, args) {
71
+ const environment = takeEnv(args, { fallback: "live", cwd: config.workspaceRoot });
72
+ const period = takeValue(args, "--period");
73
+ const json = takeFlag(args, "--json");
74
+ if (args.length > 0) fail("Usage: vatio tools [--env NAME] [--period 1h|1d|7d|30d] [--json]");
75
+
76
+ if (json) {
77
+ const { url, token } = workspaceBase(config);
78
+ const body = await new HttpClient({ baseUrl: url, token }).get("/tools", { environment, period });
79
+ process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
80
+ return;
81
+ }
82
+
83
+ const query = new URLSearchParams({ environment, ...(period ? { period } : {}) });
84
+ process.stdout.write(await fetchMarkdown(config, `/tools.md?${query}`, "the tools"));
85
+ }
86
+
61
87
  // Replays every eval case -- each flag is one -- against an environment and
62
88
  // waits for the judge. Exits 1 when a case fails, so it can gate CI.
63
89
  export async function evalCommand(config, args) {
@@ -140,6 +140,8 @@ const AGENTS_NOTE = [
140
140
  "To improve the agent from what supervisors flagged in the inbox, run",
141
141
  "`vatio flags` and follow the instructions at the top of its output. Prove",
142
142
  "each fix with `vatio eval` on a branch before opening a pull request.",
143
+ "`vatio tools --env live` shows how each tool's backend has been answering:",
144
+ "error rate, timeouts, latency and the calls that failed.",
143
145
  ""
144
146
  ].join("\n");
145
147
 
@@ -4,7 +4,7 @@
4
4
  // them is in vatio.yml.
5
5
 
6
6
  import { deployClient } from "./deploy.mjs";
7
- import { askSecret, branchDefault, fail, paint, takeValue } from "../support.mjs";
7
+ import { askSecret, branchDefault, fail, paint, takeFlag, takeValue } from "../support.mjs";
8
8
 
9
9
  // live, preview, or a branch's own like fix-pagos.
10
10
  const ENVIRONMENT_NAME = /^[a-z0-9]+(?:[-_][a-z0-9]+)*$/;
@@ -23,12 +23,22 @@ function environmentOption(args, fallback, cwd) {
23
23
 
24
24
  // Without an environment a secret is the value everywhere. On a branch, or
25
25
  // with --env, it is the value only that environment uses -- a preview pointing
26
- // its tools at staging while live keeps production.
26
+ // its tools at staging while live keeps production. With --tests, the value
27
+ // every environment but live and every eval replay uses, branches included,
28
+ // so it never takes the branch's environment.
27
29
  export async function secrets(config, args) {
28
30
  const workspace = config.resolveWorkspaceRequired();
29
31
  const client = deployClient(config, workspace);
30
- const environment = environmentOption(args, null, config.workspaceRoot);
31
- const where = environment ? `${environment} on workspace ${workspace}` : `workspace ${workspace}`;
32
+ const tests = takeFlag(args, "--tests");
33
+ if (tests && (takeValue(args, "--env") ?? takeValue(args, "--environment") ?? takeValue(args, "-e"))) {
34
+ fail("--tests is the value for every environment but live; drop --env");
35
+ }
36
+ const environment = tests ? null : environmentOption(args, null, config.workspaceRoot);
37
+ const where = tests
38
+ ? `workspace ${workspace}, for tests`
39
+ : environment
40
+ ? `${environment} on workspace ${workspace}`
41
+ : `workspace ${workspace}`;
32
42
  const sub = args.shift() ?? "list";
33
43
 
34
44
  if (sub === "list") {
@@ -44,31 +54,31 @@ export async function secrets(config, args) {
44
54
  // into a terminal scrollback.
45
55
  console.log(`Secrets on ${where} (${keys.length}):`);
46
56
  for (const secret of keys) {
47
- const scope = secret.environment ? ` (only ${secret.environment})` : "";
57
+ const scope = secret.tests ? " (tests)" : secret.environment ? ` (only ${secret.environment})` : "";
48
58
  console.log(` ${secret.key ?? secret}${scope}`);
49
59
  }
50
60
  return;
51
61
  }
52
62
  if (sub === "set") {
53
63
  const key = args.shift();
54
- if (!key) fail("Usage: vatio secrets set KEY [VALUE] [--env NAME]");
64
+ if (!key) fail("Usage: vatio secrets set KEY [VALUE] [--env NAME | --tests]");
55
65
  // Asked for rather than typed on the command line, where it would stay in
56
66
  // shell history.
57
67
  const value = args.shift() ?? (await askSecret(`${paint("green", "?")} Paste your secret: `));
58
68
  if (value === "") fail(`No value given for ${key}`);
59
- await client.upsertSecret({ key, value, environment });
69
+ await client.upsertSecret({ key, value, environment, tests });
60
70
  console.log(`${paint("green", "✓")} Set secret ${key} on ${where}`);
61
71
  return;
62
72
  }
63
73
  if (sub === "rm" || sub === "remove" || sub === "delete") {
64
74
  const key = args.shift();
65
- if (!key) fail("Usage: vatio secrets rm KEY [--env NAME]");
66
- await client.deleteSecret({ key, environment });
75
+ if (!key) fail("Usage: vatio secrets rm KEY [--env NAME | --tests]");
76
+ await client.deleteSecret({ key, environment, tests });
67
77
  console.log(`Removed ${key} from ${where}`);
68
78
  return;
69
79
  }
70
80
 
71
- fail("Usage: vatio secrets list|set KEY [VALUE]|rm KEY [--env NAME]");
81
+ fail("Usage: vatio secrets list|set KEY [VALUE]|rm KEY [--env NAME | --tests]");
72
82
  }
73
83
 
74
84
  // Repeatable: `--origin https://a.com --origin https://b.com`.
package/lib/help.mjs CHANGED
@@ -51,10 +51,15 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
51
51
  The chat you are in, or any chat in the workspace
52
52
  (web, WhatsApp, ...): messages, tool calls, channel
53
53
 
54
- Improve it from what supervisors flagged:
54
+ Improve it from what supervisors flagged and how its tools answered:
55
55
  flags [--save [PATH]|--json]
56
56
  Every open flag: the conversation, what each tool
57
57
  answered, and what should have happened
58
+ tools [--env NAME] [--period 1h|1d|7d|30d] [--json]
59
+ The console's Tools page: each tool's calls, error
60
+ rate, timeouts and p50/p95, then the calls that
61
+ failed with what your backend answered. Default:
62
+ the branch's environment, else live; 7d
58
63
  eval [--env NAME] [--agent SLUG] [--no-wait]
59
64
  Replay every eval case (each flag is one) and
60
65
  judge it; exits 1 if one fails. Default: the
@@ -91,10 +96,12 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
91
96
  business hours --clear No hours: someone is always available
92
97
 
93
98
  Credentials:
94
- secrets list|set KEY [VALUE]|rm KEY [--env NAME]
99
+ secrets list|set KEY [VALUE]|rm KEY [--env NAME | --tests]
95
100
  Without VALUE, set asks for it with the input hidden.
96
101
  On a branch or with --env, the value only that
97
- environment uses; otherwise the value everywhere
102
+ environment uses; otherwise the value everywhere.
103
+ With --tests, the value every environment but live
104
+ and every eval uses, like a staging URL
98
105
  tokens list|create [--env NAME] [--label NAME] [--origin URL]...|revoke PREFIX
99
106
  tokens origins PREFIX URL...
100
107
  The origins a token is accepted from (replaces the list)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.65.0",
3
+ "version": "0.67.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",