@vatio-ai/cli 0.54.0 → 0.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -102,3 +102,11 @@ Whether a workspace is valid is answered by Vatio, not by this package: `push`
102
102
  and `tools check` send the directory and read the result back. So the rules that
103
103
  judge your workspace are always the deployed ones, and this CLI never has to be
104
104
  upgraded to understand a new one.
105
+
106
+ ## Issues
107
+
108
+ Bugs and questions go to [issues](https://github.com/vatio-ai/cli/issues), or
109
+ `vatio issue` from inside a workspace. This repository is a read-only mirror of
110
+ the CLI as it ships inside Vatio, so a pull request cannot be merged here: open
111
+ an issue describing the change instead. Release notes live in the
112
+ [changelog](https://docs.vatio.ai/changelog).
package/bin/vatio.mjs CHANGED
@@ -24,6 +24,7 @@ import { chat } from "../lib/commands/chat.mjs";
24
24
  import { kb } from "../lib/commands/knowledge.mjs";
25
25
  import { instagram, whatsapp } from "../lib/commands/channels.mjs";
26
26
  import { docs, issue } from "../lib/commands/support-commands.mjs";
27
+ import { evalCommand, flags } from "../lib/commands/improve.mjs";
27
28
  import { mcp } from "../lib/commands/mcp.mjs";
28
29
  import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
29
30
 
@@ -93,6 +94,10 @@ async function run(command, args) {
93
94
  return await whatsapp(config, args);
94
95
  case "instagram":
95
96
  return await instagram(config, args);
97
+ case "flags":
98
+ return await flags(config, args);
99
+ case "eval":
100
+ return await evalCommand(config, args);
96
101
  case "docs":
97
102
  return await docs(config, args);
98
103
  case "issue":
@@ -0,0 +1,112 @@
1
+ // flags / eval -- what supervisors flagged, and proving a fix for it.
2
+ //
3
+ // Both print markdown, because the reader is usually a coding agent: `vatio
4
+ // flags` is the document it fixes the agent from, and `vatio eval` is the
5
+ // evidence that the fix worked on a branch before anyone merges it.
6
+
7
+ import { mkdirSync, writeFileSync } from "node:fs";
8
+ import { dirname, resolve } from "node:path";
9
+
10
+ import { HttpClient } from "../http.mjs";
11
+ import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
12
+
13
+ const POLL_MS = 3_000;
14
+ const DEFAULT_TIMEOUT_S = 900;
15
+
16
+ function workspaceBase(config) {
17
+ const base = config.resolveBaseUrl();
18
+ const token = config.resolveToken();
19
+ if (!token) fail(`Run \`vatio login\` first (no token in ${config.configPath})`);
20
+ return { url: `${base}/api/v1/${config.resolveWorkspaceRequired()}`, token };
21
+ }
22
+
23
+ async function fetchMarkdown(config, path, what) {
24
+ const { url, token } = workspaceBase(config);
25
+ const response = await fetch(`${url}${path}`, {
26
+ headers: { Accept: "text/markdown, text/plain", Authorization: `Bearer ${token}` },
27
+ signal: AbortSignal.timeout(30_000)
28
+ });
29
+ if (!response.ok) fail(`Could not read ${what} (HTTP ${response.status})`);
30
+ return await response.text();
31
+ }
32
+
33
+ // The open flags as one document: each flagged reply with the conversation
34
+ // before it, every tool call of that turn with what your backend answered,
35
+ // and what the supervisor says it should have said.
36
+ export async function flags(config, args) {
37
+ const json = takeFlag(args, "--json");
38
+ const save = takeFlag(args, "--save");
39
+ const savePath = save ? args.shift() ?? null : null;
40
+
41
+ if (json) {
42
+ const { url, token } = workspaceBase(config);
43
+ const body = await new HttpClient({ baseUrl: url, token }).get("/flags");
44
+ process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
45
+ return;
46
+ }
47
+
48
+ const markdown = await fetchMarkdown(config, "/flags.md", "the flags");
49
+ if (!save) {
50
+ process.stdout.write(markdown);
51
+ return;
52
+ }
53
+
54
+ const target = resolve(savePath ?? "vatio-flags.md");
55
+ mkdirSync(dirname(target), { recursive: true });
56
+ writeFileSync(target, markdown);
57
+ console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
58
+ }
59
+
60
+ // Replays every eval case -- each flag is one -- against an environment and
61
+ // waits for the judge. Exits 1 when a case fails, so it can gate CI.
62
+ export async function evalCommand(config, args) {
63
+ const show = takeValue(args, "--show");
64
+ if (show) return await evalShow(config, show);
65
+
66
+ const environment = takeEnv(args, { fallback: "preview", cwd: config.workspaceRoot });
67
+ const agent = takeValue(args, "--agent");
68
+ const noWait = takeFlag(args, "--no-wait");
69
+ const timeoutS = Number(takeValue(args, "--timeout") ?? DEFAULT_TIMEOUT_S);
70
+
71
+ const { url, token } = workspaceBase(config);
72
+ const client = new HttpClient({ baseUrl: url, token });
73
+ const started = await client.post("/evals/runs", { environment, agent: agent ?? undefined });
74
+
75
+ for (const slug of started.missing_agents ?? []) {
76
+ console.error(`vatio: no agent \`${slug}\` on ${environment}; its cases were skipped`);
77
+ }
78
+ const runs = started.runs ?? [];
79
+ if (runs.length === 0) {
80
+ console.log("No eval cases to run. Flag a reply in the inbox, or add a case in the console.");
81
+ return;
82
+ }
83
+
84
+ const total = runs.reduce((sum, run) => sum + run.cases, 0);
85
+ console.error(`vatio: replaying ${total} case${total === 1 ? "" : "s"} on ${environment}…`);
86
+ if (noWait) {
87
+ for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio eval --show ${run.id}`);
88
+ return;
89
+ }
90
+
91
+ const deadline = Date.now() + timeoutS * 1000;
92
+ const finished = [];
93
+ for (const run of runs) {
94
+ let row = run;
95
+ while (!row.finished) {
96
+ if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio eval --show ${run.id}`);
97
+ await sleep(POLL_MS);
98
+ row = await client.get(`/evals/runs/${run.id}`);
99
+ }
100
+ finished.push(row);
101
+ }
102
+
103
+ for (const run of finished) {
104
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `eval run ${run.id}`));
105
+ }
106
+ if (finished.some((run) => run.failed > 0)) process.exitCode = 1;
107
+ }
108
+
109
+ export async function evalShow(config, id) {
110
+ if (!id) fail("Usage: vatio eval --show RUN_ID");
111
+ process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `eval run ${id}`));
112
+ }
@@ -45,6 +45,23 @@ const COMMANDS = {
45
45
  "way to find out whether an edit is deployable; push runs the same check anyway.",
46
46
  annotations: { readOnlyHint: true }
47
47
  },
48
+ flags: {
49
+ usage: "[--json]",
50
+ summary:
51
+ "Every reply a supervisor flagged and that live has not fixed yet, as markdown: the " +
52
+ "conversation before it, each tool call of that turn with what the developer's backend " +
53
+ "answered, and what the reply should have been. Start here when asked to improve the agent.",
54
+ annotations: { readOnlyHint: true }
55
+ },
56
+ eval: {
57
+ usage: "[--env NAME] [--agent SLUG] [--no-wait] | --show RUN_ID",
58
+ summary:
59
+ "Replays every eval case — each flag is one — against an environment (the branch's by " +
60
+ "default) and has a judge compare each reply with what was expected. A fix is proven when " +
61
+ "its flags pass and nothing that passed on live now fails. Calls the workspace's tools for real. " +
62
+ "Many cases outlast this tool's 180s limit: use --no-wait, then --show each run id.",
63
+ annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true }
64
+ },
48
65
  status: {
49
66
  usage: "",
50
67
  summary: "Preview and live deployment state for this workspace.",
@@ -1,6 +1,6 @@
1
1
  // version / doctor / config / init.
2
2
 
3
- import { existsSync, writeFileSync } from "node:fs";
3
+ import { existsSync, readFileSync, writeFileSync } from "node:fs";
4
4
  import { join } from "node:path";
5
5
 
6
6
  import { KEYS, SLUG_FORMAT } from "../config.mjs";
@@ -112,6 +112,8 @@ export async function init(config, args) {
112
112
  writeFileSync(manifest, starterManifest(slug));
113
113
 
114
114
  console.log(`Wrote ${manifest}`);
115
+ const agentsNote = writeAgentsNote(target);
116
+ if (agentsNote) console.log(agentsNote);
115
117
  if (!authorized) console.log(`Local only for now — \`vatio push\` is what creates ${slug} on the platform.`);
116
118
  if (!authorized && String(name ?? "").trim() !== "") {
117
119
  console.log("The display name is set in the console (the workspace's Brand page) once it exists.");
@@ -122,6 +124,39 @@ export async function init(config, args) {
122
124
  console.log(` docs: ${DOCS_URL}`);
123
125
  }
124
126
 
127
+ // A pointer, not a copy of the docs: whatever coding agent works in this
128
+ // directory reads AGENTS.md, and these lines are how it learns that `vatio
129
+ // docs` is the contract and `vatio flags` is where fixing the agent starts.
130
+ // The instructions themselves come from the platform at run time, so nothing
131
+ // written here goes stale. An AGENTS.md that already mentions Vatio is left
132
+ // alone; any other gets the section appended.
133
+ const AGENTS_NOTE = [
134
+ "## Vatio",
135
+ "",
136
+ "This directory is a Vatio workspace (`vatio.yml`). `vatio docs` prints the",
137
+ "contract for the manifest, tools, knowledge and the CLI; read it before",
138
+ "changing anything here.",
139
+ "",
140
+ "To improve the agent from what supervisors flagged in the inbox, run",
141
+ "`vatio flags` and follow the instructions at the top of its output. Prove",
142
+ "each fix with `vatio eval` on a branch before opening a pull request.",
143
+ ""
144
+ ].join("\n");
145
+
146
+ function writeAgentsNote(target) {
147
+ const path = join(target, "AGENTS.md");
148
+ if (!existsSync(path)) {
149
+ writeFileSync(path, `# Agents\n\n${AGENTS_NOTE}`);
150
+ return `Wrote ${path}`;
151
+ }
152
+
153
+ const current = readFileSync(path, "utf8");
154
+ if (/vatio/i.test(current)) return null;
155
+
156
+ writeFileSync(path, `${current.replace(/\n*$/, "\n\n")}${AGENTS_NOTE}`);
157
+ return `Added a Vatio section to ${path}`;
158
+ }
159
+
125
160
  // The smallest file that deploys: which workspace, and what the agent does.
126
161
  // The slug is already validated as a slug, so it needs no quoting. The display
127
162
  // name is not here: it is a workspace setting, edited in the console.
package/lib/help.mjs CHANGED
@@ -48,6 +48,16 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
48
48
  its channel, environment, deployment and tool calls
49
49
  chat destroy CHAT_ID
50
50
 
51
+ Improve it from what supervisors flagged:
52
+ flags [--save [PATH]|--json]
53
+ Every open flag: the conversation, what each tool
54
+ answered, and what the reply should have been
55
+ eval [--env NAME] [--agent SLUG] [--no-wait]
56
+ Replay every eval case (each flag is one) and
57
+ judge it; exits 1 if one fails. Default: the
58
+ branch's environment, else preview
59
+ eval --show RUN_ID One run's report
60
+
51
61
  Knowledge:
52
62
  kb [list] Bases, and whether a deployed agent reads each
53
63
  kb show NAME One base: its sites and its entries
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vatio-ai/cli",
3
- "version": "0.54.0",
3
+ "version": "0.55.0",
4
4
  "description": "Vatio CLI — deploy and manage Vatio agent workspaces",
5
5
  "keywords": [
6
6
  "vatio",
@@ -10,7 +10,7 @@
10
10
  "deploy"
11
11
  ],
12
12
  "homepage": "https://vatio.ai",
13
- "bugs": "https://docs.vatio.ai",
13
+ "bugs": "https://github.com/vatio-ai/cli/issues",
14
14
  "license": "SEE LICENSE IN LICENSE.txt",
15
15
  "author": "Vatio",
16
16
  "type": "module",
@@ -34,7 +34,6 @@
34
34
  },
35
35
  "repository": {
36
36
  "type": "git",
37
- "url": "git+https://github.com/urcalab/vatio.git",
38
- "directory": "cli"
37
+ "url": "https://github.com/vatio-ai/cli"
39
38
  }
40
39
  }