@vatio-ai/cli 0.54.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/bin/vatio.mjs +5 -0
- package/lib/commands/improve.mjs +112 -0
- package/lib/commands/mcp.mjs +17 -0
- package/lib/commands/misc.mjs +36 -1
- package/lib/help.mjs +10 -0
- package/package.json +3 -4
package/README.md
CHANGED
|
@@ -102,3 +102,11 @@ Whether a workspace is valid is answered by Vatio, not by this package: `push`
|
|
|
102
102
|
and `tools check` send the directory and read the result back. So the rules that
|
|
103
103
|
judge your workspace are always the deployed ones, and this CLI never has to be
|
|
104
104
|
upgraded to understand a new one.
|
|
105
|
+
|
|
106
|
+
## Issues
|
|
107
|
+
|
|
108
|
+
Bugs and questions go to [issues](https://github.com/vatio-ai/cli/issues), or
|
|
109
|
+
`vatio issue` from inside a workspace. This repository is a read-only mirror of
|
|
110
|
+
the CLI as it ships inside Vatio, so a pull request cannot be merged here: open
|
|
111
|
+
an issue describing the change instead. Release notes live in the
|
|
112
|
+
[changelog](https://docs.vatio.ai/changelog).
|
package/bin/vatio.mjs
CHANGED
|
@@ -24,6 +24,7 @@ import { chat } from "../lib/commands/chat.mjs";
|
|
|
24
24
|
import { kb } from "../lib/commands/knowledge.mjs";
|
|
25
25
|
import { instagram, whatsapp } from "../lib/commands/channels.mjs";
|
|
26
26
|
import { docs, issue } from "../lib/commands/support-commands.mjs";
|
|
27
|
+
import { evalCommand, flags } from "../lib/commands/improve.mjs";
|
|
27
28
|
import { mcp } from "../lib/commands/mcp.mjs";
|
|
28
29
|
import { printUpdateNotice, startUpdateCheck } from "../lib/update-notice.mjs";
|
|
29
30
|
|
|
@@ -93,6 +94,10 @@ async function run(command, args) {
|
|
|
93
94
|
return await whatsapp(config, args);
|
|
94
95
|
case "instagram":
|
|
95
96
|
return await instagram(config, args);
|
|
97
|
+
case "flags":
|
|
98
|
+
return await flags(config, args);
|
|
99
|
+
case "eval":
|
|
100
|
+
return await evalCommand(config, args);
|
|
96
101
|
case "docs":
|
|
97
102
|
return await docs(config, args);
|
|
98
103
|
case "issue":
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// flags / eval -- what supervisors flagged, and proving a fix for it.
|
|
2
|
+
//
|
|
3
|
+
// Both print markdown, because the reader is usually a coding agent: `vatio
|
|
4
|
+
// flags` is the document it fixes the agent from, and `vatio eval` is the
|
|
5
|
+
// evidence that the fix worked on a branch before anyone merges it.
|
|
6
|
+
|
|
7
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
8
|
+
import { dirname, resolve } from "node:path";
|
|
9
|
+
|
|
10
|
+
import { HttpClient } from "../http.mjs";
|
|
11
|
+
import { fail, sleep, takeEnv, takeFlag, takeValue } from "../support.mjs";
|
|
12
|
+
|
|
13
|
+
const POLL_MS = 3_000;
|
|
14
|
+
const DEFAULT_TIMEOUT_S = 900;
|
|
15
|
+
|
|
16
|
+
function workspaceBase(config) {
|
|
17
|
+
const base = config.resolveBaseUrl();
|
|
18
|
+
const token = config.resolveToken();
|
|
19
|
+
if (!token) fail(`Run \`vatio login\` first (no token in ${config.configPath})`);
|
|
20
|
+
return { url: `${base}/api/v1/${config.resolveWorkspaceRequired()}`, token };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function fetchMarkdown(config, path, what) {
|
|
24
|
+
const { url, token } = workspaceBase(config);
|
|
25
|
+
const response = await fetch(`${url}${path}`, {
|
|
26
|
+
headers: { Accept: "text/markdown, text/plain", Authorization: `Bearer ${token}` },
|
|
27
|
+
signal: AbortSignal.timeout(30_000)
|
|
28
|
+
});
|
|
29
|
+
if (!response.ok) fail(`Could not read ${what} (HTTP ${response.status})`);
|
|
30
|
+
return await response.text();
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// The open flags as one document: each flagged reply with the conversation
|
|
34
|
+
// before it, every tool call of that turn with what your backend answered,
|
|
35
|
+
// and what the supervisor says it should have said.
|
|
36
|
+
export async function flags(config, args) {
|
|
37
|
+
const json = takeFlag(args, "--json");
|
|
38
|
+
const save = takeFlag(args, "--save");
|
|
39
|
+
const savePath = save ? args.shift() ?? null : null;
|
|
40
|
+
|
|
41
|
+
if (json) {
|
|
42
|
+
const { url, token } = workspaceBase(config);
|
|
43
|
+
const body = await new HttpClient({ baseUrl: url, token }).get("/flags");
|
|
44
|
+
process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const markdown = await fetchMarkdown(config, "/flags.md", "the flags");
|
|
49
|
+
if (!save) {
|
|
50
|
+
process.stdout.write(markdown);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const target = resolve(savePath ?? "vatio-flags.md");
|
|
55
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
56
|
+
writeFileSync(target, markdown);
|
|
57
|
+
console.log(`Wrote ${target}. Hand it to your coding agent; re-run \`vatio flags --save\` to refresh.`);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Replays every eval case -- each flag is one -- against an environment and
|
|
61
|
+
// waits for the judge. Exits 1 when a case fails, so it can gate CI.
|
|
62
|
+
export async function evalCommand(config, args) {
|
|
63
|
+
const show = takeValue(args, "--show");
|
|
64
|
+
if (show) return await evalShow(config, show);
|
|
65
|
+
|
|
66
|
+
const environment = takeEnv(args, { fallback: "preview", cwd: config.workspaceRoot });
|
|
67
|
+
const agent = takeValue(args, "--agent");
|
|
68
|
+
const noWait = takeFlag(args, "--no-wait");
|
|
69
|
+
const timeoutS = Number(takeValue(args, "--timeout") ?? DEFAULT_TIMEOUT_S);
|
|
70
|
+
|
|
71
|
+
const { url, token } = workspaceBase(config);
|
|
72
|
+
const client = new HttpClient({ baseUrl: url, token });
|
|
73
|
+
const started = await client.post("/evals/runs", { environment, agent: agent ?? undefined });
|
|
74
|
+
|
|
75
|
+
for (const slug of started.missing_agents ?? []) {
|
|
76
|
+
console.error(`vatio: no agent \`${slug}\` on ${environment}; its cases were skipped`);
|
|
77
|
+
}
|
|
78
|
+
const runs = started.runs ?? [];
|
|
79
|
+
if (runs.length === 0) {
|
|
80
|
+
console.log("No eval cases to run. Flag a reply in the inbox, or add a case in the console.");
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const total = runs.reduce((sum, run) => sum + run.cases, 0);
|
|
85
|
+
console.error(`vatio: replaying ${total} case${total === 1 ? "" : "s"} on ${environment}…`);
|
|
86
|
+
if (noWait) {
|
|
87
|
+
for (const run of runs) console.log(`Run ${run.id} (${run.agent}): vatio eval --show ${run.id}`);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const deadline = Date.now() + timeoutS * 1000;
|
|
92
|
+
const finished = [];
|
|
93
|
+
for (const run of runs) {
|
|
94
|
+
let row = run;
|
|
95
|
+
while (!row.finished) {
|
|
96
|
+
if (Date.now() > deadline) fail(`Timed out after ${timeoutS}s; the run keeps going. See it with: vatio eval --show ${run.id}`);
|
|
97
|
+
await sleep(POLL_MS);
|
|
98
|
+
row = await client.get(`/evals/runs/${run.id}`);
|
|
99
|
+
}
|
|
100
|
+
finished.push(row);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
for (const run of finished) {
|
|
104
|
+
process.stdout.write(await fetchMarkdown(config, `/evals/runs/${run.id}.md`, `eval run ${run.id}`));
|
|
105
|
+
}
|
|
106
|
+
if (finished.some((run) => run.failed > 0)) process.exitCode = 1;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export async function evalShow(config, id) {
|
|
110
|
+
if (!id) fail("Usage: vatio eval --show RUN_ID");
|
|
111
|
+
process.stdout.write(await fetchMarkdown(config, `/evals/runs/${encodeURIComponent(id)}.md`, `eval run ${id}`));
|
|
112
|
+
}
|
package/lib/commands/mcp.mjs
CHANGED
|
@@ -45,6 +45,23 @@ const COMMANDS = {
|
|
|
45
45
|
"way to find out whether an edit is deployable; push runs the same check anyway.",
|
|
46
46
|
annotations: { readOnlyHint: true }
|
|
47
47
|
},
|
|
48
|
+
flags: {
|
|
49
|
+
usage: "[--json]",
|
|
50
|
+
summary:
|
|
51
|
+
"Every reply a supervisor flagged and that live has not fixed yet, as markdown: the " +
|
|
52
|
+
"conversation before it, each tool call of that turn with what the developer's backend " +
|
|
53
|
+
"answered, and what the reply should have been. Start here when asked to improve the agent.",
|
|
54
|
+
annotations: { readOnlyHint: true }
|
|
55
|
+
},
|
|
56
|
+
eval: {
|
|
57
|
+
usage: "[--env NAME] [--agent SLUG] [--no-wait] | --show RUN_ID",
|
|
58
|
+
summary:
|
|
59
|
+
"Replays every eval case — each flag is one — against an environment (the branch's by " +
|
|
60
|
+
"default) and has a judge compare each reply with what was expected. A fix is proven when " +
|
|
61
|
+
"its flags pass and nothing that passed on live now fails. Calls the workspace's tools for real. " +
|
|
62
|
+
"Many cases outlast this tool's 180s limit: use --no-wait, then --show each run id.",
|
|
63
|
+
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true }
|
|
64
|
+
},
|
|
48
65
|
status: {
|
|
49
66
|
usage: "",
|
|
50
67
|
summary: "Preview and live deployment state for this workspace.",
|
package/lib/commands/misc.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// version / doctor / config / init.
|
|
2
2
|
|
|
3
|
-
import { existsSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
|
|
6
6
|
import { KEYS, SLUG_FORMAT } from "../config.mjs";
|
|
@@ -112,6 +112,8 @@ export async function init(config, args) {
|
|
|
112
112
|
writeFileSync(manifest, starterManifest(slug));
|
|
113
113
|
|
|
114
114
|
console.log(`Wrote ${manifest}`);
|
|
115
|
+
const agentsNote = writeAgentsNote(target);
|
|
116
|
+
if (agentsNote) console.log(agentsNote);
|
|
115
117
|
if (!authorized) console.log(`Local only for now — \`vatio push\` is what creates ${slug} on the platform.`);
|
|
116
118
|
if (!authorized && String(name ?? "").trim() !== "") {
|
|
117
119
|
console.log("The display name is set in the console (the workspace's Brand page) once it exists.");
|
|
@@ -122,6 +124,39 @@ export async function init(config, args) {
|
|
|
122
124
|
console.log(` docs: ${DOCS_URL}`);
|
|
123
125
|
}
|
|
124
126
|
|
|
127
|
+
// A pointer, not a copy of the docs: whatever coding agent works in this
|
|
128
|
+
// directory reads AGENTS.md, and these lines are how it learns that `vatio
|
|
129
|
+
// docs` is the contract and `vatio flags` is where fixing the agent starts.
|
|
130
|
+
// The instructions themselves come from the platform at run time, so nothing
|
|
131
|
+
// written here goes stale. An AGENTS.md that already mentions Vatio is left
|
|
132
|
+
// alone; any other gets the section appended.
|
|
133
|
+
const AGENTS_NOTE = [
|
|
134
|
+
"## Vatio",
|
|
135
|
+
"",
|
|
136
|
+
"This directory is a Vatio workspace (`vatio.yml`). `vatio docs` prints the",
|
|
137
|
+
"contract for the manifest, tools, knowledge and the CLI; read it before",
|
|
138
|
+
"changing anything here.",
|
|
139
|
+
"",
|
|
140
|
+
"To improve the agent from what supervisors flagged in the inbox, run",
|
|
141
|
+
"`vatio flags` and follow the instructions at the top of its output. Prove",
|
|
142
|
+
"each fix with `vatio eval` on a branch before opening a pull request.",
|
|
143
|
+
""
|
|
144
|
+
].join("\n");
|
|
145
|
+
|
|
146
|
+
function writeAgentsNote(target) {
|
|
147
|
+
const path = join(target, "AGENTS.md");
|
|
148
|
+
if (!existsSync(path)) {
|
|
149
|
+
writeFileSync(path, `# Agents\n\n${AGENTS_NOTE}`);
|
|
150
|
+
return `Wrote ${path}`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const current = readFileSync(path, "utf8");
|
|
154
|
+
if (/vatio/i.test(current)) return null;
|
|
155
|
+
|
|
156
|
+
writeFileSync(path, `${current.replace(/\n*$/, "\n\n")}${AGENTS_NOTE}`);
|
|
157
|
+
return `Added a Vatio section to ${path}`;
|
|
158
|
+
}
|
|
159
|
+
|
|
125
160
|
// The smallest file that deploys: which workspace, and what the agent does.
|
|
126
161
|
// The slug is already validated as a slug, so it needs no quoting. The display
|
|
127
162
|
// name is not here: it is a workspace setting, edited in the console.
|
package/lib/help.mjs
CHANGED
|
@@ -48,6 +48,16 @@ and the one its pull request deploys to. Elsewhere commands default as listed.
|
|
|
48
48
|
its channel, environment, deployment and tool calls
|
|
49
49
|
chat destroy CHAT_ID
|
|
50
50
|
|
|
51
|
+
Improve it from what supervisors flagged:
|
|
52
|
+
flags [--save [PATH]|--json]
|
|
53
|
+
Every open flag: the conversation, what each tool
|
|
54
|
+
answered, and what the reply should have been
|
|
55
|
+
eval [--env NAME] [--agent SLUG] [--no-wait]
|
|
56
|
+
Replay every eval case (each flag is one) and
|
|
57
|
+
judge it; exits 1 if one fails. Default: the
|
|
58
|
+
branch's environment, else preview
|
|
59
|
+
eval --show RUN_ID One run's report
|
|
60
|
+
|
|
51
61
|
Knowledge:
|
|
52
62
|
kb [list] Bases, and whether a deployed agent reads each
|
|
53
63
|
kb show NAME One base: its sites and its entries
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vatio-ai/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.55.0",
|
|
4
4
|
"description": "Vatio CLI — deploy and manage Vatio agent workspaces",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"vatio",
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"deploy"
|
|
11
11
|
],
|
|
12
12
|
"homepage": "https://vatio.ai",
|
|
13
|
-
"bugs": "https://
|
|
13
|
+
"bugs": "https://github.com/vatio-ai/cli/issues",
|
|
14
14
|
"license": "SEE LICENSE IN LICENSE.txt",
|
|
15
15
|
"author": "Vatio",
|
|
16
16
|
"type": "module",
|
|
@@ -34,7 +34,6 @@
|
|
|
34
34
|
},
|
|
35
35
|
"repository": {
|
|
36
36
|
"type": "git",
|
|
37
|
-
"url": "
|
|
38
|
-
"directory": "cli"
|
|
37
|
+
"url": "https://github.com/vatio-ai/cli"
|
|
39
38
|
}
|
|
40
39
|
}
|