@beryl-so/cli 0.64.0 → 0.73.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -8
- package/dist/adapters/mcp.js +6 -4
- package/dist/beryl-test-skill.js +84 -91
- package/dist/commands/accounts.js +96 -19
- package/dist/commands/config-vars.js +1 -1
- package/dist/commands/environments.js +5 -0
- package/dist/commands/health.js +3 -1
- package/dist/commands/init.js +18 -18
- package/dist/commands/jobs.js +125 -0
- package/dist/commands/projects.js +74 -21
- package/dist/commands/runs.js +3 -2
- package/dist/commands/tests.js +60 -317
- package/dist/commands/tracker.js +4 -1
- package/dist/commands/watch.js +36 -0
- package/dist/context.js +1 -2
- package/dist/lint.js +0 -6
- package/dist/local-exec.js +4 -4
- package/dist/registry/index.js +2 -0
- package/package.json +2 -2
|
@@ -26,7 +26,7 @@ function accountTable(rows) {
|
|
|
26
26
|
return table(rows.map((a) => ({
|
|
27
27
|
label: a.identity_label + (a.is_default ? " *" : ""),
|
|
28
28
|
email: a.email,
|
|
29
|
-
type: a.type,
|
|
29
|
+
type: a.type + (a.lifetime ? ` (${a.lifetime})` : ""),
|
|
30
30
|
login: a.login_method,
|
|
31
31
|
status: a.status,
|
|
32
32
|
id: a.id,
|
|
@@ -51,21 +51,40 @@ export const testAccountCommands = [
|
|
|
51
51
|
if (rows.length === 0)
|
|
52
52
|
return {
|
|
53
53
|
data: rows,
|
|
54
|
-
human: dim("No test accounts
|
|
55
|
-
"
|
|
56
|
-
"{{inbox_address}} and deletes the account in `after`."),
|
|
54
|
+
human: dim("No test accounts, so {{login_email}} and {{login_password}} have nothing to " +
|
|
55
|
+
"resolve to. Add one with `beryl accounts create`. A flow that must sign up " +
|
|
56
|
+
"fresh cites {{inbox_address}} and deletes the account in `after`."),
|
|
57
57
|
};
|
|
58
58
|
return { data: rows, human: accountTable(rows) };
|
|
59
59
|
},
|
|
60
60
|
},
|
|
61
|
+
{
|
|
62
|
+
name: "accounts get",
|
|
63
|
+
summary: "Show one test account: origin, lifetime, sign-in state",
|
|
64
|
+
description: "One test account by id. `type` is its origin: captured (a session from the Chrome " +
|
|
65
|
+
"plugin), provisioned (Beryl signed it up; `lifetime` permanent or throwaway), or " +
|
|
66
|
+
"provided (your own account).",
|
|
67
|
+
scope: "project",
|
|
68
|
+
args: [{ name: "account-id", description: "Account id from `beryl accounts list`", required: true }],
|
|
69
|
+
examples: ["beryl accounts get acc_123"],
|
|
70
|
+
async run(ctx, input) {
|
|
71
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
72
|
+
const account = (await ctx.client.get(`${accountsPath(workspaceId, projectId)}/${arg(input, "account-id")}`));
|
|
73
|
+
return { data: account, human: accountTable([account]) };
|
|
74
|
+
},
|
|
75
|
+
},
|
|
61
76
|
{
|
|
62
77
|
name: "accounts create",
|
|
63
78
|
summary: "Add a test account: the customer's own, or one Beryl signs up",
|
|
64
|
-
description: "Two kinds. `--type
|
|
65
|
-
"the site: pass --email and --password, and it is usable immediately.
|
|
66
|
-
"reserves one Beryl will sign up itself, addressed at the project
|
|
67
|
-
"`
|
|
68
|
-
"
|
|
79
|
+
description: "Two kinds. `--type provided` records a dedicated account you already have on " +
|
|
80
|
+
"the site: pass --email and --password, and it is usable immediately. " +
|
|
81
|
+
"`--type provisioned` reserves one Beryl will sign up itself, addressed at the project " +
|
|
82
|
+
"mailbox; `--lifetime` says whether it is kept (permanent, the default) or a " +
|
|
83
|
+
"throwaway. It starts `pending` and becomes usable after `beryl accounts signup`. " +
|
|
84
|
+
"A captured account (a signed-in session from the Beryl Chrome extension) is not " +
|
|
85
|
+
"created here: capture it with the extension, then give it a probe with " +
|
|
86
|
+
"`beryl accounts set-login --probe`. " +
|
|
87
|
+
"Works for Google-only apps too: bank a dedicated Google test account as provided " +
|
|
69
88
|
"and author its sign-in plan through Google's form. " +
|
|
70
89
|
"The first account an environment gets is its default whatever you pass.",
|
|
71
90
|
scope: "project",
|
|
@@ -73,19 +92,25 @@ export const testAccountCommands = [
|
|
|
73
92
|
{
|
|
74
93
|
name: "type",
|
|
75
94
|
type: "string",
|
|
76
|
-
enum: ["
|
|
77
|
-
description: "Whose account this is (default:
|
|
95
|
+
enum: ["provided", "provisioned"],
|
|
96
|
+
description: "Whose account this is (default: provided when --password is given)",
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
name: "lifetime",
|
|
100
|
+
type: "string",
|
|
101
|
+
enum: ["permanent", "throwaway"],
|
|
102
|
+
description: "provisioned only: keep it, or a throwaway (default permanent)",
|
|
78
103
|
},
|
|
79
104
|
{
|
|
80
105
|
name: "label",
|
|
81
106
|
type: "string",
|
|
82
107
|
description: "Identity label a plan's auth_label names (default 'default')",
|
|
83
108
|
},
|
|
84
|
-
{ name: "email", type: "string", description: "
|
|
109
|
+
{ name: "email", type: "string", description: "provided only: the account's email" },
|
|
85
110
|
{
|
|
86
111
|
name: "password",
|
|
87
112
|
type: "string",
|
|
88
|
-
description: "
|
|
113
|
+
description: "provided only: stored encrypted as a project secret, never returned",
|
|
89
114
|
},
|
|
90
115
|
{
|
|
91
116
|
name: "login-method",
|
|
@@ -98,7 +123,7 @@ export const testAccountCommands = [
|
|
|
98
123
|
],
|
|
99
124
|
examples: [
|
|
100
125
|
"beryl accounts create --email qa@acme.test --password 'hunter2'",
|
|
101
|
-
"beryl accounts create --type
|
|
126
|
+
"beryl accounts create --type provisioned --login-method otp",
|
|
102
127
|
"beryl accounts create --email admin@acme.test --password 'hunter2' --label admin",
|
|
103
128
|
"beryl accounts create --email qa@gmail.com --password 'hunter2' # Google-only app",
|
|
104
129
|
],
|
|
@@ -106,7 +131,8 @@ export const testAccountCommands = [
|
|
|
106
131
|
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
107
132
|
const password = flagStr(input, "password");
|
|
108
133
|
const account = (await ctx.client.post(accountsPath(workspaceId, projectId), {
|
|
109
|
-
type: flagStr(input, "type") ?? (password ? "
|
|
134
|
+
type: flagStr(input, "type") ?? (password ? "provided" : "provisioned"),
|
|
135
|
+
lifetime: flagStr(input, "lifetime") ?? null,
|
|
110
136
|
identity_label: flagStr(input, "label") ?? "default",
|
|
111
137
|
email: flagStr(input, "email") ?? null,
|
|
112
138
|
password: password ?? null,
|
|
@@ -124,6 +150,54 @@ export const testAccountCommands = [
|
|
|
124
150
|
};
|
|
125
151
|
},
|
|
126
152
|
},
|
|
153
|
+
{
|
|
154
|
+
name: "accounts capture",
|
|
155
|
+
summary: "Store a signed-in session as a captured test account",
|
|
156
|
+
description: "Usually done by the Beryl Chrome extension from a signed-in tab; this takes the " +
|
|
157
|
+
"same session JSON from a file: {schema_version: 2, final_url, cookies (CDP " +
|
|
158
|
+
"shape), local_storage, session_storage, indexed_db}. Upserts by --label on the " +
|
|
159
|
+
"environment, so capturing an existing label replaces its session. Cookies for " +
|
|
160
|
+
"other domains are dropped and listed. A captured account cannot sign itself in: " +
|
|
161
|
+
"give it a probe with `beryl accounts set-login --probe`, then prove it with " +
|
|
162
|
+
"`beryl accounts check`. Its session is never served back to a client.",
|
|
163
|
+
scope: "project",
|
|
164
|
+
flags: [
|
|
165
|
+
{
|
|
166
|
+
name: "file",
|
|
167
|
+
type: "string",
|
|
168
|
+
required: true,
|
|
169
|
+
description: "The captured session, as JSON (path, or - for stdin)",
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
name: "label",
|
|
173
|
+
type: "string",
|
|
174
|
+
description: "Identity label a plan's auth_label names (default 'default')",
|
|
175
|
+
},
|
|
176
|
+
{ name: "email", type: "string", description: "The signed-in user's email" },
|
|
177
|
+
{ name: "env", type: "string", description: "Environment id (default: the project's)" },
|
|
178
|
+
{ name: "default", type: "boolean", description: "Make this the environment's default" },
|
|
179
|
+
],
|
|
180
|
+
examples: ["beryl accounts capture --file session.json --label admin"],
|
|
181
|
+
async run(ctx, input) {
|
|
182
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
183
|
+
const account = (await ctx.client.put(`${accountsPath(workspaceId, projectId)}/captured`, {
|
|
184
|
+
session: readJsonFlag(input, "file"),
|
|
185
|
+
identity_label: flagStr(input, "label") ?? "default",
|
|
186
|
+
email: flagStr(input, "email") ?? null,
|
|
187
|
+
environment_id: flagStr(input, "env") ?? null,
|
|
188
|
+
is_default: flagBool(input, "default"),
|
|
189
|
+
}));
|
|
190
|
+
const ignored = account.ignored_cookie_domains.length
|
|
191
|
+
? yellow(`\nDropped cookies for other domains: ${account.ignored_cookie_domains.join(", ")}`)
|
|
192
|
+
: "";
|
|
193
|
+
return {
|
|
194
|
+
data: account,
|
|
195
|
+
human: `${green("Captured")} ${account.identity_label} (${account.id})${ignored}\n` +
|
|
196
|
+
dim(`Next: beryl accounts set-login ${account.id} --probe probe.json, then ` +
|
|
197
|
+
`beryl accounts check ${account.id}`),
|
|
198
|
+
};
|
|
199
|
+
},
|
|
200
|
+
},
|
|
127
201
|
{
|
|
128
202
|
name: "accounts signup",
|
|
129
203
|
summary: "Store a test account's signup plan and sign it up on its environment",
|
|
@@ -177,6 +251,8 @@ export const testAccountCommands = [
|
|
|
177
251
|
"match on a redirect all pass against a logged-out page, so without a positive " +
|
|
178
252
|
"signal a dead session would run every test logged-out and still report the run " +
|
|
179
253
|
"green.\n\n" +
|
|
254
|
+
"A captured account (from the Beryl Chrome extension) takes --probe alone: it rides " +
|
|
255
|
+
"the captured session and cannot sign itself in, so there is no --file.\n\n" +
|
|
180
256
|
"Storing only stores. Run `beryl accounts check` to prove it against the live app.",
|
|
181
257
|
scope: "project",
|
|
182
258
|
args: [
|
|
@@ -186,8 +262,8 @@ export const testAccountCommands = [
|
|
|
186
262
|
{
|
|
187
263
|
name: "file",
|
|
188
264
|
type: "string",
|
|
189
|
-
|
|
190
|
-
|
|
265
|
+
description: "The sign-in ActionPlan, as JSON (path, or - for stdin). Required except for a " +
|
|
266
|
+
"captured account",
|
|
191
267
|
},
|
|
192
268
|
{
|
|
193
269
|
name: "probe",
|
|
@@ -206,12 +282,13 @@ export const testAccountCommands = [
|
|
|
206
282
|
examples: [
|
|
207
283
|
"beryl accounts set-login acc_123 --file signin.json --probe probe.json",
|
|
208
284
|
"beryl accounts set-login acc_123 --file signin.json --probe probe.json --base-hash 9f2c…",
|
|
285
|
+
"beryl accounts set-login acc_456 --probe probe.json # captured account",
|
|
209
286
|
],
|
|
210
287
|
async run(ctx, input) {
|
|
211
288
|
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
212
289
|
try {
|
|
213
290
|
const account = (await ctx.client.put(`${accountsPath(workspaceId, projectId)}/${arg(input, "account-id")}/login`, {
|
|
214
|
-
plan: readJsonFlag(input, "file"),
|
|
291
|
+
plan: flagStr(input, "file") ? readJsonFlag(input, "file") : null,
|
|
215
292
|
probe: readJsonFlag(input, "probe"),
|
|
216
293
|
base_hash: flagStr(input, "base-hash") ?? null,
|
|
217
294
|
}));
|
|
@@ -240,7 +317,7 @@ export const testAccountCommands = [
|
|
|
240
317
|
async run(ctx, input) {
|
|
241
318
|
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
242
319
|
const login = (await ctx.client.get(`${accountsPath(workspaceId, projectId)}/${arg(input, "account-id")}/login`));
|
|
243
|
-
if (!login.plan)
|
|
320
|
+
if (!login.plan && !login.probe)
|
|
244
321
|
return {
|
|
245
322
|
data: login,
|
|
246
323
|
human: dim("No login plan stored. Author one by driving the sign-in in a browser, then " +
|
|
@@ -86,7 +86,7 @@ export const configCommands = [
|
|
|
86
86
|
},
|
|
87
87
|
{
|
|
88
88
|
name: "config secrets list",
|
|
89
|
-
summary: "List the project's secrets (values are never returned)",
|
|
89
|
+
summary: "List the project's secrets (values are never returned); a test account's password is managed on the account, not here",
|
|
90
90
|
scope: "project",
|
|
91
91
|
async run(ctx, input) {
|
|
92
92
|
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
@@ -29,6 +29,11 @@ export const environmentCommands = [
|
|
|
29
29
|
{
|
|
30
30
|
name: "envs get",
|
|
31
31
|
summary: "Show one environment",
|
|
32
|
+
description: "`site_health_runs_reset_fields` and `site_health_tracker_reset_fields` name stored Site " +
|
|
33
|
+
"Health settings that were invalid and fell back to defaults (`*` = all of them). " +
|
|
34
|
+
"Scheduled and CI runs, or ticket filing, auto-close and the digest, stop until they are " +
|
|
35
|
+
"saved again: `envs update` with any `--health-*` flag rewrites the run settings, any " +
|
|
36
|
+
"`--tracker-*` flag the tracker settings.",
|
|
32
37
|
scope: "project",
|
|
33
38
|
args: [{ name: "env-id", description: "Environment id", required: true }],
|
|
34
39
|
async run(ctx, input) {
|
package/dist/commands/health.js
CHANGED
|
@@ -277,7 +277,9 @@ export const healthCommands = [
|
|
|
277
277
|
"and on desktop separately (`speeds`, each with its own Core Web Vitals) and takes " +
|
|
278
278
|
"the worse of the two; one that couldn't be measured is left out. A category's `unchecked` lists checks this run " +
|
|
279
279
|
"couldn't determine (e.g. robots.txt timed out): they neither pass nor fail. Pass --report for a single-page check started with " +
|
|
280
|
-
"`health run --path`."
|
|
280
|
+
"`health run --path`. `site_health_runs_reset_fields` and `site_health_tracker_reset_fields` " +
|
|
281
|
+
"name stored settings that were invalid and fell back to defaults (`*` = all of them): " +
|
|
282
|
+
"scheduled and CI runs, or ticket filing, stop until they are saved again with `envs update`.",
|
|
281
283
|
scope: "project",
|
|
282
284
|
flags: [
|
|
283
285
|
ENV_FLAG,
|
package/dist/commands/init.js
CHANGED
|
@@ -20,8 +20,8 @@ const MCP_SERVER_ENTRY = {
|
|
|
20
20
|
command: "npx",
|
|
21
21
|
args: ["-y", "@beryl-so/cli@latest", "mcp"],
|
|
22
22
|
};
|
|
23
|
-
// Headless: the coding agent drives this browser to
|
|
24
|
-
// window, and a headed default breaks on CI / headless boxes. Matches how most devs
|
|
23
|
+
// Headless: the coding agent drives this browser to set up sign-ins and repair plans; nobody
|
|
24
|
+
// watches the window, and a headed default breaks on CI / headless boxes. Matches how most devs
|
|
25
25
|
// already wire their own user-scoped playwright.
|
|
26
26
|
const PLAYWRIGHT_SERVER_ENTRY = {
|
|
27
27
|
command: "npx",
|
|
@@ -194,7 +194,7 @@ const outcomeLabel = (o) => o === "installed"
|
|
|
194
194
|
? "updated (refreshed to this CLI's version)"
|
|
195
195
|
: "already current";
|
|
196
196
|
const skillLeaf = (root) => path.join(root, BERYL_TEST_SKILL_DIR, BERYL_TEST_SKILL_FILENAME);
|
|
197
|
-
// The
|
|
197
|
+
// The beryl-test skill follows the `--scope` flag, same as the MCP servers: under `user`
|
|
198
198
|
// (the default) it lands in the HOME `.agents/skills/` + `.claude/skills/` dirs, so the
|
|
199
199
|
// knowledge travels with the user into every session the user-scoped MCP tools do —
|
|
200
200
|
// otherwise an agent outside this repo has all the tools and none of the guide. Under
|
|
@@ -230,8 +230,7 @@ export function refreshInstalledSkills(cwd) {
|
|
|
230
230
|
}
|
|
231
231
|
return refreshed;
|
|
232
232
|
}
|
|
233
|
-
// Local
|
|
234
|
-
// first, then bank the plan) depends on it — so the install is mandatory, not offered:
|
|
233
|
+
// Local runs drive a real browser, so the install is mandatory, not offered:
|
|
235
234
|
// missing means install now, no prompt, no opt-out. The runner ships with the CLI; the
|
|
236
235
|
// browser is the per-machine half, fetched for the runner's own revision. Never throws — a
|
|
237
236
|
// failed install must not fail `init`, which has already done its wiring; it prints the
|
|
@@ -253,7 +252,7 @@ async function ensureLocalPlaywright(ctx, cwd) {
|
|
|
253
252
|
}
|
|
254
253
|
catch (err) {
|
|
255
254
|
ctx.err(`${red("✗")} Playwright install failed: ${err.message}`);
|
|
256
|
-
ctx.err(`${dim("•")} Finish the install by hand
|
|
255
|
+
ctx.err(`${dim("•")} Finish the install by hand; local runs need it:\n` +
|
|
257
256
|
` ${cyan(installCommandsFor(gaps, cwd))}`);
|
|
258
257
|
}
|
|
259
258
|
}
|
|
@@ -292,7 +291,7 @@ export const initCommands = [
|
|
|
292
291
|
name: "init",
|
|
293
292
|
summary: "Set up Beryl in this repo: sign in and wire up your coding agent",
|
|
294
293
|
description: "Signs you in, wires the beryl and playwright MCP servers into the coding agent you " +
|
|
295
|
-
"pick, writes the beryl-test
|
|
294
|
+
"pick, writes the beryl-test skill, and installs Chromium if it is missing. " +
|
|
296
295
|
"Pick an agent with --claude, --cursor, --vscode and/or --opencode. With no flag it " +
|
|
297
296
|
"prompts (Enter picks Claude); with no flag and no TTY it wires nothing and prints the " +
|
|
298
297
|
"flags. --claude also wires the Claude DESKTOP app when installed, which needs a quit " +
|
|
@@ -507,7 +506,7 @@ export const initCommands = [
|
|
|
507
506
|
const where = skill.outcome === "updated"
|
|
508
507
|
? `${skillWhere} (refreshed to this CLI's version)`
|
|
509
508
|
: skillWhere;
|
|
510
|
-
ctx.err(`${green("✓")}
|
|
509
|
+
ctx.err(`${green("✓")} beryl-test skill ${what} ${dim(where)}`);
|
|
511
510
|
}
|
|
512
511
|
await ensureLocalPlaywright(ctx, cwd);
|
|
513
512
|
// Name the agent actually wired, not the raw flag: the picker sets `clients` too.
|
|
@@ -519,7 +518,7 @@ export const initCommands = [
|
|
|
519
518
|
? { open: "open VS Code", agent: "Copilot" }
|
|
520
519
|
: { open: "open opencode", agent: "opencode" };
|
|
521
520
|
const nextSteps = clients.size === 0
|
|
522
|
-
? `\n${bold("Beryl is signed in and the
|
|
521
|
+
? `\n${bold("Beryl is signed in and the beryl-test skill is installed.")}\n` +
|
|
523
522
|
` ${dim("• No MCP server is wired yet, so no agent can reach Beryl.")}\n` +
|
|
524
523
|
` ${dim(`• Using ${CLIENT_LABEL_LIST}? Re-run and pick it:`)}\n` +
|
|
525
524
|
` ${cyan("npx @beryl-so/cli@latest init")}\n` +
|
|
@@ -527,10 +526,10 @@ export const initCommands = [
|
|
|
527
526
|
`${cyan(MANUAL_CONFIG_SNIPPET)}\n` +
|
|
528
527
|
` ${dim("Some clients name that key `servers` or `mcp`.")}\n` +
|
|
529
528
|
` ${yellow(`Details: ${MANUAL_SETUP_DOCS}`)}\n`
|
|
530
|
-
: `\n${bold(`Beryl is set up. Now ${ed.open} and ask ${ed.agent} to
|
|
531
|
-
` ${dim(`• Say: "
|
|
532
|
-
` ${dim("
|
|
533
|
-
` ${dim(`•
|
|
529
|
+
: `\n${bold(`Beryl is set up. Now ${ed.open} and ask ${ed.agent} to create tests.`)}\n` +
|
|
530
|
+
` ${dim(`• Say: "create tests for https://example.com from docs/checkout.md" and ${ed.agent} starts`)}\n` +
|
|
531
|
+
` ${dim(" a cloud test-creation job that authors them from that document.")}\n` +
|
|
532
|
+
` ${dim(`• Then ask ${ed.agent} to run them on this machine (local runs) or in the cloud.`)}\n`;
|
|
534
533
|
// The .mcp.json entry is pinned to @latest, but a global install / old npx cache
|
|
535
534
|
// still wins resolution — so tell the user when the CLI they just ran is stale.
|
|
536
535
|
await warnIfStale(cliVersion(), (msg) => ctx.err(yellow(msg)));
|
|
@@ -549,11 +548,12 @@ export const initCommands = [
|
|
|
549
548
|
},
|
|
550
549
|
{
|
|
551
550
|
name: "guide",
|
|
552
|
-
summary: "Print the Beryl test
|
|
553
|
-
description: "The
|
|
554
|
-
"assertions, natural-language intent, test accounts and the
|
|
555
|
-
"({{login_email}}, {{mailbox_address}}, {{inbox_address}} +
|
|
556
|
-
"local run-fix loop. Same content as the beryl-test skill
|
|
551
|
+
summary: "Print the Beryl test guide",
|
|
552
|
+
description: "The guide to durable, healable tests: where new tests come from (`jobs create`), the " +
|
|
553
|
+
"ActionPlan shape, outcome assertions, natural-language intent, test accounts and the " +
|
|
554
|
+
"project mailbox ({{login_email}}, {{mailbox_address}}, {{inbox_address}} + " +
|
|
555
|
+
"await_email), and the local run-fix loop. Same content as the beryl-test skill " +
|
|
556
|
+
"`beryl init` installs. " +
|
|
557
557
|
`Skip if a loaded beryl-test skill states v${cliVersion()}; else call this first ` +
|
|
558
558
|
"(works without logging in).",
|
|
559
559
|
examples: ["beryl guide"],
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { UsageError } from "../errors.js";
|
|
4
|
+
import { arg, flagBool, flagStr, projectPath } from "./util.js";
|
|
5
|
+
const jobsPath = (ws, p) => `${projectPath(ws, p)}/test-creation-jobs`;
|
|
6
|
+
const LLM_NOTICE = "The document is sent to the LLM that drives the agent, so leave out anything you " +
|
|
7
|
+
"would not share with it.";
|
|
8
|
+
function readDocument(file) {
|
|
9
|
+
try {
|
|
10
|
+
return new Uint8Array(fs.readFileSync(file));
|
|
11
|
+
}
|
|
12
|
+
catch (err) {
|
|
13
|
+
const code = err.code;
|
|
14
|
+
if (code === "ENOENT")
|
|
15
|
+
throw new UsageError(`file not found: ${file}`);
|
|
16
|
+
throw new UsageError(`cannot read ${file} (${code ?? err.message})`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function jobHuman(job) {
|
|
20
|
+
const lines = [`Job ${job.id}: ${job.status}`];
|
|
21
|
+
if (job.error)
|
|
22
|
+
lines.push(`Error: ${job.error}`);
|
|
23
|
+
for (const t of job.tests)
|
|
24
|
+
lines.push(` ${t.title ?? t.id} [${t.surface}] ${t.id}`);
|
|
25
|
+
if (!job.tests.length)
|
|
26
|
+
lines.push(" No tests yet.");
|
|
27
|
+
const skipped = job.skipped_flows ?? [];
|
|
28
|
+
if (skipped.length)
|
|
29
|
+
lines.push("Skipped scenarios:");
|
|
30
|
+
for (const s of skipped)
|
|
31
|
+
lines.push(` ${s.handle}: ${s.reason}`);
|
|
32
|
+
return lines.join("\n");
|
|
33
|
+
}
|
|
34
|
+
export const jobCommands = [
|
|
35
|
+
{
|
|
36
|
+
name: "jobs create",
|
|
37
|
+
groupSummary: "Test-creation jobs: the cloud agent reads a document and authors tests from it.",
|
|
38
|
+
summary: "Create tests from a document (PRD, release notes) on Beryl's cloud",
|
|
39
|
+
description: "Uploads a document (Markdown, plain text, PDF or DOCX, or --text) and queues a " +
|
|
40
|
+
"job: the agent explores the environment, drafts one flow per scenario the document " +
|
|
41
|
+
"describes, and authors one test per enabled surface of the project. New tests get " +
|
|
42
|
+
"no groups. A job runs once; to run it again, use `jobs clone`. " +
|
|
43
|
+
LLM_NOTICE,
|
|
44
|
+
scope: "project",
|
|
45
|
+
args: [{ name: "file", description: "Path to the document" }],
|
|
46
|
+
flags: [
|
|
47
|
+
{ name: "text", type: "string", description: "The document as text, instead of a file" },
|
|
48
|
+
{
|
|
49
|
+
name: "env",
|
|
50
|
+
type: "string",
|
|
51
|
+
description: "Environment id to explore (defaults to the project's default)",
|
|
52
|
+
},
|
|
53
|
+
],
|
|
54
|
+
examples: ["beryl jobs create ./release-notes.md --env <env-id>"],
|
|
55
|
+
async run(ctx, input) {
|
|
56
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
57
|
+
const file = input.args.file;
|
|
58
|
+
const text = flagStr(input, "text");
|
|
59
|
+
if (Boolean(file) === Boolean(text)) {
|
|
60
|
+
throw new UsageError("pass a document file or --text, not both");
|
|
61
|
+
}
|
|
62
|
+
const form = new FormData();
|
|
63
|
+
if (file)
|
|
64
|
+
form.set("file", new Blob([readDocument(file)]), path.basename(file));
|
|
65
|
+
else
|
|
66
|
+
form.set("text", text);
|
|
67
|
+
const envId = flagStr(input, "env");
|
|
68
|
+
if (envId)
|
|
69
|
+
form.set("environment_id", envId);
|
|
70
|
+
const job = (await ctx.client.request("POST", jobsPath(workspaceId, projectId), {
|
|
71
|
+
form,
|
|
72
|
+
}));
|
|
73
|
+
return {
|
|
74
|
+
data: job,
|
|
75
|
+
human: `Job ${job.id} queued. Check it with \`beryl jobs get ${job.id}\`.`,
|
|
76
|
+
};
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
name: "jobs list",
|
|
81
|
+
summary: "List the project's test-creation jobs",
|
|
82
|
+
scope: "project",
|
|
83
|
+
async run(ctx, input) {
|
|
84
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
85
|
+
return { data: await ctx.client.get(jobsPath(workspaceId, projectId)) };
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
name: "jobs get",
|
|
90
|
+
summary: "Show a job's status and the tests it created",
|
|
91
|
+
scope: "project",
|
|
92
|
+
args: [{ name: "job-id", description: "Job id", required: true }],
|
|
93
|
+
async run(ctx, input) {
|
|
94
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
95
|
+
const job = (await ctx.client.get(`${jobsPath(workspaceId, projectId)}/${arg(input, "job-id")}`));
|
|
96
|
+
return { data: job, human: jobHuman(job) };
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
name: "jobs clone",
|
|
101
|
+
summary: "Run a job again: queue a new job with the same document and environment",
|
|
102
|
+
description: LLM_NOTICE,
|
|
103
|
+
scope: "project",
|
|
104
|
+
args: [{ name: "job-id", description: "Job id to clone", required: true }],
|
|
105
|
+
async run(ctx, input) {
|
|
106
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
107
|
+
const job = (await ctx.client.post(`${jobsPath(workspaceId, projectId)}/${arg(input, "job-id")}/clone`));
|
|
108
|
+
return { data: job, human: `Job ${job.id} queued.` };
|
|
109
|
+
},
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
name: "jobs delete",
|
|
113
|
+
summary: "Delete a job; the flows and tests it created stay",
|
|
114
|
+
scope: "project",
|
|
115
|
+
args: [{ name: "job-id", description: "Job id", required: true }],
|
|
116
|
+
flags: [{ name: "force", type: "boolean", description: "Skip the confirmation prompt" }],
|
|
117
|
+
async run(ctx, input) {
|
|
118
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
119
|
+
const jobId = arg(input, "job-id");
|
|
120
|
+
await ctx.confirm(`Delete job ${jobId}? Its tests stay.`, flagBool(input, "force"));
|
|
121
|
+
await ctx.client.del(`${jobsPath(workspaceId, projectId)}/${jobId}`);
|
|
122
|
+
return { human: "Deleted." };
|
|
123
|
+
},
|
|
124
|
+
},
|
|
125
|
+
];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { detectAuthGating } from "../detect.js";
|
|
2
2
|
import { UsageError } from "../errors.js";
|
|
3
3
|
import { dim, green, table, timeAgo } from "../output.js";
|
|
4
|
-
import { arg, flagBool, flagNum, flagStr, projectPath } from "./util.js";
|
|
4
|
+
import { arg, argList, flagBool, flagNum, flagStr, projectPath } from "./util.js";
|
|
5
5
|
import { pollCurrentExploration, watchExploration } from "./watch.js";
|
|
6
6
|
// The API treats the auth choice as the visitor's answer and never infers it — a null
|
|
7
7
|
// choice leaves the project waiting on a login capture that never comes — so resolve it
|
|
@@ -37,7 +37,7 @@ export const projectCommands = [
|
|
|
37
37
|
summary: "List projects in the workspace",
|
|
38
38
|
scope: "workspace",
|
|
39
39
|
groupDefault: true,
|
|
40
|
-
groupSummary: "Create and manage projects: a site
|
|
40
|
+
groupSummary: "Create and manage projects: a site you author tests for and run.",
|
|
41
41
|
async run(ctx, input) {
|
|
42
42
|
const ws = await ctx.requireWorkspace(input);
|
|
43
43
|
const rows = (await ctx.client.get(`/workspaces/${ws}/projects`));
|
|
@@ -70,7 +70,7 @@ export const projectCommands = [
|
|
|
70
70
|
},
|
|
71
71
|
{
|
|
72
72
|
name: "projects create",
|
|
73
|
-
summary: "Create a project
|
|
73
|
+
summary: "Create a project for a URL, or an empty one with just --name",
|
|
74
74
|
scope: "workspace",
|
|
75
75
|
args: [
|
|
76
76
|
{
|
|
@@ -84,7 +84,7 @@ export const projectCommands = [
|
|
|
84
84
|
name: "name",
|
|
85
85
|
type: "string",
|
|
86
86
|
description: "Name for an empty project when no URL is given. Add a URL later with " +
|
|
87
|
-
"`beryl envs update <env-id
|
|
87
|
+
"`beryl envs update <env-id>`",
|
|
88
88
|
},
|
|
89
89
|
{
|
|
90
90
|
name: "auth",
|
|
@@ -95,41 +95,45 @@ export const projectCommands = [
|
|
|
95
95
|
},
|
|
96
96
|
{ name: "force-new-login", type: "boolean", description: "Ignore any reusable saved login" },
|
|
97
97
|
{
|
|
98
|
-
name: "
|
|
98
|
+
name: "explore",
|
|
99
99
|
type: "boolean",
|
|
100
|
-
description: "
|
|
101
|
-
"
|
|
100
|
+
description: "Also start a cloud exploration that authors tests for you. Off by default: create " +
|
|
101
|
+
"tests from a document with `beryl jobs create`",
|
|
102
|
+
mcpDescription: "Leave unset: create tests from a document with `jobs create`. Set only when the " +
|
|
103
|
+
"user explicitly asks for a cloud exploration",
|
|
102
104
|
},
|
|
103
|
-
{ name: "watch", type: "boolean", description: "
|
|
105
|
+
{ name: "watch", type: "boolean", description: "Start the exploration (implies --explore) and stream it live" },
|
|
104
106
|
{ name: "timeout", type: "number", description: "With --watch: max minutes to wait" },
|
|
105
107
|
],
|
|
106
108
|
examples: [
|
|
107
|
-
"beryl projects create https://app.example.com
|
|
109
|
+
"beryl projects create https://app.example.com",
|
|
108
110
|
"beryl projects create https://app.example.com --auth gated",
|
|
109
|
-
"beryl projects create https://app.example.com --auth public --no-explore",
|
|
110
111
|
'beryl projects create --name "Acme production"',
|
|
111
112
|
],
|
|
112
113
|
async run(ctx, input) {
|
|
113
|
-
const
|
|
114
|
-
if (
|
|
114
|
+
const watch = flagBool(input, "watch");
|
|
115
|
+
if (input.flags.explore === false && watch)
|
|
115
116
|
throw new UsageError("--no-explore cannot be combined with --watch");
|
|
117
|
+
const explore = flagBool(input, "explore") || watch;
|
|
116
118
|
const ws = await ctx.requireWorkspace(input);
|
|
117
119
|
const rawUrl = input.args.url;
|
|
118
120
|
const url = typeof rawUrl === "string" ? rawUrl : "";
|
|
119
121
|
const name = flagStr(input, "name");
|
|
120
122
|
// Name-only: create a real empty project (no URL, no exploration). Set a URL
|
|
121
|
-
// later with `beryl envs update <env-id
|
|
123
|
+
// later with `beryl envs update <env-id>`.
|
|
122
124
|
if (!url) {
|
|
123
125
|
if (!name)
|
|
124
|
-
throw new UsageError("Provide a URL
|
|
126
|
+
throw new UsageError("Provide a URL, or --name to create an empty project");
|
|
127
|
+
if (explore)
|
|
128
|
+
throw new UsageError("--explore and --watch need a URL to explore");
|
|
125
129
|
const created = (await ctx.client.post(`/workspaces/${ws}/projects`, {
|
|
126
130
|
name,
|
|
127
131
|
}));
|
|
128
132
|
return {
|
|
129
133
|
data: created,
|
|
130
134
|
human: `${green("Project created")}: ${created.project_id} ${dim("(empty)")}\n` +
|
|
131
|
-
`Add a site with \`beryl envs update <env-id>\`,
|
|
132
|
-
`\`beryl
|
|
135
|
+
`Add a site with \`beryl envs update <env-id>\`, then create tests with ` +
|
|
136
|
+
`\`beryl jobs create <document>\`.`,
|
|
133
137
|
};
|
|
134
138
|
}
|
|
135
139
|
const auth = flagStr(input, "auth") ?? (await resolveAuthChoice(ctx, ws, url));
|
|
@@ -137,13 +141,13 @@ export const projectCommands = [
|
|
|
137
141
|
root_url: url,
|
|
138
142
|
requires_auth_choice: auth,
|
|
139
143
|
force_new_login: flagBool(input, "force-new-login"),
|
|
140
|
-
skip_exploration:
|
|
144
|
+
skip_exploration: !explore,
|
|
141
145
|
}));
|
|
142
|
-
if (
|
|
146
|
+
if (!explore)
|
|
143
147
|
return {
|
|
144
148
|
data: created,
|
|
145
|
-
human: `${green("Project created")}: ${created.project_id}
|
|
146
|
-
`
|
|
149
|
+
human: `${green("Project created")}: ${created.project_id}\n` +
|
|
150
|
+
`Create tests from a document describing the flows: \`beryl jobs create <file>\`.`,
|
|
147
151
|
};
|
|
148
152
|
// The agent cannot start until a login is captured, so there is nothing to watch.
|
|
149
153
|
if (created.auth_status === "needs_capture")
|
|
@@ -223,6 +227,55 @@ export const projectCommands = [
|
|
|
223
227
|
return { data: await ctx.client.patch(projectPath(workspaceId, projectId), body) };
|
|
224
228
|
},
|
|
225
229
|
},
|
|
230
|
+
{
|
|
231
|
+
name: "projects surfaces",
|
|
232
|
+
summary: "Set which surfaces are enabled for this project",
|
|
233
|
+
description: "Replaces the project's enabled surfaces with the ones listed: web-desktop, " +
|
|
234
|
+
"web-tablet, web-mobile. A new project has web-desktop only. At least one is " +
|
|
235
|
+
"required; `projects get` shows the current set. Turning a surface off removes " +
|
|
236
|
+
"no tests: delete a surface's tests to stop running them.",
|
|
237
|
+
scope: "project",
|
|
238
|
+
args: [
|
|
239
|
+
{
|
|
240
|
+
name: "surfaces",
|
|
241
|
+
description: "web-desktop | web-tablet | web-mobile, one or more",
|
|
242
|
+
required: true,
|
|
243
|
+
variadic: true,
|
|
244
|
+
},
|
|
245
|
+
],
|
|
246
|
+
examples: ["beryl projects surfaces web-desktop web-mobile", "beryl projects surfaces web-desktop"],
|
|
247
|
+
async run(ctx, input) {
|
|
248
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
249
|
+
const data = (await ctx.client.patch(projectPath(workspaceId, projectId), {
|
|
250
|
+
enabled_surfaces: argList(input, "surfaces"),
|
|
251
|
+
}));
|
|
252
|
+
return { data, human: `Enabled surfaces: ${data.enabled_surfaces.join(", ")}` };
|
|
253
|
+
},
|
|
254
|
+
},
|
|
255
|
+
{
|
|
256
|
+
name: "projects findings-gate",
|
|
257
|
+
summary: "Turn the findings gate on each run on or off",
|
|
258
|
+
description: "With the gate on (new-since-green), every run also reports a findings gate: failed " +
|
|
259
|
+
"when the environment's Site Health has a finding first seen after the last green run " +
|
|
260
|
+
"or came back after it (every current finding if there is no green run yet), unknown " +
|
|
261
|
+
"when Site Health has not run there or is not recording findings (its issue tracker " +
|
|
262
|
+
"must be connected). The gate sits next to the test verdicts and never changes one, nor the " +
|
|
263
|
+
"exit code of `runs watch`; read it from `findings_gate` on `runs get`. Off by default.",
|
|
264
|
+
scope: "project",
|
|
265
|
+
args: [{ name: "mode", description: "off | new-since-green", required: true }],
|
|
266
|
+
examples: ["beryl projects findings-gate new-since-green", "beryl projects findings-gate off"],
|
|
267
|
+
async run(ctx, input) {
|
|
268
|
+
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
269
|
+
const mode = arg(input, "mode").toLowerCase().replace(/-/g, "_");
|
|
270
|
+
if (mode !== "off" && mode !== "new_since_green") {
|
|
271
|
+
throw new UsageError(`Expected "off" or "new-since-green", got "${arg(input, "mode")}"`);
|
|
272
|
+
}
|
|
273
|
+
const data = (await ctx.client.patch(projectPath(workspaceId, projectId), {
|
|
274
|
+
findings_gate: mode,
|
|
275
|
+
}));
|
|
276
|
+
return { data, human: `Findings gate: ${data.findings_gate.replace(/_/g, "-")}` };
|
|
277
|
+
},
|
|
278
|
+
},
|
|
226
279
|
{
|
|
227
280
|
name: "projects delete",
|
|
228
281
|
summary: "Delete a project and all its tests and runs",
|
|
@@ -254,7 +307,7 @@ export const projectCommands = [
|
|
|
254
307
|
},
|
|
255
308
|
{
|
|
256
309
|
name: "projects report",
|
|
257
|
-
summary: "Aggregate quality report across recent runs (pass rates,
|
|
310
|
+
summary: "Aggregate quality report across recent runs (pass rates, per-test history, trend)",
|
|
258
311
|
scope: "project",
|
|
259
312
|
flags: [
|
|
260
313
|
{ name: "runs", type: "number", description: "How many recent runs to aggregate" },
|