@beryl-so/cli 0.17.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -7
- package/dist/adapters/cli.js +15 -1
- package/dist/adapters/mcp.js +44 -5
- package/dist/beryl-test-skill.js +275 -90
- package/dist/commands/accounts.js +331 -0
- package/dist/commands/environments.js +0 -2
- package/dist/commands/init.js +43 -27
- package/dist/commands/mailboxes.js +159 -0
- package/dist/commands/mcp.js +25 -0
- package/dist/commands/projects.js +0 -6
- package/dist/commands/runs.js +159 -46
- package/dist/commands/tests.js +29 -6
- package/dist/email-pump.js +5 -2
- package/dist/http.js +4 -1
- package/dist/local-exec.js +86 -27
- package/dist/local-run.js +21 -3
- package/dist/playwright-install.js +118 -7
- package/dist/registry/index.js +4 -2
- package/dist/schema.generated.js +32 -0
- package/package.json +2 -2
- package/dist/commands/inboxes.js +0 -145
package/dist/commands/mcp.js
CHANGED
|
@@ -1,4 +1,29 @@
|
|
|
1
|
+
import { cliVersion, fetchLatestVersion, isBehind } from "../version-check.js";
|
|
1
2
|
export const mcpCommands = [
|
|
3
|
+
{
|
|
4
|
+
name: "version",
|
|
5
|
+
summary: "Show the running CLI version, API URL, and Node version",
|
|
6
|
+
description: "Answers \"which build am I actually talking to?\" — the one question a long-lived " +
|
|
7
|
+
"`beryl mcp` process can't otherwise answer, since it loads source at spawn and never " +
|
|
8
|
+
"hot-reloads. Needs no login, so it still works when a token is missing or broken. " +
|
|
9
|
+
"Also reports whether the running build is behind npm's `latest` (best-effort: " +
|
|
10
|
+
"`update_available` is null when the registry can't be reached, and the check is " +
|
|
11
|
+
"skipped entirely under BERYL_NO_UPDATE_CHECK).",
|
|
12
|
+
examples: ["beryl version", "beryl version --json"],
|
|
13
|
+
async run(ctx) {
|
|
14
|
+
const current = cliVersion();
|
|
15
|
+
const latest = process.env.BERYL_NO_UPDATE_CHECK ? null : await fetchLatestVersion();
|
|
16
|
+
return {
|
|
17
|
+
data: {
|
|
18
|
+
cli_version: current,
|
|
19
|
+
api_url: ctx.config.apiUrl,
|
|
20
|
+
node_version: process.version,
|
|
21
|
+
latest_version: latest,
|
|
22
|
+
update_available: latest === null ? null : isBehind(current, latest),
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
},
|
|
26
|
+
},
|
|
2
27
|
{
|
|
3
28
|
name: "mcp",
|
|
4
29
|
summary: "Run the Beryl MCP server (stdio) — every CLI command as an agent tool",
|
|
@@ -93,11 +93,6 @@ export const projectCommands = [
|
|
|
93
93
|
description: "Whether the site needs a login (gated) or not (public). Default: detected from the " +
|
|
94
94
|
"site — only asked when detection is genuinely unsure",
|
|
95
95
|
},
|
|
96
|
-
{
|
|
97
|
-
name: "allow-mutations",
|
|
98
|
-
type: "boolean",
|
|
99
|
-
description: "Let the agent perform state-changing actions while exploring",
|
|
100
|
-
},
|
|
101
96
|
{ name: "force-new-login", type: "boolean", description: "Ignore any reusable saved login" },
|
|
102
97
|
{
|
|
103
98
|
name: "no-explore",
|
|
@@ -141,7 +136,6 @@ export const projectCommands = [
|
|
|
141
136
|
const created = (await ctx.client.post(`/workspaces/${ws}/projects`, {
|
|
142
137
|
root_url: url,
|
|
143
138
|
requires_auth_choice: auth,
|
|
144
|
-
mutation_choice: flagBool(input, "allow-mutations") ? "allow_mutations" : null,
|
|
145
139
|
force_new_login: flagBool(input, "force-new-login"),
|
|
146
140
|
skip_exploration: noExplore,
|
|
147
141
|
}));
|
package/dist/commands/runs.js
CHANGED
|
@@ -2,14 +2,36 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { downloadRunArtifacts, failureImages, isFailing, resultsOf, } from "../artifacts.js";
|
|
4
4
|
import { CliError, UsageError } from "../errors.js";
|
|
5
|
-
import { buildImportForm, executeLocalSpec, toRunEntry, } from "../local-exec.js";
|
|
5
|
+
import { buildImportForm, establishAccountSession, executeLocalSpec, IMPORT_MAX_ERROR_LEN, toRunEntry, } from "../local-exec.js";
|
|
6
6
|
import { countPlannedFrames, countWrittenFrames, PlaywrightMissingError, } from "../local-run.js";
|
|
7
7
|
import { dim, green, red, yellow } from "../output.js";
|
|
8
|
-
import { confirmInstall, installPlaywright } from "../playwright-install.js";
|
|
8
|
+
import { anyGap, confirmInstall, describeGaps, installCommandsFor, installPlaywright, playwrightGaps, } from "../playwright-install.js";
|
|
9
9
|
import { ProgressBar } from "../progress.js";
|
|
10
10
|
import { arg, flagBool, flagNum, flagStr, projectPath } from "./util.js";
|
|
11
11
|
import { watchRun } from "./watch.js";
|
|
12
12
|
const MAX_FAILURE_SCREENSHOTS = 5;
|
|
13
|
+
// One filesystem check before a single spec is fetched. A missing browser binary otherwise
|
|
14
|
+
// fails EVERY test with the same launch error, which reads as a broken suite instead of a
|
|
15
|
+
// box that can't run tests. On a TTY it's an offer; over MCP / CI it's the exact commands.
|
|
16
|
+
async function ensureRunnableLocally(ctx) {
|
|
17
|
+
const gaps = playwrightGaps(process.cwd());
|
|
18
|
+
if (!anyGap(gaps))
|
|
19
|
+
return;
|
|
20
|
+
const commands = installCommandsFor(gaps);
|
|
21
|
+
const hint = `Local runs need ${describeGaps(gaps)}. Install it, then re-run:\n ${commands}`;
|
|
22
|
+
if (!ctx.interactive || !(await confirmInstall(ctx.prompt, commands)))
|
|
23
|
+
throw new CliError(hint);
|
|
24
|
+
try {
|
|
25
|
+
await installPlaywright(process.cwd(), (line) => ctx.err(dim(line)));
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
throw new CliError(`${err instanceof Error ? err.message : String(err)}\n\nFinish it by hand:\n ${commands}`);
|
|
29
|
+
}
|
|
30
|
+
const left = playwrightGaps(process.cwd());
|
|
31
|
+
if (anyGap(left))
|
|
32
|
+
throw new CliError(hint);
|
|
33
|
+
ctx.err(green(`✓ ${describeGaps(gaps)} installed`));
|
|
34
|
+
}
|
|
13
35
|
export function parseHeaders(raw) {
|
|
14
36
|
if (!raw || raw.length === 0)
|
|
15
37
|
return null;
|
|
@@ -75,20 +97,24 @@ export const runCommands = [
|
|
|
75
97
|
name: "runs local",
|
|
76
98
|
summary: "Run tests on your machine with your own Playwright; results sync to Beryl",
|
|
77
99
|
description: "Unlike `runs trigger`, the browser runs on YOUR machine — each test's rendered spec is " +
|
|
78
|
-
"fetched and run with your local @playwright/test
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
100
|
+
"fetched and run with your local @playwright/test and the Chromium binary it drives. " +
|
|
101
|
+
"Both are checked once before any spec is fetched, so a machine that can't run tests " +
|
|
102
|
+
"says so once instead of failing every test (on a terminal the CLI offers to install " +
|
|
103
|
+
"whichever half is missing; over MCP it prints the exact install commands). " +
|
|
104
|
+
"Signup/OTP flows work: the CLI answers the spec's await_email steps over the API " +
|
|
105
|
+
"against the same mailbox the cloud runner uses. Authenticated tests work too: a plan " +
|
|
106
|
+
"that signs itself in with {{login_email}}/{{login_password}} has its password fetched " +
|
|
107
|
+
"once over the logged secret-reveal route, handed to the spec the way the cloud runner " +
|
|
108
|
+
"does, and scrubbed from any error text or DOM snapshot before results upload. When the " +
|
|
109
|
+
"run finishes, the results and replay artifacts are imported into Beryl as a normal run " +
|
|
86
110
|
"(trigger source `local`) — history, replay, and reports all work; pass --no-sync to " +
|
|
87
|
-
"keep a run entirely off the record while iterating.
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
111
|
+
"keep a run entirely off the record while iterating. Session-mode tests behave as " +
|
|
112
|
+
"in the cloud: their account signs in once per invocation and every session-mode " +
|
|
113
|
+
"test rides that session; a failed sign-in fails those tests with the same " +
|
|
114
|
+
"SESSION_* reason a cloud run reports. Only a test that depends on a session Beryl " +
|
|
115
|
+
"captured server-side is skipped, with a note. Point " +
|
|
116
|
+
"--url-override at a local dev server or preview, and --dir to keep specs, artifacts, " +
|
|
117
|
+
"and reports on disk. Exits 0 only if every executed test passed.",
|
|
92
118
|
scope: "project",
|
|
93
119
|
args: [
|
|
94
120
|
{
|
|
@@ -129,6 +155,7 @@ export const runCommands = [
|
|
|
129
155
|
"beryl runs local 4f… --no-sync --dir ./beryl-local",
|
|
130
156
|
],
|
|
131
157
|
async run(ctx, input) {
|
|
158
|
+
await ensureRunnableLocally(ctx);
|
|
132
159
|
const { workspaceId, projectId } = await ctx.requireProject(input);
|
|
133
160
|
// Deduped: a repeated id would put the same test twice in one imported run,
|
|
134
161
|
// which the import manifest rejects.
|
|
@@ -149,19 +176,33 @@ export const runCommands = [
|
|
|
149
176
|
const ids = all ? listed.filter((t) => t.is_active !== false).map((t) => t.id) : explicitIds;
|
|
150
177
|
if (ids.length === 0)
|
|
151
178
|
throw new CliError("This project has no tests to run.");
|
|
179
|
+
const fetchScript = (id) => ctx.client.get(`${projectPath(workspaceId, projectId)}/tests/${id}/script`, {
|
|
180
|
+
// frames only matter when the run will be imported: they become the replay.
|
|
181
|
+
// environment_id keeps the baked login_email on the same environment the
|
|
182
|
+
// LOGIN_PASSWORD reveal below is scoped to.
|
|
183
|
+
base_url: urlOverride,
|
|
184
|
+
frames: sync ? true : undefined,
|
|
185
|
+
environment_id: flagStr(input, "env"),
|
|
186
|
+
});
|
|
187
|
+
const toLocalSpec = (id, script) => ({
|
|
188
|
+
id,
|
|
189
|
+
title: titles.get(id) ?? id.slice(0, 8),
|
|
190
|
+
content: script.content,
|
|
191
|
+
requiresAuth: Boolean(script.requires_auth),
|
|
192
|
+
usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(script.content),
|
|
193
|
+
usesLoginPassword: Boolean(script.uses_login_password),
|
|
194
|
+
inlinedLoginSteps: script.inlined_login_steps ?? 0,
|
|
195
|
+
...(script.session_mode && script.account_id
|
|
196
|
+
? { sessionMode: true, accountId: script.account_id }
|
|
197
|
+
: {}),
|
|
198
|
+
...(script.email ? { email: script.email } : {}),
|
|
199
|
+
...(script.login_config_error ? { loginConfigError: script.login_config_error } : {}),
|
|
200
|
+
});
|
|
152
201
|
const specs = [];
|
|
153
202
|
for (const id of ids) {
|
|
154
203
|
let script;
|
|
155
204
|
try {
|
|
156
|
-
script =
|
|
157
|
-
// frames only matter when the run will be imported: they become the replay.
|
|
158
|
-
// environment_id keeps the baked login_email on the same environment the
|
|
159
|
-
// LOGIN_PASSWORD reveal below is scoped to.
|
|
160
|
-
{
|
|
161
|
-
base_url: urlOverride,
|
|
162
|
-
frames: sync ? true : undefined,
|
|
163
|
-
environment_id: flagStr(input, "env"),
|
|
164
|
-
}));
|
|
205
|
+
script = await fetchScript(id);
|
|
165
206
|
}
|
|
166
207
|
catch (err) {
|
|
167
208
|
// With --all an unrenderable test (no plan yet) is a skip, not an abort;
|
|
@@ -173,32 +214,66 @@ export const runCommands = [
|
|
|
173
214
|
}
|
|
174
215
|
throw err;
|
|
175
216
|
}
|
|
176
|
-
specs.push(
|
|
177
|
-
id,
|
|
178
|
-
title: titles.get(id) ?? id.slice(0, 8),
|
|
179
|
-
content: script.content,
|
|
180
|
-
requiresAuth: Boolean(script.requires_auth),
|
|
181
|
-
usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(script.content),
|
|
182
|
-
usesLoginPassword: Boolean(script.uses_login_password),
|
|
183
|
-
...(script.login_config_error ? { loginConfigError: script.login_config_error } : {}),
|
|
184
|
-
});
|
|
217
|
+
specs.push(toLocalSpec(id, script));
|
|
185
218
|
}
|
|
186
219
|
if (specs.length === 0)
|
|
187
220
|
throw new CliError("No runnable tests were found.");
|
|
188
|
-
// A
|
|
189
|
-
//
|
|
190
|
-
//
|
|
191
|
-
//
|
|
192
|
-
|
|
193
|
-
|
|
221
|
+
// A test that leans on a session this CLI cannot establish gets no session here,
|
|
222
|
+
// so skip it rather than run a spec doomed at the login wall. A session-mode test
|
|
223
|
+
// is NOT in that set: its account's session is established below, once, exactly as
|
|
224
|
+
// a cloud run's setup phase does. A test whose account is missing its password
|
|
225
|
+
// would type a literal placeholder into the page: skip, with the server's fix-it
|
|
226
|
+
// message.
|
|
227
|
+
const skipped = specs.filter((s) => (s.requiresAuth && !(s.sessionMode && s.accountId)) || s.loginConfigError);
|
|
228
|
+
const runnable = specs.filter((s) => !skipped.includes(s));
|
|
194
229
|
for (const s of skipped) {
|
|
195
|
-
ctx.err(yellow(s.
|
|
196
|
-
? `! ${s.title}:
|
|
197
|
-
|
|
198
|
-
|
|
230
|
+
ctx.err(yellow(s.loginConfigError
|
|
231
|
+
? `! ${s.title}: skipped — ${s.loginConfigError}`
|
|
232
|
+
: `! ${s.title}: needs a session only Beryl's cloud holds — skipped ` +
|
|
233
|
+
`(use \`beryl runs trigger\`).`));
|
|
199
234
|
}
|
|
200
235
|
if (runnable.length === 0)
|
|
201
236
|
throw new CliError("None of the selected tests can run locally.");
|
|
237
|
+
// Once per account, before any test runs — the local counterpart of the cloud
|
|
238
|
+
// worker's run-setup phase, produced by the same server-side establish (stored-
|
|
239
|
+
// session cache, login replay, proof). A failure here fails only that account's
|
|
240
|
+
// tests, with the same stable-prefixed reason a cloud run reports; an account
|
|
241
|
+
// proven unable to carry a session gets its tests re-rendered to sign themselves
|
|
242
|
+
// in, the same degraded mode a cloud run applies.
|
|
243
|
+
const sessions = new Map();
|
|
244
|
+
const sessionErrors = new Map();
|
|
245
|
+
const accountIds = [
|
|
246
|
+
...new Set(runnable.filter((s) => s.sessionMode && s.accountId).map((s) => s.accountId)),
|
|
247
|
+
];
|
|
248
|
+
for (const accountId of accountIds) {
|
|
249
|
+
ctx.err(dim("Establishing the test account's session (once per run)…"));
|
|
250
|
+
const established = await establishAccountSession(ctx.client, projectPath(workspaceId, projectId), accountId);
|
|
251
|
+
if (established.session) {
|
|
252
|
+
sessions.set(accountId, established.session);
|
|
253
|
+
}
|
|
254
|
+
else if (established.unsupported) {
|
|
255
|
+
// The server just latched UNSUPPORTED, so a re-fetch renders these specs with
|
|
256
|
+
// their account's sign-in steps inlined — they run self-signing like any other.
|
|
257
|
+
// Whole-object replacement, not a merge: the inlined render carries no
|
|
258
|
+
// session_mode, and a merge would leave the stale sessionMode/accountId behind.
|
|
259
|
+
ctx.err(yellow(`! ${established.error} — signing in per test.`));
|
|
260
|
+
try {
|
|
261
|
+
for (const [i, s] of runnable.entries()) {
|
|
262
|
+
if (s.accountId !== accountId)
|
|
263
|
+
continue;
|
|
264
|
+
runnable[i] = toLocalSpec(s.id, await fetchScript(s.id));
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
catch (err) {
|
|
268
|
+
// A failed re-fetch fails only this account's tests, like any other
|
|
269
|
+
// establish failure — not the whole invocation.
|
|
270
|
+
sessionErrors.set(accountId, err instanceof Error ? err.message : String(err));
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
else {
|
|
274
|
+
sessionErrors.set(accountId, established.error ?? "SESSION_UNAVAILABLE");
|
|
275
|
+
}
|
|
276
|
+
}
|
|
202
277
|
// Revealed once for the batch over the member-gated, logged route — the same
|
|
203
278
|
// secret the cloud runner banks into run-config.json; never in the spec source.
|
|
204
279
|
let loginPassword;
|
|
@@ -233,16 +308,51 @@ export const runCommands = [
|
|
|
233
308
|
if (!bar.active)
|
|
234
309
|
ctx.err(dim(`Running ${spec.title}…`));
|
|
235
310
|
const testStarted = new Date().toISOString();
|
|
311
|
+
// No proven session for this account: running the spec would open the page
|
|
312
|
+
// logged out and pass every assertion that tolerates that — fail it in the
|
|
313
|
+
// setup phase with the establish error, exactly as a cloud run does.
|
|
314
|
+
const sessionError = spec.accountId ? sessionErrors.get(spec.accountId) : undefined;
|
|
315
|
+
if (spec.sessionMode && sessionError !== undefined) {
|
|
316
|
+
entries.push({
|
|
317
|
+
test_case_id: spec.id,
|
|
318
|
+
status: "failed",
|
|
319
|
+
phase: "setup",
|
|
320
|
+
error_message: sessionError.slice(0, IMPORT_MAX_ERROR_LEN),
|
|
321
|
+
started_at: testStarted,
|
|
322
|
+
completed_at: new Date().toISOString(),
|
|
323
|
+
frames: [],
|
|
324
|
+
frame_urls: [],
|
|
325
|
+
frame_durations_ms: [],
|
|
326
|
+
files: [],
|
|
327
|
+
});
|
|
328
|
+
done += 1;
|
|
329
|
+
failed += 1;
|
|
330
|
+
bar.clear();
|
|
331
|
+
ctx.err(`${red("✗")} ${spec.title}\n ${dim(sessionError.split("\n")[0] ?? "")}`);
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
const session = spec.sessionMode ? sessions.get(spec.accountId ?? "") : undefined;
|
|
236
335
|
let outcome;
|
|
237
336
|
let runError;
|
|
337
|
+
let lastStep = 0;
|
|
238
338
|
const attempt = () => executeLocalSpec({ client: ctx.client, workspaceId, projectId }, {
|
|
239
339
|
spec,
|
|
240
340
|
dir: dir ? path.join(dir, spec.id) : undefined,
|
|
241
341
|
harvest: sync,
|
|
242
342
|
loginPassword,
|
|
243
|
-
|
|
343
|
+
session,
|
|
344
|
+
// Clear the rewriting bar before a mid-test log line, or the line is
|
|
345
|
+
// appended onto the live bar and fossilises it into scrollback.
|
|
346
|
+
onEvent: (line) => {
|
|
347
|
+
bar.clear();
|
|
348
|
+
ctx.err(dim(line));
|
|
349
|
+
bar.update({ ...state(), step: lastStep });
|
|
350
|
+
},
|
|
244
351
|
onSpawn: (runDir) => {
|
|
245
|
-
const ticker = setInterval(() =>
|
|
352
|
+
const ticker = setInterval(() => {
|
|
353
|
+
lastStep = countWrittenFrames(runDir);
|
|
354
|
+
bar.update({ ...state(), step: lastStep });
|
|
355
|
+
}, 300);
|
|
246
356
|
return () => clearInterval(ticker);
|
|
247
357
|
},
|
|
248
358
|
});
|
|
@@ -282,7 +392,10 @@ export const runCommands = [
|
|
|
282
392
|
// result, not an aborted batch: the remaining tests still deserve their run.
|
|
283
393
|
runError = err instanceof Error ? err.message : String(err);
|
|
284
394
|
}
|
|
285
|
-
const entry = toRunEntry(spec, outcome, runError, testStarted, entries.length,
|
|
395
|
+
const entry = toRunEntry(spec, outcome, runError, testStarted, entries.length, [
|
|
396
|
+
...(spec.usesLoginPassword && loginPassword ? [loginPassword] : []),
|
|
397
|
+
...(session?.secrets ?? []),
|
|
398
|
+
]);
|
|
286
399
|
entries.push(entry);
|
|
287
400
|
done += 1;
|
|
288
401
|
if (entry.status === "passed")
|
package/dist/commands/tests.js
CHANGED
|
@@ -2,7 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import { CliError, UsageError } from "../errors.js";
|
|
3
3
|
import { ApiError } from "../http.js";
|
|
4
4
|
import { lintPlan } from "../lint.js";
|
|
5
|
-
import { buildImportForm, executeLocalSpec, toRunEntry, } from "../local-exec.js";
|
|
5
|
+
import { buildImportForm, establishAccountSession, executeLocalSpec, toRunEntry, } from "../local-exec.js";
|
|
6
6
|
import { PlaywrightMissingError } from "../local-run.js";
|
|
7
7
|
import { dim, green, red, table, yellow } from "../output.js";
|
|
8
8
|
import { confirmInstall, installPlaywright } from "../playwright-install.js";
|
|
@@ -162,9 +162,11 @@ export const testCommands = [
|
|
|
162
162
|
"terminal the CLI offers to install it). " +
|
|
163
163
|
"A red replay banks NOTHING: the failure evidence comes back (over MCP the screenshot is " +
|
|
164
164
|
"image content), you fix the plan file and re-run. The proving run is imported as the " +
|
|
165
|
-
"test's first run (--no-sync to skip). A plan that signs in with a
|
|
166
|
-
"replay locally (
|
|
167
|
-
"verification automatically.
|
|
165
|
+
"test's first run (--no-sync to skip). A plan that signs in with a session Beryl captured " +
|
|
166
|
+
"server-side cannot replay locally (that session never leaves Beryl's cloud) — it falls " +
|
|
167
|
+
"back to server-side verification automatically. A session-mode plan replays locally " +
|
|
168
|
+
"fine: the server renders it with its account's stored sign-in steps in front, so " +
|
|
169
|
+
"the same identity is exercised on your machine. Optional `before` and `after` arrays hold setup and teardown " +
|
|
168
170
|
"steps: `after` runs even when a main step fails, which is how a create/update/delete test " +
|
|
169
171
|
"cleans up the record it made on the runs that go red.",
|
|
170
172
|
scope: "project",
|
|
@@ -241,9 +243,27 @@ export const testCommands = [
|
|
|
241
243
|
frames: sync ? true : undefined,
|
|
242
244
|
});
|
|
243
245
|
let compiled = await compile();
|
|
246
|
+
// Session-mode plan: establish the account's session once (server-side, reused
|
|
247
|
+
// from the stored one when it still proves live) and replay against it — the
|
|
248
|
+
// cloud shape. An unsupported account is re-compiled to sign itself in.
|
|
249
|
+
let session;
|
|
250
|
+
if (compiled.session_mode && compiled.account_id) {
|
|
251
|
+
ctx.err(dim("Establishing the test account's session…"));
|
|
252
|
+
const established = await establishAccountSession(ctx.client, projectPath(workspaceId, projectId), compiled.account_id);
|
|
253
|
+
if (established.session) {
|
|
254
|
+
session = established.session;
|
|
255
|
+
}
|
|
256
|
+
else if (established.unsupported) {
|
|
257
|
+
ctx.err(yellow(`! ${established.error} — the replay signs in itself.`));
|
|
258
|
+
compiled = await compile();
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
throw new CliError(established.error ?? "SESSION_UNAVAILABLE");
|
|
262
|
+
}
|
|
263
|
+
}
|
|
244
264
|
// A captured session lives encrypted in Beryl's cloud and is never handed to
|
|
245
265
|
// this machine — the cloud is the only place this plan can be proven.
|
|
246
|
-
if (compiled.requires_auth) {
|
|
266
|
+
if (compiled.requires_auth && !session) {
|
|
247
267
|
ctx.err(yellow("! This plan signs in with a captured session, so it can only be verified " +
|
|
248
268
|
"in Beryl's cloud — verifying server-side instead."));
|
|
249
269
|
try {
|
|
@@ -279,6 +299,8 @@ export const testCommands = [
|
|
|
279
299
|
requiresAuth: false,
|
|
280
300
|
usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(content),
|
|
281
301
|
usesLoginPassword: compiled.uses_login_password,
|
|
302
|
+
inlinedLoginSteps: compiled.inlined_login_steps ?? 0,
|
|
303
|
+
email: compiled.email ?? undefined,
|
|
282
304
|
});
|
|
283
305
|
const startedAt = new Date().toISOString();
|
|
284
306
|
const attempts = 1 + Math.max(0, compiled.policy.error_retries);
|
|
@@ -300,6 +322,7 @@ export const testCommands = [
|
|
|
300
322
|
dir: flagStr(input, "dir"),
|
|
301
323
|
harvest: true,
|
|
302
324
|
loginPassword,
|
|
325
|
+
session,
|
|
303
326
|
onEvent: (line) => ctx.err(dim(line)),
|
|
304
327
|
}));
|
|
305
328
|
}
|
|
@@ -366,7 +389,7 @@ export const testCommands = [
|
|
|
366
389
|
}
|
|
367
390
|
let runId;
|
|
368
391
|
if (sync) {
|
|
369
|
-
const entry = toRunEntry({ ...specOf(compiled.content), id: String(created.id) }, outcome, undefined, startedAt, 0, loginPassword);
|
|
392
|
+
const entry = toRunEntry({ ...specOf(compiled.content), id: String(created.id) }, outcome, undefined, startedAt, 0, [...(loginPassword ? [loginPassword] : []), ...(session?.secrets ?? [])]);
|
|
370
393
|
const imported = (await ctx.client.request("POST", `${projectPath(workspaceId, projectId)}/runs/import`, {
|
|
371
394
|
form: buildImportForm([entry], {
|
|
372
395
|
environmentId: env,
|
package/dist/email-pump.js
CHANGED
|
@@ -10,14 +10,17 @@ export function startEmailPump(opts) {
|
|
|
10
10
|
let stopped = false;
|
|
11
11
|
const served = new Set();
|
|
12
12
|
const consumed = new Set();
|
|
13
|
-
let
|
|
13
|
+
let storedAfter = opts.storedAfter;
|
|
14
|
+
let since;
|
|
14
15
|
const fetchLatest = async (req, timeoutS) => {
|
|
15
16
|
try {
|
|
16
|
-
return (await opts.client.get(`/workspaces/${opts.workspaceId}/
|
|
17
|
+
return (await opts.client.get(`/workspaces/${opts.workspaceId}/mailboxes/${opts.inboxId}/emails/latest`, {
|
|
17
18
|
timeout_s: timeoutS,
|
|
18
19
|
since,
|
|
20
|
+
stored_after: storedAfter,
|
|
19
21
|
from_contains: req.from_contains,
|
|
20
22
|
subject_contains: req.subject_contains,
|
|
23
|
+
recipient_contains: opts.recipientContains,
|
|
21
24
|
}));
|
|
22
25
|
}
|
|
23
26
|
catch (err) {
|
package/dist/http.js
CHANGED
|
@@ -52,7 +52,10 @@ export class ApiClient {
|
|
|
52
52
|
}
|
|
53
53
|
if (response.status === 401) {
|
|
54
54
|
throw new AuthError(this.token
|
|
55
|
-
? "Authentication failed — the token is invalid, expired, or revoked. Run
|
|
55
|
+
? "Authentication failed — the token is invalid, expired, or revoked. Run " +
|
|
56
|
+
"`beryl login`. If you just did, and this is an MCP server started with an " +
|
|
57
|
+
"explicit BERYL_API_KEY/BERYL_TOKEN, that env var wins over the login — " +
|
|
58
|
+
"re-register the server to pick up the new token."
|
|
56
59
|
: "Not logged in. Run `beryl login` or set BERYL_API_KEY.");
|
|
57
60
|
}
|
|
58
61
|
if (!response.ok) {
|
package/dist/local-exec.js
CHANGED
|
@@ -7,13 +7,16 @@ import { PlaywrightMissingError, runSpecLocally, } from "./local-run.js";
|
|
|
7
7
|
export const IMPORT_MAX_ERROR_LEN = 5000;
|
|
8
8
|
export const IMPORT_MAX_FILE_BYTES = 14 * 1024 * 1024;
|
|
9
9
|
export const IMPORT_MAX_TOTAL_BYTES = 180 * 1024 * 1024;
|
|
10
|
+
export const IMPORT_MAX_FILES = 3600;
|
|
10
11
|
export function toRunEntry(spec, outcome, runError, startedAt, ordinal, redact) {
|
|
11
|
-
// Mirrors the cloud runner's redact_result: the spec types
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
|
|
12
|
+
// Mirrors the cloud runner's redact_result: the spec types secrets into the page
|
|
13
|
+
// and an injected session authenticates its requests, so error text and the DOM
|
|
14
|
+
// snapshot can echo them back — scrub before the bytes leave this machine. Frames
|
|
15
|
+
// and screenshots are pixels; nothing to scrub.
|
|
16
|
+
const values = (Array.isArray(redact) ? redact : [redact]).filter((v) => Boolean(v));
|
|
17
|
+
const scrub = (text) => values.reduce((out, v) => out.split(v).join("***"), text);
|
|
18
|
+
const scrubBytes = (bytes) => values.some((v) => bytes.includes(v))
|
|
19
|
+
? Buffer.from(scrub(bytes.toString("utf8")), "utf8")
|
|
17
20
|
: bytes;
|
|
18
21
|
const completedAt = new Date().toISOString();
|
|
19
22
|
const base = {
|
|
@@ -51,14 +54,19 @@ export function toRunEntry(spec, outcome, runError, startedAt, ordinal, redact)
|
|
|
51
54
|
return entry;
|
|
52
55
|
// A frame over the per-file cap would force dropping mid-list and shift the
|
|
53
56
|
// index-aligned url/duration sidecars — drop the whole filmstrip instead.
|
|
54
|
-
|
|
55
|
-
|
|
57
|
+
// Drop the inlined sign-in's frames before anything is index-aligned to the stored
|
|
58
|
+
// plan: `runs explain` zips the plan's steps against these lists, so an off-by-K here
|
|
59
|
+
// shows every step's screenshot next to the wrong step.
|
|
60
|
+
const skip = Math.min(spec.inlinedLoginSteps ?? 0, harvested.frames.length);
|
|
61
|
+
const frames = harvested.frames.slice(skip);
|
|
62
|
+
if (frames.every((f) => f.length <= IMPORT_MAX_FILE_BYTES)) {
|
|
63
|
+
frames.forEach((bytes, i) => {
|
|
56
64
|
const name = `r${ordinal}-frame-${String(i).padStart(3, "0")}.png`;
|
|
57
65
|
entry.frames.push(name);
|
|
58
66
|
entry.files.push({ name, bytes });
|
|
59
67
|
});
|
|
60
|
-
entry.frame_urls = harvested.frameUrls;
|
|
61
|
-
entry.frame_durations_ms = harvested.frameDurationsMs;
|
|
68
|
+
entry.frame_urls = harvested.frameUrls.slice(skip);
|
|
69
|
+
entry.frame_durations_ms = harvested.frameDurationsMs.slice(skip);
|
|
62
70
|
}
|
|
63
71
|
if (harvested.screenshot && harvested.screenshot.length <= IMPORT_MAX_FILE_BYTES) {
|
|
64
72
|
entry.screenshot = `r${ordinal}-screenshot.png`;
|
|
@@ -90,6 +98,22 @@ export function buildImportForm(entries, opts) {
|
|
|
90
98
|
total -= frameBytes;
|
|
91
99
|
opts.onNote?.(`replay frames for test ${entry.test_case_id} dropped (upload size cap)`);
|
|
92
100
|
}
|
|
101
|
+
// File-count budget: the server parses at most RUN_IMPORT_MAX_FILES multipart
|
|
102
|
+
// parts, so a big suite's filmstrips can overflow on count with bytes to spare.
|
|
103
|
+
let fileCount = entries.reduce((n, e) => n + e.files.length, 0);
|
|
104
|
+
for (const entry of entries) {
|
|
105
|
+
if (fileCount <= IMPORT_MAX_FILES)
|
|
106
|
+
break;
|
|
107
|
+
const frameFiles = entry.files.filter((f) => entry.frames.includes(f.name)).length;
|
|
108
|
+
if (frameFiles === 0)
|
|
109
|
+
continue;
|
|
110
|
+
entry.files = entry.files.filter((f) => !entry.frames.includes(f.name));
|
|
111
|
+
entry.frames = [];
|
|
112
|
+
entry.frame_urls = [];
|
|
113
|
+
entry.frame_durations_ms = [];
|
|
114
|
+
fileCount -= frameFiles;
|
|
115
|
+
opts.onNote?.(`replay frames for test ${entry.test_case_id} dropped (upload file-count cap)`);
|
|
116
|
+
}
|
|
93
117
|
const manifest = {
|
|
94
118
|
environment_id: opts.environmentId ?? null,
|
|
95
119
|
target_url_override: opts.targetUrlOverride ?? null,
|
|
@@ -107,10 +131,33 @@ export function buildImportForm(entries, opts) {
|
|
|
107
131
|
}
|
|
108
132
|
return form;
|
|
109
133
|
}
|
|
134
|
+
/** Establish (or reuse) an account's proven session server-side — the same establish a
|
|
135
|
+
* cloud run's setup phase does, so one sign-in serves cloud and local alike. */
|
|
136
|
+
export async function establishAccountSession(client, projectBase, accountId) {
|
|
137
|
+
let res;
|
|
138
|
+
try {
|
|
139
|
+
res = (await client.request("POST", `${projectBase}/test-accounts/${accountId}/session`));
|
|
140
|
+
}
|
|
141
|
+
catch (err) {
|
|
142
|
+
return { error: err instanceof Error ? err.message : String(err) };
|
|
143
|
+
}
|
|
144
|
+
if (res.ok && res.storage_state) {
|
|
145
|
+
return {
|
|
146
|
+
session: {
|
|
147
|
+
storageState: res.storage_state,
|
|
148
|
+
initScripts: res.init_scripts ?? [],
|
|
149
|
+
secrets: res.secrets ?? [],
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
error: res.error ?? "SESSION_UNAVAILABLE",
|
|
155
|
+
unsupported: Boolean(res.unsupported),
|
|
156
|
+
};
|
|
157
|
+
}
|
|
110
158
|
/**
|
|
111
|
-
* Run one rendered spec on this machine with its full service harness:
|
|
112
|
-
* inbox
|
|
113
|
-
* inbox/login sidecars, pump `await_email` requests over the API while Playwright
|
|
159
|
+
* Run one rendered spec on this machine with its full service harness: write the
|
|
160
|
+
* inbox/login sidecars and pump `await_email` requests over the API while Playwright
|
|
114
161
|
* runs. The one local-execution seam shared by `runs local` and the local-verify
|
|
115
162
|
* `tests create`. {@link PlaywrightMissingError} propagates (environmental — the
|
|
116
163
|
* caller decides whether to offer an install); any other throw is captured as
|
|
@@ -118,11 +165,18 @@ export function buildImportForm(entries, opts) {
|
|
|
118
165
|
*/
|
|
119
166
|
export async function executeLocalSpec(deps, opts) {
|
|
120
167
|
const { spec } = opts;
|
|
121
|
-
const inbox = spec.
|
|
122
|
-
?
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
168
|
+
const inbox = spec.email
|
|
169
|
+
? {
|
|
170
|
+
id: spec.email.inbox_id,
|
|
171
|
+
address: spec.email.address,
|
|
172
|
+
recipientContains: spec.email.recipient_contains ?? undefined,
|
|
173
|
+
// NOW, not the mailbox's creation time: it already holds every earlier run's
|
|
174
|
+
// mail, and an older code served as this run's would pass the step with a value
|
|
175
|
+
// the app then rejects. Sent as stored_after, which fences on the server's own
|
|
176
|
+
// INGESTION time and is clamped server-side — `since` compares against the
|
|
177
|
+
// sender's Date: header, so a fast local clock would fence out mail that arrived.
|
|
178
|
+
storedAfter: new Date().toISOString(),
|
|
179
|
+
}
|
|
126
180
|
: undefined;
|
|
127
181
|
try {
|
|
128
182
|
const outcome = await runSpecLocally({
|
|
@@ -130,7 +184,18 @@ export async function executeLocalSpec(deps, opts) {
|
|
|
130
184
|
testName: spec.title,
|
|
131
185
|
dir: opts.dir,
|
|
132
186
|
harvest: opts.harvest,
|
|
133
|
-
redact:
|
|
187
|
+
redact: [
|
|
188
|
+
...(spec.usesLoginPassword && opts.loginPassword ? [opts.loginPassword] : []),
|
|
189
|
+
...(opts.session?.secrets ?? []),
|
|
190
|
+
],
|
|
191
|
+
...(opts.session
|
|
192
|
+
? {
|
|
193
|
+
authSession: {
|
|
194
|
+
storageState: opts.session.storageState,
|
|
195
|
+
initScripts: opts.session.initScripts,
|
|
196
|
+
},
|
|
197
|
+
}
|
|
198
|
+
: {}),
|
|
134
199
|
setup: inbox || (spec.usesLoginPassword && opts.loginPassword !== undefined)
|
|
135
200
|
? (runDir) => {
|
|
136
201
|
if (inbox)
|
|
@@ -146,7 +211,8 @@ export async function executeLocalSpec(deps, opts) {
|
|
|
146
211
|
client: deps.client,
|
|
147
212
|
workspaceId: deps.workspaceId,
|
|
148
213
|
inboxId: inbox.id,
|
|
149
|
-
|
|
214
|
+
storedAfter: inbox.storedAfter,
|
|
215
|
+
recipientContains: inbox.recipientContains,
|
|
150
216
|
onEvent: opts.onEvent,
|
|
151
217
|
})
|
|
152
218
|
: undefined;
|
|
@@ -166,11 +232,4 @@ export async function executeLocalSpec(deps, opts) {
|
|
|
166
232
|
// caller records, not an aborted command.
|
|
167
233
|
return { runError: err instanceof Error ? err.message : String(err) };
|
|
168
234
|
}
|
|
169
|
-
finally {
|
|
170
|
-
if (inbox) {
|
|
171
|
-
await deps.client
|
|
172
|
-
.del(`/workspaces/${deps.workspaceId}/inboxes/${inbox.id}`)
|
|
173
|
-
.catch(() => undefined);
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
235
|
}
|