@beryl-so/cli 0.67.0 → 0.77.1

package/dist/commands/runs.js

@@ -1,38 +1,10 @@
 import fs from "node:fs";
-import path from "node:path";
 import { downloadRunArtifacts, failureImages, isFailing, resultsOf, } from "../artifacts.js";
-import { AuthError, CliError, UsageError } from "../errors.js";
-import { ApiError } from "../http.js";
-import { buildImportForm, establishAccountSession, executeLocalSpec, IMPORT_MAX_ERROR_LEN, specUsesThirdParty, THIRD_PARTY_USE_TRIGGER, toRunEntry, } from "../local-exec.js";
-import { countPlannedFrames, countWrittenFrames, PlaywrightMissingError, } from "../local-run.js";
-import { dim, green, red, yellow } from "../output.js";
-import { anyGap, confirmInstall, describeGaps, installCommandsFor, installHint, installPlaywright, playwrightGaps, } from "../playwright-install.js";
-import { ProgressBar } from "../progress.js";
+import { UsageError } from "../errors.js";
+import { dim, yellow } from "../output.js";
 import { arg, flagBool, flagNum, flagStr, idsInGroup, projectPath } from "./util.js";
 import { watchRun } from "./watch.js";
 const MAX_FAILURE_SCREENSHOTS = 5;
-// One filesystem check before a single spec is fetched. A missing browser binary otherwise
-// fails EVERY test with the same launch error, which reads as a broken suite instead of a
-// box that can't run tests. On a TTY it's an offer; over MCP / CI it's the exact commands.
-async function ensureRunnableLocally(ctx) {
-    const gaps = playwrightGaps(process.cwd());
-    if (!anyGap(gaps))
-        return;
-    const commands = installCommandsFor(gaps, process.cwd());
-    const hint = installHint(process.cwd(), gaps);
-    if (!ctx.interactive || !(await confirmInstall(ctx.prompt, commands)))
-        throw new CliError(hint);
-    try {
-        await installPlaywright(process.cwd(), (line) => ctx.err(dim(line)));
-    }
-    catch (err) {
-        throw new CliError(`${err instanceof Error ? err.message : String(err)}\n\nFinish it by hand:\n  ${commands}`);
-    }
-    const left = playwrightGaps(process.cwd());
-    if (anyGap(left))
-        throw new CliError(hint);
-    ctx.err(green(`✓ ${describeGaps(gaps)} installed`));
-}
 export function parseHeaders(raw) {
     if (!raw || raw.length === 0)
         return null;
@@ -131,406 +103,6 @@ export const runCommands = [
             });
         },
     },
-    {
-        name: "runs local",
-        summary: "Run tests on your machine with Playwright; results sync to Beryl",
-        description: "Unlike `runs trigger`, the browser runs on YOUR machine: each test's rendered spec is " +
-            "fetched and run with the Playwright runner bundled in the CLI (your repo's own " +
-            "@playwright/test wins when present) and the Chromium binary it drives. " +
-            "Both are checked once before any spec is fetched, so a machine that can't run tests " +
-            "says so once instead of failing every test (on a terminal the CLI offers to install " +
-            "the missing browser; over MCP it prints the exact install command). " +
-            "Signup/OTP flows work: the CLI answers the spec's await_email steps over the API " +
-            "against the same mailbox the cloud runner uses. Authenticated tests work too: for a plan " +
-            "that signs itself in with {{login_email}}/{{login_password}}, the email is baked into " +
-            "the fetched spec and the password is fetched " +
-            "once over the logged secret-reveal route, handed to the spec the way the cloud runner " +
-            "does, and scrubbed from any error text or DOM snapshot before results upload; a test " +
-            "whose account has no stored password is skipped with the exact fix-it command. When the " +
-            "run finishes, the results and replay artifacts are imported into Beryl as a normal run " +
-            "(trigger source `local`), so history, replay, and reports all work; pass --no-sync to " +
-            "keep a run entirely off the record while iterating. Session-mode tests behave as " +
-            "in the cloud: their account signs in once per invocation and every session-mode " +
-            "test rides that session; a failed sign-in fails those tests with the same " +
-            "SESSION_* reason a cloud run reports. Only a test that depends on a session Beryl " +
-            "captured server-side, or that has third party integration steps (cloud only), is " +
-            "skipped, with a note. Run those with `runs trigger`. Point " +
-            "--url-override at a local dev server or preview, and --dir to keep specs, artifacts, " +
-            "and reports on disk. With no ids, a test with no plan yet is skipped quietly; one " +
-            "whose script can't be fetched is listed under `skipped` and fails the run. Exits 0 " +
-            "only if every executed test passed and every script was fetched.",
-        scope: "project",
-        args: [
-            {
-                name: "test-ids",
-                description: "Test ids to run (from `beryl tests list`); omit to run every active test",
-                variadic: true,
-            },
-        ],
-        flags: [
-            {
-                name: "all",
-                type: "boolean",
-                description: "Run every active test in the project (the default when no ids are given)",
-            },
-            {
-                name: "group",
-                type: "string",
-                description: "Run only the active tests in this group (by name). Cannot be combined with test ids.",
-            },
-            {
-                name: "url-override",
-                type: "string",
-                description: "Run against this base URL instead of the environment's (e.g. http://localhost:3000)",
-            },
-            {
-                name: "env",
-                type: "string",
-                description: "Environment id to run against and attach the imported run to. Use for a " +
-                    "standing environment; --url-override is for a throwaway host",
-            },
-            {
-                name: "sync",
-                type: "boolean",
-                default: true,
-                description: "Import the results into Beryl as a run when finished (--no-sync: local only, " +
-                    "nothing recorded)",
-            },
-            {
-                name: "dir",
-                type: "string",
-                description: "Write each test's spec, artifacts, and JSON report under this directory",
-            },
-        ],
-        examples: [
-            "beryl runs local",
-            "beryl runs local 4f… 9a…",
-            "beryl runs local --url-override http://localhost:3000",
-            "beryl runs local 4f… --no-sync --dir ./beryl-local",
-        ],
-        async run(ctx, input) {
-            // Argument validation runs BEFORE the Chromium/Playwright precondition: a bad flag
-            // combination should say so on any machine, not report a missing browser.
-            // Deduped: a repeated id would put the same test twice in one imported run,
-            // which the import manifest rejects.
-            const explicitIds = [...new Set(input.args["test-ids"] ?? [])];
-            const group = flagStr(input, "group");
-            if (group && explicitIds.length > 0)
-                throw new UsageError("--group cannot be combined with explicit test ids");
-            if (explicitIds.length > 0 && flagBool(input, "all"))
-                throw new UsageError("--all cannot be combined with explicit test ids");
-            if (group && flagBool(input, "all"))
-                throw new UsageError("--all cannot be combined with --group");
-            // No ids and no group means the whole suite — `beryl runs local` alone is a
-            // complete local run.
-            const all = explicitIds.length === 0 && !group;
-            await ensureRunnableLocally(ctx);
-            const { workspaceId, projectId } = await ctx.requireProject(input);
-            const sync = input.flags.sync !== false;
-            const urlOverride = flagStr(input, "url-override");
-            const dir = flagStr(input, "dir");
-            const listed = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/tests`, {
-                environment_id: flagStr(input, "env"),
-            }));
-            // The list rows carry the authored name as nl_title (title is the customer's
-            // rename, usually unset) — fall through so the terminal shows names, not ids.
-            const titles = new Map(listed.map((t) => [t.id, t.title || t.nl_title || t.id.slice(0, 8)]));
-            const ids = all
-                ? listed.filter((t) => t.is_active !== false).map((t) => t.id)
-                : group
-                    ? idsInGroup(listed, group)
-                    : explicitIds;
-            if (ids.length === 0)
-                throw group
-                    ? new UsageError(`No active tests in group '${group}'.`)
-                    : new CliError("This project has no tests to run.");
-            const fetchScript = (id) => ctx.client.get(`${projectPath(workspaceId, projectId)}/tests/${id}/script`, {
-                // frames only matter when the run will be imported: they become the replay.
-                // environment_id keeps the baked login_email on the same environment the
-                // LOGIN_PASSWORD reveal below is scoped to.
-                base_url: urlOverride,
-                frames: sync ? true : undefined,
-                environment_id: flagStr(input, "env"),
-            });
-            const toLocalSpec = (id, script) => ({
-                id,
-                title: titles.get(id) ?? id.slice(0, 8),
-                content: script.content,
-                requiresAuth: Boolean(script.requires_auth),
-                usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(script.content),
-                usesLoginPassword: Boolean(script.uses_login_password),
-                usesThirdParty: specUsesThirdParty(script.content),
-                inlinedLoginSteps: script.inlined_login_steps ?? 0,
-                ...(script.session_mode && script.account_id
-                    ? { sessionMode: true, accountId: script.account_id }
-                    : {}),
-                ...(script.email ? { email: script.email } : {}),
-                ...(script.login_config_error ? { loginConfigError: script.login_config_error } : {}),
-            });
-            const specs = [];
-            // With --all, a test the server can't render (no plan yet, 409) is a quiet skip;
-            // any other fetch failure is listed and fails the run. An explicitly-requested id
-            // failing to render is the caller's problem.
-            const fetchFailed = [];
-            for (const id of ids) {
-                let script;
-                try {
-                    script = await fetchScript(id);
-                }
-                catch (err) {
-                    if (!all || err instanceof AuthError)
-                        throw err;
-                    const detail = err instanceof Error ? err.message : String(err);
-                    if (err instanceof ApiError && err.status === 409) {
-                        ctx.err(yellow(`! ${titles.get(id) ?? id}: no runnable script — skipped (${detail})`));
-                    }
-                    else {
-                        fetchFailed.push(id);
-                        ctx.err(red(`✗ ${titles.get(id) ?? id}: could not fetch its script (${detail})`));
-                    }
-                    continue;
-                }
-                specs.push(toLocalSpec(id, script));
-            }
-            if (specs.length === 0)
-                throw new CliError(fetchFailed.length
-                    ? `Could not fetch the script for ${fetchFailed.length} test(s); nothing ran.`
-                    : "No runnable tests were found.");
-            // A test that leans on a session this CLI cannot establish gets no session here,
-            // so skip it rather than run a spec doomed at the login wall. A session-mode test
-            // is NOT in that set: its account's session is established below, once, exactly as
-            // a cloud run's setup phase does. A test whose account is missing its password
-            // would type a literal placeholder into the page: skip, with the server's fix-it
-            // message.
-            const skipped = specs.filter((s) => (s.requiresAuth && !(s.sessionMode && s.accountId)) ||
-                s.loginConfigError ||
-                s.usesThirdParty);
-            const runnable = specs.filter((s) => !skipped.includes(s));
-            for (const s of skipped) {
-                ctx.err(yellow(s.loginConfigError
-                    ? `! ${s.title}: skipped — ${s.loginConfigError}`
-                    : s.usesThirdParty
-                        ? `! ${s.title}: skipped, ${THIRD_PARTY_USE_TRIGGER}.`
-                        : `! ${s.title}: needs a session only Beryl's cloud holds, skipped ` +
-                            `(use \`beryl runs trigger\`).`));
-            }
-            if (runnable.length === 0)
-                throw new CliError("None of the selected tests can run locally.");
-            // Once per account, before any test runs — the local counterpart of the cloud
-            // worker's run-setup phase, produced by the same server-side establish (stored-
-            // session cache, login replay, proof). A failure here fails only that account's
-            // tests, with the same stable-prefixed reason a cloud run reports; an account
-            // proven unable to carry a session gets its tests re-rendered to sign themselves
-            // in, the same degraded mode a cloud run applies.
-            const sessions = new Map();
-            const sessionErrors = new Map();
-            const accountIds = [
-                ...new Set(runnable.filter((s) => s.sessionMode && s.accountId).map((s) => s.accountId)),
-            ];
-            for (const accountId of accountIds) {
-                ctx.err(dim("Establishing the test account's session (once per run)…"));
-                const established = await establishAccountSession(ctx.client, projectPath(workspaceId, projectId), accountId);
-                if (established.session) {
-                    sessions.set(accountId, established.session);
-                }
-                else if (established.unsupported) {
-                    // The server just latched UNSUPPORTED, so a re-fetch renders these specs with
-                    // their account's sign-in steps inlined — they run self-signing like any other.
-                    // Whole-object replacement, not a merge: the inlined render carries no
-                    // session_mode, and a merge would leave the stale sessionMode/accountId behind.
-                    ctx.err(yellow(`! ${established.error} — signing in per test.`));
-                    try {
-                        for (const [i, s] of runnable.entries()) {
-                            if (s.accountId !== accountId)
-                                continue;
-                            runnable[i] = toLocalSpec(s.id, await fetchScript(s.id));
-                        }
-                    }
-                    catch (err) {
-                        // A failed re-fetch fails only this account's tests, like any other
-                        // establish failure — not the whole invocation.
-                        sessionErrors.set(accountId, err instanceof Error ? err.message : String(err));
-                    }
-                }
-                else {
-                    sessionErrors.set(accountId, established.error ?? "SESSION_UNAVAILABLE");
-                }
-            }
-            // Revealed once for the batch over the member-gated, logged route — the same
-            // secret the cloud runner banks into run-config.json; never in the spec source.
-            let loginPassword;
-            if (runnable.some((s) => s.usesLoginPassword)) {
-                try {
-                    const secret = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/config/secrets/LOGIN_PASSWORD/value`, { environment_id: flagStr(input, "env") }));
-                    loginPassword = secret.value;
-                }
-                catch (err) {
-                    throw new CliError("Could not reveal the LOGIN_PASSWORD secret for the saved login: " +
-                        (err instanceof Error ? err.message : String(err)));
-                }
-            }
-            const startedAt = new Date().toISOString();
-            const entries = [];
-            const bar = new ProgressBar();
-            let installOffered = false;
-            let done = 0;
-            let passed = 0;
-            let failed = 0;
-            for (const spec of runnable) {
-                const stepTotal = countPlannedFrames(spec.content);
-                const state = () => ({
-                    total: runnable.length,
-                    done,
-                    passed,
-                    failed,
-                    title: spec.title,
-                    stepTotal,
-                });
-                bar.update(state());
-                if (!bar.active)
-                    ctx.err(dim(`Running ${spec.title}…`));
-                const testStarted = new Date().toISOString();
-                // No proven session for this account: running the spec would open the page
-                // logged out and pass every assertion that tolerates that — fail it in the
-                // setup phase with the establish error, exactly as a cloud run does.
-                const sessionError = spec.accountId ? sessionErrors.get(spec.accountId) : undefined;
-                if (spec.sessionMode && sessionError !== undefined) {
-                    entries.push({
-                        test_case_id: spec.id,
-                        status: "failed",
-                        phase: "setup",
-                        error_message: sessionError.slice(0, IMPORT_MAX_ERROR_LEN),
-                        started_at: testStarted,
-                        completed_at: new Date().toISOString(),
-                        frames: [],
-                        frame_urls: [],
-                        frame_durations_ms: [],
-                        files: [],
-                    });
-                    done += 1;
-                    failed += 1;
-                    bar.clear();
-                    ctx.err(`${red("✗")} ${spec.title}\n    ${dim(sessionError.split("\n")[0] ?? "")}`);
-                    continue;
-                }
-                const session = spec.sessionMode ? sessions.get(spec.accountId ?? "") : undefined;
-                let outcome;
-                let runError;
-                let lastStep = 0;
-                const attempt = () => executeLocalSpec({ client: ctx.client, workspaceId, projectId }, {
-                    spec,
-                    dir: dir ? path.join(dir, spec.id) : undefined,
-                    harvest: sync,
-                    loginPassword,
-                    session,
-                    // Clear the rewriting bar before a mid-test log line, or the line is
-                    // appended onto the live bar and fossilises it into scrollback.
-                    onEvent: (line) => {
-                        bar.clear();
-                        ctx.err(dim(line));
-                        bar.update({ ...state(), step: lastStep });
-                    },
-                    onSpawn: (runDir) => {
-                        const ticker = setInterval(() => {
-                            lastStep = countWrittenFrames(runDir);
-                            bar.update({ ...state(), step: lastStep });
-                        }, 300);
-                        return () => clearInterval(ticker);
-                    },
-                });
-                try {
-                    try {
-                        ({ outcome, runError } = await attempt());
-                    }
-                    catch (err) {
-                        // Local Playwright missing: on a TTY offer to install it once and retry,
-                        // instead of only printing a hint the user then has to act on by hand.
-                        if (err instanceof PlaywrightMissingError && ctx.interactive && !installOffered) {
-                            installOffered = true;
-                            bar.clear();
-                            const cwd = process.cwd();
-                            const commands = installCommandsFor(playwrightGaps(cwd), cwd);
-                            if (!(await confirmInstall(ctx.prompt, commands)))
-                                throw err;
-                            try {
-                                await installPlaywright(process.cwd(), (line) => ctx.err(dim(line)));
-                            }
-                            catch (installErr) {
-                                throw new CliError(installErr instanceof Error ? installErr.message : String(installErr));
-                            }
-                            ctx.err(green("✓ Local Playwright installed — running the test…"));
-                            ({ outcome, runError } = await attempt());
-                        }
-                        else {
-                            throw err;
-                        }
-                    }
-                }
-                catch (err) {
-                    // Environmental failures abort the batch — nothing later would fare better.
-                    if (err instanceof PlaywrightMissingError || err instanceof CliError) {
-                        bar.clear();
-                        throw err instanceof CliError ? err : new CliError(err.message);
-                    }
-                    // The spec never ran and the executor could not classify it — an errored
-                    // result, not an aborted batch: the remaining tests still deserve their run.
-                    runError = err instanceof Error ? err.message : String(err);
-                }
-                const entry = toRunEntry(spec, outcome, runError, testStarted, entries.length, [
-                    ...(spec.usesLoginPassword && loginPassword ? [loginPassword] : []),
-                    ...(session?.secrets ?? []),
-                ]);
-                entries.push(entry);
-                done += 1;
-                if (entry.status === "passed")
-                    passed += 1;
-                else
-                    failed += 1;
-                bar.clear();
-                ctx.err(`${entry.status === "passed" ? green("✓") : red("✗")} ${spec.title}` +
-                    `${entry.duration_ms ? dim(` (${(entry.duration_ms / 1000).toFixed(1)}s)`) : ""}` +
-                    `${entry.error_message ? `\n    ${dim(entry.error_message.split("\n")[0] ?? "")}` : ""}`);
-            }
-            bar.clear();
-            let runId;
-            if (sync && entries.length > 0) {
-                ctx.err(dim("Importing results into Beryl…"));
-                const imported = (await ctx.client.request("POST", `${projectPath(workspaceId, projectId)}/runs/import`, {
-                    form: buildImportForm(entries, {
-                        environmentId: flagStr(input, "env"),
-                        targetUrlOverride: urlOverride,
-                        startedAt,
-                        completedAt: new Date().toISOString(),
-                        onNote: (line) => ctx.err(yellow(`! ${line}`)),
-                    }),
-                }));
-                runId = imported.id;
-            }
-            const summaryLine = failed === 0
-                ? green(`All tests passed (${passed}/${entries.length})`)
-                : red(`${failed} test(s) failed`) + ` (${passed} passed)`;
-            const human = `${summaryLine}` +
-                (skipped.length ? `\n${yellow(`${skipped.length} skipped (see notes above)`)}` : "") +
-                (fetchFailed.length
-                    ? `\n${red(`${fetchFailed.length} could not be fetched (see notes above)`)}`
-                    : "") +
-                (runId ? `\nSynced to Beryl as run ${runId} — \`beryl runs get ${runId}\`` : "") +
-                (!sync ? `\n${dim("--no-sync: nothing recorded in Beryl")}` : "") +
-                (dir ? `\n${dim(`Specs + artifacts + reports under ${path.resolve(dir)}`)}` : "");
-            return {
-                data: {
-                    passed,
-                    failed,
-                    skipped: [...skipped.map((s) => s.id), ...fetchFailed],
-                    results: entries.map(({ files: _files, ...rest }) => rest),
-                    ...(runId ? { run_id: runId } : {}),
-                },
-                human,
-                // Exit codes are a CI contract: any failing/errored or unfetchable test → exit 1.
-                ...(failed > 0 || fetchFailed.length > 0 ? { exitCode: 1 } : {}),
-            };
-        },
-    },
     {
         name: "runs list",
         summary: "List recent runs",
@@ -581,7 +153,8 @@ export const runCommands = [
             "carries `heal` when Beryl healed it: its steps were rewritten with the intent " +
             "unchanged, proved in a browser and it re-ran green; `version_no` is the revision it " +
             "wrote (see `tests diff`) and `rationale` the agent's reason. A heal that did not turn " +
-            "the test green is not reported: a failure is a failure. " +
+            "the test green is not reported: a failure is a failure. A red that Beryl did not try " +
+            "to heal carries `heal_skip_reason` saying why. " +
             "`runs report` returns the generated report document, `runs explain` an AI explanation " +
             "of one failed result.",
         scope: "project",
@@ -607,7 +180,7 @@ export const runCommands = [
                 name: "status",
                 type: "string",
                 description: "With --page, show only this bucket: passed, failed, errored, cancelled, " +
-                    "quarantined, or unstable (currently unstable in the run's environment)",
+                    "or quarantined",
             },
             {
                 name: "group",