@beryl-so/cli 0.11.1 → 0.17.0

package/dist/commands/projects.js

@@ -1,6 +1,6 @@
 import { detectAuthGating } from "../detect.js";
 import { UsageError } from "../errors.js";
-import { dim, green } from "../output.js";
+import { dim, green, table, timeAgo } from "../output.js";
 import { arg, flagBool, flagNum, flagStr, projectPath } from "./util.js";
 import { pollCurrentExploration, watchExploration } from "./watch.js";
 // The API treats the auth choice as the visitor's answer and never infers it — a null
@@ -40,7 +40,23 @@ export const projectCommands = [
         groupSummary: "Create and manage projects — a site Beryl explores, authors tests for, and runs.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
-            return { data: await ctx.client.get(`/workspaces/${ws}/projects`) };
+            const rows = (await ctx.client.get(`/workspaces/${ws}/projects`));
+            // Full rows stay in `data` for --json/MCP; the terminal gets the columns a
+            // human scans a project list for, with readable ages instead of raw ISO.
+            const human = table(rows.map((p) => {
+                const last = p.last_execution;
+                return {
+                    name: p.name ?? p.root_url ?? p.id,
+                    tests: p.test_count ?? 0,
+                    status: p.status,
+                    last_run: last
+                        ? `${last.status} ${last.passed_count ?? 0}/${last.total_tests ?? "?"} · ${timeAgo(last.started_at)}`
+                        : "",
+                    updated: timeAgo(p.updated_at ?? p.created_at),
+                    id: p.id,
+                };
+            }));
+            return { data: rows, human };
         },
     },
     {
@@ -140,9 +156,9 @@ export const projectCommands = [
                 return {
                     data: created,
                     human: `${green("Project created")}: ${created.project_id}\n` +
-                        `The site needs a login before the agent can explore it. Capture one with ` +
-                        `\`beryl credentials capture --project ${created.project_id}\` — the exploration ` +
-                        `starts as soon as you do.`,
+                        `The site needs a login before the agent can explore it. Capture one in the ` +
+                        `Beryl webapp (open the project — it walks you through the login) — the ` +
+                        `exploration starts as soon as you do.`,
                 };
             if (!flagBool(input, "watch"))
                 return { data: created };

package/dist/commands/runs.js

@@ -1,9 +1,12 @@
 import fs from "node:fs";
+import path from "node:path";
 import { downloadRunArtifacts, failureImages, isFailing, resultsOf, } from "../artifacts.js";
 import { CliError, UsageError } from "../errors.js";
-import { PlaywrightMissingError, runSpecLocally } from "../local-run.js";
+import { buildImportForm, executeLocalSpec, toRunEntry, } from "../local-exec.js";
+import { countPlannedFrames, countWrittenFrames, PlaywrightMissingError, } from "../local-run.js";
 import { dim, green, red, yellow } from "../output.js";
 import { confirmInstall, installPlaywright } from "../playwright-install.js";
+import { ProgressBar } from "../progress.js";
 import { arg, flagBool, flagNum, flagStr, projectPath } from "./util.js";
 import { watchRun } from "./watch.js";
 const MAX_FAILURE_SCREENSHOTS = 5;
@@ -70,99 +73,261 @@ export const runCommands = [
     },
     {
         name: "runs local",
-        summary: "Run a banked test locally with your own Playwright (public flows)",
-        description: "Unlike `runs trigger`, this runs on YOUR machine, not Beryl's cloud — fetches the test's " +
-            "rendered spec, then runs it with your local @playwright/test. On a terminal it offers to " +
-            "install @playwright/test + chromium for you the first time they're missing (over MCP it " +
-            "prints the install commands instead). Point --url-override at a local " +
-            "dev server or preview, and --dir to keep the spec, artifacts, and JSON report on disk so " +
-            "an agent can run-fix-run. v1 targets public/unauthenticated flows: an authenticated test " +
-            "refuses to run locally (those run in Beryl's cloud, which holds the session) — no session " +
-            "is ever decrypted to your disk. Exits 0 if every test passed, 1 on a test failure.",
+        summary: "Run tests on your machine with your own Playwright; results sync to Beryl",
+        description: "Unlike `runs trigger`, the browser runs on YOUR machine — each test's rendered spec is " +
+            "fetched and run with your local @playwright/test (on a terminal the CLI offers to " +
+            "install it the first time it's missing; over MCP it prints the install commands). " +
+            "Signup/OTP flows work: the CLI mints a fresh run inbox and answers the spec's " +
+            "await_email steps over the API, exactly as the cloud runner would. Saved-login tests " +
+            "({{login_email}}/{{login_password}}) work too: the password is fetched once over the " +
+            "logged secret-reveal route, handed to the spec the way the cloud runner does, and " +
+            "scrubbed from any error text or DOM snapshot before results upload. When the run " +
+            "finishes, the results and replay artifacts are imported into Beryl as a normal run " +
+            "(trigger source `local`) — history, replay, and reports all work; pass --no-sync to " +
+            "keep a run entirely off the record while iterating. A captured-session test still " +
+            "refuses to run locally (its session lives encrypted in Beryl's cloud and is never " +
+            "decrypted to your disk) — it is skipped with a note. Point --url-override at a local " +
+            "dev server or preview, and --dir to keep specs, artifacts, and reports on disk. " +
+            "Exits 0 only if every executed test passed.",
         scope: "project",
-        args: [{ name: "test-id", description: "Test id to run (from `beryl tests list`)", required: true }],
+        args: [
+            {
+                name: "test-ids",
+                description: "Test ids to run (from `beryl tests list`); omit to run every active test",
+                variadic: true,
+            },
+        ],
         flags: [
+            {
+                name: "all",
+                type: "boolean",
+                description: "Run every active test in the project (the default when no ids are given)",
+            },
             {
                 name: "url-override",
                 type: "string",
                 description: "Run against this base URL instead of the environment's (e.g. http://localhost:3000)",
             },
+            { name: "env", type: "string", description: "Environment id to attach the imported run to" },
+            {
+                name: "sync",
+                type: "boolean",
+                default: true,
+                description: "Import the results into Beryl as a run when finished (--no-sync: local only, " +
+                    "nothing recorded)",
+            },
             {
                 name: "dir",
                 type: "string",
-                description: "Write the spec, artifacts, and JSON report here (default: a temp dir)",
+                description: "Write each test's spec, artifacts, and JSON report under this directory",
             },
         ],
         examples: [
-            "beryl runs local 4f…",
-            "beryl runs local 4f… --url-override http://localhost:3000 --dir ./beryl-local",
+            "beryl runs local",
+            "beryl runs local 4f… 9a…",
+            "beryl runs local --url-override http://localhost:3000",
+            "beryl runs local 4f… --no-sync --dir ./beryl-local",
         ],
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
-            const testId = arg(input, "test-id");
-            const script = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/tests/${testId}/script`, { base_url: flagStr(input, "url-override") }));
-            // v1 is public flows only. A gated test's session lives (encrypted) in the cloud and is
-            // never handed to a local runner, so refuse rather than run a spec doomed to fail at the
-            // login wall. Exit 1: it's a failure to run this test locally, not a usage error.
-            if (script.requires_auth) {
-                throw new CliError("This test signs in first, so it can only run in Beryl's cloud (which holds the " +
-                    "session) — local runs are for public/unauthenticated flows. Run it with " +
-                    "`beryl runs trigger`.");
+            // Deduped: a repeated id would put the same test twice in one imported run,
+            // which the import manifest rejects.
+            const explicitIds = [...new Set(input.args["test-ids"] ?? [])];
+            // No ids means the whole suite — `beryl runs local` alone is a complete local run.
+            const all = explicitIds.length === 0;
+            if (explicitIds.length > 0 && flagBool(input, "all"))
+                throw new UsageError("--all cannot be combined with explicit test ids");
+            const sync = input.flags.sync !== false;
+            const urlOverride = flagStr(input, "url-override");
+            const dir = flagStr(input, "dir");
+            const listed = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/tests`, {
+                environment_id: flagStr(input, "env"),
+            }));
+            // The list rows carry the authored name as nl_title (title is the customer's
+            // rename, usually unset) — fall through so the terminal shows names, not ids.
+            const titles = new Map(listed.map((t) => [t.id, t.title || t.nl_title || t.id.slice(0, 8)]));
+            const ids = all ? listed.filter((t) => t.is_active !== false).map((t) => t.id) : explicitIds;
+            if (ids.length === 0)
+                throw new CliError("This project has no tests to run.");
+            const specs = [];
+            for (const id of ids) {
+                let script;
+                try {
+                    script = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/tests/${id}/script`, 
+                    // frames only matter when the run will be imported: they become the replay.
+                    // environment_id keeps the baked login_email on the same environment the
+                    // LOGIN_PASSWORD reveal below is scoped to.
+                    {
+                        base_url: urlOverride,
+                        frames: sync ? true : undefined,
+                        environment_id: flagStr(input, "env"),
+                    }));
+                }
+                catch (err) {
+                    // With --all an unrenderable test (no plan yet) is a skip, not an abort;
+                    // an explicitly-requested id failing to render is the caller's problem.
+                    if (all) {
+                        const detail = err instanceof Error ? err.message : String(err);
+                        ctx.err(yellow(`! ${titles.get(id) ?? id}: no runnable script — skipped (${detail})`));
+                        continue;
+                    }
+                    throw err;
+                }
+                specs.push({
+                    id,
+                    title: titles.get(id) ?? id.slice(0, 8),
+                    content: script.content,
+                    requiresAuth: Boolean(script.requires_auth),
+                    usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(script.content),
+                    usesLoginPassword: Boolean(script.uses_login_password),
+                    ...(script.login_config_error ? { loginConfigError: script.login_config_error } : {}),
+                });
             }
-            const runOnce = () => runSpecLocally({
-                spec: script.content,
-                testName: testId,
-                dir: flagStr(input, "dir"),
-                onProgress: (line) => ctx.err(dim(line)),
-            });
-            let outcome;
-            try {
+            if (specs.length === 0)
+                throw new CliError("No runnable tests were found.");
+            // A captured-session test's login lives encrypted in Beryl's cloud and is never
+            // handed to a local runner — skip it rather than run a spec doomed at the wall.
+            // A saved-login test missing its backing config would type a literal placeholder
+            // into the page: same treatment, with the server's fix-it message.
+            const skipped = specs.filter((s) => s.requiresAuth || s.loginConfigError);
+            const runnable = specs.filter((s) => !s.requiresAuth && !s.loginConfigError);
+            for (const s of skipped) {
+                ctx.err(yellow(s.requiresAuth
+                    ? `! ${s.title}: signs in with a captured session, so it only runs in Beryl's ` +
+                        `cloud — skipped (use \`beryl runs trigger\`).`
+                    : `! ${s.title}: skipped — ${s.loginConfigError}`));
+            }
+            if (runnable.length === 0)
+                throw new CliError("None of the selected tests can run locally.");
+            // Revealed once for the batch over the member-gated, logged route — the same
+            // secret the cloud runner banks into run-config.json; never in the spec source.
+            let loginPassword;
+            if (runnable.some((s) => s.usesLoginPassword)) {
                 try {
-                    outcome = await runOnce();
+                    const secret = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/config/secrets/LOGIN_PASSWORD/value`, { environment_id: flagStr(input, "env") }));
+                    loginPassword = secret.value;
                 }
                 catch (err) {
-                    // Local Playwright missing: on a TTY offer to install it and retry, instead of only
-                    // printing a hint the user then has to act on by hand. Non-interactively we can't
-                    // prompt, so we re-throw and the hint surfaces as before (no unprompted install).
-                    if (err instanceof PlaywrightMissingError && ctx.interactive) {
-                        // confirmInstall swallows a prompt failure into `false`, so a broken prompt falls
-                        // back to re-throwing the actionable missing-Playwright hint, not the prompt's error.
-                        if (!(await confirmInstall(ctx.prompt)))
+                    throw new CliError("Could not reveal the LOGIN_PASSWORD secret for the saved login: " +
+                        (err instanceof Error ? err.message : String(err)));
+                }
+            }
+            const startedAt = new Date().toISOString();
+            const entries = [];
+            const bar = new ProgressBar();
+            let installOffered = false;
+            let done = 0;
+            let passed = 0;
+            let failed = 0;
+            for (const spec of runnable) {
+                const stepTotal = countPlannedFrames(spec.content);
+                const state = () => ({
+                    total: runnable.length,
+                    done,
+                    passed,
+                    failed,
+                    title: spec.title,
+                    stepTotal,
+                });
+                bar.update(state());
+                if (!bar.active)
+                    ctx.err(dim(`Running ${spec.title}…`));
+                const testStarted = new Date().toISOString();
+                let outcome;
+                let runError;
+                const attempt = () => executeLocalSpec({ client: ctx.client, workspaceId, projectId }, {
+                    spec,
+                    dir: dir ? path.join(dir, spec.id) : undefined,
+                    harvest: sync,
+                    loginPassword,
+                    onEvent: (line) => ctx.err(dim(line)),
+                    onSpawn: (runDir) => {
+                        const ticker = setInterval(() => bar.update({ ...state(), step: countWrittenFrames(runDir) }), 300);
+                        return () => clearInterval(ticker);
+                    },
+                });
+                try {
+                    try {
+                        ({ outcome, runError } = await attempt());
+                    }
+                    catch (err) {
+                        // Local Playwright missing: on a TTY offer to install it once and retry,
+                        // instead of only printing a hint the user then has to act on by hand.
+                        if (err instanceof PlaywrightMissingError && ctx.interactive && !installOffered) {
+                            installOffered = true;
+                            bar.clear();
+                            if (!(await confirmInstall(ctx.prompt)))
+                                throw err;
+                            try {
+                                await installPlaywright(process.cwd(), (line) => ctx.err(dim(line)));
+                            }
+                            catch (installErr) {
+                                throw new CliError(installErr instanceof Error ? installErr.message : String(installErr));
+                            }
+                            ctx.err(green("✓ Local Playwright installed — running the test…"));
+                            ({ outcome, runError } = await attempt());
+                        }
+                        else {
                             throw err;
-                        await installPlaywright(process.cwd(), (line) => ctx.err(dim(line)));
-                        ctx.err(green("✓ Local Playwright installed — running the test…"));
-                        outcome = await runOnce();
+                        }
                     }
-                    else {
-                        throw err;
+                }
+                catch (err) {
+                    // Environmental failures abort the batch — nothing later would fare better.
+                    if (err instanceof PlaywrightMissingError || err instanceof CliError) {
+                        bar.clear();
+                        throw err instanceof CliError ? err : new CliError(err.message);
                     }
+                    // The spec never ran and the executor could not classify it — an errored
+                    // result, not an aborted batch: the remaining tests still deserve their run.
+                    runError = err instanceof Error ? err.message : String(err);
                 }
+                const entry = toRunEntry(spec, outcome, runError, testStarted, entries.length, spec.usesLoginPassword ? loginPassword : undefined);
+                entries.push(entry);
+                done += 1;
+                if (entry.status === "passed")
+                    passed += 1;
+                else
+                    failed += 1;
+                bar.clear();
+                ctx.err(`${entry.status === "passed" ? green("✓") : red("✗")} ${spec.title}` +
+                    `${entry.duration_ms ? dim(` (${(entry.duration_ms / 1000).toFixed(1)}s)`) : ""}` +
+                    `${entry.error_message ? `\n    ${dim(entry.error_message.split("\n")[0] ?? "")}` : ""}`);
             }
-            catch (err) {
-                // Every failure to run the spec (missing Playwright, a compile error, a customer
-                // config filtering our spec away) already carries an actionable message — surface it
-                // as a CliError so the user sees that, not a raw Node stack trace.
-                throw new CliError(err instanceof Error ? err.message : String(err));
+            bar.clear();
+            let runId;
+            if (sync && entries.length > 0) {
+                ctx.err(dim("Importing results into Beryl…"));
+                const imported = (await ctx.client.request("POST", `${projectPath(workspaceId, projectId)}/runs/import`, {
+                    form: buildImportForm(entries, {
+                        environmentId: flagStr(input, "env"),
+                        targetUrlOverride: urlOverride,
+                        startedAt,
+                        completedAt: new Date().toISOString(),
+                        onNote: (line) => ctx.err(yellow(`! ${line}`)),
+                    }),
+                }));
+                runId = imported.id;
             }
-            const summary = { passed: outcome.passed, failed: outcome.failed, results: outcome.results };
-            const marker = (status) => status === "passed" || status === "expected"
-                ? green("✓")
-                : status === "skipped"
-                    ? dim("○")
-                    : red("✗");
-            const lines = outcome.results.map((r) => `  ${marker(r.status)} ${r.name}${r.error ? `\n    ${dim(r.error.split("\n")[0] ?? "")}` : ""}`);
-            const human = `${outcome.failed === 0 ? green("All tests passed") : red(`${outcome.failed} test(s) failed`)} ` +
-                `(${outcome.passed} passed, ${outcome.failed} failed)\n${lines.join("\n")}` +
-                (flagStr(input, "dir") ? `\n${dim(`Spec + artifacts + report in ${outcome.directory}`)}` : "");
-            const persisted = Boolean(flagStr(input, "dir"));
+            const summaryLine = failed === 0
+                ? green(`All tests passed (${passed}/${entries.length})`)
+                : red(`${failed} test(s) failed`) + ` (${passed} passed)`;
+            const human = `${summaryLine}` +
+                (skipped.length ? `\n${yellow(`${skipped.length} skipped (see notes above)`)}` : "") +
+                (runId ? `\nSynced to Beryl as run ${runId} — \`beryl runs get ${runId}\`` : "") +
+                (!sync ? `\n${dim("--no-sync: nothing recorded in Beryl")}` : "") +
+                (dir ? `\n${dim(`Specs + artifacts + reports under ${path.resolve(dir)}`)}` : "");
             return {
-                // Only surface on-disk paths when --dir kept them; without it the run dir is deleted.
-                data: persisted
-                    ? { ...summary, directory: outcome.directory, report: outcome.report }
-                    : summary,
+                data: {
+                    passed,
+                    failed,
+                    skipped: skipped.map((s) => s.id),
+                    results: entries.map(({ files: _files, ...rest }) => rest),
+                    ...(runId ? { run_id: runId } : {}),
+                },
                 human,
-                // Exit codes are a CI contract: any failing test → exit 1.
-                ...(outcome.failed > 0 ? { exitCode: 1 } : {}),
+                // Exit codes are a CI contract: any failing/errored test → exit 1.
+                ...(failed > 0 ? { exitCode: 1 } : {}),
             };
         },
     },