@beryl-so/cli 0.1.0 → 0.2.0

package/README.md

@@ -122,6 +122,8 @@ Every non-interactive command below is exposed as an MCP tool with the same name
 
 ### projects
 
+`beryl projects` with no subcommand runs `projects list`.
+
 | Command | Summary |
 | --- | --- |
 | `beryl projects list` | List projects in the workspace |
@@ -154,13 +156,19 @@ Every non-interactive command below is exposed as an MCP tool with the same name
 
 ### tests
 
+`beryl tests` with no subcommand runs `tests list`.
+
 | Command | Summary |
 | --- | --- |
+| `beryl tests lint` | Validate a plan JSON file offline, before sending it to the server |
 | `beryl tests list` | List the project's tests with their latest result |
 | `beryl tests get <test-id>` | Show one test |
 | `beryl tests plan <test-id>` | Print a test's current step plan (JSON) |
 | `beryl tests create` | Create a test case from a JSON action plan — for tests authored locally, e.g. by your coding agent |
 | `beryl tests set-plan <test-id>` | Replace a test's step plan from a JSON file (creates a new version) |
+| `beryl tests rename <test-id> <title>` | Rename a test |
+| `beryl tests quarantine <test-id> <state>` | Mute a flaky test: it keeps running, but its failures stop failing the run |
+| `beryl tests delete <test-id>` | Delete a test, its version history, and its results |
 | `beryl tests recompile <test-id>` | Validate + verify an edited plan against the live site before persisting |
 | `beryl tests versions <test-id>` | List a test's version history |
 | `beryl tests version <test-id> <version-no>` | Show one specific version of a test (including its plan) |
@@ -174,6 +182,8 @@ Every non-interactive command below is exposed as an MCP tool with the same name
 
 ### runs
 
+`beryl runs` with no subcommand runs `runs list`.
+
 | Command | Summary |
 | --- | --- |
 | `beryl runs trigger` | Trigger a test run (whole suite, a subset, or one environment) |
@@ -182,7 +192,7 @@ Every non-interactive command below is exposed as an MCP tool with the same name
 | `beryl runs watch <run-id>` | Attach to a run and stream progress until it finishes |
 | `beryl runs cancel <run-id>` | Cancel an in-flight run |
 | `beryl runs report <run-id>` | Show the generated report for a run |
-| `beryl runs download <run-id>` | Download a run's full results as JSON |
+| `beryl runs download <run-id>` | Download a run's results, with its artifacts, to disk |
 | `beryl runs explain <result-id>` | AI explanation of why a test result failed |
 
 ### explorations

package/dist/adapters/cli.js

@@ -90,6 +90,10 @@ export function parseArgv(spec, tokens) {
             throw new UsageError(`Unknown flag --${name} for \`beryl ${spec.name}\``);
         }
     }
+    // `--help` wins over every completeness check below: you ask for help precisely when you
+    // don't yet know which flags/args the command needs.
+    if (help)
+        return { input: { args: {}, flags }, json, apiUrl, token, help };
     for (const f of specFlags) {
         if (flags[f.name] === undefined && f.default !== undefined)
             flags[f.name] = f.default;
@@ -184,7 +188,8 @@ export function rootHelp() {
 function groupHelp(group, specs) {
     const lines = [`${bold("beryl " + group)} — subcommands:`, ""];
     for (const spec of specs) {
-        lines.push(`  ${cyan(spec.name.padEnd(28))} ${spec.summary}`);
+        const suffix = spec.groupDefault ? dim(`  (default — \`beryl ${group}\` runs this)`) : "";
+        lines.push(`  ${cyan(spec.name.padEnd(28))} ${spec.summary}${suffix}`);
     }
     lines.push("", `Run ${cyan(`beryl ${group} <subcommand> --help`)} for details.`);
     return lines.join("\n");
@@ -207,15 +212,25 @@ export async function runCli(argv) {
     }
     if (words.length === 0) {
         process.stdout.write(rootHelp() + "\n");
-        return argv.length === 0 || argv.includes("--help") || argv.includes("-h")
+        return rest.length === 0 || rest.includes("--help") || rest.includes("-h")
             ? EXIT_OK
             : EXIT_USAGE;
     }
+    // A group of subcommands, as opposed to a single-word command like `init`, which
+    // commandGroups() also keys under its own name.
+    const groupSpecs = words.length === 1 ? commandGroups().get(words[0]) : undefined;
+    const group = groupSpecs && !(groupSpecs.length === 1 && groupSpecs[0].name === words[0])
+        ? groupSpecs
+        : undefined;
+    // `beryl tests --help` asks about the group, not about the subcommand `beryl tests`
+    // happens to default to — so group help wins over the default's own help.
+    if (group && (rest.includes("--help") || rest.includes("-h"))) {
+        process.stdout.write(groupHelp(words[0], group) + "\n");
+        return EXIT_OK;
+    }
     const found = findCommand(words);
     if (!found) {
-        const groups = commandGroups();
-        const group = groups.get(words[0]);
-        if (group && words.length === 1) {
+        if (group) {
             process.stdout.write(groupHelp(words[0], group) + "\n");
             return EXIT_OK;
         }
@@ -245,7 +260,7 @@ export async function runCli(argv) {
     if (parsed.token)
         config.token = parsed.token;
     const client = new ApiClient(config.apiUrl, config.token);
-    const ctx = createContext({ client, config, json: parsed.json });
+    const ctx = createContext({ client, config, json: parsed.json, mcp: false });
     try {
         const result = (await spec.run(ctx, parsed.input)) ?? {};
         if (parsed.json) {

package/dist/adapters/mcp.js

@@ -31,6 +31,24 @@ export function toolInputSchema(spec) {
     }
     return { type: "object", properties, ...(required.length ? { required } : {}) };
 }
+export function toolResult(result, lines) {
+    const parts = [...lines];
+    if (result.data !== undefined)
+        parts.push(JSON.stringify(result.data, null, 2));
+    else if (result.human)
+        parts.push(result.human);
+    return {
+        content: [
+            { type: "text", text: parts.join("\n") || "ok" },
+            ...(result.images ?? []).map((image) => ({
+                type: "image",
+                data: image.data,
+                mimeType: image.mimeType,
+            })),
+        ],
+        isError: result.exitCode !== undefined && result.exitCode !== 0,
+    };
+}
 function toInput(spec, params) {
     const args = {};
     const flags = {};
@@ -76,20 +94,13 @@ export async function serveMcp(baseCtx) {
             config: baseCtx.config,
             json: true,
             interactive: false,
+            mcp: true,
             out: push,
             err: push,
         });
         try {
             const result = (await spec.run(ctx, toInput(spec, request.params.arguments ?? {}))) ?? {};
-            const parts = [...lines];
-            if (result.data !== undefined)
-                parts.push(JSON.stringify(result.data, null, 2));
-            else if (result.human)
-                parts.push(result.human);
-            return {
-                content: [{ type: "text", text: parts.join("\n") || "ok" }],
-                isError: result.exitCode !== undefined && result.exitCode !== 0,
-            };
+            return toolResult(result, lines);
         }
         catch (err) {
             const message = err instanceof CliError ? err.message : String(err);

package/dist/artifacts.js

@@ -0,0 +1,132 @@
+import fs from "node:fs";
+import path from "node:path";
+import { CliError } from "./errors.js";
+// Base64 inflates by 4/3, and the block is spent from the agent's context window —
+// a few MB of screenshot would blow it. Skipped shots are reported, never dropped.
+export const MAX_IMAGE_BYTES = 1_500_000;
+export const FAILING_STATUSES = new Set(["failed", "errored", "error"]);
+export function isFailing(status) {
+    return FAILING_STATUSES.has((status ?? "").toLowerCase());
+}
+export function resultsOf(run) {
+    return run.tests ?? run.test_results ?? [];
+}
+/** Directory name for one result. Sanitised: the id reaches us over the wire, and it
+ *  is joined onto a user-supplied path — a `../` in it must not escape the output dir. */
+export function resultId(result) {
+    const raw = result.test_result_id ?? result.id ?? result.test_case_id ?? "result";
+    const safe = raw.replace(/[^A-Za-z0-9._-]/g, "-").replace(/^[.]+/, "");
+    return safe || "result";
+}
+function extensionFor(url, fallback) {
+    // Presigned S3 URLs carry the signature in the query string — the extension is
+    // on the path, so parse rather than regex the whole URL.
+    try {
+        const ext = path.extname(new URL(url).pathname);
+        return ext || fallback;
+    }
+    catch {
+        return fallback;
+    }
+}
+function mimeFor(url) {
+    const ext = extensionFor(url, ".png").toLowerCase();
+    if (ext === ".jpg" || ext === ".jpeg")
+        return "image/jpeg";
+    if (ext === ".webp")
+        return "image/webp";
+    return "image/png";
+}
+async function fetchBytes(url) {
+    // Presigned URLs authenticate via their own signature — never attach the API
+    // bearer token, and never route them through ApiClient (it forces the API base).
+    let response;
+    try {
+        response = await fetch(url);
+    }
+    catch (err) {
+        throw new CliError(`Could not fetch artifact: ${err.message}`);
+    }
+    if (!response.ok) {
+        throw new CliError(`Could not fetch artifact (HTTP ${response.status}) — presigned URLs are ` +
+            "short-lived; re-run the command to get fresh ones.");
+    }
+    return Buffer.from(await response.arrayBuffer());
+}
+export async function fetchImage(url) {
+    const bytes = await fetchBytes(url);
+    if (bytes.byteLength > MAX_IMAGE_BYTES) {
+        throw new CliError(`Screenshot is ${bytes.byteLength} bytes — too large to inline ` +
+            `(limit ${MAX_IMAGE_BYTES}); fetch it from its URL instead.`);
+    }
+    return { data: bytes.toString("base64"), mimeType: mimeFor(url) };
+}
+/** Failure screenshots as MCP image content. Never fails the command — an agent
+ *  losing a picture must still get the run JSON — but it does say what it lost. */
+export async function failureImages(run, limit) {
+    const failing = resultsOf(run).filter((r) => isFailing(r.status) && r.screenshot_url);
+    const skipped = [];
+    if (failing.length > limit) {
+        skipped.push(`${failing.length - limit} more failure screenshot(s) not attached (limit ${limit}) — ` +
+            "fetch them from the screenshot_url values above.");
+    }
+    const settled = await Promise.allSettled(failing.slice(0, limit).map((r) => fetchImage(r.screenshot_url)));
+    const images = [];
+    settled.forEach((s, i) => {
+        if (s.status === "fulfilled")
+            images.push(s.value);
+        else
+            skipped.push(`${resultId(failing[i])}: ${s.reason.message}`);
+    });
+    return { images, skipped };
+}
+function pendingFor(result, withFrames) {
+    const id = resultId(result);
+    const items = [];
+    const add = (url, kind, name) => {
+        if (url)
+            items.push({ url, file: path.join(id, name), kind, testResultId: id });
+    };
+    const ext = (url, fallback) => extensionFor(url ?? "", fallback);
+    add(result.screenshot_url, "screenshot", `screenshot${ext(result.screenshot_url, ".png")}`);
+    add(result.dom_snapshot_url, "dom_snapshot", `dom-snapshot${ext(result.dom_snapshot_url, ".html")}`);
+    add(result.trace_url, "trace", `trace${ext(result.trace_url, ".zip")}`);
+    if (withFrames) {
+        (result.frame_urls ?? []).forEach((url, i) => {
+            const n = String(i + 1).padStart(3, "0");
+            add(url, "frame", path.join("frames", `${n}${extensionFor(url, ".png")}`));
+        });
+    }
+    return items;
+}
+const DOWNLOAD_CONCURRENCY = 6;
+export async function downloadRunArtifacts(run, directory, options = {}) {
+    const manifest = path.join(directory, "run.json");
+    fs.mkdirSync(directory, { recursive: true });
+    fs.writeFileSync(manifest, JSON.stringify(run, null, 2) + "\n");
+    // A green test's filmstrip is bulk nobody reads, and every URL here expires within
+    // the hour — so by default only failing tests bring their frames down.
+    const pending = resultsOf(run).flatMap((r) => pendingFor(r, options.allFrames === true || isFailing(r.status)));
+    const artifacts = [];
+    const failures = [];
+    let next = 0;
+    const worker = async () => {
+        while (next < pending.length) {
+            const item = pending[next++];
+            const target = path.join(directory, item.file);
+            try {
+                const bytes = await fetchBytes(item.url);
+                fs.mkdirSync(path.dirname(target), { recursive: true });
+                fs.writeFileSync(target, bytes);
+                artifacts.push({ test_result_id: item.testResultId, kind: item.kind, path: target });
+                options.onProgress?.(`${item.file} (${bytes.byteLength} bytes)`);
+            }
+            catch (err) {
+                // One expired/missing artifact must not cost us the rest of the bundle.
+                failures.push({ url: item.url, error: err.message });
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(DOWNLOAD_CONCURRENCY, pending.length) }, worker));
+    return { directory, manifest, artifacts, failures };
+}

package/dist/commands/auth.js

@@ -1,9 +1,53 @@
+import { execFileSync } from "node:child_process";
 import os from "node:os";
 import { saveGlobalConfig } from "../config.js";
 import { CliError, UsageError } from "../errors.js";
 import { ApiClient } from "../http.js";
 import { dim, green } from "../output.js";
 import { arg, flagBool, flagStr } from "./util.js";
+function gitEmail() {
+    try {
+        const email = execFileSync("git", ["config", "--get", "user.email"], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+        return email.includes("@") ? email : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+// The address the user most likely wants: whoever the stored token belongs to, else the
+// git identity. Only ever a prompt default — never signs anyone in without a confirm.
+async function suggestedEmail(ctx) {
+    if (ctx.config.token) {
+        try {
+            const me = (await ctx.client.get("/account/"));
+            if (me.email)
+                return me.email;
+        }
+        catch {
+            // a stale/invalid token is exactly why they're logging in again
+        }
+    }
+    return gitEmail();
+}
+async function promptForEmail(ctx) {
+    const suggestion = await suggestedEmail(ctx);
+    const answer = await ctx.prompt(suggestion ? `Email [${suggestion}]: ` : "Email: ");
+    return answer || suggestion || "";
+}
+export async function promptForOtpCode(ctx) {
+    let code = "";
+    for (let attempt = 0; attempt < 3 && !/^\d{6}$/.test(code); attempt++) {
+        if (attempt > 0)
+            ctx.err("The code is the 6 digits from the email.");
+        code = (await ctx.prompt("Code: ")).replace(/\s/g, "");
+    }
+    if (!/^\d{6}$/.test(code))
+        throw new UsageError("No valid 6-digit code entered");
+    return code;
+}
 export const authCommands = [
     {
         name: "login",
@@ -14,7 +58,12 @@ export const authCommands = [
         interactive: true,
         flags: [
             { name: "token", type: "string", description: "Use an existing personal access token" },
-            { name: "email", type: "string", description: "Email for the one-time code sign-in" },
+            {
+                name: "email",
+                type: "string",
+                description: "Email for the one-time code sign-in (default: your last sign-in, or your git " +
+                    "user.email — offered as the prompt default)",
+            },
             {
                 name: "token-name",
                 type: "string",
@@ -26,13 +75,15 @@ export const authCommands = [
             const apiUrl = ctx.client.baseUrl;
             let token = flagStr(input, "token");
             if (!token) {
-                const email = flagStr(input, "email") ?? (await ctx.prompt("Email: "));
+                const email = flagStr(input, "email") ?? (await promptForEmail(ctx));
                 if (!email.includes("@"))
                     throw new UsageError(`"${email}" is not an email address`);
                 const anon = new ApiClient(apiUrl);
                 await anon.post("/auth/request-login-otp", { email });
                 ctx.err(dim(`Sent a 6-digit code to ${email}`));
-                const code = await ctx.prompt("Code: ");
+                ctx.err(dim("Not arriving? Check spam; if you're new to Beryl, sign up at https://beryl.so first — " +
+                    "or use `beryl login --token` with a token from beryl.so → Account → API tokens."));
+                const code = await promptForOtpCode(ctx);
                 const login = (await anon.post("/auth/verify-otp", { email, code }));
                 if (!login.access_token)
                     throw new CliError("Login did not return an access token");
@@ -44,13 +95,33 @@ export const authCommands = [
             }
             const authed = new ApiClient(apiUrl, token);
             const me = (await authed.get("/account/"));
+            let sole;
+            let workspaceCount = 0;
+            try {
+                const workspaces = (await authed.get("/workspaces/"));
+                workspaceCount = workspaces.length;
+                if (workspaces.length === 1)
+                    sole = workspaces[0];
+            }
+            catch {
+                // pinning a default is a nicety — never fail the login over it
+            }
             const saved = saveGlobalConfig({
                 token,
                 api_url: apiUrl === "https://api.beryl.so" ? undefined : apiUrl,
+                ...(sole && sole.id !== ctx.config.workspace
+                    ? { workspace: sole.id, project: undefined }
+                    : {}),
             });
+            let human = `${green("Logged in")} as ${me.name} <${me.email}>`;
+            if (sole)
+                human += `\n${dim(`Default workspace: ${sole.name} (${sole.id})`)}`;
+            else if (workspaceCount > 1 && !ctx.config.workspace)
+                human += `\n${dim("Pick a default workspace with `beryl workspaces use <id>`")}`;
+            human += `\n${dim(`Token saved to ${saved}`)}`;
             return {
-                data: { email: me.email, name: me.name, config: saved },
-                human: `${green("Logged in")} as ${me.name} <${me.email}>\n${dim(`Token saved to ${saved}`)}`,
+                data: { email: me.email, name: me.name, workspace: sole?.id ?? null, config: saved },
+                human,
             };
         },
     },

package/dist/commands/init.js

@@ -86,7 +86,8 @@ export const initCommands = [
                 name: "local",
                 type: "boolean",
                 description: "Also wire the Playwright MCP so your coding agent can drive a local browser " +
-                    "(for authoring tests yourself)",
+                    "(for authoring tests yourself). Default: on whenever a coding agent is wired; " +
+                    "pass --no-local to skip it",
             },
         ],
         examples: [
@@ -161,7 +162,10 @@ export const initCommands = [
                     : choice === "none"
                         ? []
                         : [choice];
-            const local = flagBool(input, "local");
+            // Wiring an editor's MCP config at all implies the user wants to author tests there,
+            // and local authoring needs the Playwright MCP — so default it on. `--no-local` (parsed
+            // as an explicit false) opts out.
+            const local = input.flags.local ?? editors.length > 0;
             for (const editor of editors) {
                 const file = editor === "claude-code"
                     ? path.join(cwd, ".mcp.json")

package/dist/commands/projects.js

@@ -1,13 +1,43 @@
 import { saveGlobalConfig } from "../config.js";
+import { detectAuthGating } from "../detect.js";
 import { UsageError } from "../errors.js";
 import { dim, green } from "../output.js";
 import { arg, flagBool, flagNum, flagStr, projectPath } from "./util.js";
 import { pollCurrentExploration, watchExploration } from "./watch.js";
+// The API treats the auth choice as the visitor's answer and never infers it — a null
+// choice leaves the project waiting on a login capture that never comes — so resolve it
+// here. A saved login for the host settles it; otherwise probe the site.
+async function resolveAuthChoice(ctx, workspaceId, url) {
+    let reusable;
+    try {
+        reusable = (await ctx.client.get(`/workspaces/${workspaceId}/projects/reusable-auth`, {
+            url,
+        }));
+    }
+    catch {
+        // a probe is still worth trying if the lookup fails
+    }
+    if (reusable?.available) {
+        ctx.err(dim("Detected a gated site — a saved login for this host is already on file."));
+        return "gated";
+    }
+    const detected = await detectAuthGating(url);
+    if (detected.choice) {
+        ctx.err(dim(`Detected a ${detected.choice} site — ${detected.reason}.`));
+        return detected.choice;
+    }
+    const question = `Could not tell whether ${url} needs a login — ${detected.reason}.`;
+    if (!ctx.interactive)
+        throw new UsageError(`${question}\n\nPass --auth public or --auth gated to say which it is.`);
+    const answer = await ctx.prompt(`${question}\nDoes testing it require signing in? [y/N] `);
+    return /^y(es)?$/i.test(answer) ? "gated" : "public";
+}
 export const projectCommands = [
     {
         name: "projects list",
         summary: "List projects in the workspace",
         scope: "workspace",
+        groupDefault: true,
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             return { data: await ctx.client.get(`/workspaces/${ws}/projects`) };
@@ -32,7 +62,8 @@ export const projectCommands = [
                 name: "auth",
                 type: "string",
                 enum: ["public", "gated"],
-                description: "Whether the site needs a login (gated) or not (public)",
+                description: "Whether the site needs a login (gated) or not (public). Default: detected from the " +
+                    "site — only asked when detection is genuinely unsure",
             },
             {
                 name: "allow-mutations",
@@ -50,7 +81,7 @@ export const projectCommands = [
             { name: "timeout", type: "number", description: "With --watch: max minutes to wait" },
         ],
         examples: [
-            "beryl projects create https://app.example.com --auth public --watch",
+            "beryl projects create https://app.example.com --watch",
             "beryl projects create https://app.example.com --auth gated",
             "beryl projects create https://app.example.com --auth public --no-explore",
         ],
@@ -59,10 +90,11 @@ export const projectCommands = [
             if (noExplore && flagBool(input, "watch"))
                 throw new UsageError("--no-explore cannot be combined with --watch");
             const ws = await ctx.requireWorkspace(input);
-            const auth = flagStr(input, "auth");
+            const url = arg(input, "url");
+            const auth = flagStr(input, "auth") ?? (await resolveAuthChoice(ctx, ws, url));
             const created = (await ctx.client.post(`/workspaces/${ws}/projects`, {
-                root_url: arg(input, "url"),
-                requires_auth_choice: auth ?? null,
+                root_url: url,
+                requires_auth_choice: auth,
                 mutation_choice: flagBool(input, "allow-mutations") ? "allow_mutations" : null,
                 force_new_login: flagBool(input, "force-new-login"),
                 skip_exploration: noExplore,
@@ -73,6 +105,15 @@ export const projectCommands = [
                     human: `${green("Project created")}: ${created.project_id} ${dim("(no exploration started)")}\n` +
                         `Ready for your own tests — author a plan and push it with \`beryl tests create\`.`,
                 };
+            // The agent cannot start until a login is captured, so there is nothing to watch.
+            if (created.auth_status === "needs_capture")
+                return {
+                    data: created,
+                    human: `${green("Project created")}: ${created.project_id}\n` +
+                        `The site needs a login before the agent can explore it. Capture one with ` +
+                        `\`beryl credentials capture --project ${created.project_id}\` — the exploration ` +
+                        `starts as soon as you do.`,
+                };
             if (!flagBool(input, "watch"))
                 return { data: created };
             ctx.err(dim(`project ${created.project_id} created — waiting for the agent to start…`));