@beryl-so/cli 0.1.0 → 0.5.0

package/dist/adapters/mcp.js

@@ -11,6 +11,12 @@ export function toolName(spec) {
 export function mcpTools() {
     return commands.filter((c) => !c.interactive && !c.hidden && c.name !== "mcp");
 }
+const mcpToolNames = new Set(mcpTools().map(toolName));
+/** The tool name a command is exposed as under `beryl mcp`, or undefined if it isn't exposed. */
+export function mcpToolFor(spec) {
+    const name = toolName(spec);
+    return mcpToolNames.has(name) ? name : undefined;
+}
 export function toolInputSchema(spec) {
     const properties = {};
     const required = [];
@@ -31,6 +37,24 @@ export function toolInputSchema(spec) {
     }
     return { type: "object", properties, ...(required.length ? { required } : {}) };
 }
+export function toolResult(result, lines) {
+    const parts = [...lines];
+    if (result.data !== undefined)
+        parts.push(JSON.stringify(result.data, null, 2));
+    else if (result.human)
+        parts.push(result.human);
+    return {
+        content: [
+            { type: "text", text: parts.join("\n") || "ok" },
+            ...(result.images ?? []).map((image) => ({
+                type: "image",
+                data: image.data,
+                mimeType: image.mimeType,
+            })),
+        ],
+        isError: result.exitCode !== undefined && result.exitCode !== 0,
+    };
+}
 function toInput(spec, params) {
     const args = {};
     const flags = {};
@@ -76,20 +100,13 @@ export async function serveMcp(baseCtx) {
             config: baseCtx.config,
             json: true,
             interactive: false,
+            mcp: true,
             out: push,
             err: push,
         });
         try {
             const result = (await spec.run(ctx, toInput(spec, request.params.arguments ?? {}))) ?? {};
-            const parts = [...lines];
-            if (result.data !== undefined)
-                parts.push(JSON.stringify(result.data, null, 2));
-            else if (result.human)
-                parts.push(result.human);
-            return {
-                content: [{ type: "text", text: parts.join("\n") || "ok" }],
-                isError: result.exitCode !== undefined && result.exitCode !== 0,
-            };
+            return toolResult(result, lines);
         }
         catch (err) {
             const message = err instanceof CliError ? err.message : String(err);

package/dist/artifacts.js

@@ -0,0 +1,132 @@
+import fs from "node:fs";
+import path from "node:path";
+import { CliError } from "./errors.js";
+// Base64 inflates by 4/3, and the block is spent from the agent's context window —
+// a few MB of screenshot would blow it. Skipped shots are reported, never dropped.
+export const MAX_IMAGE_BYTES = 1_500_000;
+export const FAILING_STATUSES = new Set(["failed", "errored", "error"]);
+export function isFailing(status) {
+    return FAILING_STATUSES.has((status ?? "").toLowerCase());
+}
+export function resultsOf(run) {
+    return run.tests ?? run.test_results ?? [];
+}
+/** Directory name for one result. Sanitised: the id reaches us over the wire, and it
+ *  is joined onto a user-supplied path — a `../` in it must not escape the output dir. */
+export function resultId(result) {
+    const raw = result.test_result_id ?? result.id ?? result.test_case_id ?? "result";
+    const safe = raw.replace(/[^A-Za-z0-9._-]/g, "-").replace(/^[.]+/, "");
+    return safe || "result";
+}
+function extensionFor(url, fallback) {
+    // Presigned S3 URLs carry the signature in the query string — the extension is
+    // on the path, so parse rather than regex the whole URL.
+    try {
+        const ext = path.extname(new URL(url).pathname);
+        return ext || fallback;
+    }
+    catch {
+        return fallback;
+    }
+}
+function mimeFor(url) {
+    const ext = extensionFor(url, ".png").toLowerCase();
+    if (ext === ".jpg" || ext === ".jpeg")
+        return "image/jpeg";
+    if (ext === ".webp")
+        return "image/webp";
+    return "image/png";
+}
+async function fetchBytes(url) {
+    // Presigned URLs authenticate via their own signature — never attach the API
+    // bearer token, and never route them through ApiClient (it forces the API base).
+    let response;
+    try {
+        response = await fetch(url);
+    }
+    catch (err) {
+        throw new CliError(`Could not fetch artifact: ${err.message}`);
+    }
+    if (!response.ok) {
+        throw new CliError(`Could not fetch artifact (HTTP ${response.status}) — presigned URLs are ` +
+            "short-lived; re-run the command to get fresh ones.");
+    }
+    return Buffer.from(await response.arrayBuffer());
+}
+export async function fetchImage(url) {
+    const bytes = await fetchBytes(url);
+    if (bytes.byteLength > MAX_IMAGE_BYTES) {
+        throw new CliError(`Screenshot is ${bytes.byteLength} bytes — too large to inline ` +
+            `(limit ${MAX_IMAGE_BYTES}); fetch it from its URL instead.`);
+    }
+    return { data: bytes.toString("base64"), mimeType: mimeFor(url) };
+}
+/** Failure screenshots as MCP image content. Never fails the command — an agent
+ *  losing a picture must still get the run JSON — but it does say what it lost. */
+export async function failureImages(run, limit) {
+    const failing = resultsOf(run).filter((r) => isFailing(r.status) && r.screenshot_url);
+    const skipped = [];
+    if (failing.length > limit) {
+        skipped.push(`${failing.length - limit} more failure screenshot(s) not attached (limit ${limit}) — ` +
+            "fetch them from the screenshot_url values above.");
+    }
+    const settled = await Promise.allSettled(failing.slice(0, limit).map((r) => fetchImage(r.screenshot_url)));
+    const images = [];
+    settled.forEach((s, i) => {
+        if (s.status === "fulfilled")
+            images.push(s.value);
+        else
+            skipped.push(`${resultId(failing[i])}: ${s.reason.message}`);
+    });
+    return { images, skipped };
+}
+function pendingFor(result, withFrames) {
+    const id = resultId(result);
+    const items = [];
+    const add = (url, kind, name) => {
+        if (url)
+            items.push({ url, file: path.join(id, name), kind, testResultId: id });
+    };
+    const ext = (url, fallback) => extensionFor(url ?? "", fallback);
+    add(result.screenshot_url, "screenshot", `screenshot${ext(result.screenshot_url, ".png")}`);
+    add(result.dom_snapshot_url, "dom_snapshot", `dom-snapshot${ext(result.dom_snapshot_url, ".html")}`);
+    add(result.trace_url, "trace", `trace${ext(result.trace_url, ".zip")}`);
+    if (withFrames) {
+        (result.frame_urls ?? []).forEach((url, i) => {
+            const n = String(i + 1).padStart(3, "0");
+            add(url, "frame", path.join("frames", `${n}${extensionFor(url, ".png")}`));
+        });
+    }
+    return items;
+}
+const DOWNLOAD_CONCURRENCY = 6;
+export async function downloadRunArtifacts(run, directory, options = {}) {
+    const manifest = path.join(directory, "run.json");
+    fs.mkdirSync(directory, { recursive: true });
+    fs.writeFileSync(manifest, JSON.stringify(run, null, 2) + "\n");
+    // A green test's filmstrip is bulk nobody reads, and every URL here expires within
+    // the hour — so by default only failing tests bring their frames down.
+    const pending = resultsOf(run).flatMap((r) => pendingFor(r, options.allFrames === true || isFailing(r.status)));
+    const artifacts = [];
+    const failures = [];
+    let next = 0;
+    const worker = async () => {
+        while (next < pending.length) {
+            const item = pending[next++];
+            const target = path.join(directory, item.file);
+            try {
+                const bytes = await fetchBytes(item.url);
+                fs.mkdirSync(path.dirname(target), { recursive: true });
+                fs.writeFileSync(target, bytes);
+                artifacts.push({ test_result_id: item.testResultId, kind: item.kind, path: target });
+                options.onProgress?.(`${item.file} (${bytes.byteLength} bytes)`);
+            }
+            catch (err) {
+                // One expired/missing artifact must not cost us the rest of the bundle.
+                failures.push({ url: item.url, error: err.message });
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(DOWNLOAD_CONCURRENCY, pending.length) }, worker));
+    return { directory, manifest, artifacts, failures };
+}

package/dist/beryl-test-skill.js

@@ -0,0 +1,136 @@
+// The `beryl-test` authoring skill, installed by `beryl init` into
+// `.agents/skills/beryl-test/SKILL.md` (vendor-neutral, editor-agnostic). Kept as an
+// embedded string so it ships in the published package (`files: ["dist"]`) with no
+// build-time asset copy, and so there is ONE source for the guidance — not a copy in
+// the CLI and another in the docs. Edit here; `init` writes it verbatim.
+export const BERYL_TEST_SKILL_FILENAME = "SKILL.md";
+export const BERYL_TEST_SKILL_DIR = "beryl-test";
+export const BERYL_TEST_SKILL = `---
+name: beryl-test
+description: Author durable, healable end-to-end tests for a web app with Beryl. Use when writing, running, or fixing a Beryl test locally with your own coding agent — drafting the plan over the Playwright MCP, writing the natural-language intent, and running it with \`beryl runs local\`.
+---
+
+# Authoring Beryl tests
+
+Beryl is automated web-app testing: a test drives a real browser through a flow and
+asserts the flow reached its meaningful outcome. \`beryl init\` has already wired two MCP
+servers for you — **beryl** (create/list/run tests) and **playwright** (drive a real
+browser). Your job is to author tests that keep passing as the app's markup drifts,
+because Beryl can **heal** them — but only when you give it what it needs to.
+
+Read this before authoring. The three ideas that make a test durable: a real **outcome
+assertion**, a strong **natural-language intent**, and the **local run-fix loop**.
+
+## 1. Author locally over the Playwright MCP
+
+1. **Drive the flow in a real browser first.** Use the Playwright MCP to open the app and
+   walk the flow by hand — log in, fill the form, submit, whatever the flow is. You act on
+   elements by their accessibility ref from the latest page snapshot, not a guessed
+   selector. Watch what actually happens; don't author from imagination.
+2. **Write it as an ActionPlan** — a JSON object whose \`steps\` are
+   \`{action, selector, url, value, ...}\`. Two structural rules the plan must satisfy:
+   - the **first executed step is a \`goto\`** (the flow has to start by navigating somewhere), and
+   - **at least one step is an \`expect\`** (a test that asserts nothing is not a test).
+   Optional \`before\` / \`after\` arrays hold setup and teardown; \`after\` runs even when a
+   main step fails, so a create/update/delete flow can clean up the record it made.
+3. **Validate offline, then create:**
+   \`\`\`
+   beryl tests lint --file plan.json        # check the plan against the schema, no network
+   beryl tests create --title "Log in" --file plan.json \\
+     --description "<the intent — see §3>"
+   \`\`\`
+   By default \`create\` verifies the plan in a real browser before accepting it. The full
+   ActionPlan JSON Schema is at
+   https://api.beryl.so/api/v1/schemas/action-plan.schema.json.
+
+### The outcome assertion is the whole game
+
+A flow is only worth banking if you can point at the **success signal** — the one
+observable proof the flow worked. Get this right and everything else follows.
+
+- The signal must be **true only if the flow succeeded**. A confirmation message that
+  appeared, an element that showed up or disappeared, content unique to where the flow
+  landed.
+- **Never assert global chrome** — the nav bar, logo, footer, or cookie banner is on every
+  page, so asserting it tests nothing. "Was there anyway" means site-wide chrome, NOT the
+  destination's own distinctive content.
+- For a **navigation** flow, the strongest signal is that the destination actually
+  **rendered**: assert its unique heading or a piece of content specific to that page (for
+  \`/pricing\`, the "Pricing" H1 or a plan name). Prefer that over the URL alone — a bare
+  "the URL is /pricing" passes even on a blank or broken page that never rendered.
+  Reserve a URL-only assertion for when the URL *is* the outcome (a form that lands on
+  \`/thank-you\`) and no distinctive destination content is available.
+- \`expect\` kinds: \`text\` (a string is visible), \`url\` (the URL contains a value), \`gone\`
+  (an element disappeared — e.g. a spinner, or the item you just deleted).
+- **If you can't name a success signal, the flow is not test-worthy.** Don't bank a test
+  that verifies nothing. Explore a different flow instead.
+- **Don't work around a real app failure to make a test go green.** If the flow is
+  genuinely broken in the app, that's a finding to report — not something to paper over
+  with a weaker assertion.
+
+## 2. What "durable" and "healable" mean here
+
+Beryl's cloud runs your test on a schedule. When the app's markup drifts and a selector
+stops matching, a heal-vs-fail agent decides whether to **heal** the test (silently
+re-derive the selector/trajectory and keep it green) or **fail** it (surface a real
+regression). It decides that against your test's **intent**:
+
+- **The natural-language intent is the immutable anchor. Beryl never rewrites it.** It's
+  the description of what the test proves — the load-bearing statement the heal agent
+  judges every future run against.
+- **Selectors and the trajectory are the healable "how".** A button moved, a class name
+  changed, a step needs an extra click — those are mechanics Beryl can re-derive on its
+  own, because your intent tells it what the flow was *for*.
+- **A failed outcome assertion is a real regression Beryl will NOT silently heal green.**
+  If the success signal from §1 stops holding — the confirmation never appears, the page
+  never renders — that's the app breaking, and the test fails loudly. That is the point.
+
+So a test is *healable* exactly when you gave it **a strong intent + a real outcome
+assertion**. A test with a vague intent and a chrome-only assertion is brittle: Beryl
+can't tell a real regression from cosmetic drift, so it either heals over real breakage or
+fails on noise.
+
+## 3. Writing the natural-language intent
+
+Pass the intent as \`--description\` on \`beryl tests create\` (or \`tests set-plan\` when you
+re-author). 1–3 sentences. This is the immutable anchor from §2 — write it well.
+
+- **State the purpose, not the steps.** Not "clicks Sign in, types email and password,
+  clicks submit" — that's the trajectory, which Beryl already has and which will change.
+  Instead: *what does a green run prove is true about the app?*
+- **Name the one observable outcome** that is true only if the flow worked — the same
+  success signal you asserted in §1, in words.
+- **Never describe global chrome.** The intent is about the flow's destination and
+  outcome, not "the header is present".
+
+Good:
+> "Proves a returning user can sign in: after submitting valid credentials, the
+> authenticated dashboard with the user's name in the header loads."
+
+Weak (describes steps + asserts nothing meaningful):
+> "Goes to /login, fills the form, and checks the page loaded."
+
+## 4. The local run-fix loop
+
+Iterate on your machine before you rely on the cloud. \`beryl runs local\` fetches the
+test's rendered spec and runs it with your local \`@playwright/test\` — no cloud, no waiting
+for a scheduled run.
+
+\`\`\`
+npm i -D @playwright/test && npx playwright install   # once
+beryl runs local <test-id> --url-override http://localhost:3000 --dir ./beryl-local
+\`\`\`
+
+- \`--url-override\` points the run at a local dev server or a preview deploy.
+- \`--dir\` keeps the **spec, artifacts, and a JSON \`report.json\`** on disk so you (or your
+  coding agent) can read exactly what happened and iterate: read the report, see which step
+  or assertion failed and why, fix the plan, \`beryl tests set-plan\`, run again.
+- It exits **0** if every test passed, **1** on a failure — so it drops straight into a
+  run-fix-run loop.
+- **v1 is public / unauthenticated flows only.** A test that signs in first runs only in
+  Beryl's cloud (which holds the encrypted session — it's never handed to your disk);
+  \`runs local\` refuses it. Run those with \`beryl runs trigger\`.
+
+Once the test passes locally against a real outcome, it's ready to bank and let Beryl run
+and heal it.
+`;

package/dist/commands/account.js

@@ -4,6 +4,7 @@ export const accountCommands = [
     {
         name: "account get",
         summary: "Show your account profile",
+        groupSummary: "View and update your personal account profile.",
         async run(ctx) {
             return { data: await ctx.client.get("/account/") };
         },
@@ -36,6 +37,7 @@ export const accountCommands = [
     {
         name: "feedback send",
         summary: "Send product feedback to the Beryl team",
+        groupSummary: "Send product feedback to the Beryl team.",
         args: [{ name: "message", description: "Your feedback", required: true }],
         async run(ctx, input) {
             return { data: await ctx.client.post("/feedback", { message: arg(input, "message") }) };
@@ -45,6 +47,7 @@ export const accountCommands = [
         name: "billing usage",
         summary: "Show plan usage: services and weekly AI units",
         scope: "workspace",
+        groupSummary: "Review a workspace's plan usage, subscription, and invoices.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             return { data: await ctx.client.get(`/workspaces/${ws}/billing/usage`) };

package/dist/commands/auth.js

@@ -1,9 +1,53 @@
+import { execFileSync } from "node:child_process";
 import os from "node:os";
 import { saveGlobalConfig } from "../config.js";
 import { CliError, UsageError } from "../errors.js";
 import { ApiClient } from "../http.js";
 import { dim, green } from "../output.js";
 import { arg, flagBool, flagStr } from "./util.js";
+function gitEmail() {
+    try {
+        const email = execFileSync("git", ["config", "--get", "user.email"], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+        return email.includes("@") ? email : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+// The address the user most likely wants: whoever the stored token belongs to, else the
+// git identity. Only ever a prompt default — never signs anyone in without a confirm.
+async function suggestedEmail(ctx) {
+    if (ctx.config.token) {
+        try {
+            const me = (await ctx.client.get("/account/"));
+            if (me.email)
+                return me.email;
+        }
+        catch {
+            // a stale/invalid token is exactly why they're logging in again
+        }
+    }
+    return gitEmail();
+}
+async function promptForEmail(ctx) {
+    const suggestion = await suggestedEmail(ctx);
+    const answer = await ctx.prompt(suggestion ? `Email [${suggestion}]: ` : "Email: ");
+    return answer || suggestion || "";
+}
+export async function promptForOtpCode(ctx) {
+    let code = "";
+    for (let attempt = 0; attempt < 3 && !/^\d{6}$/.test(code); attempt++) {
+        if (attempt > 0)
+            ctx.err("The code is the 6 digits from the email.");
+        code = (await ctx.prompt("Code: ")).replace(/\s/g, "");
+    }
+    if (!/^\d{6}$/.test(code))
+        throw new UsageError("No valid 6-digit code entered");
+    return code;
+}
 export const authCommands = [
     {
         name: "login",
@@ -14,7 +58,12 @@ export const authCommands = [
         interactive: true,
         flags: [
             { name: "token", type: "string", description: "Use an existing personal access token" },
-            { name: "email", type: "string", description: "Email for the one-time code sign-in" },
+            {
+                name: "email",
+                type: "string",
+                description: "Email for the one-time code sign-in (default: your last sign-in, or your git " +
+                    "user.email — offered as the prompt default)",
+            },
             {
                 name: "token-name",
                 type: "string",
@@ -26,13 +75,15 @@ export const authCommands = [
             const apiUrl = ctx.client.baseUrl;
             let token = flagStr(input, "token");
             if (!token) {
-                const email = flagStr(input, "email") ?? (await ctx.prompt("Email: "));
+                const email = flagStr(input, "email") ?? (await promptForEmail(ctx));
                 if (!email.includes("@"))
                     throw new UsageError(`"${email}" is not an email address`);
                 const anon = new ApiClient(apiUrl);
                 await anon.post("/auth/request-login-otp", { email });
                 ctx.err(dim(`Sent a 6-digit code to ${email}`));
-                const code = await ctx.prompt("Code: ");
+                ctx.err(dim("Not arriving? Check spam; if you're new to Beryl, sign up at https://beryl.so first — " +
+                    "or use `beryl login --token` with a token from beryl.so → Account → API tokens."));
+                const code = await promptForOtpCode(ctx);
                 const login = (await anon.post("/auth/verify-otp", { email, code }));
                 if (!login.access_token)
                     throw new CliError("Login did not return an access token");
@@ -44,13 +95,33 @@ export const authCommands = [
             }
             const authed = new ApiClient(apiUrl, token);
             const me = (await authed.get("/account/"));
+            let sole;
+            let workspaceCount = 0;
+            try {
+                const workspaces = (await authed.get("/workspaces/"));
+                workspaceCount = workspaces.length;
+                if (workspaces.length === 1)
+                    sole = workspaces[0];
+            }
+            catch {
+                // pinning a default is a nicety — never fail the login over it
+            }
             const saved = saveGlobalConfig({
                 token,
                 api_url: apiUrl === "https://api.beryl.so" ? undefined : apiUrl,
+                ...(sole && sole.id !== ctx.config.workspace
+                    ? { workspace: sole.id, project: undefined }
+                    : {}),
             });
+            let human = `${green("Logged in")} as ${me.name} <${me.email}>`;
+            if (sole)
+                human += `\n${dim(`Default workspace: ${sole.name} (${sole.id})`)}`;
+            else if (workspaceCount > 1 && !ctx.config.workspace)
+                human += `\n${dim("Pick a default workspace with `beryl workspaces use <id>`")}`;
+            human += `\n${dim(`Token saved to ${saved}`)}`;
             return {
-                data: { email: me.email, name: me.name, config: saved },
-                human: `${green("Logged in")} as ${me.name} <${me.email}>\n${dim(`Token saved to ${saved}`)}`,
+                data: { email: me.email, name: me.name, workspace: sole?.id ?? null, config: saved },
+                human,
             };
         },
     },
@@ -98,6 +169,7 @@ export const authCommands = [
     {
         name: "tokens list",
         summary: "List your personal access tokens",
+        groupSummary: "Manage the personal access tokens that authenticate the CLI and CI.",
         async run(ctx) {
             const tokens = (await ctx.client.get("/account/tokens"));
             return { data: tokens };

package/dist/commands/config-vars.js

@@ -17,6 +17,7 @@ export const configCommands = [
         name: "config vars list",
         summary: "List the project's config variables (visible to the agent during runs)",
         scope: "project",
+        groupSummary: "Manage the variables, secrets, and files the agent can use while exploring and running.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.get(`${configPath(workspaceId, projectId)}/variables`) };

package/dist/commands/credentials.js

@@ -6,6 +6,7 @@ export const credentialCommands = [
         name: "credentials list",
         summary: "List the workspace's saved logins",
         scope: "workspace",
+        groupSummary: "Manage saved logins Beryl reuses to test behind authentication, and attach them to projects.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             return { data: await ctx.client.get(`/workspaces/${ws}/credentials`) };
@@ -103,6 +104,7 @@ export const credentialCommands = [
         name: "auth-capture start",
         summary: "Start a login-capture browser session for the project (non-interactive)",
         scope: "project",
+        groupSummary: "Drive a browser session that captures a target-site login for Beryl to reuse.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.post(capturePath(workspaceId, projectId)) };

package/dist/commands/environments.js

@@ -5,6 +5,7 @@ export const environmentCommands = [
         name: "envs list",
         summary: "List a project's environments",
         scope: "project",
+        groupSummary: "Manage a project's environments — the URLs and auth Beryl runs tests against.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.get(`${projectPath(workspaceId, projectId)}/environments`) };
@@ -95,6 +96,7 @@ export const environmentCommands = [
         name: "schedule get",
         summary: "Show the project's daily/weekly run schedule",
         scope: "project",
+        groupSummary: "View and set the schedule on which Beryl runs a project's tests automatically.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.get(`${projectPath(workspaceId, projectId)}/schedule`) };

package/dist/commands/explorations.js

@@ -3,6 +3,7 @@ import { watchExploration } from "./watch.js";
 export const explorationCommands = [
     {
         name: "explorations list",
+        groupSummary: "Inspect the agent's exploration runs — how it crawled a site and authored its tests.",
         summary: "List the agent's exploration passes for a project",
         scope: "project",
         async run(ctx, input) {