@beryl-so/cli 0.2.0 → 0.6.0

package/dist/beryl-test-skill.js

@@ -0,0 +1,136 @@
+// The `beryl-test` authoring skill, installed by `beryl init` into
+// `.agents/skills/beryl-test/SKILL.md` (vendor-neutral, editor-agnostic). Kept as an
+// embedded string so it ships in the published package (`files: ["dist"]`) with no
+// build-time asset copy, and so there is ONE source for the guidance — not a copy in
+// the CLI and another in the docs. Edit here; `init` writes it verbatim.
+export const BERYL_TEST_SKILL_FILENAME = "SKILL.md";
+export const BERYL_TEST_SKILL_DIR = "beryl-test";
+export const BERYL_TEST_SKILL = `---
+name: beryl-test
+description: Author durable, healable end-to-end tests for a web app with Beryl. Use when writing, running, or fixing a Beryl test locally with your own coding agent — drafting the plan over the Playwright MCP, writing the natural-language intent, and running it with \`beryl runs local\`.
+---
+
+# Authoring Beryl tests
+
+Beryl is automated web-app testing: a test drives a real browser through a flow and
+asserts the flow reached its meaningful outcome. \`beryl init\` has already wired two MCP
+servers for you — **beryl** (create/list/run tests) and **playwright** (drive a real
+browser). Your job is to author tests that keep passing as the app's markup drifts,
+because Beryl can **heal** them — but only when you give it what it needs to.
+
+Read this before authoring. The three ideas that make a test durable: a real **outcome
+assertion**, a strong **natural-language intent**, and the **local run-fix loop**.
+
+## 1. Author locally over the Playwright MCP
+
+1. **Drive the flow in a real browser first.** Use the Playwright MCP to open the app and
+   walk the flow by hand — log in, fill the form, submit, whatever the flow is. You act on
+   elements by their accessibility ref from the latest page snapshot, not a guessed
+   selector. Watch what actually happens; don't author from imagination.
+2. **Write it as an ActionPlan** — a JSON object whose \`steps\` are
+   \`{action, selector, url, value, ...}\`. Two structural rules the plan must satisfy:
+   - the **first executed step is a \`goto\`** (the flow has to start by navigating somewhere), and
+   - **at least one step is an \`expect\`** (a test that asserts nothing is not a test).
+   Optional \`before\` / \`after\` arrays hold setup and teardown; \`after\` runs even when a
+   main step fails, so a create/update/delete flow can clean up the record it made.
+3. **Validate offline, then create:**
+   \`\`\`
+   beryl tests lint --file plan.json        # check the plan against the schema, no network
+   beryl tests create --title "Log in" --file plan.json \\
+     --description "<the intent — see §3>"
+   \`\`\`
+   By default \`create\` verifies the plan in a real browser before accepting it. The full
+   ActionPlan JSON Schema is at
+   https://api.beryl.so/api/v1/schemas/action-plan.schema.json.
+
+### The outcome assertion is the whole game
+
+A flow is only worth banking if you can point at the **success signal** — the one
+observable proof the flow worked. Get this right and everything else follows.
+
+- The signal must be **true only if the flow succeeded**. A confirmation message that
+  appeared, an element that showed up or disappeared, content unique to where the flow
+  landed.
+- **Never assert global chrome** — the nav bar, logo, footer, or cookie banner is on every
+  page, so asserting it tests nothing. "Was there anyway" means site-wide chrome, NOT the
+  destination's own distinctive content.
+- For a **navigation** flow, the strongest signal is that the destination actually
+  **rendered**: assert its unique heading or a piece of content specific to that page (for
+  \`/pricing\`, the "Pricing" H1 or a plan name). Prefer that over the URL alone — a bare
+  "the URL is /pricing" passes even on a blank or broken page that never rendered.
+  Reserve a URL-only assertion for when the URL *is* the outcome (a form that lands on
+  \`/thank-you\`) and no distinctive destination content is available.
+- \`expect\` kinds: \`text\` (a string is visible), \`url\` (the URL contains a value), \`gone\`
+  (an element disappeared — e.g. a spinner, or the item you just deleted).
+- **If you can't name a success signal, the flow is not test-worthy.** Don't bank a test
+  that verifies nothing. Explore a different flow instead.
+- **Don't work around a real app failure to make a test go green.** If the flow is
+  genuinely broken in the app, that's a finding to report — not something to paper over
+  with a weaker assertion.
+
+## 2. What "durable" and "healable" mean here
+
+Beryl's cloud runs your test on a schedule. When the app's markup drifts and a selector
+stops matching, a heal-vs-fail agent decides whether to **heal** the test (silently
+re-derive the selector/trajectory and keep it green) or **fail** it (surface a real
+regression). It decides that against your test's **intent**:
+
+- **The natural-language intent is the immutable anchor. Beryl never rewrites it.** It's
+  the description of what the test proves — the load-bearing statement the heal agent
+  judges every future run against.
+- **Selectors and the trajectory are the healable "how".** A button moved, a class name
+  changed, a step needs an extra click — those are mechanics Beryl can re-derive on its
+  own, because your intent tells it what the flow was *for*.
+- **A failed outcome assertion is a real regression Beryl will NOT silently heal green.**
+  If the success signal from §1 stops holding — the confirmation never appears, the page
+  never renders — that's the app breaking, and the test fails loudly. That is the point.
+
+So a test is *healable* exactly when you gave it **a strong intent + a real outcome
+assertion**. A test with a vague intent and a chrome-only assertion is brittle: Beryl
+can't tell a real regression from cosmetic drift, so it either heals over real breakage or
+fails on noise.
+
+## 3. Writing the natural-language intent
+
+Pass the intent as \`--description\` on \`beryl tests create\` (or \`tests set-plan\` when you
+re-author). 1–3 sentences. This is the immutable anchor from §2 — write it well.
+
+- **State the purpose, not the steps.** Not "clicks Sign in, types email and password,
+  clicks submit" — that's the trajectory, which Beryl already has and which will change.
+  Instead: *what does a green run prove is true about the app?*
+- **Name the one observable outcome** that is true only if the flow worked — the same
+  success signal you asserted in §1, in words.
+- **Never describe global chrome.** The intent is about the flow's destination and
+  outcome, not "the header is present".
+
+Good:
+> "Proves a returning user can sign in: after submitting valid credentials, the
+> authenticated dashboard with the user's name in the header loads."
+
+Weak (describes steps + asserts nothing meaningful):
+> "Goes to /login, fills the form, and checks the page loaded."
+
+## 4. The local run-fix loop
+
+Iterate on your machine before you rely on the cloud. \`beryl runs local\` fetches the
+test's rendered spec and runs it with your local \`@playwright/test\` — no cloud, no waiting
+for a scheduled run.
+
+\`\`\`
+npm i -D @playwright/test && npx playwright install   # once
+beryl runs local <test-id> --url-override http://localhost:3000 --dir ./beryl-local
+\`\`\`
+
+- \`--url-override\` points the run at a local dev server or a preview deploy.
+- \`--dir\` keeps the **spec, artifacts, and a JSON \`report.json\`** on disk so you (or your
+  coding agent) can read exactly what happened and iterate: read the report, see which step
+  or assertion failed and why, fix the plan, \`beryl tests set-plan\`, run again.
+- It exits **0** if every test passed, **1** on a failure — so it drops straight into a
+  run-fix-run loop.
+- **v1 is public / unauthenticated flows only.** A test that signs in first runs only in
+  Beryl's cloud (which holds the encrypted session — it's never handed to your disk);
+  \`runs local\` refuses it. Run those with \`beryl runs trigger\`.
+
+Once the test passes locally against a real outcome, it's ready to bank and let Beryl run
+and heal it.
+`;

package/dist/commands/account.js

@@ -4,6 +4,7 @@ export const accountCommands = [
     {
         name: "account get",
         summary: "Show your account profile",
+        groupSummary: "View and update your personal account profile.",
         async run(ctx) {
             return { data: await ctx.client.get("/account/") };
         },
@@ -36,6 +37,7 @@ export const accountCommands = [
     {
         name: "feedback send",
         summary: "Send product feedback to the Beryl team",
+        groupSummary: "Send product feedback to the Beryl team.",
         args: [{ name: "message", description: "Your feedback", required: true }],
         async run(ctx, input) {
             return { data: await ctx.client.post("/feedback", { message: arg(input, "message") }) };
@@ -45,6 +47,7 @@ export const accountCommands = [
         name: "billing usage",
         summary: "Show plan usage: services and weekly AI units",
         scope: "workspace",
+        groupSummary: "Review a workspace's plan usage, subscription, and invoices.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             return { data: await ctx.client.get(`/workspaces/${ws}/billing/usage`) };

package/dist/commands/auth.js

@@ -54,7 +54,9 @@ export const authCommands = [
         summary: "Authenticate the CLI with your Beryl account",
         description: "Signs in with an emailed one-time code and mints a personal access token, " +
             "which is stored in the CLI config. Pass --token to use an existing token " +
-            "from Account → API tokens instead. In CI, prefer the BERYL_API_KEY environment variable.",
+            "from Account → API tokens instead. Pass --email plus --code (the 6 digits " +
+            "from the email, e.g. read from a `beryl inbox`) to complete the OTP flow " +
+            "without a prompt. In CI, prefer the BERYL_API_KEY environment variable.",
         interactive: true,
         flags: [
             { name: "token", type: "string", description: "Use an existing personal access token" },
@@ -64,26 +66,48 @@ export const authCommands = [
                 description: "Email for the one-time code sign-in (default: your last sign-in, or your git " +
                     "user.email — offered as the prompt default)",
             },
+            {
+                name: "code",
+                type: "string",
+                description: "The emailed 6-digit code — skips the prompt for non-interactive use " +
+                    "(requires --email; also skips sending a fresh code, so pair it with a " +
+                    "code already requested via `beryl login`, `beryl signup`, or the API)",
+            },
             {
                 name: "token-name",
                 type: "string",
                 description: "Name for the minted token (default: CLI on <hostname>)",
             },
         ],
-        examples: ["beryl login", "beryl login --token beryl_pat_…", "beryl login --email you@example.com"],
+        examples: [
+            "beryl login",
+            "beryl login --token beryl_pat_…",
+            "beryl login --email you@example.com",
+            "beryl login --email agent@example.com --code 123456 --json",
+        ],
         async run(ctx, input) {
             const apiUrl = ctx.client.baseUrl;
             let token = flagStr(input, "token");
             if (!token) {
+                const codeFlag = flagStr(input, "code")?.replace(/\s/g, "");
+                if (codeFlag !== undefined) {
+                    if (!/^\d{6}$/.test(codeFlag))
+                        throw new UsageError("--code must be the 6 digits from the email");
+                    if (!flagStr(input, "email"))
+                        throw new UsageError("--code requires --email");
+                }
                 const email = flagStr(input, "email") ?? (await promptForEmail(ctx));
                 if (!email.includes("@"))
                     throw new UsageError(`"${email}" is not an email address`);
                 const anon = new ApiClient(apiUrl);
-                await anon.post("/auth/request-login-otp", { email });
-                ctx.err(dim(`Sent a 6-digit code to ${email}`));
-                ctx.err(dim("Not arriving? Check spam; if you're new to Beryl, sign up at https://beryl.so first — " +
-                    "or use `beryl login --token` with a token from beryl.so → Account → API tokens."));
-                const code = await promptForOtpCode(ctx);
+                let code = codeFlag;
+                if (!code) {
+                    await anon.post("/auth/request-login-otp", { email });
+                    ctx.err(dim(`Sent a 6-digit code to ${email}`));
+                    ctx.err(dim("Not arriving? Check spam; if you're new to Beryl, sign up at https://beryl.so first — " +
+                        "or use `beryl login --token` with a token from beryl.so → Account → API tokens."));
+                    code = await promptForOtpCode(ctx);
+                }
                 const login = (await anon.post("/auth/verify-otp", { email, code }));
                 if (!login.access_token)
                     throw new CliError("Login did not return an access token");
@@ -125,6 +149,36 @@ export const authCommands = [
             };
         },
     },
+    {
+        name: "signup",
+        summary: "Register a new Beryl account (emails a 6-digit verification code)",
+        description: "Creates a passwordless account for the email and sends it a 6-digit code. " +
+            "Finish with `beryl login --email <addr> --code <the 6 digits>` — that verifies " +
+            "the account, creates its workspace, and signs the CLI in. With an inbox from " +
+            "`beryl inbox create` as the address, an agent can provision a fresh account " +
+            "end-to-end with no human at a prompt.",
+        flags: [
+            {
+                name: "email",
+                type: "string",
+                required: true,
+                description: "Email address for the new account",
+            },
+        ],
+        examples: ["beryl signup --email agent+ci@example.com --json"],
+        async run(ctx, input) {
+            const email = flagStr(input, "email") ?? "";
+            if (!email.includes("@"))
+                throw new UsageError(`"${email}" is not an email address`);
+            const anon = new ApiClient(ctx.client.baseUrl);
+            const created = (await anon.post("/auth/signup", { email }));
+            return {
+                data: created,
+                human: `${green("Signup started")} for ${email}\n` +
+                    dim(`Sent a 6-digit code to ${email} — finish with: beryl login --email ${email} --code <code>`),
+            };
+        },
+    },
     {
         name: "logout",
         summary: "Remove the stored token from the CLI config",
@@ -169,6 +223,7 @@ export const authCommands = [
     {
         name: "tokens list",
         summary: "List your personal access tokens",
+        groupSummary: "Manage the personal access tokens that authenticate the CLI and CI.",
         async run(ctx) {
             const tokens = (await ctx.client.get("/account/tokens"));
             return { data: tokens };

package/dist/commands/config-vars.js

@@ -17,6 +17,7 @@ export const configCommands = [
         name: "config vars list",
         summary: "List the project's config variables (visible to the agent during runs)",
         scope: "project",
+        groupSummary: "Manage the variables, secrets, and files the agent can use while exploring and running.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.get(`${configPath(workspaceId, projectId)}/variables`) };

package/dist/commands/credentials.js

@@ -1,11 +1,13 @@
+import { UsageError } from "../errors.js";
 import { dim, green, yellow } from "../output.js";
-import { arg, flagBool } from "./util.js";
+import { arg, flagBool, flagStr } from "./util.js";
 const capturePath = (ws, p) => `/auth-capture/workspaces/${ws}/projects/${p}/sessions`;
 export const credentialCommands = [
     {
         name: "credentials list",
         summary: "List the workspace's saved logins",
         scope: "workspace",
+        groupSummary: "Manage saved logins Beryl reuses to test behind authentication, and attach them to projects.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             return { data: await ctx.client.get(`/workspaces/${ws}/credentials`) };
@@ -103,11 +105,43 @@ export const credentialCommands = [
         name: "auth-capture start",
         summary: "Start a login-capture browser session for the project (non-interactive)",
         scope: "project",
+        groupSummary: "Drive a browser session that captures a target-site login for Beryl to reuse.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.post(capturePath(workspaceId, projectId)) };
         },
     },
+    {
+        name: "auth-capture login",
+        summary: "Log into the target site headlessly with credentials (no human at the browser)",
+        description: "Drives the login inside the capture session started by `auth-capture start`, so " +
+            "an agent can complete start → login → capture with zero human intervention. The " +
+            "credentials are sent to the server, typed into the target site over the wire, and " +
+            "never stored, logged, or returned — the captured session stays encrypted " +
+            "server-side. Follow with `auth-capture capture` to snapshot the authenticated session.",
+        scope: "project",
+        args: [
+            { name: "session-id", description: "Session id from auth-capture start", required: true },
+        ],
+        flags: [
+            { name: "username", type: "string", description: "Login username / email", required: true },
+            { name: "password", type: "string", description: "Login password", required: true },
+            {
+                name: "login-url",
+                type: "string",
+                description: "Explicit login page URL (defaults to the session's current page)",
+            },
+        ],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const username = flagStr(input, "username");
+            const password = flagStr(input, "password");
+            if (!username || !password)
+                throw new UsageError("--username and --password are required");
+            await ctx.client.post(`${capturePath(workspaceId, projectId)}/${arg(input, "session-id")}/login`, { username, password, login_url: flagStr(input, "login-url") ?? null });
+            return { human: "Logged in." };
+        },
+    },
     {
         name: "auth-capture capture",
         summary: "Capture the session after the user has logged in via the live-view URL",

package/dist/commands/environments.js

@@ -5,6 +5,7 @@ export const environmentCommands = [
         name: "envs list",
         summary: "List a project's environments",
         scope: "project",
+        groupSummary: "Manage a project's environments — the URLs and auth Beryl runs tests against.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.get(`${projectPath(workspaceId, projectId)}/environments`) };
@@ -95,6 +96,7 @@ export const environmentCommands = [
         name: "schedule get",
         summary: "Show the project's daily/weekly run schedule",
         scope: "project",
+        groupSummary: "View and set the schedule on which Beryl runs a project's tests automatically.",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return { data: await ctx.client.get(`${projectPath(workspaceId, projectId)}/schedule`) };

package/dist/commands/explorations.js

@@ -3,6 +3,7 @@ import { watchExploration } from "./watch.js";
 export const explorationCommands = [
     {
         name: "explorations list",
+        groupSummary: "Inspect the agent's exploration runs — how it crawled a site and authored its tests.",
         summary: "List the agent's exploration passes for a project",
         scope: "project",
         async run(ctx, input) {
@@ -34,6 +35,20 @@ export const explorationCommands = [
             };
         },
     },
+    {
+        name: "explorations cancel",
+        summary: "Cancel an in-flight exploration",
+        description: "Stops a running exploration without touching the project or its tests — the " +
+            "lever for a runaway/non-convergent pass burning agent budget.",
+        scope: "project",
+        args: [{ name: "exploration-id", description: "Exploration id", required: true }],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            return {
+                data: await ctx.client.post(`${projectPath(workspaceId, projectId)}/explorations/${arg(input, "exploration-id")}/cancel`),
+            };
+        },
+    },
     {
         name: "explorations watch",
         summary: "Stream an exploration live — watch the agent explore and author tests",

package/dist/commands/inboxes.js

@@ -0,0 +1,117 @@
+import { dim, green } from "../output.js";
+import { arg, flagBool, flagNum, flagStr } from "./util.js";
+// Mirrors the API's own extractor (email_inbox/step_resolver.py): a labelled digit run
+// ("your code is 654321") beats the bare fenced pattern, because a real sign-in mail is
+// full of innocent 4-8 digit runs — "© 2026", a support number — and the bare fence
+// would happily return the first of them.
+const CODE_PATTERN = /(?<!\d)(\d{4,8})(?!\d)/;
+const LABELLED_CODE_PATTERN = /(?:verification|security|one[\s-]?time|login|sign[\s-]?in|access|confirmation)?\s*(?:code|otp|passcode|pin)\b[^0-9]{0,20}(?<!\d)(\d{4,8})(?!\d)/i;
+function visibleText(html) {
+    return html
+        .replace(/<(style|script|head)\b[\s\S]*?<\/\1>/gi, " ")
+        .replace(/<[^>]+>/g, " ");
+}
+function extractCode(email) {
+    const body = email.body_text || (email.body_html ? visibleText(email.body_html) : "");
+    for (const pattern of [LABELLED_CODE_PATTERN, CODE_PATTERN]) {
+        for (const text of [body, email.subject ?? ""]) {
+            const match = text.match(pattern);
+            if (match)
+                return match[1];
+        }
+    }
+    return null;
+}
+export const inboxCommands = [
+    {
+        name: "inbox create",
+        summary: "Mint a disposable email inbox that Beryl receives mail for",
+        groupSummary: "Disposable email inboxes for testing flows that send mail — signups, OTPs, receipts.",
+        description: "Creates a receiving address under Beryl's inbound email domain and returns it. " +
+            "Use it wherever a test needs a real, readable mailbox — e.g. as the --email for " +
+            "`beryl signup`, then read the code back with `beryl inbox read --extract-code`.",
+        scope: "workspace",
+        flags: [
+            {
+                name: "ttl-hours",
+                type: "number",
+                description: "Hours before the inbox expires and stops receiving (1-168, default 24)",
+            },
+            { name: "project", type: "string", description: "Attach the inbox to a project id" },
+        ],
+        examples: ["beryl inbox create --json", "beryl inbox create --ttl-hours 2"],
+        async run(ctx, input) {
+            const ws = await ctx.requireWorkspace(input);
+            const inbox = (await ctx.client.post(`/workspaces/${ws}/inboxes`, {
+                ttl_hours: flagNum(input, "ttl-hours") ?? 24,
+                project_id: flagStr(input, "project") ?? null,
+            }));
+            return {
+                data: inbox,
+                human: `${green("Created")} inbox ${inbox.id}\n\n  ${inbox.address}\n\n` +
+                    dim(`Read it with: beryl inbox read ${inbox.id}`),
+            };
+        },
+    },
+    {
+        name: "inbox read",
+        summary: "Read the latest email from an inbox (waits for one to arrive)",
+        description: "Waits up to --timeout-s for a matching email and returns it (one blocking request; " +
+            "the server caps the wait at 50s — re-run to keep waiting). With --extract-code, " +
+            "also pulls the one-time code (4-8 digits) out of the body/subject — handy for " +
+            "completing `beryl login --email <addr> --code <code>` unattended. Exits non-zero " +
+            "if nothing arrives before the timeout.",
+        scope: "workspace",
+        args: [{ name: "inbox-id", description: "Inbox id from `beryl inbox create`", required: true }],
+        flags: [
+            {
+                name: "timeout-s",
+                type: "number",
+                description: "Seconds to wait for a matching email (0 = don't wait; max 50, default 30)",
+            },
+            { name: "since", type: "string", description: "Only emails received after this ISO timestamp" },
+            { name: "from-contains", type: "string", description: "Only emails whose sender contains this" },
+            {
+                name: "subject-contains",
+                type: "string",
+                description: "Only emails whose subject contains this",
+            },
+            {
+                name: "extract-code",
+                type: "boolean",
+                description: "Also return the one-time code found in the email as `code`",
+            },
+        ],
+        examples: [
+            "beryl inbox read ibx_123 --timeout-s 45 --json",
+            "beryl inbox read ibx_123 --subject-contains code --extract-code --json",
+        ],
+        async run(ctx, input) {
+            const ws = await ctx.requireWorkspace(input);
+            const email = (await ctx.client.get(`/workspaces/${ws}/inboxes/${arg(input, "inbox-id")}/emails/latest`, {
+                timeout_s: flagNum(input, "timeout-s"),
+                since: flagStr(input, "since"),
+                from_contains: flagStr(input, "from-contains"),
+                subject_contains: flagStr(input, "subject-contains"),
+            }));
+            if (!flagBool(input, "extract-code"))
+                return { data: email };
+            return { data: { ...email, code: extractCode(email) } };
+        },
+    },
+    {
+        name: "inbox emails",
+        summary: "List the emails an inbox has received",
+        scope: "workspace",
+        args: [{ name: "inbox-id", description: "Inbox id from `beryl inbox create`", required: true }],
+        flags: [
+            { name: "since", type: "string", description: "Only emails received after this ISO timestamp" },
+        ],
+        async run(ctx, input) {
+            const ws = await ctx.requireWorkspace(input);
+            return {
+                data: await ctx.client.get(`/workspaces/${ws}/inboxes/${arg(input, "inbox-id")}/emails`, { since: flagStr(input, "since") }),
+            };
+        },
+    },
+];