@beryl-so/cli 0.2.0 → 0.6.0

package/dist/commands/init.js

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
+import { BERYL_TEST_SKILL, BERYL_TEST_SKILL_DIR, BERYL_TEST_SKILL_FILENAME, } from "../beryl-test-skill.js";
 import { LOCAL_CONFIG_FILENAME, loadConfig } from "../config.js";
 import { CliError, UsageError } from "../errors.js";
 import { ApiClient } from "../http.js";
@@ -14,6 +15,22 @@ const PLAYWRIGHT_SERVER_ENTRY = {
     command: "npx",
     args: ["@playwright/mcp@latest"],
 };
+const ACTION_PLAN_SCHEMA_URL = "https://api.beryl.so/api/v1/schemas/action-plan.schema.json";
+// The authoring fork only appears when we create a fresh project with no tests yet; an
+// explicit flag wins, otherwise a TTY prompt defaults to local on enter, and a
+// non-interactive run defaults to local so CI never burns server exploration on a project
+// the user means to hand-author.
+async function resolveAuthoring(ctx, flag) {
+    if (flag === "local" || flag === "agent")
+        return flag;
+    if (!ctx.interactive)
+        return "local";
+    ctx.err(`\n${bold("How do you want to author tests for this project?")}`);
+    ctx.err(`  ${green("1")}. Local / manual — with your own coding agent (Playwright MCP) or by hand ${dim("(default)")}`);
+    ctx.err(`  2. Beryl agent explores the site and authors tests for you`);
+    const answer = await ctx.prompt("Authoring [1-2]: ");
+    return answer.trim() === "2" ? "agent" : "local";
+}
 async function pick(ctx, kind, items) {
     if (items.length === 1)
         return items[0];
@@ -62,19 +79,53 @@ function detectEditors(cwd) {
         editors.push("cursor");
     return editors;
 }
+// The authoring skill lives in the vendor-neutral `.agents/skills/` dir (mirroring
+// Momentic), NOT `.claude/` — any coding agent that reads `.agents/skills/` picks it up.
+// Idempotent: an identical copy is left alone; a customer-EDITED copy is never clobbered —
+// we notice and skip so their changes survive a re-run.
+function writeSkill(cwd) {
+    const file = path.join(cwd, ".agents", "skills", BERYL_TEST_SKILL_DIR, BERYL_TEST_SKILL_FILENAME);
+    if (fs.existsSync(file)) {
+        // Compare with line endings normalized so a CRLF checkout of our own content still
+        // reads as unchanged (not falsely "customized") — we always write LF.
+        const norm = (s) => s.replace(/\r\n/g, "\n");
+        const current = fs.readFileSync(file, "utf8");
+        if (norm(current) === norm(BERYL_TEST_SKILL))
+            return { outcome: "unchanged", file };
+        return { outcome: "customized", file };
+    }
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.writeFileSync(file, BERYL_TEST_SKILL);
+    return { outcome: "wrote", file };
+}
 export const initCommands = [
     {
         name: "init",
         summary: "Set up Beryl in this repo — sign in, pin a project, wire up your coding agent",
         description: "One-command onboarding: signs you in (emailed one-time code), pins this repo to a " +
-            "workspace and project via .beryl.json (offering to create the project — the agent " +
-            "starts exploring and authoring tests immediately), and writes the MCP server config " +
-            "for your coding agent (.mcp.json for Claude Code, .cursor/mcp.json for Cursor). " +
-            "Safe to re-run; every step skips what is already set up.",
+            "workspace and project via .beryl.json (offering to create the project), and writes the " +
+            "MCP server config for your coding agent (.mcp.json for Claude Code, .cursor/mcp.json for " +
+            "Cursor). When creating a fresh project it asks how you want to author tests — locally with " +
+            "your own coding agent or by hand (the default), or by letting Beryl's agent explore and " +
+            "author them for you. Safe to re-run; every step skips what is already set up.",
         interactive: true,
         flags: [
             { name: "workspace", type: "string", description: "Workspace id or name to pin" },
-            { name: "project", type: "string", description: "Project id, name, or URL to pin" },
+            {
+                name: "project",
+                type: "string",
+                description: "Project id, name, or URL to pin. With no projects yet, a URL here is the root URL of " +
+                    "the project to create — supply it (plus --authoring) to run init non-interactively in CI",
+            },
+            {
+                name: "authoring",
+                type: "string",
+                enum: ["local", "agent"],
+                description: "How to author tests for a newly created project: `local` (author yourself with your " +
+                    "coding agent or by hand — the project is created without server exploration) or " +
+                    "`agent` (Beryl's agent explores the site and authors tests). Default: local. " +
+                    "Pass this to skip the interactive prompt in CI",
+            },
             {
                 name: "editor-tools",
                 type: "string",
@@ -93,7 +144,8 @@ export const initCommands = [
         examples: [
             "npx @beryl-so/cli@latest init",
             "beryl init --editor-tools claude-code",
-            "beryl init --project https://app.example.com --editor-tools none",
+            "beryl init --project https://app.example.com --authoring agent",
+            "beryl init --project https://app.example.com --authoring local --editor-tools none",
         ],
         async run(ctx, input) {
             const cwd = process.cwd();
@@ -112,6 +164,7 @@ export const initCommands = [
             }
             let workspaceId;
             let projectId;
+            let authoredLocally = false;
             if (!flagBool(input, "no-pin")) {
                 const workspaces = (await client.get("/workspaces/"));
                 if (workspaces.length === 0)
@@ -125,7 +178,9 @@ export const initCommands = [
                 workspaceId = workspace.id;
                 const projects = (await client.get(`/workspaces/${workspaceId}/projects`));
                 const wantedProject = flagStr(input, "project");
-                if (wantedProject) {
+                // With projects already present, --project pins an existing one; with none, it's the
+                // root URL of the project to create — which is what makes a fresh init CI-drivable.
+                if (wantedProject && projects.length > 0) {
                     projectId = (await ctx.requireProject({
                         args: {},
                         flags: { workspace: workspaceId, project: wantedProject },
@@ -135,13 +190,19 @@ export const initCommands = [
                     projectId = (await pick(ctx, "project", projects)).id;
                 }
                 else {
-                    const url = await ctx.prompt("No projects yet. Root URL of the site to test: ");
+                    const url = wantedProject ?? (await ctx.prompt("No projects yet. Root URL of the site to test: "));
+                    const authoring = await resolveAuthoring(ctx, flagStr(input, "authoring"));
+                    authoredLocally = authoring === "local";
                     const created = (await client.post(`/workspaces/${workspaceId}/projects`, {
                         root_url: url,
+                        skip_exploration: authoredLocally,
                     }));
                     projectId = created.project_id;
-                    ctx.err(`${green("✓")} Project created — the agent is exploring ${url} and authoring tests ` +
-                        `(watch with \`beryl explorations watch\`)`);
+                    if (authoredLocally)
+                        ctx.err(`${green("✓")} Project created for ${url} ${dim("(no server exploration — you'll author the tests)")}`);
+                    else
+                        ctx.err(`${green("✓")} Project created — the agent is exploring ${url} and authoring tests ` +
+                            `(watch with \`beryl explorations watch\`)`);
                 }
                 const localFile = path.join(cwd, LOCAL_CONFIG_FILENAME);
                 let local = {};
@@ -177,16 +238,34 @@ export const initCommands = [
             }
             if (choice === "auto" && editors.length === 0)
                 ctx.err(dim("No coding agent detected — pass --editor-tools claude-code|cursor to wire one."));
-            return {
-                data: { workspace: workspaceId ?? null, project: projectId ?? null, editors },
-                human: `\n${bold("Beryl is set up.")} Try:\n` +
+            // Editor-agnostic: the authoring skill goes to `.agents/skills/` regardless of which
+            // (if any) editor MCP config we wrote, so any `.agents/skills/`-aware harness gets it.
+            const skill = writeSkill(cwd);
+            const skillRel = path.relative(cwd, skill.file);
+            if (skill.outcome === "customized")
+                ctx.err(`${dim("•")} Beryl authoring skill left as-is ${dim(`(${skillRel} — you edited it; delete it to reinstall)`)}`);
+            else
+                ctx.err(`${green("✓")} Beryl authoring skill ${skill.outcome === "wrote" ? "installed" : "already installed"} ${dim(skillRel)}`);
+            const nextSteps = authoredLocally
+                ? `\n${bold("Beryl is set up — author your tests locally.")} Next steps:\n` +
+                    (local
+                        ? `  ${dim("• Playwright MCP is wired — point your coding agent at it to drive a real browser and draft a plan.")}\n`
+                        : `  ${dim("• Re-run with --editor-tools to wire the Playwright MCP for your coding agent, or author a plan by hand.")}\n`) +
+                    `  ${cyan("beryl tests lint --file plan.json")}     validate a plan offline against the schema\n` +
+                    `  ${cyan("beryl tests create --file plan.json")}   push a test from your plan\n` +
+                    `  ${cyan("beryl runs trigger --watch")}            run the suite\n\n` +
+                    `Author against the ActionPlan JSON Schema: ${cyan(ACTION_PLAN_SCHEMA_URL)}`
+                : `\n${bold("Beryl is set up.")} Try:\n` +
                     `  ${cyan("beryl runs trigger --watch")}     run the suite\n` +
                     `  ${cyan("beryl explorations watch")}       watch the agent work\n` +
                     `  ${cyan("beryl tests list")}               see authored tests` +
                     (local
                         ? `\n\nPlaywright MCP is wired — your coding agent can explore the site locally ` +
                             `and push tests with ${cyan("beryl tests create")}.`
-                        : ""),
+                        : "");
+            return {
+                data: { workspace: workspaceId ?? null, project: projectId ?? null, editors },
+                human: nextSteps,
             };
         },
     },

package/dist/commands/projects.js

@@ -38,6 +38,7 @@ export const projectCommands = [
         summary: "List projects in the workspace",
         scope: "workspace",
         groupDefault: true,
+        groupSummary: "Create and manage projects — a site Beryl explores, authors tests for, and runs.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             return { data: await ctx.client.get(`/workspaces/${ws}/projects`) };

package/dist/commands/runs.js

@@ -1,9 +1,24 @@
 import fs from "node:fs";
 import { downloadRunArtifacts, failureImages, isFailing, resultsOf, } from "../artifacts.js";
-import { dim, yellow } from "../output.js";
+import { CliError, UsageError } from "../errors.js";
+import { runSpecLocally } from "../local-run.js";
+import { dim, green, red, yellow } from "../output.js";
 import { arg, flagBool, flagNum, flagStr, projectPath } from "./util.js";
 import { watchRun } from "./watch.js";
 const MAX_FAILURE_SCREENSHOTS = 5;
+export function parseHeaders(raw) {
+    if (!raw || raw.length === 0)
+        return null;
+    const headers = {};
+    for (const entry of raw) {
+        const eq = entry.indexOf("=");
+        const key = eq > 0 ? entry.slice(0, eq).trim() : "";
+        if (!key)
+            throw new UsageError(`--header must be KEY=VALUE, got: ${entry}`);
+        headers[key] = entry.slice(eq + 1);
+    }
+    return headers;
+}
 export const runCommands = [
     {
         name: "runs trigger",
@@ -15,6 +30,12 @@ export const runCommands = [
             { name: "test", type: "strings", description: "Run only these test ids (repeatable)" },
             { name: "env", type: "string", description: "Environment id to run against" },
             { name: "url-override", type: "string", description: "Replace the base URL (preview deploys)" },
+            {
+                name: "header",
+                type: "strings",
+                description: "Send a custom request header on every navigation, KEY=VALUE (repeatable). " +
+                    "Reaches auth-walled preview deploys, e.g. --header x-vercel-protection-bypass=<token>",
+            },
             { name: "watch", type: "boolean", description: "Stream progress and exit non-zero on failure" },
             { name: "timeout", type: "number", description: "With --watch: max minutes to wait" },
             {
@@ -26,28 +47,109 @@ export const runCommands = [
         examples: [
             "beryl runs trigger --watch",
             "beryl runs trigger --url-override https://preview-123.example.com --watch --timeout 30",
+            "beryl runs trigger --url-override https://preview-123.example.com --header x-vercel-protection-bypass=<token> --watch",
             "beryl runs trigger --test 4f… --test 9a…",
             "beryl runs trigger --retries 0 --watch",
         ],
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             const tests = input.flags.test;
+            const extraHeaders = parseHeaders(input.flags.header);
             const created = (await ctx.client.post(`${projectPath(workspaceId, projectId)}/runs`, {
                 test_case_ids: tests && tests.length > 0 ? tests : null,
                 environment_id: flagStr(input, "env") ?? null,
                 target_url_override: flagStr(input, "url-override") ?? null,
                 max_retries: flagNum(input, "retries") ?? null,
+                extra_headers: extraHeaders,
             }));
             if (!flagBool(input, "watch"))
                 return { data: created };
             return await watchRun(ctx, workspaceId, projectId, created.id, flagNum(input, "timeout"));
         },
     },
+    {
+        name: "runs local",
+        summary: "Run a banked test locally with your own Playwright (public flows)",
+        description: "Unlike `runs trigger`, this runs on YOUR machine, not Beryl's cloud — fetches the test's " +
+            "rendered spec, then runs it with your local @playwright/test (install it once with " +
+            "`npm i -D @playwright/test && npx playwright install`). Point --url-override at a local " +
+            "dev server or preview, and --dir to keep the spec, artifacts, and JSON report on disk so " +
+            "an agent can run-fix-run. v1 targets public/unauthenticated flows: an authenticated test " +
+            "refuses to run locally (those run in Beryl's cloud, which holds the session) — no session " +
+            "is ever decrypted to your disk. Exits 0 if every test passed, 1 on a test failure.",
+        scope: "project",
+        args: [{ name: "test-id", description: "Test id to run (from `beryl tests list`)", required: true }],
+        flags: [
+            {
+                name: "url-override",
+                type: "string",
+                description: "Run against this base URL instead of the environment's (e.g. http://localhost:3000)",
+            },
+            {
+                name: "dir",
+                type: "string",
+                description: "Write the spec, artifacts, and JSON report here (default: a temp dir)",
+            },
+        ],
+        examples: [
+            "beryl runs local 4f…",
+            "beryl runs local 4f… --url-override http://localhost:3000 --dir ./beryl-local",
+        ],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const testId = arg(input, "test-id");
+            const script = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/tests/${testId}/script`, { base_url: flagStr(input, "url-override") }));
+            // v1 is public flows only. A gated test's session lives (encrypted) in the cloud and is
+            // never handed to a local runner, so refuse rather than run a spec doomed to fail at the
+            // login wall. Exit 1: it's a failure to run this test locally, not a usage error.
+            if (script.requires_auth) {
+                throw new CliError("This test signs in first, so it can only run in Beryl's cloud (which holds the " +
+                    "session) — local runs are for public/unauthenticated flows. Run it with " +
+                    "`beryl runs trigger`.");
+            }
+            let outcome;
+            try {
+                outcome = await runSpecLocally({
+                    spec: script.content,
+                    testName: testId,
+                    dir: flagStr(input, "dir"),
+                    onProgress: (line) => ctx.err(dim(line)),
+                });
+            }
+            catch (err) {
+                // Every failure to run the spec (missing Playwright, a compile error, a customer
+                // config filtering our spec away) already carries an actionable message — surface it
+                // as a CliError so the user sees that, not a raw Node stack trace.
+                throw new CliError(err instanceof Error ? err.message : String(err));
+            }
+            const summary = { passed: outcome.passed, failed: outcome.failed, results: outcome.results };
+            const marker = (status) => status === "passed" || status === "expected"
+                ? green("✓")
+                : status === "skipped"
+                    ? dim("○")
+                    : red("✗");
+            const lines = outcome.results.map((r) => `  ${marker(r.status)} ${r.name}${r.error ? `\n    ${dim(r.error.split("\n")[0] ?? "")}` : ""}`);
+            const human = `${outcome.failed === 0 ? green("All tests passed") : red(`${outcome.failed} test(s) failed`)} ` +
+                `(${outcome.passed} passed, ${outcome.failed} failed)\n${lines.join("\n")}` +
+                (flagStr(input, "dir") ? `\n${dim(`Spec + artifacts + report in ${outcome.directory}`)}` : "");
+            const persisted = Boolean(flagStr(input, "dir"));
+            return {
+                // Only surface on-disk paths when --dir kept them; without it the run dir is deleted.
+                data: persisted
+                    ? { ...summary, directory: outcome.directory, report: outcome.report }
+                    : summary,
+                human,
+                // Exit codes are a CI contract: any failing test → exit 1.
+                ...(outcome.failed > 0 ? { exitCode: 1 } : {}),
+            };
+        },
+    },
     {
         name: "runs list",
         summary: "List recent runs",
         scope: "project",
         groupDefault: true,
+        groupSummary: "Trigger a run of a project's tests (e.g. in CI), then watch, inspect, and download results.",
         flags: [{ name: "env", type: "string", description: "Filter by environment id" }],
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
@@ -192,7 +294,7 @@ export const runCommands = [
     },
     {
         name: "runs explain",
-        summary: "AI explanation of why a test result failed",
+        summary: "Explain, with AI, why a test result failed",
         scope: "project",
         args: [{ name: "result-id", description: "Test result id (from `beryl runs get`)", required: true }],
         async run(ctx, input) {

package/dist/commands/slack.js

@@ -0,0 +1,82 @@
+import { arg, flagBool, projectPath } from "./util.js";
+const slackPath = (ws, p) => `${projectPath(ws, p)}/slack`;
+export const slackCommands = [
+    {
+        name: "slack show",
+        summary: "Show the project's Slack alert config (webhook is masked)",
+        groupSummary: "Send run outcomes to a Slack channel via a per-project incoming webhook.",
+        scope: "project",
+        groupDefault: true,
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            return { data: await ctx.client.get(slackPath(workspaceId, projectId)) };
+        },
+    },
+    {
+        name: "slack set-webhook",
+        summary: "Set the Slack incoming-webhook URL run alerts post to",
+        scope: "project",
+        args: [
+            {
+                name: "url",
+                description: "Slack incoming-webhook URL (https://hooks.slack.com/…)",
+                required: true,
+            },
+        ],
+        flags: [
+            {
+                name: "all-runs",
+                type: "boolean",
+                description: "Post on every run, not only on a new regression",
+            },
+            {
+                name: "failures-only",
+                type: "boolean",
+                description: "Post only on a new regression (the default)",
+            },
+        ],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const path = slackPath(workspaceId, projectId);
+            // Preserve the stored notify preference unless the caller states one, so
+            // rotating the webhook doesn't silently reset an existing all-runs opt-in.
+            let notifyAllRuns;
+            if (flagBool(input, "all-runs")) {
+                notifyAllRuns = true;
+            }
+            else if (flagBool(input, "failures-only")) {
+                notifyAllRuns = false;
+            }
+            else {
+                const current = (await ctx.client.get(path));
+                notifyAllRuns = current?.notify_all_runs ?? false;
+            }
+            return {
+                data: await ctx.client.put(path, {
+                    webhook_url: arg(input, "url"),
+                    notify_all_runs: notifyAllRuns,
+                }),
+            };
+        },
+    },
+    {
+        name: "slack clear",
+        summary: "Remove the project's Slack webhook (stops all alerts)",
+        scope: "project",
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            await ctx.client.del(slackPath(workspaceId, projectId));
+            return { human: "Cleared." };
+        },
+    },
+    {
+        name: "slack test",
+        summary: "Post a sample alert to the configured webhook",
+        scope: "project",
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            await ctx.client.post(`${slackPath(workspaceId, projectId)}/test`);
+            return { human: "Sent a test message to Slack." };
+        },
+    },
+];

package/dist/commands/tests.js

@@ -35,6 +35,7 @@ export const testCommands = [
         summary: "List the project's tests with their latest result",
         scope: "project",
         groupDefault: true,
+        groupSummary: "Author, inspect, version, and heal a project's tests — the checks Beryl runs on each run.",
         flags: [{ name: "env", type: "string", description: "Filter by environment id" }],
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
@@ -80,13 +81,23 @@ export const testCommands = [
         flags: [
             { name: "title", type: "string", required: true, description: "Title for the new test" },
             { name: "file", type: "string", required: true, description: "Plan JSON file, or - for stdin" },
+            {
+                name: "description",
+                type: "string",
+                description: "1–3 sentences stating what this test proves — the immutable outcome Beryl's healing " +
+                    "checks against. State the purpose, not the steps; the one observable signal that's true " +
+                    "only if the flow worked.",
+            },
             {
                 name: "no-verify",
                 type: "boolean",
                 description: "Skip the compile-time browser/AI verification — trust the authored plan as-is",
             },
         ],
-        examples: ['beryl tests create --title "Checkout happy path" --file plan.json'],
+        examples: [
+            'beryl tests create --title "Checkout happy path" --file plan.json',
+            'beryl tests create --title "Checkout happy path" --file plan.json --description "Proves a shopper can buy a product: after paying, an order-confirmation page with an order number appears."',
+        ],
         async run(ctx, input) {
             const title = flagStr(input, "title");
             if (!title)
@@ -97,6 +108,7 @@ export const testCommands = [
                     title,
                     plan: readJsonFlag(input, "file"),
                     verify: !flagBool(input, "no-verify"),
+                    description: flagStr(input, "description"),
                 }),
             };
         },
@@ -106,17 +118,25 @@ export const testCommands = [
         summary: "Replace a test's step plan from a JSON file (creates a new version)",
         description: "Accepts the same plan shape as `tests create`, including the optional `before` and " +
             "`after` sections — `after` runs on pass and on fail, so cleanup happens even when the " +
-            "test goes red.",
+            "test goes red. Pass `--description` when the re-authored plan changes what the test " +
+            "proves; omit it to keep the test's existing intent.",
         scope: "project",
         args: [{ name: "test-id", description: "Test id", required: true }],
         flags: [
             { name: "file", type: "string", required: true, description: "Plan JSON file, or - for stdin" },
+            {
+                name: "description",
+                type: "string",
+                description: "1–3 sentences stating what this test proves — the immutable outcome Beryl's healing " +
+                    "checks against. State the purpose, not the steps; the one observable signal that's true " +
+                    "only if the flow worked. Omit to keep the test's existing intent.",
+            },
         ],
         examples: ["beryl tests plan 4f… > plan.json  # edit, then:", "beryl tests set-plan 4f… --file plan.json"],
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
             return {
-                data: await ctx.client.patch(`${testPath(workspaceId, projectId, arg(input, "test-id"))}/plan`, { json_plan: readJsonFlag(input, "file") }),
+                data: await ctx.client.patch(`${testPath(workspaceId, projectId, arg(input, "test-id"))}/plan`, { json_plan: readJsonFlag(input, "file"), description: flagStr(input, "description") }),
             };
         },
     },

package/dist/commands/workspaces.js

@@ -6,6 +6,7 @@ export const workspaceCommands = [
     {
         name: "workspaces list",
         summary: "List workspaces you belong to",
+        groupSummary: "Create and manage workspaces, and pick the one your commands act on by default.",
         async run(ctx) {
             const rows = (await ctx.client.get("/workspaces/"));
             return { data: rows };
@@ -115,6 +116,7 @@ export const workspaceCommands = [
     {
         name: "members list",
         summary: "List workspace members",
+        groupSummary: "Manage who belongs to a workspace and their roles.",
         scope: "workspace",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
@@ -154,6 +156,7 @@ export const workspaceCommands = [
     {
         name: "invites send",
         summary: "Invite someone to the workspace by email",
+        groupSummary: "Send, list, and revoke workspace invitations, and accept ones sent to you.",
         scope: "workspace",
         args: [{ name: "email", description: "Invitee email", required: true }],
         flags: [