@beryl-so/cli 0.63.2 → 0.67.0

package/README.md

@@ -2,7 +2,7 @@
 
 # beryl CLI
 
-Beryl on the command line: create projects, watch the agent explore and author tests,
+Beryl on the command line: create projects, author tests,
 trigger runs in CI, manage everything the web app can, plus an MCP server (`beryl mcp`)
 that exposes every command to coding agents.
 
@@ -166,7 +166,7 @@ Send, list, and revoke workspace invitations, and accept ones sent to you.
 
 ### projects
 
-Create and manage projects: a site Beryl explores, authors tests for, and runs.
+Create and manage projects: a site you author tests for and run.
 
 `beryl projects` with no subcommand runs `projects list`.
 
@@ -174,7 +174,7 @@ Create and manage projects: a site Beryl explores, authors tests for, and runs.
 | --- | --- | --- |
 | `beryl projects list` | List projects in the workspace | `projects_list` |
 | `beryl projects get` | Show one project, including its current exploration state | `projects_get` |
-| `beryl projects create [url]` | Create a project. With a URL the agent starts exploring; with just --name an empty one | `projects_create` |
+| `beryl projects create [url]` | Create a project for a URL, or an empty one with just --name | `projects_create` |
 | `beryl projects rename <name>` | Rename a project | `projects_rename` |
 | `beryl projects quarantine [state]` | Turn auto quarantine on or off and tune when it mutes and releases a test | `projects_quarantine` |
 | `beryl projects delete` | Delete a project and all its tests and runs | `projects_delete` |
@@ -354,6 +354,7 @@ Durable identities on the site under test: what an authenticated test signs in a
 | Command | Summary | MCP tool |
 | --- | --- | --- |
 | `beryl accounts list` | List the test accounts an environment's tests sign in as | `accounts_list` |
+| `beryl accounts get <account-id>` | Show one test account: origin, lifetime, sign-in state | `accounts_get` |
 | `beryl accounts create` | Add a test account: the customer's own, or one Beryl signs up | `accounts_create` |
 | `beryl accounts signup <account-id>` | Store a test account's signup plan and sign it up on its environment | `accounts_signup` |
 | `beryl accounts set-login <account-id>` | Store the sign-in plan a run replays once, plus the probe that proves it | `accounts_set_login` |

package/dist/beryl-test-skill.js

@@ -115,7 +115,9 @@ Set up (once): if you cannot call the Beryl tools yet, run \`beryl init\` in a t
 install the browser and wire the tools. Then, before the first plan:
 
 - \`envs_list\`: the root URL gotos resolve against (§2). If it is empty, set it with
-  \`envs_update\`.
+  \`envs_update\` to a URL Beryl's cloud can reach, never \`localhost\`, \`127.0.0.1\`
+  or a private IP (refused). For a local dev server, pass \`url-override\` to
+  \`tests_create\` and \`runs_local\` instead.
 - \`accounts_list\`: who do authenticated tests sign in as (§1)? Store its sign-in with
   \`accounts_set_login\`, prove it with \`accounts_check\`.
 - \`mailbox_get\`: the address the account receives mail at (§5).
@@ -143,9 +145,9 @@ An authenticated test **signs in as an account that already exists** — it does
 one up. Start every authoring session with \`accounts_list\`:
 
 \`\`\`
-label      email                          type           login     status
-default *  qa@acme.test                   user_provided  password  ready
-admin      qa+admin@x7k2p9.email.beryl.so beryl          otp       ready
+label      email                          type                     login     status
+default *  qa@acme.test                   provided                 password  ready
+admin      qa+admin@x7k2p9.email.beryl.so provisioned (permanent)  otp       ready
 \`\`\`
 
 Author the sign-in as ordinary opening steps citing the reserved handles — fill

package/dist/commands/accounts.js

@@ -26,7 +26,7 @@ function accountTable(rows) {
     return table(rows.map((a) => ({
         label: a.identity_label + (a.is_default ? " *" : ""),
         email: a.email,
-        type: a.type,
+        type: a.type + (a.lifetime ? ` (${a.lifetime})` : ""),
         login: a.login_method,
         status: a.status,
         id: a.id,
@@ -58,14 +58,31 @@ export const testAccountCommands = [
             return { data: rows, human: accountTable(rows) };
         },
     },
+    {
+        name: "accounts get",
+        summary: "Show one test account: origin, lifetime, sign-in state",
+        description: "One test account by id. `type` is its origin: captured (a session from the Chrome " +
+            "plugin), provisioned (Beryl signed it up; `lifetime` permanent or throwaway), or " +
+            "provided (your own account).",
+        scope: "project",
+        args: [{ name: "account-id", description: "Account id from `beryl accounts list`", required: true }],
+        examples: ["beryl accounts get acc_123"],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const account = (await ctx.client.get(`${accountsPath(workspaceId, projectId)}/${arg(input, "account-id")}`));
+            return { data: account, human: accountTable([account]) };
+        },
+    },
     {
         name: "accounts create",
         summary: "Add a test account: the customer's own, or one Beryl signs up",
-        description: "Two kinds. `--type user_provided` records a dedicated account you already have on " +
-            "the site: pass --email and --password, and it is usable immediately. `--type beryl` " +
-            "reserves one Beryl will sign up itself, addressed at the project mailbox. It starts " +
-            "`pending` and becomes usable after `beryl accounts signup`. " +
-            "Works for Google-only apps too: bank a dedicated Google test account as user_provided " +
+        description: "Two kinds. `--type provided` records a dedicated account you already have on " +
+            "the site: pass --email and --password, and it is usable immediately. " +
+            "`--type provisioned` reserves one Beryl will sign up itself, addressed at the project " +
+            "mailbox; `--lifetime` says whether it is kept (permanent, the default) or a " +
+            "throwaway. It starts `pending` and becomes usable after `beryl accounts signup`. " +
+            "A captured account (a signed-in session from the Chrome plugin) is not created here. " +
+            "Works for Google-only apps too: bank a dedicated Google test account as provided " +
             "and author its sign-in plan through Google's form. " +
             "The first account an environment gets is its default whatever you pass.",
         scope: "project",
@@ -73,19 +90,25 @@ export const testAccountCommands = [
             {
                 name: "type",
                 type: "string",
-                enum: ["user_provided", "beryl"],
-                description: "Whose account this is (default: user_provided when --password is given)",
+                enum: ["provided", "provisioned"],
+                description: "Whose account this is (default: provided when --password is given)",
+            },
+            {
+                name: "lifetime",
+                type: "string",
+                enum: ["permanent", "throwaway"],
+                description: "provisioned only: keep it, or a throwaway (default permanent)",
             },
             {
                 name: "label",
                 type: "string",
                 description: "Identity label a plan's auth_label names (default 'default')",
             },
-            { name: "email", type: "string", description: "user_provided only: the account's email" },
+            { name: "email", type: "string", description: "provided only: the account's email" },
             {
                 name: "password",
                 type: "string",
-                description: "user_provided only: stored encrypted as a project secret, never returned",
+                description: "provided only: stored encrypted as a project secret, never returned",
             },
             {
                 name: "login-method",
@@ -98,7 +121,7 @@ export const testAccountCommands = [
         ],
         examples: [
             "beryl accounts create --email qa@acme.test --password 'hunter2'",
-            "beryl accounts create --type beryl --login-method otp",
+            "beryl accounts create --type provisioned --login-method otp",
             "beryl accounts create --email admin@acme.test --password 'hunter2' --label admin",
             "beryl accounts create --email qa@gmail.com --password 'hunter2'  # Google-only app",
         ],
@@ -106,7 +129,8 @@ export const testAccountCommands = [
             const { workspaceId, projectId } = await ctx.requireProject(input);
             const password = flagStr(input, "password");
             const account = (await ctx.client.post(accountsPath(workspaceId, projectId), {
-                type: flagStr(input, "type") ?? (password ? "user_provided" : "beryl"),
+                type: flagStr(input, "type") ?? (password ? "provided" : "provisioned"),
+                lifetime: flagStr(input, "lifetime") ?? null,
                 identity_label: flagStr(input, "label") ?? "default",
                 email: flagStr(input, "email") ?? null,
                 password: password ?? null,

package/dist/commands/environments.js

@@ -29,6 +29,11 @@ export const environmentCommands = [
     {
         name: "envs get",
         summary: "Show one environment",
+        description: "`site_health_runs_reset_fields` and `site_health_tracker_reset_fields` name stored Site " +
+            "Health settings that were invalid and fell back to defaults (`*` = all of them). " +
+            "Scheduled and CI runs, or ticket filing, auto-close and the digest, stop until they are " +
+            "saved again: `envs update` with any `--health-*` flag rewrites the run settings, any " +
+            "`--tracker-*` flag the tracker settings.",
         scope: "project",
         args: [{ name: "env-id", description: "Environment id", required: true }],
         async run(ctx, input) {

package/dist/commands/health.js

@@ -277,7 +277,9 @@ export const healthCommands = [
             "and on desktop separately (`speeds`, each with its own Core Web Vitals) and takes " +
             "the worse of the two; one that couldn't be measured is left out. A category's `unchecked` lists checks this run " +
             "couldn't determine (e.g. robots.txt timed out): they neither pass nor fail. Pass --report for a single-page check started with " +
-            "`health run --path`.",
+            "`health run --path`. `site_health_runs_reset_fields` and `site_health_tracker_reset_fields` " +
+            "name stored settings that were invalid and fell back to defaults (`*` = all of them): " +
+            "scheduled and CI runs, or ticket filing, stop until they are saved again with `envs update`.",
         scope: "project",
         flags: [
             ENV_FLAG,

package/dist/commands/projects.js

@@ -37,7 +37,7 @@ export const projectCommands = [
         summary: "List projects in the workspace",
         scope: "workspace",
         groupDefault: true,
-        groupSummary: "Create and manage projects: a site Beryl explores, authors tests for, and runs.",
+        groupSummary: "Create and manage projects: a site you author tests for and run.",
         async run(ctx, input) {
             const ws = await ctx.requireWorkspace(input);
             const rows = (await ctx.client.get(`/workspaces/${ws}/projects`));
@@ -70,7 +70,7 @@ export const projectCommands = [
     },
     {
         name: "projects create",
-        summary: "Create a project. With a URL the agent starts exploring; with just --name an empty one",
+        summary: "Create a project for a URL, or an empty one with just --name",
         scope: "workspace",
         args: [
             {
@@ -95,24 +95,26 @@ export const projectCommands = [
             },
             { name: "force-new-login", type: "boolean", description: "Ignore any reusable saved login" },
             {
-                name: "no-explore",
+                name: "explore",
                 type: "boolean",
-                description: "Create the project without starting the cloud exploration. Author tests yourself " +
-                    "via `beryl tests create` or your coding agent over MCP",
+                description: "Also start a cloud exploration that authors tests for you. Off by default: author " +
+                    "tests with `beryl tests create` or your coding agent over MCP",
+                mcpDescription: "Leave unset: you author the tests yourself with `tests create`. Set only when the " +
+                    "user explicitly asks for a cloud exploration",
             },
-            { name: "watch", type: "boolean", description: "Stream the agent's exploration live" },
+            { name: "watch", type: "boolean", description: "Start the exploration (implies --explore) and stream it live" },
             { name: "timeout", type: "number", description: "With --watch: max minutes to wait" },
         ],
         examples: [
-            "beryl projects create https://app.example.com --watch",
+            "beryl projects create https://app.example.com",
             "beryl projects create https://app.example.com --auth gated",
-            "beryl projects create https://app.example.com --auth public --no-explore",
             'beryl projects create --name "Acme production"',
         ],
         async run(ctx, input) {
-            const noExplore = flagBool(input, "no-explore");
-            if (noExplore && flagBool(input, "watch"))
+            const watch = flagBool(input, "watch");
+            if (input.flags.explore === false && watch)
                 throw new UsageError("--no-explore cannot be combined with --watch");
+            const explore = flagBool(input, "explore") || watch;
             const ws = await ctx.requireWorkspace(input);
             const rawUrl = input.args.url;
             const url = typeof rawUrl === "string" ? rawUrl : "";
@@ -121,7 +123,9 @@ export const projectCommands = [
             // later with `beryl envs update <env-id>` or author over the CLI/MCP.
             if (!url) {
                 if (!name)
-                    throw new UsageError("Provide a URL to explore, or --name to create an empty project");
+                    throw new UsageError("Provide a URL, or --name to create an empty project");
+                if (explore)
+                    throw new UsageError("--explore and --watch need a URL to explore");
                 const created = (await ctx.client.post(`/workspaces/${ws}/projects`, {
                     name,
                 }));
@@ -137,13 +141,13 @@ export const projectCommands = [
                 root_url: url,
                 requires_auth_choice: auth,
                 force_new_login: flagBool(input, "force-new-login"),
-                skip_exploration: noExplore,
+                skip_exploration: !explore,
             }));
-            if (noExplore)
+            if (!explore)
                 return {
                     data: created,
-                    human: `${green("Project created")}: ${created.project_id} ${dim("(no exploration started)")}\n` +
-                        `Ready for your own tests — author a plan and push it with \`beryl tests create\`.`,
+                    human: `${green("Project created")}: ${created.project_id}\n` +
+                        `Ready for your own tests: author a plan and push it with \`beryl tests create\`.`,
                 };
             // The agent cannot start until a login is captured, so there is nothing to watch.
             if (created.auth_status === "needs_capture")

package/dist/commands/tests.js

@@ -89,7 +89,7 @@ function verifyFailureResult(err) {
 // The concise human table for `tests list` — the full TestResponse is a 24-column
 // firehose of internal ids that wraps unreadably in a normal terminal. `--wide`
 // (and `--json`) still expose every field.
-const LIST_COLUMNS = ["title", "status", "access", "id", "last_result_status", "last_run_at"];
+const LIST_COLUMNS = ["title", "surface", "status", "access", "id", "last_result_status", "last_run_at"];
 const ACCESS_VALUES = ["read", "write"];
 function accessFlag(input, fallback) {
     const raw = flagStr(input, "access") ?? fallback;
@@ -101,6 +101,7 @@ function accessFlag(input, fallback) {
 function conciseTestRow(row) {
     return {
         title: row.nl_title,
+        surface: row.surface,
         status: row.quarantined ? "quarantined" : row.is_active ? "active" : "inactive",
         access: row.access,
         id: row.id,
@@ -188,8 +189,8 @@ export const testCommands = [
             {
                 name: "status",
                 type: "string",
-                description: "With --page, show only this bucket: passed, failed, errored, cancelled, " +
-                    "quarantined, or unstable (failed no longer includes errored)",
+                description: "With --page, show only this bucket: passed, failing (failed or errored), " +
+                    "failed, errored, cancelled, quarantined, or unstable",
             },
             {
                 name: "ungrouped",
@@ -291,7 +292,8 @@ export const testCommands = [
             "reconcile with that test (`tests get` / `tests set-plan`), don't rename-and-retry; a " +
             "409 `plan_hash_mismatch` means the submitted plan is not the bytes that were replayed " +
             "(re-run `tests create`); a 429 with Retry-After 30 means the verify slots are " +
-            "saturated (wait and retry).",
+            "saturated (wait and retry). A server-side verify that could not run (verdict `flag`) " +
+            "still banks the test but returns verified:false and exits 1; prove it with `tests recompile`.",
         scope: "project",
         flags: [
             { name: "title", type: "string", required: true, description: "Title for the new test" },
@@ -374,7 +376,18 @@ export const testCommands = [
             const verifyInCloud = async (reason) => {
                 ctx.err(yellow(`! ${reason}, so this plan is verified server-side instead.`));
                 try {
-                    return { data: await bank({ verify: true, plan_hash: compiled.plan_hash }) };
+                    const created = await bank({ verify: true, plan_hash: compiled.plan_hash });
+                    if (created.verified === false) {
+                        // The replay could not run (verdict "flag"): saved, but nothing was proven.
+                        return {
+                            data: created,
+                            human: yellow(`Banked UNVERIFIED: ${created.id}. The server-side replay could not run` +
+                                (created.verify_message ? ` (${String(created.verify_message)})` : "") +
+                                "; the test is saved but unproven. Prove it with `tests recompile`."),
+                            exitCode: 1,
+                        };
+                    }
+                    return { data: created };
                 }
                 catch (err) {
                     const failure = verifyFailureResult(err);
@@ -524,8 +537,9 @@ export const testCommands = [
                 ctx.err(yellow(`! The proof ran against ${urlOverride}, not the environment's root URL — ` +
                     "the banked test is unproven against its real target until a run there passes."));
             }
+            // The server saw verify:false; the proof is the local replay, unless it ran off-target.
             return {
-                data: created,
+                data: { ...created, verified: !urlOverride, verdict: "keep", verified_by: "local" },
                 human: green(`✓ Replay passed locally — banked as ${created.id}`),
             };
         },
@@ -583,7 +597,8 @@ export const testCommands = [
         description: "Groups are project-defined labels used to filter, run, or schedule a slice of the " +
             "suite (`beryl runs trigger --group <name>`). This REPLACES the whole set: pass every " +
             "group you want, or none to clear it. Names must already exist in the project " +
-            "(`beryl groups list`); an unknown name is an error, never a new group.",
+            "(`beryl groups list`); an unknown name is an error, never a new group. Every " +
+            "surface of the test carries the same set.",
         scope: "project",
         args: [{ name: "test-id", description: "Test id", required: true }],
         flags: [
@@ -611,7 +626,8 @@ export const testCommands = [
         summary: "Mark a test read or write",
         description: "read: the test only reads the app. write: it creates, edits, renames, deletes or " +
             "toggles something that persists. The label is yours to set; Beryl does not infer it " +
-            "from the plan. A write test must clean up in `after`, ending with a proof expect.",
+            "from the plan. A write test must clean up in `after`, ending with a proof expect. " +
+            "The label covers every surface of the test, so each surface's plan must clean up.",
         scope: "project",
         args: [
             { name: "test-id", description: "Test id", required: true },

package/dist/commands/tracker.js

@@ -195,7 +195,10 @@ export const trackerCommands = [
             "revoked; ticket changes queue until you reconnect), the number of queued changes, " +
             "open tickets (filed in the tracker), `unfiled_tickets` the tracker hasn't accepted " +
             "yet with its `last_sync_error`, digest findings, and `last_sync`: what the latest run filed, " +
-            "updated, closed or skipped. Change settings with `envs update --tracker-*`.",
+            "updated, closed or skipped. `reset_fields` names stored settings that were invalid and " +
+            "fell back to defaults (`*` = all of them); filing, auto-close and the digest stop " +
+            "(`last_sync.skipped: settings_invalid`) until they are saved again. Change settings " +
+            "with `envs update --tracker-*`.",
         scope: "project",
         flags: [ENV_FLAG],
         async run(ctx, input) {

package/dist/context.js

@@ -133,8 +133,7 @@ export function createContext(options) {
                 projectId = projects[0].id;
             }
             else if (projects.length === 0) {
-                throw new CliError("This workspace has no project yet — create one with " +
-                    "`beryl projects create <url>` (add --no-explore to author tests yourself).");
+                throw new CliError("This workspace has no project yet. Create one with `beryl projects create <url>`.");
             }
             else {
                 projectId = await pickOne(ctx, projects, "project", ambiguousMessage(ctx, "project", projects));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.63.2",
-  "description": "Beryl on the command line: projects, runs, the exploring agent, and an MCP server over the same commands.",
+  "version": "0.67.0",
+  "description": "Beryl on the command line: projects, tests, runs, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",
   "homepage": "https://beryl.so/docs/cli",