@beryl-so/cli 0.42.0 → 0.44.0

package/README.md

@@ -317,6 +317,7 @@ The project's standing email addresses, where its tests receive sign-in mail.
 | --- | --- | --- |
 | `beryl mailbox get` | The project's mailbox address | `mailbox_get` |
 | `beryl mailbox list` | List the project's mailboxes | `mailbox_list` |
+| `beryl mailbox identities` | The +tag addresses Beryl minted for this project's tests, and how each cleanup ended | `mailbox_identities` |
 | `beryl mailbox create` | Add a second mailbox to the project | `mailbox_create` |
 | `beryl mailbox delete <mailbox-id>` | Delete a mailbox and every email it has received | `mailbox_delete` |
 | `beryl mailbox read <mailbox-id>` | Read the latest email in a mailbox (waits for one to arrive) | `mailbox_read` |

package/dist/beryl-test-skill.js

@@ -46,6 +46,12 @@ export const BERYL_TEST_SKILL_OTP_EXAMPLE_PLAN = {
         { action: "click", selector: "text=Verify" },
         { action: "expect", expect_kind: "visible", selector: "text=Welcome" },
     ],
+    after: [
+        { action: "goto", url: "/account" },
+        { action: "wait_for", selector: "text=Delete account", optional: true },
+        { action: "click", selector: "text=Delete account" },
+        { action: "click", selector: "[role=dialog] button:has-text(\"Delete\")" },
+    ],
 };
 export const BERYL_TEST_SKILL = `---
 name: beryl-test
@@ -238,7 +244,12 @@ never leaves Beryl's cloud).
    handoff, a magic link on another domain).
    Optional \`before\` / \`after\` arrays hold setup and teardown; \`after\` runs even
    when a main step fails, so a create flow can clean up its record — and a deletion
-   flow creates its own record in \`before\` first.
+   flow creates its own record in \`before\` first. A step in \`after\` may carry
+   \`"optional": true\`: if it misses (the record was never created, or is already
+   gone) the teardown ends quietly right there instead of failing the test. Put it on
+   the \`wait_for\` or \`expect\` that probes for the record, so the delete steps
+   after it run only when there is something to delete; an optional \`click\` or
+   \`fill\` must be the last step of \`after\`.
 
    The full action vocabulary:
 
@@ -392,19 +403,20 @@ receipt — is testable with the \`await_email\` action. No setup, no configurat
 flag to turn on.
 
 **The project has one permanent mailbox and all of its mail arrives there.** Two
-handles put an address on the page; the one you cite decides which *identity* the test
-acts as:
+handles put an address on the page; pick by what the test is about:
 
-| Handle | Renders as | Use it for |
+| The test... | Cite | Then |
 |---|---|---|
-| \`{{mailbox_address}}\` | the mailbox's own address, the same every run | signing in as the project's standing test account (§1) |
-| \`{{inbox_address}}\` | a \`+tag\` alias of it, fresh every run | tests whose subject IS getting a NEW identity — a signup, an invited teammate |
-
-An alias is a real address the site has never issued, so a signup is repeatable run
-after run; the mail lands in the same mailbox and Beryl reads only the alias's own
-mail. Nothing expires; there is no second inbox to manage. Default to
-\`{{mailbox_address}}\`; reach for \`{{inbox_address}}\` only when an existing account
-would be rejected — a signup form, an invite accepted as a second person.
+| signs in as the project's standing account, or only needs a code to arrive somewhere (a wrong code, a lockout, a resend countdown, the code screen) | \`{{mailbox_address}}\` | nothing to clean up |
+| must hand the site an address it has never seen but creates no account (a code sent to a new address, never verified) | \`{{inbox_address}}\` | nothing on Beryl's side; the site may still keep a row |
+| creates an account or an invitation (a signup, an accepted invite) | \`{{inbox_address}}\` | an \`after\` section that deletes it through the app, its steps marked \`optional\` |
+
+\`{{inbox_address}}\` renders a \`+tag\` alias of the mailbox, fresh every run, so a
+signup is repeatable; the mail lands in the same mailbox and Beryl reads only the
+alias's own mail. Beryl records every alias it mints (\`beryl mailbox identities\`)
+together with how the teardown ended. A plan that signs up with \`{{inbox_address}}\`
+and reads the mail is refused at save time unless \`after\` deletes the account; one
+that mints but never reads mail is flagged by \`tests lint\`. Mail is kept for 48 hours.
 
 The wiring is a three-part chain:
 

package/dist/commands/accounts.js

@@ -52,7 +52,8 @@ export const testAccountCommands = [
                 return {
                     data: rows,
                     human: dim("No test accounts. Authenticated tests fall back to the LOGIN_EMAIL variable " +
-                        "and LOGIN_PASSWORD secret, or mint a throwaway with {{inbox_address}}."),
+                        "and LOGIN_PASSWORD secret. A flow that must sign up fresh cites " +
+                        "{{inbox_address}} and deletes the account in `after`."),
                 };
             return { data: rows, human: accountTable(rows) };
         },

package/dist/commands/mailboxes.js

@@ -11,7 +11,9 @@ export const mailboxCommands = [
         description: "Returns the address {{mailbox_address}} resolves to, creating it on first ask. " +
             "Every test that reads mail receives here. A test needing an address the site has " +
             "never seen cites {{inbox_address}} instead. That renders a `+tag` alias of this " +
-            "same mailbox, so a signup stays repeatable without a second address to manage.",
+            "same mailbox, so a signup stays repeatable without a second address to manage. " +
+            "Every alias minted is listed by `mailbox identities`, and a plan that signs up " +
+            "with one must delete the account in its `after` section.",
         scope: "project",
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
@@ -42,6 +44,47 @@ export const mailboxCommands = [
             };
         },
     },
+    {
+        name: "mailbox identities",
+        summary: "The +tag addresses Beryl minted for this project's tests, and how each cleanup ended",
+        description: "Every {{inbox_address}} a run rendered is recorded: which test and run minted it, and " +
+            "what the runner saw of its teardown (clean, skipped, failed, unverified, or none when " +
+            "the plan has no `after` section). This is the list of identities that may still exist " +
+            "on the site under test. Rows are kept for 30 days.",
+        scope: "project",
+        flags: [
+            { name: "run", type: "string", description: "Only addresses minted by this run id" },
+            { name: "test", type: "string", description: "Only addresses minted by this test id" },
+            { name: "limit", type: "number", description: "Newest N rows (default 100, max 500)" },
+        ],
+        examples: ["beryl mailbox identities", "beryl mailbox identities --run run_123 --json"],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const rows = (await ctx.client.get(`${mailboxPath(workspaceId)}/identities`, {
+                project_id: projectId,
+                run_id: flagStr(input, "run"),
+                test_id: flagStr(input, "test"),
+                limit: flagNum(input, "limit"),
+            }));
+            if (rows.length === 0) {
+                return {
+                    data: rows,
+                    human: dim("No minted addresses yet. Tests that sign up with {{inbox_address}} appear here."),
+                };
+            }
+            return {
+                data: rows,
+                human: table(rows.map((r) => ({
+                    address: r.address,
+                    teardown: r.teardown_outcome ?? "open",
+                    source: r.source,
+                    test: r.test_case_id ?? "",
+                    run: r.test_run_id ?? "",
+                    minted: r.minted_at,
+                }))),
+            };
+        },
+    },
     {
         name: "mailbox create",
         summary: "Add a second mailbox to the project",

package/dist/commands/runs.js

@@ -552,8 +552,13 @@ export const runCommands = [
         description: "Over MCP the failure screenshots come back as viewable image content, so an agent can " +
             "look at the page that broke instead of guessing from the error string. Set screenshots " +
             "to false to skip fetching them. Ignored outside MCP (the terminal cannot show an image). " +
-            "Returns the run row with its per-test results. `runs report` returns the generated " +
-            "report document, `runs explain` an AI explanation of one failed result.",
+            "Returns the run row with every per-test result unless --page is given; pass --page to " +
+            "read a large run a slice at a time, optionally narrowed by --status, --group or --q, " +
+            "which keeps the output readable rather than saving a round trip. With --page the " +
+            "match total and page counts come back under results_page. Paged results are the run's " +
+            "settled tests only: anything still running, healing or queued is left out. " +
+            "`runs report` returns the generated report document, `runs explain` an AI explanation " +
+            "of one failed result.",
         scope: "project",
         args: [{ name: "run-id", description: "Run id", required: true }],
         flags: [
@@ -563,10 +568,62 @@ export const runCommands = [
                 default: true,
                 description: "Attach failure screenshots as image content (MCP only; default true)",
             },
+            {
+                name: "page",
+                type: "number",
+                description: "Return only this 1-indexed page of results instead of every result",
+            },
+            {
+                name: "page-size",
+                type: "number",
+                description: "Results per page when --page is given (default 20, max 100)",
+            },
+            {
+                name: "status",
+                type: "string",
+                description: "With --page, show only this bucket: passed, failed, errored, cancelled, or " +
+                    "quarantined",
+            },
+            {
+                name: "group",
+                type: "string",
+                description: "With --page, show only results whose test is in this group (by name; see " +
+                    "`beryl groups list`)",
+            },
+            {
+                name: "ungrouped",
+                type: "boolean",
+                description: "With --page, show only results whose test carries no group",
+            },
+            {
+                name: "q",
+                type: "string",
+                description: "With --page, show only results whose test title contains this text " +
+                    "(case-insensitive)",
+            },
         ],
         async run(ctx, input) {
             const { workspaceId, projectId } = await ctx.requireProject(input);
-            const data = (await ctx.client.get(`${projectPath(workspaceId, projectId)}/runs/${arg(input, "run-id")}`));
+            const runPath = `${projectPath(workspaceId, projectId)}/runs/${arg(input, "run-id")}`;
+            const page = flagNum(input, "page");
+            // The run row (header, counters) only exists on the unpaged route, so --page still
+            // fetches it and then swaps test_results for the requested slice. This trims what
+            // the caller has to read, not what the api has to send.
+            const data = (await ctx.client.get(runPath));
+            if (page !== undefined) {
+                const { items, ...meta } = (await ctx.client.get(`${runPath}/results/page`, {
+                    page,
+                    page_size: flagNum(input, "page-size"),
+                    status: flagStr(input, "status"),
+                    group: flagBool(input, "ungrouped") ? "__ungrouped__" : flagStr(input, "group"),
+                    q: flagStr(input, "q"),
+                }));
+                const paged = data;
+                paged.test_results = items;
+                // total / page / page_size / status_totals / group_counts ride along: without
+                // them a caller cannot tell whether another page exists short of asking for it.
+                paged.results_page = meta;
+            }
             // Only the MCP adapter renders images; fetching bytes for a plain CLI run
             // would be wasted network.
             if (!ctx.mcp || input.flags.screenshots === false)

package/dist/commands/tests.js

@@ -82,7 +82,10 @@ export const testCommands = [
         summary: "Validate a plan JSON file offline, before sending it to the server",
         description: "Checks a plan against the published ActionPlan JSON Schema: every action's required " +
             "fields, plus the two structural rules (the first EXECUTED step must be a goto, and at " +
-            "least one step across before + steps must be an expect). Runs entirely locally, so a " +
+            "least one step across before + steps must be an expect). Also the identity rules: a " +
+            "plan that signs up with {{inbox_address}} and reads the mail must delete the account " +
+            "in `after` (an error; the server refuses it too), and one that mints an address it " +
+            "never reads gets a warning to cite {{mailbox_address}}. Runs entirely locally, so a " +
             "malformed plan fails here instead of costing a server round-trip. " +
             `Schema: ${ACTION_PLAN_SCHEMA.$id}`,
         scope: "none",
@@ -92,14 +95,14 @@ export const testCommands = [
         examples: ["beryl tests lint --file plan.json"],
         async run(_ctx, input) {
             const issues = lintPlan(readJsonFlag(input, "file"));
-            if (issues.length === 0) {
-                return { data: { valid: true, issues: [] }, human: "Plan is valid." };
+            const lines = issues.map((i) => `${i.severity === "warning" ? "warning" : "error"} ${i.path}: ${i.message}`);
+            if (issues.every((i) => i.severity === "warning")) {
+                return {
+                    data: { valid: true, issues },
+                    human: ["Plan is valid.", ...lines].join("\n"),
+                };
             }
-            return {
-                data: { valid: false, issues },
-                human: issues.map((i) => `${i.path}: ${i.message}`).join("\n"),
-                exitCode: 1,
-            };
+            return { data: { valid: false, issues }, human: lines.join("\n"), exitCode: 1 };
         },
     },
     {

package/dist/lint.js

@@ -1,4 +1,16 @@
 import { ACTION_PLAN_SCHEMA } from "./schema.generated.js";
+// Word for word what the server's save gate says (api/app/ta_shared/schemas.py);
+// keep them identical.
+export const MINTED_IDENTITY_NEEDS_TEARDOWN = "This plan signs up with {{inbox_address}} and reads the mail it triggers, so it " +
+    "leaves an account behind on the site. Add an 'after' section that deletes it " +
+    "through the app (mark steps optional so a missing account ends the teardown " +
+    "quietly), or cite {{mailbox_address}} if no new account is meant to exist.";
+export const OPTIONAL_TEARDOWN_ACTION_MUST_BE_LAST = (label, action) => `${label} (${action}) is optional but not the last step of 'after'. An optional step ` +
+    "that misses ends the teardown, so the steps after it would silently not run. Guard " +
+    "them with an optional 'wait_for' or 'expect' instead, or move this step to the end.";
+export const MINTED_IDENTITY_UNUSED = "This plan cites {{inbox_address}} but never reads the mail it triggers, so it mints " +
+    "an identity it never uses. Cite {{mailbox_address}} instead, unless the site must be " +
+    "handed an address it has never seen.";
 const typeOf = (value) => {
     if (value === null)
         return "null";
@@ -120,6 +132,43 @@ function validate(value, raw, root, path) {
     }
     return issues;
 }
+const steps = (section) => Array.isArray(section)
+    ? section.filter((s) => typeof s === "object" && s !== null)
+    : [];
+const citesInboxAddress = (step) => Object.values(step).some((v) => typeof v === "string" && v.includes("{{inbox_address}}"));
+const TEARDOWN_GUARD_ACTIONS = new Set(["wait_for", "expect"]);
+function optionalPlacementIssues(after) {
+    return after.flatMap((s, i) => s.optional === true && !TEARDOWN_GUARD_ACTIONS.has(String(s.action)) && i !== after.length - 1
+        ? [
+            {
+                path: `after[${i}]`,
+                message: OPTIONAL_TEARDOWN_ACTION_MUST_BE_LAST(`after[${i}]`, String(s.action)),
+            },
+        ]
+        : []);
+}
+function mintedIdentityIssues(plan) {
+    if (typeof plan !== "object" || plan === null)
+        return [];
+    const { before, steps: main, after } = plan;
+    const placement = optionalPlacementIssues(steps(after));
+    if (placement.length > 0)
+        return placement;
+    const body = [...steps(before), ...steps(main)];
+    if (![...body, ...steps(after)].some(citesInboxAddress))
+        return [];
+    const readsMail = body.some((s) => s.action === "await_email");
+    const tearsDown = steps(after).some((s) => s.action !== "expect");
+    if (readsMail && !tearsDown) {
+        return [{ path: "(root)", message: MINTED_IDENTITY_NEEDS_TEARDOWN }];
+    }
+    if (!readsMail) {
+        return [{ path: "(root)", message: MINTED_IDENTITY_UNUSED, severity: "warning" }];
+    }
+    return [];
+}
 export function lintPlan(plan) {
-    return validate(plan, ACTION_PLAN_SCHEMA, ACTION_PLAN_SCHEMA, "");
+    const issues = validate(plan, ACTION_PLAN_SCHEMA, ACTION_PLAN_SCHEMA, "");
+    // The identity rules read a well-formed plan; a shape error is reported alone.
+    return issues.length > 0 ? issues : mintedIdentityIssues(plan);
 }

package/dist/schema.generated.js

@@ -1050,6 +1050,11 @@ export const ACTION_PLAN_SCHEMA = {
                     "default": null,
                     "title": "Option"
                 },
+                "optional": {
+                    "default": false,
+                    "title": "Optional",
+                    "type": "boolean"
+                },
                 "selector": {
                     "anyOf": [
                         {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.42.0",
+  "version": "0.44.0",
   "description": "Beryl on the command line — projects, runs, the exploring agent, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",