@beryl-so/cli 0.44.0 → 0.46.0

package/README.md

@@ -175,6 +175,7 @@ Create and manage projects: a site Beryl explores, authors tests for, and runs.
 | `beryl projects get` | Show one project, including its current exploration state | `projects_get` |
 | `beryl projects create [url]` | Create a project. With a URL the agent starts exploring; with just --name an empty one | `projects_create` |
 | `beryl projects rename <name>` | Rename a project | `projects_rename` |
+| `beryl projects quarantine [state]` | Turn auto quarantine on or off and tune when it mutes and releases a test | `projects_quarantine` |
 | `beryl projects delete` | Delete a project and all its tests and runs | `projects_delete` |
 | `beryl projects re-explore` | Send the agent back in to run/heal existing tests and discover new flows | none |
 | `beryl projects report` | Aggregate quality report across recent runs (pass rates, flaky tests, trend) | `projects_report` |

package/dist/beryl-test-skill.js

@@ -402,28 +402,27 @@ A flow that emails the user — a signup verification code, a magic sign-in link
 receipt — is testable with the \`await_email\` action. No setup, no configuration, no
 flag to turn on.
 
-**The project has one permanent mailbox and all of its mail arrives there.** Two
-handles put an address on the page; pick by what the test is about:
-
-| The test... | Cite | Then |
-|---|---|---|
-| signs in as the project's standing account, or only needs a code to arrive somewhere (a wrong code, a lockout, a resend countdown, the code screen) | \`{{mailbox_address}}\` | nothing to clean up |
-| must hand the site an address it has never seen but creates no account (a code sent to a new address, never verified) | \`{{inbox_address}}\` | nothing on Beryl's side; the site may still keep a row |
-| creates an account or an invitation (a signup, an accepted invite) | \`{{inbox_address}}\` | an \`after\` section that deletes it through the app, its steps marked \`optional\` |
-
-\`{{inbox_address}}\` renders a \`+tag\` alias of the mailbox, fresh every run, so a
-signup is repeatable; the mail lands in the same mailbox and Beryl reads only the
-alias's own mail. Beryl records every alias it mints (\`beryl mailbox identities\`)
-together with how the teardown ended. A plan that signs up with \`{{inbox_address}}\`
-and reads the mail is refused at save time unless \`after\` deletes the account; one
-that mints but never reads mail is flagged by \`tests lint\`. Mail is kept for 48 hours.
+**The project has one permanent mailbox and all of its mail arrives there.** The
+standing account is used one way only, \`requires_auth: true\` (its session is injected
+before the first step); a test never types its address (the save refuses it), only the
+account's own \`signup\` / \`set-login\` plans fill \`{{mailbox_address}}\`. Every flow that
+sends mail (a signup, a sign-in code, a wrong code, a lockout, a resend timer, an
+invitation) cites \`{{inbox_address}}\`, a \`+tag\` alias minted per execution and read by
+this test alone.
+
+| The test... | Then |
+|---|---|
+| creates an account, or signs in as one it created (a returning-user test: \`before\` signs up the alias and logs out, the steps sign in as it) | an \`after\` section that deletes it through the app, its steps marked \`optional\`; a plan that reads its signup mail is refused at save time without one |
+| only makes mail arrive, read or not, no account created | nothing to clean up; mail is kept for 48 hours |
+
+Every alias minted is listed by \`beryl mailbox identities\` with how its teardown ended.
 
 The wiring is a three-part chain:
 
-1. **Type the address into the app** — a \`fill\` with \`value: "{{inbox_address}}"\`
-   (or \`{{mailbox_address}}\`). The alias is fresh every run, so a signup is
-   repeatable by construction — no \`{{unique}}\` needed for the email itself; use it
-   for other must-not-collide values like a username. The generator handles
+1. **Type the address into the app**: a \`fill\` with \`value: "{{inbox_address}}"\`.
+   The alias is fresh every run, so a signup is repeatable by construction, no
+   \`{{unique}}\` needed for the email itself; use it for other must-not-collide
+   values like a username. The generator handles
    \`{{unique}}\`, \`{{uuid}}\`, \`{{timestamp}}\` are each minted once per execution;
    \`{{timestamp}}\` is second-resolution (a time, NOT a uniqueness guarantee —
    \`{{unique}}\` is).

package/dist/commands/mailboxes.js

@@ -9,9 +9,11 @@ export const mailboxCommands = [
         groupDefault: true,
         groupSummary: "The project's standing email addresses, where its tests receive sign-in mail.",
         description: "Returns the address {{mailbox_address}} resolves to, creating it on first ask. " +
-            "Every test that reads mail receives here. A test needing an address the site has " +
-            "never seen cites {{inbox_address}} instead. That renders a `+tag` alias of this " +
-            "same mailbox, so a signup stays repeatable without a second address to manage. " +
+            "Every test that reads mail receives here, at its own `+tag` alias of this mailbox: " +
+            "a plan cites {{inbox_address}}, minted per execution and read by that test alone, " +
+            "so parallel tests never consume each other's codes. A test never types the bare " +
+            "address (the save refuses it); only an account's own signup and sign-in plans " +
+            "(`accounts signup`, `accounts set-login`) may fill it. " +
             "Every alias minted is listed by `mailbox identities`, and a plan that signs up " +
             "with one must delete the account in its `after` section.",
         scope: "project",

package/dist/commands/projects.js

@@ -176,6 +176,53 @@ export const projectCommands = [
             };
         },
     },
+    {
+        name: "projects quarantine",
+        summary: "Turn auto quarantine on or off and tune when it mutes and releases a test",
+        description: "With auto quarantine on, the end of every run checks the tests that ran: a test that " +
+            "failed --failures times in the last --days days with a pass in between is " +
+            "quarantined (straight failures are a regression and never are), and a quarantined " +
+            "test, manual or auto, is released after --passes first-attempt passes in a row. " +
+            "Pass on|off and/or any of the numbers; `projects get` shows the current values.",
+        scope: "project",
+        args: [{ name: "state", description: "on | off; omit to only tune the numbers" }],
+        flags: [
+            { name: "failures", type: "number", description: "Failures in the window that quarantine a test (1-50, default 3)" },
+            { name: "days", type: "number", description: "Window in days (1-90, default 7)" },
+            { name: "passes", type: "number", description: "Passes in a row that release a test (1-50, default 5)" },
+        ],
+        examples: [
+            "beryl projects quarantine on",
+            "beryl projects quarantine on --failures 3 --days 7 --passes 5",
+            "beryl projects quarantine --passes 3",
+            "beryl projects quarantine off",
+        ],
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const body = {};
+            const state = input.args.state;
+            if (typeof state === "string" && state) {
+                const lower = state.toLowerCase();
+                if (lower !== "on" && lower !== "off") {
+                    throw new UsageError(`Expected "on" or "off", got "${state}"`);
+                }
+                body.auto_quarantine = lower === "on";
+            }
+            const failures = flagNum(input, "failures");
+            const days = flagNum(input, "days");
+            const passes = flagNum(input, "passes");
+            if (failures !== undefined)
+                body.quarantine_failures = failures;
+            if (days !== undefined)
+                body.quarantine_days = days;
+            if (passes !== undefined)
+                body.quarantine_release_passes = passes;
+            if (Object.keys(body).length === 0) {
+                throw new UsageError("Pass on|off or at least one of --failures, --days, --passes");
+            }
+            return { data: await ctx.client.patch(projectPath(workspaceId, projectId), body) };
+        },
+    },
     {
         name: "projects delete",
         summary: "Delete a project and all its tests and runs",

package/dist/commands/tests.js

@@ -84,17 +84,29 @@ export const testCommands = [
             "fields, plus the two structural rules (the first EXECUTED step must be a goto, and at " +
             "least one step across before + steps must be an expect). Also the identity rules: a " +
             "plan that signs up with {{inbox_address}} and reads the mail must delete the account " +
-            "in `after` (an error; the server refuses it too), and one that mints an address it " +
-            "never reads gets a warning to cite {{mailbox_address}}. Runs entirely locally, so a " +
-            "malformed plan fails here instead of costing a server round-trip. " +
+            "in `after`, and no test plan may type {{mailbox_address}} (errors; the server refuses " +
+            "both too). Pass --provisioning for an account's signup or sign-in plan, which may fill " +
+            "{{mailbox_address}}. Runs entirely locally, so a malformed plan fails here instead of " +
+            "costing a server round-trip. " +
             `Schema: ${ACTION_PLAN_SCHEMA.$id}`,
         scope: "none",
         flags: [
             { name: "file", type: "string", required: true, description: "Plan JSON file, or - for stdin" },
+            {
+                name: "provisioning",
+                type: "boolean",
+                description: "The file is an account's signup or sign-in plan (for `accounts signup` / " +
+                    "`accounts set-login`), so it may fill {{mailbox_address}}",
+            },
+        ],
+        examples: [
+            "beryl tests lint --file plan.json",
+            "beryl tests lint --file signup-plan.json --provisioning",
         ],
-        examples: ["beryl tests lint --file plan.json"],
         async run(_ctx, input) {
-            const issues = lintPlan(readJsonFlag(input, "file"));
+            const issues = lintPlan(readJsonFlag(input, "file"), {
+                provisioning: flagBool(input, "provisioning"),
+            });
             const lines = issues.map((i) => `${i.severity === "warning" ? "warning" : "error"} ${i.path}: ${i.message}`);
             if (issues.every((i) => i.severity === "warning")) {
                 return {
@@ -575,9 +587,10 @@ export const testCommands = [
             "red lands in the run's quarantined_count and gates neither the run's verdict nor exit " +
             "codes, so it can't red-light a deploy. Use " +
             "it on a persistently flaky test instead of deleting it (which destroys the history) or " +
-            "asking support to deactivate it (which stops it running at all). After 5 consecutive " +
-            "clean passes the test reports rehab_ready (advisory only, nothing un-quarantines " +
-            "itself). `off` un-quarantines.",
+            "asking support to deactivate it (which stops it running at all). After the project's " +
+            "release threshold of consecutive clean passes (default 5) the test reports " +
+            "rehab_ready: a suggestion while auto quarantine is off, the actual release once " +
+            "`projects quarantine on` has turned it on. `off` un-quarantines.",
         scope: "project",
         args: [
             { name: "test-id", description: "Test id", required: true },

package/dist/lint.js

@@ -3,14 +3,15 @@ import { ACTION_PLAN_SCHEMA } from "./schema.generated.js";
 // keep them identical.
 export const MINTED_IDENTITY_NEEDS_TEARDOWN = "This plan signs up with {{inbox_address}} and reads the mail it triggers, so it " +
     "leaves an account behind on the site. Add an 'after' section that deletes it " +
-    "through the app (mark steps optional so a missing account ends the teardown " +
-    "quietly), or cite {{mailbox_address}} if no new account is meant to exist.";
+    "through the app (mark steps optional so a missing account ends the teardown quietly).";
+export const MAILBOX_ADDRESS_IN_TEST = "This plan types the standing account's address ({{mailbox_address}}) into the app. " +
+    "A test never signs in that way: one that needs to be signed in sets requires_auth " +
+    "and rides the account's session; one that needs an address the site can mail cites " +
+    "{{inbox_address}}, an alias only this test reads. Only the account's own signup and " +
+    "sign-in plans (accounts signup, accounts set-login) may fill {{mailbox_address}}.";
 export const OPTIONAL_TEARDOWN_ACTION_MUST_BE_LAST = (label, action) => `${label} (${action}) is optional but not the last step of 'after'. An optional step ` +
     "that misses ends the teardown, so the steps after it would silently not run. Guard " +
     "them with an optional 'wait_for' or 'expect' instead, or move this step to the end.";
-export const MINTED_IDENTITY_UNUSED = "This plan cites {{inbox_address}} but never reads the mail it triggers, so it mints " +
-    "an identity it never uses. Cite {{mailbox_address}} instead, unless the site must be " +
-    "handed an address it has never seen.";
 const typeOf = (value) => {
     if (value === null)
         return "null";
@@ -135,7 +136,9 @@ function validate(value, raw, root, path) {
 const steps = (section) => Array.isArray(section)
     ? section.filter((s) => typeof s === "object" && s !== null)
     : [];
-const citesInboxAddress = (step) => Object.values(step).some((v) => typeof v === "string" && v.includes("{{inbox_address}}"));
+const cites = (handle) => (step) => Object.values(step).some((v) => typeof v === "string" && v.includes(`{{${handle}}}`));
+const citesInboxAddress = cites("inbox_address");
+const citesMailboxAddress = cites("mailbox_address");
 const TEARDOWN_GUARD_ACTIONS = new Set(["wait_for", "expect"]);
 function optionalPlacementIssues(after) {
     return after.flatMap((s, i) => s.optional === true && !TEARDOWN_GUARD_ACTIONS.has(String(s.action)) && i !== after.length - 1
@@ -147,7 +150,7 @@ function optionalPlacementIssues(after) {
         ]
         : []);
 }
-function mintedIdentityIssues(plan) {
+function identityIssues(plan, provisioning) {
     if (typeof plan !== "object" || plan === null)
         return [];
     const { before, steps: main, after } = plan;
@@ -155,20 +158,22 @@ function mintedIdentityIssues(plan) {
     if (placement.length > 0)
         return placement;
     const body = [...steps(before), ...steps(main)];
-    if (![...body, ...steps(after)].some(citesInboxAddress))
+    const all = [...body, ...steps(after)];
+    if (!provisioning && all.some(citesMailboxAddress)) {
+        return [{ path: "(root)", message: MAILBOX_ADDRESS_IN_TEST }];
+    }
+    if (!all.some(citesInboxAddress))
         return [];
     const readsMail = body.some((s) => s.action === "await_email");
     const tearsDown = steps(after).some((s) => s.action !== "expect");
-    if (readsMail && !tearsDown) {
-        return [{ path: "(root)", message: MINTED_IDENTITY_NEEDS_TEARDOWN }];
-    }
-    if (!readsMail) {
-        return [{ path: "(root)", message: MINTED_IDENTITY_UNUSED, severity: "warning" }];
-    }
-    return [];
+    return readsMail && !tearsDown
+        ? [{ path: "(root)", message: MINTED_IDENTITY_NEEDS_TEARDOWN }]
+        : [];
 }
-export function lintPlan(plan) {
+// `provisioning`: the plan is an account's own signup or sign-in plan, the one place
+// {{mailbox_address}} is legal (mirrors the server's validate_editable_plan flag).
+export function lintPlan(plan, opts = {}) {
     const issues = validate(plan, ACTION_PLAN_SCHEMA, ACTION_PLAN_SCHEMA, "");
     // The identity rules read a well-formed plan; a shape error is reported alone.
-    return issues.length > 0 ? issues : mintedIdentityIssues(plan);
+    return issues.length > 0 ? issues : identityIssues(plan, opts.provisioning === true);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.44.0",
+  "version": "0.46.0",
   "description": "Beryl on the command line — projects, runs, the exploring agent, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",