@beryl-so/cli 0.60.0 → 0.61.0

package/README.md

@@ -334,6 +334,17 @@ The project's standing email addresses, where its tests receive sign-in mail.
 | `beryl mailbox read <mailbox-id>` | Read the latest email in a mailbox (waits for one to arrive) | `mailbox_read` |
 | `beryl mailbox emails <mailbox-id>` | List the emails a mailbox has received | `mailbox_emails` |
 
+### third-party-integrations
+
+Test a site's own integrations (Connect Slack) against Beryl's accounts on those providers.
+
+`beryl third-party-integrations` with no subcommand runs `third-party-integrations list`.
+
+| Command | Summary | MCP tool |
+| --- | --- | --- |
+| `beryl third-party-integrations list` | The third party providers this project's tests can connect to | `third_party_integrations_list` |
+| `beryl third-party-integrations target` | The channel this project's tests pick when the app asks where to post | `third_party_integrations_target` |
+
 ### accounts
 
 Durable identities on the site under test: what an authenticated test signs in as.

package/dist/beryl-test-skill.js

@@ -551,4 +551,31 @@ against your test's **intent**:
 So a test is *healable* exactly when it has **a strong intent + a real outcome
 assertion**. With a vague intent and a chrome-only assertion Beryl can't tell
 regression from cosmetic drift — it either heals over real breakage or fails on noise.
+
+## 7. Third party integration tests
+
+An app's own integrations (its "Connect Slack" button, the message it then posts) are
+testable against Beryl's account on that provider. Call
+\`third_party_integrations_list\` first; if the provider is not listed, do not write the
+test.
+
+Two shapes:
+
+- **Connect from scratch**: declare \`"third_party_integrations": ["slack"]\`, click the
+  app's Connect button, then \`{ "action": "third_party_authorize", "provider": "slack" }\`
+  approves the consent page and returns to the app. Pick the channel
+  \`third_party_integrations_target\` names, trigger the app's message, then
+  \`{ "action": "await_third_party", "provider": "slack", "expect_text": "..." }\`.
+- **Integration intact**: no declaration. Trigger the app's message, then
+  \`await_third_party\`.
+
+Rules:
+
+- Always tear down what the test connected in \`after\`: disconnect the integration in
+  the app.
+- These steps run only in Beryl's cloud: \`tests_create\` verifies server-side and
+  \`runs_local\` skips them, so run them with \`runs_trigger\`.
+- A setup error saying Beryl's access "is being restored" is on Beryl's side: do not
+  edit the test.
+- Never add steps that browse Slack itself; the actions above do all of it.
 `;

package/dist/commands/runs.js

@@ -2,7 +2,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { downloadRunArtifacts, failureImages, isFailing, resultsOf, } from "../artifacts.js";
 import { CliError, UsageError } from "../errors.js";
-import { buildImportForm, establishAccountSession, executeLocalSpec, IMPORT_MAX_ERROR_LEN, toRunEntry, } from "../local-exec.js";
+import { buildImportForm, establishAccountSession, executeLocalSpec, IMPORT_MAX_ERROR_LEN, specUsesThirdParty, THIRD_PARTY_USE_TRIGGER, toRunEntry, } from "../local-exec.js";
 import { countPlannedFrames, countWrittenFrames, PlaywrightMissingError, } from "../local-run.js";
 import { dim, green, red, yellow } from "../output.js";
 import { anyGap, confirmInstall, describeGaps, installCommandsFor, installHint, installPlaywright, playwrightGaps, } from "../playwright-install.js";
@@ -152,7 +152,8 @@ export const runCommands = [
             "in the cloud: their account signs in once per invocation and every session-mode " +
             "test rides that session; a failed sign-in fails those tests with the same " +
             "SESSION_* reason a cloud run reports. Only a test that depends on a session Beryl " +
-            "captured server-side is skipped, with a note. Run those with `runs trigger`. Point " +
+            "captured server-side, or that has third party integration steps (cloud only), is " +
+            "skipped, with a note. Run those with `runs trigger`. Point " +
             "--url-override at a local dev server or preview, and --dir to keep specs, artifacts, " +
             "and reports on disk. Exits 0 only if every executed test passed.",
         scope: "project",
@@ -255,6 +256,7 @@ export const runCommands = [
                 requiresAuth: Boolean(script.requires_auth),
                 usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(script.content),
                 usesLoginPassword: Boolean(script.uses_login_password),
+                usesThirdParty: specUsesThirdParty(script.content),
                 inlinedLoginSteps: script.inlined_login_steps ?? 0,
                 ...(script.session_mode && script.account_id
                     ? { sessionMode: true, accountId: script.account_id }
@@ -288,13 +290,17 @@ export const runCommands = [
             // a cloud run's setup phase does. A test whose account is missing its password
             // would type a literal placeholder into the page: skip, with the server's fix-it
             // message.
-            const skipped = specs.filter((s) => (s.requiresAuth && !(s.sessionMode && s.accountId)) || s.loginConfigError);
+            const skipped = specs.filter((s) => (s.requiresAuth && !(s.sessionMode && s.accountId)) ||
+                s.loginConfigError ||
+                s.usesThirdParty);
             const runnable = specs.filter((s) => !skipped.includes(s));
             for (const s of skipped) {
                 ctx.err(yellow(s.loginConfigError
                     ? `! ${s.title}: skipped — ${s.loginConfigError}`
-                    : `! ${s.title}: needs a session only Beryl's cloud holds — skipped ` +
-                        `(use \`beryl runs trigger\`).`));
+                    : s.usesThirdParty
+                        ? `! ${s.title}: skipped, ${THIRD_PARTY_USE_TRIGGER}.`
+                        : `! ${s.title}: needs a session only Beryl's cloud holds, skipped ` +
+                            `(use \`beryl runs trigger\`).`));
             }
             if (runnable.length === 0)
                 throw new CliError("None of the selected tests can run locally.");

package/dist/commands/tests.js

@@ -1,8 +1,8 @@
 import fs from "node:fs";
 import { CliError, UsageError } from "../errors.js";
 import { ApiError } from "../http.js";
-import { lintPlan } from "../lint.js";
-import { establishAccountSession, executeLocalSpec, } from "../local-exec.js";
+import { lintPlan, planUsesThirdParty } from "../lint.js";
+import { establishAccountSession, executeLocalSpec, THIRD_PARTY_CLOUD_ONLY, } from "../local-exec.js";
 import { PlaywrightMissingError } from "../local-run.js";
 import { dim, green, red, table, yellow } from "../output.js";
 import { confirmInstall, installCommandsFor, installPlaywright, playwrightGaps, } from "../playwright-install.js";
@@ -281,7 +281,8 @@ export const testCommands = [
             "tests is banked, run their ids together once (`runs local`, or `runs trigger` for the " +
             "cloud) so they land as ONE run with real first results. A plan that signs in with a session Beryl captured " +
             "server-side cannot replay locally (that session never leaves Beryl's cloud). It falls " +
-            "back to server-side verification automatically, and says so. A session-mode plan replays locally " +
+            "back to server-side verification automatically, and says so; so does a plan with third party " +
+            "integration steps, which run only in Beryl's cloud. A session-mode plan replays locally " +
             "fine: the server renders it with its account's stored sign-in steps in front, so " +
             "the same identity is exercised on your machine. Optional `before` and `after` arrays hold setup and teardown " +
             "steps: `after` runs even when a main step fails, which is how a create/update/delete test " +
@@ -370,6 +371,23 @@ export const testCommands = [
                 environment_id: env,
             });
             let compiled = await compile();
+            const verifyInCloud = async (reason) => {
+                ctx.err(yellow(`! ${reason}, so this plan is verified server-side instead.`));
+                try {
+                    return { data: await bank({ verify: true, plan_hash: compiled.plan_hash }) };
+                }
+                catch (err) {
+                    const failure = verifyFailureResult(err);
+                    if (failure)
+                        return failure;
+                    const conflict = createConflictResult(err, compiled.plan_hash);
+                    if (conflict)
+                        return conflict;
+                    throw err;
+                }
+            };
+            if (planUsesThirdParty(plan))
+                return verifyInCloud(THIRD_PARTY_CLOUD_ONLY);
             // Session-mode plan: establish the account's session once (server-side, reused
             // from the stored one when it still proves live) and replay against it — the
             // cloud shape. An unsupported account is re-compiled to sign itself in.
@@ -391,20 +409,7 @@ export const testCommands = [
             // A captured session lives encrypted in Beryl's cloud and is never handed to
             // this machine — the cloud is the only place this plan can be proven.
             if (compiled.requires_auth && !session) {
-                ctx.err(yellow("! This plan signs in with a captured session, so it can only be verified " +
-                    "in Beryl's cloud — verifying server-side instead."));
-                try {
-                    return { data: await bank({ verify: true, plan_hash: compiled.plan_hash }) };
-                }
-                catch (err) {
-                    const failure = verifyFailureResult(err);
-                    if (failure)
-                        return failure;
-                    const conflict = createConflictResult(err, compiled.plan_hash);
-                    if (conflict)
-                        return conflict;
-                    throw err;
-                }
+                return verifyInCloud("A captured session never leaves Beryl's cloud");
             }
             if (compiled.login_config_error)
                 throw new CliError(compiled.login_config_error);

package/dist/commands/third-party-integrations.js

@@ -0,0 +1,59 @@
+import { UsageError } from "../errors.js";
+import { dim, table } from "../output.js";
+import { flagStr, projectPath } from "./util.js";
+const integrationsPath = (ws, p) => `${projectPath(ws, p)}/third-party-integrations`;
+export const thirdPartyIntegrationCommands = [
+    {
+        name: "third-party-integrations list",
+        summary: "The third party providers this project's tests can connect to",
+        groupDefault: true,
+        groupSummary: "Test a site's own integrations (Connect Slack) against Beryl's accounts on those providers.",
+        description: "Lists the providers set up on this Beryl deployment. A test may use " +
+            "`third_party_authorize` / `await_third_party` only for a provider listed here; " +
+            "anything else is refused at save time.",
+        scope: "project",
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const { providers } = (await ctx.client.get(integrationsPath(workspaceId, projectId)));
+            return {
+                data: providers,
+                human: providers.length
+                    ? table(providers.map((p) => ({ key: p.key, label: p.label })))
+                    : dim("No third party providers are set up on this deployment."),
+            };
+        },
+    },
+    {
+        name: "third-party-integrations target",
+        summary: "The channel this project's tests pick when the app asks where to post",
+        description: "Returns the project's own destination on Beryl's provider account (for Slack, a " +
+            "channel), creating it on first ask. When a test connects the app, it picks this " +
+            "channel in the app's settings; `await_third_party` then waits for the app's message " +
+            "there. `recorded_app_id` is the app whose first message was seen, null until then. " +
+            "A 409 means the provider is not set up on this deployment (see " +
+            "`third-party-integrations list`).",
+        scope: "project",
+        flags: [
+            {
+                name: "provider",
+                type: "string",
+                required: true,
+                description: "Provider key from `third-party-integrations list`, e.g. slack",
+            },
+        ],
+        examples: ["beryl third-party-integrations target --provider slack"],
+        async run(ctx, input) {
+            const provider = flagStr(input, "provider");
+            if (!provider)
+                throw new UsageError("--provider is required");
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const target = (await ctx.client.post(`${integrationsPath(workspaceId, projectId)}/${encodeURIComponent(provider)}/target`));
+            return {
+                data: target,
+                human: `#${target.external_name}\n\n` +
+                    dim(`Pick this channel in the app when the test connects ${target.provider}.` +
+                        (target.recorded_app_id ? ` Recorded app: ${target.recorded_app_id}` : "")),
+            };
+        },
+    },
+];

package/dist/lint.js

@@ -16,6 +16,9 @@ export const WRITE_NEEDS_CLOSING_TEARDOWN = (detail) => "A 'write' test must end
 export const OPTIONAL_TEARDOWN_ACTION_MUST_BE_LAST = (label, action) => `${label} (${action}) is optional but not the last step of 'after'. An optional step ` +
     "that misses ends the teardown, so the steps after it would silently not run. Guard " +
     "them with an optional 'wait_for' or 'expect' instead, or move this step to the end.";
+export const THIRD_PARTY_UNDECLARED = (p) => `third_party_authorize approves the install on ${p}'s consent page, which needs ` +
+    `Beryl's ${p} login: add "${p}" to third_party_integrations`;
+export const THIRD_PARTY_UNUSED = (p) => `third_party_integrations declares "${p}" but no third_party_authorize step uses it; remove it`;
 const typeOf = (value) => {
     if (value === null)
         return "null";
@@ -174,6 +177,37 @@ function identityIssues(plan, provisioning) {
         ? [{ path: "(root)", message: MINTED_IDENTITY_NEEDS_TEARDOWN }]
         : [];
 }
+export const THIRD_PARTY_AUTHORIZE = "third_party_authorize";
+export const AWAIT_THIRD_PARTY = "await_third_party";
+const executed = (plan) => [
+    ...steps(plan.before),
+    ...steps(plan.steps),
+    ...steps(plan.after),
+];
+export function planUsesThirdParty(plan) {
+    if (typeof plan !== "object" || plan === null)
+        return false;
+    return executed(plan).some((s) => s.action === THIRD_PARTY_AUTHORIZE || s.action === AWAIT_THIRD_PARTY);
+}
+// Mirrors the server's ActionPlan validator, which raises the first sorted mismatch only.
+function thirdPartyIssues(plan) {
+    if (typeof plan !== "object" || plan === null)
+        return [];
+    const record = plan;
+    const declared = new Set(Array.isArray(record.third_party_integrations)
+        ? record.third_party_integrations.map(String)
+        : []);
+    const authorized = new Set(executed(record)
+        .filter((s) => s.action === THIRD_PARTY_AUTHORIZE && typeof s.provider === "string")
+        .map((s) => s.provider));
+    const [undeclared] = [...authorized].filter((p) => !declared.has(p)).sort();
+    if (undeclared)
+        return [{ path: "(root)", message: THIRD_PARTY_UNDECLARED(undeclared) }];
+    const [unused] = [...declared].filter((p) => !authorized.has(p)).sort();
+    if (unused)
+        return [{ path: "third_party_integrations", message: THIRD_PARTY_UNUSED(unused) }];
+    return [];
+}
 const TEARDOWN_PROOF_KINDS = new Set([
     "gone",
     "hidden",
@@ -212,6 +246,7 @@ export function lintPlan(plan, opts = {}) {
     if (issues.length > 0)
         return issues;
     return [
+        ...thirdPartyIssues(plan),
         ...identityIssues(plan, opts.provisioning === true),
         ...(opts.access === undefined ? [] : accessIssues(plan, opts.access)),
     ];

package/dist/local-exec.js

@@ -8,6 +8,11 @@ export const IMPORT_MAX_ERROR_LEN = 5000;
 export const IMPORT_MAX_FILE_BYTES = 14 * 1024 * 1024;
 export const IMPORT_MAX_TOTAL_BYTES = 180 * 1024 * 1024;
 export const IMPORT_MAX_FILES = 3600;
+export const THIRD_PARTY_CLOUD_ONLY = "third party integration steps run only in Beryl's cloud";
+export const THIRD_PARTY_USE_TRIGGER = `${THIRD_PARTY_CLOUD_ONLY}; use \`beryl runs trigger\``;
+// The rendered await_third_party helper, and the authorize step's wait to leave the provider's host.
+const THIRD_PARTY_SPEC_MARKER = /__vmThirdParty\(|\.test\(u\.hostname\)/;
+export const specUsesThirdParty = (content) => THIRD_PARTY_SPEC_MARKER.test(content);
 export function toRunEntry(spec, outcome, runError, startedAt, ordinal, redact) {
     // Mirrors the cloud runner's redact_result: the spec types secrets into the page
     // and an injected session authenticates its requests, so error text and the DOM

package/dist/registry/index.js

@@ -12,6 +12,7 @@ import { mcpCommands } from "../commands/mcp.js";
 import { projectCommands } from "../commands/projects.js";
 import { runCommands } from "../commands/runs.js";
 import { testCommands } from "../commands/tests.js";
+import { thirdPartyIntegrationCommands } from "../commands/third-party-integrations.js";
 import { trackerCommands } from "../commands/tracker.js";
 import { workspaceCommands } from "../commands/workspaces.js";
 export const WORKSPACE_FLAG = {
@@ -70,6 +71,7 @@ export const commands = [
     ...explorationCommands,
     ...configCommands,
     ...mailboxCommands,
+    ...thirdPartyIntegrationCommands,
     ...testAccountCommands,
     ...accountCommands,
     ...mcpCommands,

package/dist/schema.generated.js

@@ -22,7 +22,9 @@ export const ACTION_PLAN_SCHEMA = {
                 "dialog",
                 "switch_tab",
                 "close_tab",
-                "drag"
+                "drag",
+                "third_party_authorize",
+                "await_third_party"
             ],
             "title": "ActionType",
             "type": "string"
@@ -446,6 +448,54 @@ export const ACTION_PLAN_SCHEMA = {
                         ]
                     }
                 },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "third_party_authorize"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "provider": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            }
+                        },
+                        "required": [
+                            "provider"
+                        ]
+                    }
+                },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "await_third_party"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "provider": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            }
+                        },
+                        "required": [
+                            "provider"
+                        ]
+                    }
+                },
                 {
                     "if": {
                         "properties": {
@@ -901,6 +951,25 @@ export const ACTION_PLAN_SCHEMA = {
                             "selector"
                         ]
                     }
+                },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "await_email"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "wait_s": {
+                                "maximum": 50
+                            }
+                        }
+                    }
                 }
             ],
             "properties": {
@@ -1096,6 +1165,17 @@ export const ACTION_PLAN_SCHEMA = {
                     "title": "Optional",
                     "type": "boolean"
                 },
+                "provider": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/$defs/ThirdPartyProvider"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "default": null
+                },
                 "selector": {
                     "anyOf": [
                         {
@@ -1164,7 +1244,7 @@ export const ACTION_PLAN_SCHEMA = {
                 "wait_s": {
                     "anyOf": [
                         {
-                            "maximum": 50,
+                            "maximum": 600,
                             "minimum": 1,
                             "type": "integer"
                         },
@@ -1181,6 +1261,13 @@ export const ACTION_PLAN_SCHEMA = {
             ],
             "title": "PlanStep",
             "type": "object"
+        },
+        "ThirdPartyProvider": {
+            "enum": [
+                "slack"
+            ],
+            "title": "ThirdPartyProvider",
+            "type": "string"
         }
     },
     "$id": "https://api.beryl.so/api/v1/schemas/action-plan.schema.json",
@@ -1344,6 +1431,14 @@ export const ACTION_PLAN_SCHEMA = {
             "minItems": 1,
             "title": "Steps",
             "type": "array"
+        },
+        "third_party_integrations": {
+            "description": "Providers whose consent page this test approves; the browser then carries Beryl's login for them. Declare exactly the providers of its third_party_authorize steps.",
+            "items": {
+                "$ref": "#/$defs/ThirdPartyProvider"
+            },
+            "title": "Third Party Integrations",
+            "type": "array"
         }
     },
     "required": [

package/dist/skill-tables.js

@@ -36,6 +36,8 @@ export const ACTION_DOCS = {
     switch_tab: "Make another open tab the active page; `value` picks it — see below",
     close_tab: "Close the active tab (optional `value` picks one) and fall back to the previous",
     drag: "Drag the `selector` element onto the `value` TARGET selector",
+    third_party_authorize: "Approve the `provider` install on its consent page as Beryl's account (cloud only, see third party integration tests)",
+    await_third_party: "Wait for the connected app's message on Beryl's `provider` account (cloud only)",
 };
 export const EXPECT_DOCS = {
     visible: "The element is visible",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.60.0",
+  "version": "0.61.0",
   "description": "Beryl on the command line: projects, runs, the exploring agent, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",