@beryl-so/cli 0.60.0 → 0.62.1

package/README.md

@@ -334,6 +334,17 @@ The project's standing email addresses, where its tests receive sign-in mail.
 | `beryl mailbox read <mailbox-id>` | Read the latest email in a mailbox (waits for one to arrive) | `mailbox_read` |
 | `beryl mailbox emails <mailbox-id>` | List the emails a mailbox has received | `mailbox_emails` |
 
+### third-party-integrations
+
+Test a site's own integrations (Connect Slack) against Beryl's accounts on those providers.
+
+`beryl third-party-integrations` with no subcommand runs `third-party-integrations list`.
+
+| Command | Summary | MCP tool |
+| --- | --- | --- |
+| `beryl third-party-integrations list` | The third party providers this project's tests can connect to | `third_party_integrations_list` |
+| `beryl third-party-integrations target` | The channel this project's tests pick when the app asks where to post | `third_party_integrations_target` |
+
 ### accounts
 
 Durable identities on the site under test: what an authenticated test signs in as.

package/dist/adapters/cli.js

@@ -1,4 +1,4 @@
-import { loadConfig } from "../config.js";
+import { loadConfig, withFlagOverrides } from "../config.js";
 import { createContext } from "../context.js";
 import { CliError, EXIT_OK, EXIT_USAGE, UsageError } from "../errors.js";
 import { ApiClient } from "../http.js";
@@ -317,11 +317,17 @@ export async function runCli(argv) {
         process.stdout.write(commandHelp(spec) + "\n");
         return EXIT_OK;
     }
-    const config = loadConfig();
-    if (parsed.apiUrl)
-        config.apiUrl = parsed.apiUrl;
-    if (parsed.token)
-        config.token = parsed.token;
+    let config;
+    try {
+        config = withFlagOverrides(loadConfig(), { apiUrl: parsed.apiUrl, token: parsed.token });
+    }
+    catch (err) {
+        if (err instanceof CliError) {
+            process.stderr.write(err.message + "\n");
+            return err.exitCode;
+        }
+        throw err;
+    }
     const client = new ApiClient(config.apiUrl, config.token);
     const ctx = createContext({ client, config, json: parsed.json, mcp: false });
     // The MCP server reports its own tool calls; every other invocation is one command.

package/dist/adapters/mcp.js

@@ -3,7 +3,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
 import { instrument } from "@posthog/mcp";
 import fs from "node:fs";
-import { loadConfig } from "../config.js";
+import { loadConfig, withFlagOverrides } from "../config.js";
 import { createContext } from "../context.js";
 import { AuthError, CliError } from "../errors.js";
 import { ApiClient, ApiError } from "../http.js";
@@ -222,7 +222,7 @@ export function currentAuth(fallback) {
     }
     if (authCache?.mtimeMs === mtimeMs)
         return authCache.auth;
-    const config = loadConfig();
+    const config = withFlagOverrides(loadConfig(), fallback.config.flagOverrides ?? {});
     const auth = {
         client: new ApiClient(config.apiUrl, config.token),
         config,
@@ -330,17 +330,17 @@ export async function serveMcp(baseCtx) {
             if (lines.length > 400)
                 lines.splice(0, lines.length - 400);
         };
-        const auth = currentAuth(baseCtx);
-        const ctx = createContext({
-            client: auth.client,
-            config: auth.config,
-            json: true,
-            interactive: false,
-            mcp: true,
-            out: push,
-            err: push,
-        });
         try {
+            const auth = currentAuth(baseCtx);
+            const ctx = createContext({
+                client: auth.client,
+                config: auth.config,
+                json: true,
+                interactive: false,
+                mcp: true,
+                out: push,
+                err: push,
+            });
             const input = toInput(spec, request.params.arguments ?? {});
             const result = (await withTool(request.params.name, () => spec.run(ctx, input))) ?? {};
             return toolResult(result, lines);

package/dist/beryl-test-skill.js

@@ -551,4 +551,31 @@ against your test's **intent**:
 So a test is *healable* exactly when it has **a strong intent + a real outcome
 assertion**. With a vague intent and a chrome-only assertion Beryl can't tell
 regression from cosmetic drift — it either heals over real breakage or fails on noise.
+
+## 7. Third party integration tests
+
+An app's own integrations (its "Connect Slack" button, the message it then posts) are
+testable against Beryl's account on that provider. Call
+\`third_party_integrations_list\` first; if the provider is not listed, do not write the
+test.
+
+Two shapes:
+
+- **Connect from scratch**: declare \`"third_party_integrations": ["slack"]\`, click the
+  app's Connect button, then \`{ "action": "third_party_authorize", "provider": "slack" }\`
+  approves the consent page and returns to the app. Pick the channel
+  \`third_party_integrations_target\` names, trigger the app's message, then
+  \`{ "action": "await_third_party", "provider": "slack", "expect_text": "..." }\`.
+- **Integration intact**: no declaration. Trigger the app's message, then
+  \`await_third_party\`.
+
+Rules:
+
+- Always tear down what the test connected in \`after\`: disconnect the integration in
+  the app.
+- These steps run only in Beryl's cloud: \`tests_create\` verifies server-side and
+  \`runs_local\` skips them, so run them with \`runs_trigger\`.
+- A setup error saying Beryl's access "is being restored" is on Beryl's side: do not
+  edit the test.
+- Never add steps that browse Slack itself; the actions above do all of it.
 `;

package/dist/commands/explorations.js

@@ -53,7 +53,8 @@ export const explorationCommands = [
         name: "explorations watch",
         summary: "Stream an exploration live: watch the agent explore and author tests",
         description: "Replays every recorded step on connect, then follows live until the exploration " +
-            "completes or fails.",
+            "ends. Exits 0 only if it completed; a failed or cancelled exploration, or a stream " +
+            "that keeps closing without a final status, exits 1.",
         scope: "project",
         args: [{ name: "exploration-id", description: "Exploration id", required: true }],
         flags: [{ name: "timeout", type: "number", description: "Max minutes to wait" }],

package/dist/commands/health.js

@@ -275,7 +275,8 @@ export const healthCommands = [
             "`selector`, measured values, the `viewports` it shows at and a fix hint in " +
             "`recommendation`. The `performance` category grades Google PageSpeed on mobile " +
             "and on desktop separately (`speeds`, each with its own Core Web Vitals) and takes " +
-            "the worse of the two; one that couldn't be measured is left out. Pass --report for a single-page check started with " +
+            "the worse of the two; one that couldn't be measured is left out. A category's `unchecked` lists checks this run " +
+            "couldn't determine (e.g. robots.txt timed out): they neither pass nor fail. Pass --report for a single-page check started with " +
             "`health run --path`.",
         scope: "project",
         flags: [
@@ -321,7 +322,8 @@ export const healthCommands = [
             "With --ci it starts a CI run, waits for every page, prints a Markdown summary you can " +
             "post as the PR comment, and exits 1 when findings at or above the environment's " +
             "fail-on severity exist, a page couldn't be checked, or the run failed. Builds that the " +
-            "environment's CI mode doesn't cover skip and exit 0. CI runs file tracker tickets only when --health-ci-tickets is on.",
+            "environment's CI mode doesn't cover skip and exit 0; if the stored run settings are invalid it exits 1 " +
+            "rather than skipping. CI runs file tracker tickets only when --health-ci-tickets is on.",
         scope: "project",
         flags: [
             ENV_FLAG,

package/dist/commands/init.js

@@ -77,26 +77,21 @@ function mergeMcpConfig(file, key, entries) {
     }
     return { beryl, playwright };
 }
-// A tool's config lives per-user (same as the PAT it needs, in ~/.config/beryl) — so wire the
-// MCP servers per-user too, not in a committed .mcp.json that 401s for every teammate until
-// they run `beryl login` anyway. We DON'T hand-edit ~/.claude.json (Claude Code owns it; a
-// corrupt write breaks the user's whole CLI) — we shell out to `claude mcp add` and let Claude
-// own the edit. Returns false when the `claude` binary isn't on PATH so the caller can fall
-// back to a copy-paste command.
 function claudeUserAdd(name, entry) {
     try {
         execFileSync("claude", ["mcp", "add", name, "-s", "user", "--", entry.command, ...entry.args], {
-            stdio: "ignore",
+            stdio: ["ignore", "pipe", "pipe"],
         });
-        return true;
+        return { ok: true };
     }
     catch (err) {
-        // A non-zero exit ALSO covers "already exists" — but so does a missing binary. Distinguish:
-        // ENOENT means no `claude` on PATH (fall back), anything else means it ran and declined
-        // (already configured — treat as success, nothing to do).
-        if (err.code === "ENOENT")
-            return false;
-        return true;
+        const e = err;
+        if (e.code === "ENOENT")
+            return { ok: false, reason: "missing" };
+        const output = `${e.stderr ?? ""}\n${e.stdout ?? ""}`.trim();
+        if (/already exists/i.test(output))
+            return { ok: true };
+        return { ok: false, reason: "failed", detail: output.split("\n")[0] || e.message };
     }
 }
 function claudeAddHint(name, entry) {
@@ -399,13 +394,16 @@ export const initCommands = [
             const clients = await selectClients(ctx, input.flags);
             if (clients.has("claude") && scope === "user") {
                 // Claude Code owns ~/.claude.json — shell out to `claude mcp add` rather than write it.
-                const berylOk = claudeUserAdd("beryl", MCP_SERVER_ENTRY);
-                const playwrightOk = claudeUserAdd("playwright", PLAYWRIGHT_SERVER_ENTRY);
-                if (berylOk && playwrightOk) {
+                const beryl = claudeUserAdd("beryl", MCP_SERVER_ENTRY);
+                const playwright = claudeUserAdd("playwright", PLAYWRIGHT_SERVER_ENTRY);
+                if (beryl.ok && playwright.ok) {
                     ctx.err(`${green("✓")} beryl + playwright MCP configured ${dim("(user scope)")}`);
                 }
                 else {
-                    ctx.err(yellow("• `claude` not on PATH — run these to wire user-scope MCP servers:"));
+                    const failure = [beryl, playwright].find((r) => !r.ok && r.reason === "failed");
+                    ctx.err(yellow(failure
+                        ? `• \`claude mcp add\` failed (${failure.detail}); run these to wire user-scope MCP servers:`
+                        : "• `claude` not on PATH; run these to wire user-scope MCP servers:"));
                     ctx.err(`    ${cyan(claudeAddHint("beryl", MCP_SERVER_ENTRY))}`);
                     ctx.err(`    ${cyan(claudeAddHint("playwright", PLAYWRIGHT_SERVER_ENTRY))}`);
                 }

package/dist/commands/runs.js

@@ -1,8 +1,9 @@
 import fs from "node:fs";
 import path from "node:path";
 import { downloadRunArtifacts, failureImages, isFailing, resultsOf, } from "../artifacts.js";
-import { CliError, UsageError } from "../errors.js";
-import { buildImportForm, establishAccountSession, executeLocalSpec, IMPORT_MAX_ERROR_LEN, toRunEntry, } from "../local-exec.js";
+import { AuthError, CliError, UsageError } from "../errors.js";
+import { ApiError } from "../http.js";
+import { buildImportForm, establishAccountSession, executeLocalSpec, IMPORT_MAX_ERROR_LEN, specUsesThirdParty, THIRD_PARTY_USE_TRIGGER, toRunEntry, } from "../local-exec.js";
 import { countPlannedFrames, countWrittenFrames, PlaywrightMissingError, } from "../local-run.js";
 import { dim, green, red, yellow } from "../output.js";
 import { anyGap, confirmInstall, describeGaps, installCommandsFor, installHint, installPlaywright, playwrightGaps, } from "../playwright-install.js";
@@ -152,9 +153,12 @@ export const runCommands = [
             "in the cloud: their account signs in once per invocation and every session-mode " +
             "test rides that session; a failed sign-in fails those tests with the same " +
             "SESSION_* reason a cloud run reports. Only a test that depends on a session Beryl " +
-            "captured server-side is skipped, with a note. Run those with `runs trigger`. Point " +
+            "captured server-side, or that has third party integration steps (cloud only), is " +
+            "skipped, with a note. Run those with `runs trigger`. Point " +
             "--url-override at a local dev server or preview, and --dir to keep specs, artifacts, " +
-            "and reports on disk. Exits 0 only if every executed test passed.",
+            "and reports on disk. With no ids, a test with no plan yet is skipped quietly; one " +
+            "whose script can't be fetched is listed under `skipped` and fails the run. Exits 0 " +
+            "only if every executed test passed and every script was fetched.",
         scope: "project",
         args: [
             {
@@ -255,6 +259,7 @@ export const runCommands = [
                 requiresAuth: Boolean(script.requires_auth),
                 usesEmail: /__vmInbox\(|__vmAwaitEmail\(/.test(script.content),
                 usesLoginPassword: Boolean(script.uses_login_password),
+                usesThirdParty: specUsesThirdParty(script.content),
                 inlinedLoginSteps: script.inlined_login_steps ?? 0,
                 ...(script.session_mode && script.account_id
                     ? { sessionMode: true, accountId: script.account_id }
@@ -263,38 +268,51 @@ export const runCommands = [
                 ...(script.login_config_error ? { loginConfigError: script.login_config_error } : {}),
             });
             const specs = [];
+            // With --all, a test the server can't render (no plan yet, 409) is a quiet skip;
+            // any other fetch failure is listed and fails the run. An explicitly-requested id
+            // failing to render is the caller's problem.
+            const fetchFailed = [];
             for (const id of ids) {
                 let script;
                 try {
                     script = await fetchScript(id);
                 }
                 catch (err) {
-                    // With --all an unrenderable test (no plan yet) is a skip, not an abort;
-                    // an explicitly-requested id failing to render is the caller's problem.
-                    if (all) {
-                        const detail = err instanceof Error ? err.message : String(err);
+                    if (!all || err instanceof AuthError)
+                        throw err;
+                    const detail = err instanceof Error ? err.message : String(err);
+                    if (err instanceof ApiError && err.status === 409) {
                         ctx.err(yellow(`! ${titles.get(id) ?? id}: no runnable script — skipped (${detail})`));
-                        continue;
                     }
-                    throw err;
+                    else {
+                        fetchFailed.push(id);
+                        ctx.err(red(`✗ ${titles.get(id) ?? id}: could not fetch its script (${detail})`));
+                    }
+                    continue;
                 }
                 specs.push(toLocalSpec(id, script));
             }
             if (specs.length === 0)
-                throw new CliError("No runnable tests were found.");
+                throw new CliError(fetchFailed.length
+                    ? `Could not fetch the script for ${fetchFailed.length} test(s); nothing ran.`
+                    : "No runnable tests were found.");
             // A test that leans on a session this CLI cannot establish gets no session here,
             // so skip it rather than run a spec doomed at the login wall. A session-mode test
             // is NOT in that set: its account's session is established below, once, exactly as
             // a cloud run's setup phase does. A test whose account is missing its password
             // would type a literal placeholder into the page: skip, with the server's fix-it
             // message.
-            const skipped = specs.filter((s) => (s.requiresAuth && !(s.sessionMode && s.accountId)) || s.loginConfigError);
+            const skipped = specs.filter((s) => (s.requiresAuth && !(s.sessionMode && s.accountId)) ||
+                s.loginConfigError ||
+                s.usesThirdParty);
             const runnable = specs.filter((s) => !skipped.includes(s));
             for (const s of skipped) {
                 ctx.err(yellow(s.loginConfigError
                     ? `! ${s.title}: skipped — ${s.loginConfigError}`
-                    : `! ${s.title}: needs a session only Beryl's cloud holds — skipped ` +
-                        `(use \`beryl runs trigger\`).`));
+                    : s.usesThirdParty
+                        ? `! ${s.title}: skipped, ${THIRD_PARTY_USE_TRIGGER}.`
+                        : `! ${s.title}: needs a session only Beryl's cloud holds, skipped ` +
+                            `(use \`beryl runs trigger\`).`));
             }
             if (runnable.length === 0)
                 throw new CliError("None of the selected tests can run locally.");
@@ -493,6 +511,9 @@ export const runCommands = [
                 : red(`${failed} test(s) failed`) + ` (${passed} passed)`;
             const human = `${summaryLine}` +
                 (skipped.length ? `\n${yellow(`${skipped.length} skipped (see notes above)`)}` : "") +
+                (fetchFailed.length
+                    ? `\n${red(`${fetchFailed.length} could not be fetched (see notes above)`)}`
+                    : "") +
                 (runId ? `\nSynced to Beryl as run ${runId} — \`beryl runs get ${runId}\`` : "") +
                 (!sync ? `\n${dim("--no-sync: nothing recorded in Beryl")}` : "") +
                 (dir ? `\n${dim(`Specs + artifacts + reports under ${path.resolve(dir)}`)}` : "");
@@ -500,13 +521,13 @@ export const runCommands = [
                 data: {
                     passed,
                     failed,
-                    skipped: skipped.map((s) => s.id),
+                    skipped: [...skipped.map((s) => s.id), ...fetchFailed],
                     results: entries.map(({ files: _files, ...rest }) => rest),
                     ...(runId ? { run_id: runId } : {}),
                 },
                 human,
-                // Exit codes are a CI contract: any failing/errored test → exit 1.
-                ...(failed > 0 ? { exitCode: 1 } : {}),
+                // Exit codes are a CI contract: any failing/errored or unfetchable test → exit 1.
+                ...(failed > 0 || fetchFailed.length > 0 ? { exitCode: 1 } : {}),
             };
         },
     },

package/dist/commands/tests.js

@@ -1,8 +1,8 @@
 import fs from "node:fs";
 import { CliError, UsageError } from "../errors.js";
 import { ApiError } from "../http.js";
-import { lintPlan } from "../lint.js";
-import { establishAccountSession, executeLocalSpec, } from "../local-exec.js";
+import { lintPlan, planUsesThirdParty } from "../lint.js";
+import { establishAccountSession, executeLocalSpec, THIRD_PARTY_CLOUD_ONLY, } from "../local-exec.js";
 import { PlaywrightMissingError } from "../local-run.js";
 import { dim, green, red, table, yellow } from "../output.js";
 import { confirmInstall, installCommandsFor, installPlaywright, playwrightGaps, } from "../playwright-install.js";
@@ -281,7 +281,8 @@ export const testCommands = [
             "tests is banked, run their ids together once (`runs local`, or `runs trigger` for the " +
             "cloud) so they land as ONE run with real first results. A plan that signs in with a session Beryl captured " +
             "server-side cannot replay locally (that session never leaves Beryl's cloud). It falls " +
-            "back to server-side verification automatically, and says so. A session-mode plan replays locally " +
+            "back to server-side verification automatically, and says so; so does a plan with third party " +
+            "integration steps, which run only in Beryl's cloud. A session-mode plan replays locally " +
             "fine: the server renders it with its account's stored sign-in steps in front, so " +
             "the same identity is exercised on your machine. Optional `before` and `after` arrays hold setup and teardown " +
             "steps: `after` runs even when a main step fails, which is how a create/update/delete test " +
@@ -370,6 +371,23 @@ export const testCommands = [
                 environment_id: env,
             });
             let compiled = await compile();
+            const verifyInCloud = async (reason) => {
+                ctx.err(yellow(`! ${reason}, so this plan is verified server-side instead.`));
+                try {
+                    return { data: await bank({ verify: true, plan_hash: compiled.plan_hash }) };
+                }
+                catch (err) {
+                    const failure = verifyFailureResult(err);
+                    if (failure)
+                        return failure;
+                    const conflict = createConflictResult(err, compiled.plan_hash);
+                    if (conflict)
+                        return conflict;
+                    throw err;
+                }
+            };
+            if (planUsesThirdParty(plan))
+                return verifyInCloud(THIRD_PARTY_CLOUD_ONLY);
             // Session-mode plan: establish the account's session once (server-side, reused
             // from the stored one when it still proves live) and replay against it — the
             // cloud shape. An unsupported account is re-compiled to sign itself in.
@@ -391,20 +409,7 @@ export const testCommands = [
             // A captured session lives encrypted in Beryl's cloud and is never handed to
             // this machine — the cloud is the only place this plan can be proven.
             if (compiled.requires_auth && !session) {
-                ctx.err(yellow("! This plan signs in with a captured session, so it can only be verified " +
-                    "in Beryl's cloud — verifying server-side instead."));
-                try {
-                    return { data: await bank({ verify: true, plan_hash: compiled.plan_hash }) };
-                }
-                catch (err) {
-                    const failure = verifyFailureResult(err);
-                    if (failure)
-                        return failure;
-                    const conflict = createConflictResult(err, compiled.plan_hash);
-                    if (conflict)
-                        return conflict;
-                    throw err;
-                }
+                return verifyInCloud("A captured session never leaves Beryl's cloud");
             }
             if (compiled.login_config_error)
                 throw new CliError(compiled.login_config_error);

package/dist/commands/third-party-integrations.js

@@ -0,0 +1,62 @@
+import { UsageError } from "../errors.js";
+import { dim, table } from "../output.js";
+import { flagStr, projectPath } from "./util.js";
+const integrationsPath = (ws, p) => `${projectPath(ws, p)}/third-party-integrations`;
+export const thirdPartyIntegrationCommands = [
+    {
+        name: "third-party-integrations list",
+        summary: "The third party providers this project's tests can connect to",
+        groupDefault: true,
+        groupSummary: "Test a site's own integrations (Connect Slack) against Beryl's accounts on those providers.",
+        description: "Lists the providers set up on this Beryl deployment. A test may use " +
+            "`third_party_authorize` / `await_third_party` only for a provider listed here; " +
+            "anything else is refused at save time.",
+        scope: "project",
+        async run(ctx, input) {
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const { providers } = (await ctx.client.get(integrationsPath(workspaceId, projectId)));
+            return {
+                data: providers,
+                human: providers.length
+                    ? table(providers.map((p) => ({ key: p.key, label: p.label })))
+                    : dim("No third party providers are set up on this deployment."),
+            };
+        },
+    },
+    {
+        name: "third-party-integrations target",
+        summary: "The channel this project's tests pick when the app asks where to post",
+        description: "Returns the project's own destination on Beryl's provider account (for Slack, a " +
+            "channel), creating it on first ask. When a test connects the app, it picks this " +
+            "channel in the app's settings; `await_third_party` then waits for the app's message " +
+            "there. `recorded_app_ids` maps each environment id to the app whose first message " +
+            "was seen there; an environment accepts only its own app once recorded. " +
+            "A 409 means the provider is not set up on this deployment (see " +
+            "`third-party-integrations list`).",
+        scope: "project",
+        flags: [
+            {
+                name: "provider",
+                type: "string",
+                required: true,
+                description: "Provider key from `third-party-integrations list`, e.g. slack",
+            },
+        ],
+        examples: ["beryl third-party-integrations target --provider slack"],
+        async run(ctx, input) {
+            const provider = flagStr(input, "provider");
+            if (!provider)
+                throw new UsageError("--provider is required");
+            const { workspaceId, projectId } = await ctx.requireProject(input);
+            const target = (await ctx.client.post(`${integrationsPath(workspaceId, projectId)}/${encodeURIComponent(provider)}/target`));
+            return {
+                data: target,
+                human: `#${target.external_name}\n\n` +
+                    dim(`Pick this channel in the app when the test connects ${target.provider}.`) +
+                    Object.entries(target.recorded_app_ids ?? {})
+                        .map(([env, app]) => dim(`\nRecorded app for environment ${env}: ${app}`))
+                        .join(""),
+            };
+        },
+    },
+];

package/dist/commands/watch.js

@@ -1,5 +1,6 @@
 import fs from "node:fs";
-import { CliError } from "../errors.js";
+import { AuthError, CliError } from "../errors.js";
+import { ApiError, NonJsonResponseError } from "../http.js";
 import { sseStream } from "../sse.js";
 import { dim, green, red, statusColor, yellow } from "../output.js";
 import { projectPath } from "./util.js";
@@ -34,6 +35,15 @@ function runPageUrl(apiUrl, projectId, runId) {
 }
 const escapeAnnotationData = (s) => s.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
 const escapeAnnotationProp = (s) => escapeAnnotationData(s).replace(/:/g, "%3A").replace(/,/g, "%2C");
+// Auth, client and wrong-host errors fail identically on every retry; only a network
+// blip, a timeout, throttling or a 5xx is worth waiting out.
+function isRetryable(e) {
+    if (e instanceof AuthError || e instanceof NonJsonResponseError)
+        return false;
+    if (e instanceof ApiError)
+        return e.status >= 500 || e.status === 408 || e.status === 429;
+    return true;
+}
 function sleep(ms) {
     return new Promise((r) => setTimeout(r, ms));
 }
@@ -97,9 +107,10 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
             backoffMs = 2_000;
         }
         catch (e) {
+            if (!isRetryable(e))
+                throw e;
             clearStatusLine();
-            if (!ctx.json)
-                ctx.err(dim(`watch request failed (${e.message}); retrying`));
+            ctx.err(dim(`watch request failed (${e.message}); retrying`));
             await sleep(backoffMs);
             backoffMs = Math.min(backoffMs * 2, 30_000);
             continue;
@@ -118,7 +129,11 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
                 run = (await ctx.client.get(`${path}/runs/${runId}`));
                 backoffMs = 2_000;
             }
-            catch {
+            catch (e) {
+                if (!isRetryable(e))
+                    throw e;
+                clearStatusLine();
+                ctx.err(dim(`run detail request failed (${e.message}); retrying`));
                 // Same backoff as the poll: a terminal run answers the next poll
                 // instantly, so retrying without a delay would busy-loop both endpoints.
                 await sleep(backoffMs);
@@ -148,9 +163,10 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
                 try {
                     await ctx.client.post(`${path}/runs/${runId}/cancel`);
                 }
-                catch {
-                    // Best-effort: the cancel may race the worker or the run may already be
-                    // terminal; the failure that triggered it decides the exit code anyway.
+                catch (e) {
+                    // The failure that triggered the cancel decides the exit code; this only
+                    // warns that the run may still be consuming minutes.
+                    ctx.err(yellow(`fail-fast: could not cancel run ${runId} (${e.message})`));
                 }
                 break;
             }
@@ -279,7 +295,7 @@ export async function watchExploration(ctx, ws, project, explorationId, timeoutM
     }
     return {
         data: { exploration_id: explorationId, status: finalStatus, test_run_id: handedOffRunId, error },
-        exitCode: finalStatus === "failed" ? 1 : 0,
+        exitCode: finalStatus === "completed" ? 0 : 1,
     };
 }
 export async function pollCurrentExploration(ctx, ws, project, timeoutSeconds = 180) {

package/dist/config.js

@@ -2,20 +2,44 @@ import { randomUUID } from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { CliError } from "./errors.js";
 export const DEFAULT_API_URL = "https://api.beryl.so";
+export function withFlagOverrides(config, flags) {
+    return {
+        ...config,
+        ...(flags.apiUrl ? { apiUrl: flags.apiUrl } : {}),
+        ...(flags.token ? { token: flags.token } : {}),
+        flagOverrides: flags,
+    };
+}
 export function globalConfigPath(env = process.env) {
     const base = env.BERYL_CONFIG_DIR ??
         (env.XDG_CONFIG_HOME ? path.join(env.XDG_CONFIG_HOME, "beryl") : undefined) ??
         path.join(os.homedir(), ".config", "beryl");
     return path.join(base, "config.json");
 }
+// Only a missing file means "no config". Anything else must stop the command: treating
+// it as empty would drop the token and api_url, then the next write would overwrite it.
 function readJson(file) {
+    let text;
     try {
-        return JSON.parse(fs.readFileSync(file, "utf8"));
+        text = fs.readFileSync(file, "utf8");
     }
-    catch {
-        return undefined;
+    catch (err) {
+        if (err.code === "ENOENT")
+            return undefined;
+        throw new CliError(`Could not read ${file}: ${err.message}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch (err) {
+        throw new CliError(`${file} is not valid JSON (${err.message}). Fix or delete it, then retry.`);
     }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+        throw new CliError(`${file} must hold a JSON object. Fix or delete it, then retry.`);
+    return parsed;
 }
 export function loadConfig(env = process.env) {
     const globalPath = globalConfigPath(env);

package/dist/http.js

@@ -11,6 +11,8 @@ export class ApiError extends CliError {
         this.detail = detail;
     }
 }
+export class NonJsonResponseError extends CliError {
+}
 export class ApiClient {
     baseUrl;
     token;
@@ -85,7 +87,9 @@ export class ApiClient {
             return JSON.parse(text);
         }
         catch {
-            return text;
+            const type = response.headers.get("content-type") ?? "no content-type";
+            throw new NonJsonResponseError(`${this.baseUrl} answered ${method} ${path} with non-JSON (${type}); ` +
+                "is the API URL pointing at the Beryl API?");
         }
     }
     get(path, query) {

package/dist/lint.js

@@ -16,6 +16,9 @@ export const WRITE_NEEDS_CLOSING_TEARDOWN = (detail) => "A 'write' test must end
 export const OPTIONAL_TEARDOWN_ACTION_MUST_BE_LAST = (label, action) => `${label} (${action}) is optional but not the last step of 'after'. An optional step ` +
     "that misses ends the teardown, so the steps after it would silently not run. Guard " +
     "them with an optional 'wait_for' or 'expect' instead, or move this step to the end.";
+export const THIRD_PARTY_UNDECLARED = (p) => `third_party_authorize approves the install on ${p}'s consent page, which needs ` +
+    `Beryl's ${p} login: add "${p}" to third_party_integrations`;
+export const THIRD_PARTY_UNUSED = (p) => `third_party_integrations declares "${p}" but no third_party_authorize step uses it; remove it`;
 const typeOf = (value) => {
     if (value === null)
         return "null";
@@ -174,6 +177,37 @@ function identityIssues(plan, provisioning) {
         ? [{ path: "(root)", message: MINTED_IDENTITY_NEEDS_TEARDOWN }]
         : [];
 }
+export const THIRD_PARTY_AUTHORIZE = "third_party_authorize";
+export const AWAIT_THIRD_PARTY = "await_third_party";
+const executed = (plan) => [
+    ...steps(plan.before),
+    ...steps(plan.steps),
+    ...steps(plan.after),
+];
+export function planUsesThirdParty(plan) {
+    if (typeof plan !== "object" || plan === null)
+        return false;
+    return executed(plan).some((s) => s.action === THIRD_PARTY_AUTHORIZE || s.action === AWAIT_THIRD_PARTY);
+}
+// Mirrors the server's ActionPlan validator, which raises the first sorted mismatch only.
+function thirdPartyIssues(plan) {
+    if (typeof plan !== "object" || plan === null)
+        return [];
+    const record = plan;
+    const declared = new Set(Array.isArray(record.third_party_integrations)
+        ? record.third_party_integrations.map(String)
+        : []);
+    const authorized = new Set(executed(record)
+        .filter((s) => s.action === THIRD_PARTY_AUTHORIZE && typeof s.provider === "string")
+        .map((s) => s.provider));
+    const [undeclared] = [...authorized].filter((p) => !declared.has(p)).sort();
+    if (undeclared)
+        return [{ path: "(root)", message: THIRD_PARTY_UNDECLARED(undeclared) }];
+    const [unused] = [...declared].filter((p) => !authorized.has(p)).sort();
+    if (unused)
+        return [{ path: "third_party_integrations", message: THIRD_PARTY_UNUSED(unused) }];
+    return [];
+}
 const TEARDOWN_PROOF_KINDS = new Set([
     "gone",
     "hidden",
@@ -212,6 +246,7 @@ export function lintPlan(plan, opts = {}) {
     if (issues.length > 0)
         return issues;
     return [
+        ...thirdPartyIssues(plan),
         ...identityIssues(plan, opts.provisioning === true),
         ...(opts.access === undefined ? [] : accessIssues(plan, opts.access)),
     ];

package/dist/local-exec.js

@@ -8,6 +8,11 @@ export const IMPORT_MAX_ERROR_LEN = 5000;
 export const IMPORT_MAX_FILE_BYTES = 14 * 1024 * 1024;
 export const IMPORT_MAX_TOTAL_BYTES = 180 * 1024 * 1024;
 export const IMPORT_MAX_FILES = 3600;
+export const THIRD_PARTY_CLOUD_ONLY = "third party integration steps run only in Beryl's cloud";
+export const THIRD_PARTY_USE_TRIGGER = `${THIRD_PARTY_CLOUD_ONLY}; use \`beryl runs trigger\``;
+// The rendered await_third_party helper, and the authorize step's wait to leave the provider's host.
+const THIRD_PARTY_SPEC_MARKER = /__vmThirdParty\(|\.test\(u\.hostname\)/;
+export const specUsesThirdParty = (content) => THIRD_PARTY_SPEC_MARKER.test(content);
 export function toRunEntry(spec, outcome, runError, startedAt, ordinal, redact) {
     // Mirrors the cloud runner's redact_result: the spec types secrets into the page
     // and an injected session authenticates its requests, so error text and the DOM

package/dist/registry/index.js

@@ -12,6 +12,7 @@ import { mcpCommands } from "../commands/mcp.js";
 import { projectCommands } from "../commands/projects.js";
 import { runCommands } from "../commands/runs.js";
 import { testCommands } from "../commands/tests.js";
+import { thirdPartyIntegrationCommands } from "../commands/third-party-integrations.js";
 import { trackerCommands } from "../commands/tracker.js";
 import { workspaceCommands } from "../commands/workspaces.js";
 export const WORKSPACE_FLAG = {
@@ -70,6 +71,7 @@ export const commands = [
     ...explorationCommands,
     ...configCommands,
     ...mailboxCommands,
+    ...thirdPartyIntegrationCommands,
     ...testAccountCommands,
     ...accountCommands,
     ...mcpCommands,

package/dist/schema.generated.js

@@ -22,7 +22,9 @@ export const ACTION_PLAN_SCHEMA = {
                 "dialog",
                 "switch_tab",
                 "close_tab",
-                "drag"
+                "drag",
+                "third_party_authorize",
+                "await_third_party"
             ],
             "title": "ActionType",
             "type": "string"
@@ -446,6 +448,54 @@ export const ACTION_PLAN_SCHEMA = {
                         ]
                     }
                 },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "third_party_authorize"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "provider": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            }
+                        },
+                        "required": [
+                            "provider"
+                        ]
+                    }
+                },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "await_third_party"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "provider": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            }
+                        },
+                        "required": [
+                            "provider"
+                        ]
+                    }
+                },
                 {
                     "if": {
                         "properties": {
@@ -901,6 +951,25 @@ export const ACTION_PLAN_SCHEMA = {
                             "selector"
                         ]
                     }
+                },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "await_email"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "wait_s": {
+                                "maximum": 50
+                            }
+                        }
+                    }
                 }
             ],
             "properties": {
@@ -1096,6 +1165,17 @@ export const ACTION_PLAN_SCHEMA = {
                     "title": "Optional",
                     "type": "boolean"
                 },
+                "provider": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/$defs/ThirdPartyProvider"
+                        },
+                        {
+                            "type": "null"
+                        }
+                    ],
+                    "default": null
+                },
                 "selector": {
                     "anyOf": [
                         {
@@ -1164,7 +1244,7 @@ export const ACTION_PLAN_SCHEMA = {
                 "wait_s": {
                     "anyOf": [
                         {
-                            "maximum": 50,
+                            "maximum": 600,
                             "minimum": 1,
                             "type": "integer"
                         },
@@ -1181,6 +1261,13 @@ export const ACTION_PLAN_SCHEMA = {
             ],
             "title": "PlanStep",
             "type": "object"
+        },
+        "ThirdPartyProvider": {
+            "enum": [
+                "slack"
+            ],
+            "title": "ThirdPartyProvider",
+            "type": "string"
         }
     },
     "$id": "https://api.beryl.so/api/v1/schemas/action-plan.schema.json",
@@ -1344,6 +1431,14 @@ export const ACTION_PLAN_SCHEMA = {
             "minItems": 1,
             "title": "Steps",
             "type": "array"
+        },
+        "third_party_integrations": {
+            "description": "Providers whose consent page this test approves; the browser then carries Beryl's login for them. Declare exactly the providers of its third_party_authorize steps.",
+            "items": {
+                "$ref": "#/$defs/ThirdPartyProvider"
+            },
+            "title": "Third Party Integrations",
+            "type": "array"
         }
     },
     "required": [

package/dist/skill-tables.js

@@ -36,6 +36,8 @@ export const ACTION_DOCS = {
     switch_tab: "Make another open tab the active page; `value` picks it — see below",
     close_tab: "Close the active tab (optional `value` picks one) and fall back to the previous",
     drag: "Drag the `selector` element onto the `value` TARGET selector",
+    third_party_authorize: "Approve the `provider` install on its consent page as Beryl's account (cloud only, see third party integration tests)",
+    await_third_party: "Wait for the connected app's message on Beryl's `provider` account (cloud only)",
 };
 export const EXPECT_DOCS = {
     visible: "The element is visible",

package/dist/sse.js

@@ -69,7 +69,7 @@ export async function* sseStream(client, path, signal) {
             // Clean server-side close without a terminal event: reconnect and replay.
             failures += 1;
             if (failures >= MAX_CONSECUTIVE_FAILURES)
-                return;
+                throw new CliError(`Stream closed ${failures} times without a terminal event; its status is unknown`);
             await sleep(1000, signal);
         }
         catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.60.0",
+  "version": "0.62.1",
   "description": "Beryl on the command line: projects, runs, the exploring agent, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",