@beryl-so/cli 0.73.2 → 0.78.0

package/dist/commands/watch.js

@@ -46,6 +46,13 @@ export function findingsGateLine(gate) {
     }
     return `findings gate unknown: ${gate.reason ?? "not evaluated"}`;
 }
+export function healNote(r) {
+    if (r.heal_attempt) {
+        const why = r.heal_attempt.reason ? `: ${r.heal_attempt.reason.slice(0, 300)}` : "";
+        return `Auto-heal did not fix it (${r.heal_attempt.status})${why}`;
+    }
+    return r.heal_skip_reason ?? null;
+}
 // Auth, client and wrong-host errors fail identically on every retry; only a network
 // blip, a timeout, throttling or a 5xx is worth waiting out.
 function isRetryable(e) {
@@ -102,8 +109,9 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
             ctx.err(red(detail.replace(/^/gm, "    ")));
             process.stdout.write(`::error title=${escapeAnnotationProp(r.title || r.id)}::${escapeAnnotationData(detail)}\n`);
         }
-        if (!ok && r.heal_skip_reason)
-            ctx.err(dim(`    ${r.heal_skip_reason}`));
+        const note = ok ? null : healNote(r);
+        if (note)
+            ctx.err(dim(`    ${note}`));
     };
     if (!ctx.json)
         ctx.err(dim(`watching run ${runId}`));
@@ -260,14 +268,17 @@ export async function watchRun(ctx, ws, project, runId, timeoutMinutes, opts = {
             `| ${counters.passed} | ${counters.failed} | ${counters.errored} | ${counters.cancelled} | ${counters.quarantined} | ${counters.total} |`,
             ...(gate ? ["", findingsGateLine(gate)] : []),
             ...(link ? ["", `[Open the run in Beryl](${link})`] : []),
-            ...failures.flatMap((r) => [
-                "",
-                `**✗ ${r.title || r.id}**`,
-                "```",
-                String(r.error_message ?? "").slice(0, CI_ERROR_MAX_LEN),
-                "```",
-                ...(r.heal_skip_reason ? ["", r.heal_skip_reason] : []),
-            ]),
+            ...failures.flatMap((r) => {
+                const note = healNote(r);
+                return [
+                    "",
+                    `**✗ ${r.title || r.id}**`,
+                    "```",
+                    String(r.error_message ?? "").slice(0, CI_ERROR_MAX_LEN),
+                    "```",
+                    ...(note ? ["", note] : []),
+                ];
+            }),
             "",
         ];
         try {

package/dist/lint.js

@@ -142,6 +142,13 @@ function validate(value, raw, root, path) {
         else
             issues.push(...found);
     }
+    // A branch can itself be an if/then/else (the first-step rule nests on device, then
+    // on whether `before` exists).
+    if (schema.if) {
+        const branch = matches(value, schema.if, root) ? schema.then : schema.else;
+        if (branch)
+            issues.push(...validate(value, branch, root, path));
+    }
     return issues;
 }
 const steps = (section) => Array.isArray(section)
@@ -206,6 +213,118 @@ function thirdPartyIssues(plan) {
         return [{ path: "third_party_integrations", message: THIRD_PARTY_UNUSED(unused) }];
     return [];
 }
+// Mirrors ActionPlan._validate_nonempty and _assert_app_plan in
+// api/app/ta_shared/schemas.py, which raise the first violation only.
+const APP_DEVICES = ["android", "ios"];
+const APP_PLATFORM_NAMES = {
+    android: "Android",
+    ios: "iOS",
+};
+export const APP_ACTIONS = [
+    "launch",
+    "back",
+    "fill",
+    "click",
+    "press",
+    "scroll",
+    "wait_for",
+    "expect",
+    "await_email",
+];
+export const APP_EXPECTS = [
+    "visible",
+    "hidden",
+    "checked",
+    "enabled",
+    "disabled",
+    "have_text",
+    "persisted",
+    "gone",
+];
+const APP_KEYS = {
+    android: ["Enter", "Backspace", "Tab"],
+    ios: ["Enter", "Backspace"],
+};
+const APP_ID_SELECTOR = {
+    android: "resource id",
+    ios: "accessibility identifier",
+};
+const APP_SELECTOR_RE = /^(id|text)=[\s\S]+$/;
+const APP_STRING_FIELDS = [
+    "selector",
+    "value",
+    "key",
+    "expect_text",
+    "from_contains",
+    "subject_contains",
+];
+const sortedList = (values) => [...values].sort().join(", ");
+const sectionSteps = (plan) => ["before", "steps", "after"].flatMap((section) => steps(plan[section]).map((step, i) => ({ path: `${section}[${i}]`, section, step })));
+const appStepRules = (p) => [
+    (s) => p === "ios" && s.action === "back"
+        ? "is not available on ios: iOS has no back button; tap the app's own back control."
+        : null,
+    (s) => APP_ACTIONS.includes(String(s.action))
+        ? null
+        : `is not available on ${p}. ${APP_PLATFORM_NAMES[p]} steps: ${sortedList(APP_ACTIONS)}.`,
+    (s) => s.action !== "expect" || APP_EXPECTS.includes(String(s.expect_kind))
+        ? null
+        : `expect_kind '${typeof s.expect_kind === "string" ? s.expect_kind : "?"}' is not ` +
+            `available on ${p}. ${APP_PLATFORM_NAMES[p]} expects: ${sortedList(APP_EXPECTS)}.`,
+    (s) => (s.iframe ? "sets iframe, which only exists on web pages." : null),
+    (s) => s.action === "scroll" && !s.selector
+        ? `needs a selector: on ${p}, scroll moves until that element is on screen.`
+        : null,
+    (s) => typeof s.selector === "string" && !APP_SELECTOR_RE.test(s.selector)
+        ? `selector ${pyRepr(s.selector)} is not an ${p} selector. Use ` +
+            `'id=<${APP_ID_SELECTOR[p]}>' or 'text=<visible text or accessibility label>'.`
+        : null,
+    (s) => s.expect_kind === "have_text" &&
+        typeof s.selector === "string" &&
+        s.selector &&
+        !s.selector.startsWith("id=")
+        ? "checks text, so it needs an 'id=' selector; to assert that some text is on " +
+            "screen, use expect_kind 'visible' with 'text=<text>'."
+        : null,
+    (s) => s.action === "press" && !APP_KEYS[p].includes(String(s.key))
+        ? `key ${pyRepr(s.key)} is not available on ${p}. Keys: ${APP_KEYS[p].join(", ")}.`
+        : null,
+    (s) => {
+        const field = APP_STRING_FIELDS.find((f) => {
+            const text = s[f];
+            return typeof text === "string" && text.includes("${");
+        });
+        return field
+            ? `${field} contains '\${', which the app runner would evaluate as code. Remove it.`
+            : null;
+    },
+];
+function deviceIssues(plan) {
+    if (typeof plan !== "object" || plan === null)
+        return [];
+    const record = plan;
+    const issue = (path, message) => [{ path, message }];
+    const platform = String(record.device);
+    if (!APP_DEVICES.includes(platform)) {
+        const appOnly = [...new Set(executed(record).map((s) => String(s.action)))]
+            .filter((a) => a === "launch" || a === "back")
+            .sort();
+        return appOnly.length
+            ? issue("(root)", `${appOnly.join(", ")} drives a native app, so it is only for devices 'android' ` +
+                "and 'ios'; a web plan starts with a goto")
+            : [];
+    }
+    // The server names a flat plan's steps `step[N]` and a sectioned one's `after[0]`.
+    const flat = steps(record.before).length === 0 && steps(record.after).length === 0;
+    for (const [i, { path, step }] of sectionSteps(record).entries()) {
+        for (const rule of appStepRules(platform)) {
+            const why = rule(step);
+            if (why)
+                return issue(path, `${flat ? `step[${i}]` : path} (${String(step.action)}) ${why}`);
+        }
+    }
+    return [];
+}
 const TEARDOWN_PROOF_KINDS = new Set([
     "gone",
     "hidden",
@@ -295,6 +414,7 @@ export function lintPlan(plan, opts = {}) {
     if (issues.length > 0)
         return issues;
     return [
+        ...deviceIssues(plan),
         ...thirdPartyIssues(plan),
         ...identityIssues(plan, opts.provisioning === true),
         ...(opts.access === undefined ? [] : accessIssues(plan, opts.access)),

package/dist/schema.generated.js

@@ -4,6 +4,8 @@ export const ACTION_PLAN_SCHEMA = {
         "ActionType": {
             "enum": [
                 "goto",
+                "launch",
+                "back",
                 "fill",
                 "click",
                 "press",
@@ -24,7 +26,8 @@ export const ACTION_PLAN_SCHEMA = {
                 "close_tab",
                 "drag",
                 "third_party_authorize",
-                "await_third_party"
+                "await_third_party",
+                "send_third_party"
             ],
             "title": "ActionType",
             "type": "string"
@@ -39,11 +42,13 @@ export const ACTION_PLAN_SCHEMA = {
             "type": "string"
         },
         "DeviceClass": {
-            "description": "The viewport and device a test runs at. The AUTHOR sets it when a flow is specific\nto a form factor (a mobile-only nav, a hamburger, a stacked layout); omit it and the\ntest runs at the default desktop viewport.\n\n``mobile`` / ``tablet`` are full Chromium device emulation, not a bare resize: the\nviewport, a mobile Chrome user-agent, touch, and the device pixel ratio all change\ntogether, so a control gated behind a touch or mobile-UA media query actually appears.",
+            "description": "The viewport and device a test runs at. The AUTHOR sets it when a flow is specific\nto a form factor (a mobile-only nav, a hamburger, a stacked layout); omit it and the\ntest runs at the default desktop viewport.\n\n``mobile`` / ``tablet`` are full Chromium device emulation, not a bare resize: the\nviewport, a mobile Chrome user-agent, touch, and the device pixel ratio all change\ntogether, so a control gated behind a touch or mobile-UA media query actually appears.\n``android`` and ``ios`` are not browsers: the plan drives the environment's app on\nan Android emulator or an iOS simulator.",
             "enum": [
                 "desktop",
                 "tablet",
-                "mobile"
+                "mobile",
+                "android",
+                "ios"
             ],
             "title": "DeviceClass",
             "type": "string"
@@ -80,7 +85,8 @@ export const ACTION_PLAN_SCHEMA = {
                 "have_count",
                 "persisted",
                 "gone",
-                "count_delta"
+                "count_delta",
+                "judged"
             ],
             "title": "ExpectKind",
             "type": "string"
@@ -110,6 +116,38 @@ export const ACTION_PLAN_SCHEMA = {
                         ]
                     }
                 },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "launch"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {},
+                        "required": []
+                    }
+                },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "back"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {},
+                        "required": []
+                    }
+                },
                 {
                     "if": {
                         "properties": {
@@ -496,6 +534,36 @@ export const ACTION_PLAN_SCHEMA = {
                         ]
                     }
                 },
+                {
+                    "if": {
+                        "properties": {
+                            "action": {
+                                "const": "send_third_party"
+                            }
+                        },
+                        "required": [
+                            "action"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "provider": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            },
+                            "value": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            }
+                        },
+                        "required": [
+                            "provider",
+                            "value"
+                        ]
+                    }
+                },
                 {
                     "if": {
                         "properties": {
@@ -768,6 +836,35 @@ export const ACTION_PLAN_SCHEMA = {
                         ]
                     }
                 },
+                {
+                    "if": {
+                        "properties": {
+                            "expect_kind": {
+                                "const": "judged"
+                            }
+                        },
+                        "required": [
+                            "expect_kind"
+                        ]
+                    },
+                    "then": {
+                        "properties": {
+                            "expect_text": {
+                                "not": {
+                                    "type": "null"
+                                }
+                            },
+                            "selector": {
+                                "minLength": 1,
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "expect_text",
+                            "selector"
+                        ]
+                    }
+                },
                 {
                     "if": {
                         "properties": {
@@ -1292,45 +1389,103 @@ export const ACTION_PLAN_SCHEMA = {
             }
         },
         {
-            "$comment": "The first executed step must be a goto, so the test loads a page before acting (that is before[0] when there is a setup section, else steps[0]).",
+            "$comment": "The first executed step must be a goto, so the test loads a page before acting (that is before[0] when there is a setup section, else steps[0]). An android or ios plan opens its app instead, so its first executed step is a launch.",
             "else": {
-                "properties": {
-                    "steps": {
-                        "prefixItems": [
-                            {
-                                "properties": {
-                                    "action": {
-                                        "const": "goto"
+                "else": {
+                    "properties": {
+                        "steps": {
+                            "prefixItems": [
+                                {
+                                    "properties": {
+                                        "action": {
+                                            "const": "goto"
+                                        }
                                     }
                                 }
-                            }
-                        ]
+                            ]
+                        }
+                    }
+                },
+                "if": {
+                    "properties": {
+                        "before": {
+                            "minItems": 1,
+                            "type": "array"
+                        }
+                    },
+                    "required": [
+                        "before"
+                    ]
+                },
+                "then": {
+                    "properties": {
+                        "before": {
+                            "prefixItems": [
+                                {
+                                    "properties": {
+                                        "action": {
+                                            "const": "goto"
+                                        }
+                                    }
+                                }
+                            ]
+                        }
                     }
                 }
             },
             "if": {
                 "properties": {
-                    "before": {
-                        "minItems": 1,
-                        "type": "array"
+                    "device": {
+                        "enum": [
+                            "android",
+                            "ios"
+                        ]
                     }
                 },
                 "required": [
-                    "before"
+                    "device"
                 ]
             },
             "then": {
-                "properties": {
-                    "before": {
-                        "prefixItems": [
-                            {
-                                "properties": {
-                                    "action": {
-                                        "const": "goto"
+                "else": {
+                    "properties": {
+                        "steps": {
+                            "prefixItems": [
+                                {
+                                    "properties": {
+                                        "action": {
+                                            "const": "launch"
+                                        }
                                     }
                                 }
-                            }
-                        ]
+                            ]
+                        }
+                    }
+                },
+                "if": {
+                    "properties": {
+                        "before": {
+                            "minItems": 1,
+                            "type": "array"
+                        }
+                    },
+                    "required": [
+                        "before"
+                    ]
+                },
+                "then": {
+                    "properties": {
+                        "before": {
+                            "prefixItems": [
+                                {
+                                    "properties": {
+                                        "action": {
+                                            "const": "launch"
+                                        }
+                                    }
+                                }
+                            ]
+                        }
                     }
                 }
             }
@@ -1417,7 +1572,7 @@ export const ACTION_PLAN_SCHEMA = {
         "device": {
             "$ref": "#/$defs/DeviceClass",
             "default": "desktop",
-            "description": "Viewport and device to run at. 'mobile' and 'tablet' apply full Chromium device emulation (viewport, mobile user-agent, touch, pixel ratio); 'desktop' (default) runs at the standard desktop viewport."
+            "description": "Viewport and device to run at. 'mobile' and 'tablet' apply full Chromium device emulation (viewport, mobile user-agent, touch, pixel ratio); 'desktop' (default) runs at the standard desktop viewport; 'android' and 'ios' run the environment's app on an Android emulator or iOS simulator."
         },
         "requires_auth": {
             "default": false,

package/dist/skill-tables.js

@@ -17,6 +17,8 @@ function expectRequires(kind) {
 }
 export const ACTION_DOCS = {
     goto: "Navigate to `url` — a path on your app, absolute only for another origin",
+    launch: "App plans only (android, ios), and their first step: open the app (a clean start unless the test rides a session, a relaunch keeping its data after)",
+    back: "Android only: press the system back button (iOS has none)",
     fill: "Type `value` into the `selector` element",
     click: "Click the `selector` element",
     press: "Press keyboard `key` (e.g. `Enter`) on the `selector` element",
@@ -38,6 +40,7 @@ export const ACTION_DOCS = {
     drag: "Drag the `selector` element onto the `value` TARGET selector",
     third_party_authorize: "Approve the `provider` install on its consent page as Beryl's account (cloud only, see third party integration tests)",
     await_third_party: "Wait for the connected app's message on Beryl's `provider` account (cloud only)",
+    send_third_party: "Mention the connected app's bot with `value` as Beryl's `provider` account (cloud only)",
 };
 export const EXPECT_DOCS = {
     visible: "The element is visible",
@@ -54,6 +57,7 @@ export const EXPECT_DOCS = {
     persisted: "The record just created is present — a visible match for the `{{unique}}`-named row",
     gone: "Zero matches — absent from the DOM, stronger than `hidden`",
     count_delta: "The match count moved by signed `expect_delta` vs the `capture_ref` baseline",
+    judged: "A model judges the LAST match's text, once it stops changing, against the plain-language criteria in `expect_text`: for answers that vary in wording (a chatbot reply)",
 };
 export const EXTRACT_DOCS = {
     code: "the one-time code",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beryl-so/cli",
-  "version": "0.73.2",
+  "version": "0.78.0",
   "description": "Beryl on the command line: projects, tests, runs, and an MCP server over the same commands.",
   "license": "MIT",
   "type": "module",
@@ -31,8 +31,7 @@
     "docs": "tsx scripts/gen-docs.ts"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.29.0",
-    "@playwright/test": "^1.61.1",
+    "@modelcontextprotocol/sdk": "^1.31.0",
     "@posthog/mcp": "^0.11.6",
     "posthog-node": "^5.49.1"
   },

package/dist/email-pump.js

@@ -1,138 +0,0 @@
-import fs from "node:fs";
-import { ApiError } from "./http.js";
-const SIDECAR_POLL_MS = 500;
-// The server caps one blocking wait at 50s; stay under it and loop.
-const SERVER_WAIT_MAX_S = 45;
-const DEFAULT_WAIT_S = 30;
-const EXTRACT_RETRY_MS = 1000;
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-export function startEmailPump(opts) {
-    let stopped = false;
-    const served = new Set();
-    const consumed = new Set();
-    let storedAfter = opts.storedAfter;
-    let since;
-    const fetchLatest = async (req, timeoutS) => {
-        try {
-            return (await opts.client.get(`/workspaces/${opts.workspaceId}/mailboxes/${opts.inboxId}/emails/latest`, {
-                timeout_s: timeoutS,
-                since,
-                stored_after: storedAfter,
-                from_contains: req.from_contains,
-                subject_contains: req.subject_contains,
-                recipient_contains: opts.recipientContains,
-            }));
-        }
-        catch (err) {
-            if (err instanceof ApiError && err.status === 404)
-                return null;
-            throw err;
-        }
-    };
-    // A 422 is a real verdict — "this email carries no artifact" — and sends the wait on
-    // to the next email; anything else is transport trouble, retried until the step's own
-    // deadline does the failing.
-    const extractOnServer = async (emailId, req, deadline) => {
-        while (!stopped && Date.now() < deadline) {
-            try {
-                const res = (await opts.client.post(`/workspaces/${opts.workspaceId}/mailboxes/${opts.inboxId}/emails/${emailId}/extract`, { extract: req.extract ?? "code", extract_pattern: req.pattern }));
-                return { ok: true, value: res.value };
-            }
-            catch (err) {
-                // 424 is fatal; 422 walks on; the rest retries.
-                if (err instanceof ApiError && (err.status === 422 || err.status === 424)) {
-                    const detail = typeof err.detail === "string" ? err.detail : err.message;
-                    return {
-                        ok: false,
-                        fatal: err.status === 424,
-                        error: detail.replace(/^EMAIL_(EXTRACTION_FAILED|EXTRACTOR_UNAVAILABLE):\s*/, ""),
-                    };
-                }
-                await sleep(EXTRACT_RETRY_MS);
-            }
-        }
-        return { ok: false, error: "extraction did not complete before the wait deadline" };
-    };
-    const resolveRequest = async (req) => {
-        // Wait exactly wait_s, like the cloud resolver — the spec's own grace window
-        // covers this side's poll tick + round trip.
-        const waitS = Math.max(1, Number(req.wait_s) || DEFAULT_WAIT_S);
-        const deadline = Date.now() + waitS * 1000;
-        let lastExtractError;
-        while (!stopped && Date.now() < deadline) {
-            const remainingS = Math.ceil((deadline - Date.now()) / 1000);
-            const email = await fetchLatest(req, Math.min(SERVER_WAIT_MAX_S, remainingS));
-            if (email === null)
-                continue;
-            if (consumed.has(email.id)) {
-                // The fence rides received_at, which comes from the sender's Date header — a
-                // lagging mailer clock can re-serve a consumed mail. Nudge past it instead.
-                since = new Date(new Date(email.received_at).getTime() + 1000).toISOString();
-                continue;
-            }
-            consumed.add(email.id);
-            since = email.received_at;
-            opts.onEvent?.(`await_email: matched "${email.subject ?? "(no subject)"}"`);
-            const extracted = await extractOnServer(email.id, req, deadline);
-            if (extracted.ok)
-                return extracted.value;
-            if (extracted.fatal)
-                throw new Error(extracted.error);
-            // Same move as the cloud resolver: an email without the artifact (a welcome mail
-            // racing the code mail) is consumed and the wait continues for the next one.
-            lastExtractError = extracted.error;
-            opts.onEvent?.(`await_email: ${extracted.error} — waiting for the next email`);
-        }
-        throw new Error(lastExtractError ?? `no matching email arrived in the run inbox within ${waitS}s`);
-    };
-    const writeResponse = (response) => {
-        let box = {};
-        try {
-            box = JSON.parse(fs.readFileSync(opts.sidecarPath, "utf8"));
-        }
-        catch {
-            // keep at least the response readable
-        }
-        box.response = response;
-        // Write-then-rename so the spec never reads a half-written response.
-        const tmp = `${opts.sidecarPath}.tmp`;
-        fs.writeFileSync(tmp, JSON.stringify(box), "utf8");
-        fs.renameSync(tmp, opts.sidecarPath);
-    };
-    void (async () => {
-        while (!stopped) {
-            await sleep(SIDECAR_POLL_MS);
-            let box;
-            try {
-                box = JSON.parse(fs.readFileSync(opts.sidecarPath, "utf8"));
-            }
-            catch {
-                continue; // torn read or not yet written — the spec re-polls
-            }
-            const req = box.request;
-            if (!req || typeof req.id !== "string" || served.has(req.id))
-                continue;
-            served.add(req.id);
-            const response = { id: req.id };
-            try {
-                response.value = await resolveRequest(req);
-            }
-            catch (err) {
-                response.error = err instanceof Error ? err.message : String(err);
-            }
-            if (stopped)
-                return;
-            try {
-                writeResponse(response);
-            }
-            catch {
-                // run dir vanished mid-write — the run is over anyway
-            }
-        }
-    })();
-    return {
-        stop: () => {
-            stopped = true;
-        },
-    };
-}