@shipvo/cli 0.1.4 → 0.1.6

package/dist/run-bridge.js

@@ -0,0 +1,728 @@
+// The page script of `shipvo run`. The frame page serves it at `/__shipvo/bridge.js`. It connects to the agent
+// channel as role `screen` and answers `tree` requests from the same-origin app iframe. No Playwright: the tree is
+// built from the DOM, in the shape of the simulator's `- role "name" [ref=eN]` lines (slice 3b of #448).
+// Differences from the simulator tree are listed in docs/run-bridge.md.
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+/**
+ * The shared screen rules (`@shipvo/sim`), as source that the page can run. Same copy the simulator uses.
+ * Only plain `export function` / `export const` lines are supported: `export default` and `export {…}` would not be stripped.
+ */
+export function screenRulesSource() {
+    const path = createRequire(import.meta.url).resolve("@shipvo/sim/screen-rules.mjs");
+    const src = readFileSync(path, "utf8");
+    if (/^export\s+(default\b|\{|\*)/m.test(src))
+        throw new Error("screen-rules.mjs may only use `export function` or `export const`");
+    return src.replace(/^export /gm, "");
+}
+/** Runs in the page. Everything it needs is nested inside, because it is serialised with `String(fn)`. */
+function bridgeMain(win, token) {
+    const doc = win.document;
+    // A ref names one element with one role and name. If the element's content changes, it gets a new ref and the old one is stale.
+    const refs = new WeakMap();
+    const byRef = new Map();
+    let counter = 0;
+    const refOf = (el, sig) => {
+        let r = refs.get(el);
+        if (!r || r.sig !== sig) {
+            r = { n: ++counter, sig };
+            refs.set(el, r);
+        }
+        byRef.set("e" + r.n, { el, sig });
+        return "e" + r.n;
+    };
+    const INPUT_ROLE = { checkbox: "checkbox", radio: "radio", range: "slider", button: "button", submit: "button", reset: "button", image: "button", search: "searchbox", number: "spinbutton" };
+    const TAG_ROLE = { button: "button", textarea: "textbox", select: "combobox", img: "img", ul: "list", ol: "list", li: "listitem", nav: "navigation", main: "main", dialog: "dialog", form: "form", table: "table", tr: "row", td: "cell", th: "columnheader", p: "paragraph", progress: "progressbar", article: "article", aside: "complementary" };
+    const FROM_CONTENT = new Set(["button", "link", "heading", "tab", "checkbox", "radio", "switch", "menuitem", "option", "cell", "columnheader"]);
+    const norm = (s) => (s ?? "").replace(/\s+/g, " ").trim();
+    const hidden = (el, cs) => el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true" || cs.display === "none" || cs.visibility === "hidden" || cs.visibility === "collapse";
+    function roleOf(el) {
+        const explicit = norm(el.getAttribute("role")).split(" ")[0];
+        if (explicit && explicit !== "presentation" && explicit !== "none")
+            return explicit;
+        const tag = el.tagName.toLowerCase();
+        if (tag === "a")
+            return el.hasAttribute("href") ? "link" : "";
+        if (/^h[1-6]$/.test(tag))
+            return "heading";
+        if (tag === "input") {
+            const t = (el.getAttribute("type") || "text").toLowerCase();
+            return t === "hidden" ? "" : INPUT_ROLE[t] || "textbox";
+        }
+        if (tag === "section")
+            return el.hasAttribute("aria-label") || el.hasAttribute("aria-labelledby") ? "region" : "";
+        if (tag === "img" && el.getAttribute("alt") === "")
+            return "";
+        return TAG_ROLE[tag] || "";
+    }
+    function textOf(node) {
+        if (node.nodeType === 3)
+            return node.nodeValue || "";
+        if (node.nodeType !== 1)
+            return "";
+        const tag = node.tagName.toLowerCase();
+        if (tag === "script" || tag === "style" || tag === "noscript")
+            return "";
+        if (hidden(node, win.getComputedStyle(node)))
+            return "";
+        if (tag === "img")
+            return node.getAttribute("alt") || "";
+        const own = node.getAttribute("aria-label");
+        if (own && roleOf(node))
+            return own;
+        let out = "";
+        for (const c of node.childNodes)
+            out += textOf(c) + (c.nodeType === 1 && win.getComputedStyle(c).display !== "inline" ? " " : "");
+        return out;
+    }
+    /** Accessible name and whether it came from the element's own content. */
+    function nameOf(el, role) {
+        const by = norm(el.getAttribute("aria-labelledby"));
+        if (by) {
+            const t = by.split(" ").map((id) => { const r = doc.getElementById(id); return r ? norm(textOf(r)) : ""; }).filter(Boolean).join(" ");
+            if (t)
+                return { name: t, fromContent: false };
+        }
+        const label = norm(el.getAttribute("aria-label"));
+        if (label)
+            return { name: label, fromContent: false };
+        const tag = el.tagName.toLowerCase();
+        if (tag === "input" || tag === "textarea" || tag === "select") {
+            const ls = el.labels ? [...el.labels] : [];
+            const t = norm(ls.map((l) => textOf(l)).join(" "));
+            if (t)
+                return { name: t, fromContent: false };
+            if (tag === "input" && /^(button|submit|reset)$/.test(el.type) && el.value)
+                return { name: norm(el.value), fromContent: false };
+            const ph = norm(el.getAttribute("placeholder")) || norm(el.getAttribute("title"));
+            return { name: ph, fromContent: false };
+        }
+        if (tag === "img")
+            return { name: norm(el.getAttribute("alt")) || norm(el.getAttribute("title")), fromContent: false };
+        if (FROM_CONTENT.has(role)) {
+            const t = norm(textOf(el));
+            if (t)
+                return { name: t, fromContent: true };
+        }
+        return { name: norm(el.getAttribute("title")), fromContent: false };
+    }
+    function states(el, role) {
+        let s = "";
+        const aria = (n) => el.getAttribute(n);
+        const on = (n) => aria(n) === "true";
+        if (role === "heading") {
+            const l = aria("aria-level") || (/^h([1-6])$/i.exec(el.tagName)?.[1] ?? "2");
+            s += " [level=" + l + "]";
+        }
+        if (/^(checkbox|radio|switch|menuitemcheckbox|menuitemradio)$/.test(role)) {
+            if (el.tagName === "INPUT" ? el.checked : on("aria-checked"))
+                s += " [checked]";
+            else if (aria("aria-checked") === "mixed")
+                s += " [checked=mixed]";
+        }
+        if (el.disabled === true || on("aria-disabled") || (el.closest && el.closest("fieldset[disabled]")))
+            s += " [disabled]";
+        if (on("aria-expanded"))
+            s += " [expanded]";
+        if (/^(tab|option|treeitem|row|gridcell)$/.test(role) && on("aria-selected"))
+            s += " [selected]";
+        if (on("aria-pressed"))
+            s += " [pressed]";
+        return s;
+    }
+    const quote = (s) => JSON.stringify(s);
+    // The simulator's snapshot is YAML-like: a key with ": " inside is wrapped in single quotes, a value with ": " in double quotes.
+    const needs = (t) => /: | #|:$|^[\[\]{}&*!|>'"%@`#?,-]/.test(t);
+    const keyLine = (pad, key, tail) => pad + "- " + (needs(key) ? "'" + key.replace(/'/g, "''") + "'" : key) + tail;
+    const val = (t) => (needs(t) ? JSON.stringify(t) : t);
+    function walk(el, depth, out) {
+        const tag = el.tagName.toLowerCase();
+        if (tag === "script" || tag === "style" || tag === "noscript" || tag === "template")
+            return;
+        if (hidden(el, win.getComputedStyle(el)))
+            return;
+        const role = roleOf(el);
+        const pad = "  ".repeat(depth);
+        if (role) {
+            const { name, fromContent } = nameOf(el, role);
+            const key = role + (name ? " " + quote(name) : "") + states(el, role) + " [ref=" + refOf(el, role + "|" + name) + "]";
+            if (tag === "input" || tag === "textarea") {
+                const v = el.value;
+                out.push(keyLine(pad, key, v && !/^(checkbox|radio|button|submit|reset)$/.test(el.type) ? ": " + val(v) : ""));
+                return;
+            }
+            if (fromContent || tag === "img") {
+                out.push(keyLine(pad, key, ""));
+                return;
+            }
+            const mark = out.length;
+            out.push("");
+            for (const c of el.childNodes)
+                child(c, depth + 1, out);
+            out[mark] = keyLine(pad, key, out.length > mark + 1 ? ":" : "");
+            return;
+        }
+        // No role: keep the text, drop the wrapper. A wrapper with only text becomes one generic line.
+        const hasEl = [...el.children].length > 0;
+        if (!hasEl) {
+            const t = norm(el.textContent);
+            if (t)
+                out.push(pad + "- generic [ref=" + refOf(el, "generic|" + t) + "]: " + val(t));
+            return;
+        }
+        for (const c of el.childNodes)
+            child(c, depth, out);
+    }
+    function child(c, depth, out) {
+        if (c.nodeType === 1)
+            walk(c, depth, out);
+        else if (c.nodeType === 3) {
+            const t = norm(c.nodeValue);
+            if (t)
+                out.push("  ".repeat(depth) + "- text: " + val(t));
+        }
+    }
+    function tree() {
+        const frame = doc.getElementById("app");
+        const d = frame && frame.contentDocument;
+        if (!d || !d.body)
+            throw new Error("the app is not loaded yet");
+        globalThis.__shipvoRules.clearScreenRules(d);
+        globalThis.__shipvoRules.applyScreenRules(d);
+        try {
+            const out = [];
+            for (const c of d.body.childNodes)
+                child(c, 0, out);
+            return globalThis.__shipvoRules.filterTree(out.join("\n"));
+        }
+        finally {
+            globalThis.__shipvoRules.clearScreenRules(d);
+        }
+    }
+    /** Same code and message shape as the simulator's typed errors. */
+    class ActError extends Error {
+        code;
+        constructor(code, msg) { super(msg); this.code = code; }
+    }
+    const sigOf = (el) => { const role = roleOf(el); return role ? role + "|" + nameOf(el, role).name : "generic|" + norm(el.textContent); };
+    const appDoc = () => { const f = doc.getElementById("app"); return f && f.contentDocument; };
+    /** The element for a ref from a snapshot, or `stale_ref` if it is unknown, gone, left the page, or changed role or name. */
+    function resolve(ref) {
+        const stale = () => new ActError("stale_ref", `stale ref ${ref}: the screen changed. Take a new snapshot (action "screen") and use a ref from it.`);
+        const d = appDoc();
+        const rec = byRef.get(String(ref));
+        if (!d || !rec || !rec.el.isConnected || rec.el.ownerDocument !== d)
+            throw stale();
+        if (sigOf(rec.el) !== rec.sig || refs.get(rec.el)?.sig !== rec.sig)
+            throw stale();
+        return rec.el;
+    }
+    /** Say why an element cannot receive a tap. Nothing has reached the app when this throws. */
+    function ready(el, ref) {
+        const fail = (code, why) => new ActError(code, `${ref} ${why}, so nothing was sent to the app.`);
+        if (hidden(el, win.getComputedStyle(el)) || el.getClientRects().length === 0)
+            throw fail("not_visible", "is not visible");
+        if (el.disabled === true || el.getAttribute("aria-disabled") === "true" || (el.closest && el.closest("fieldset[disabled]")))
+            throw fail("disabled", "is disabled");
+        el.scrollIntoView({ block: "center", inline: "center" });
+        const r = el.getBoundingClientRect();
+        const w = el.ownerDocument.defaultView;
+        if (r.right <= 0 || r.bottom <= 0 || r.left >= w.innerWidth || r.top >= w.innerHeight)
+            throw fail("not_visible", "is not visible");
+        const x = r.left + r.width / 2, y = r.top + r.height / 2;
+        const top = el.ownerDocument.elementFromPoint(x, y);
+        if (top && top !== el && !el.contains(top) && !top.contains(el))
+            throw fail("covered", "is covered by another element");
+        return { x, y, target: top && el.contains(top) ? top : el };
+    }
+    function tap(el, ref) {
+        const { x, y, target } = ready(el, ref);
+        mark(x, y, el);
+        const w = el.ownerDocument.defaultView;
+        const init = { bubbles: true, cancelable: true, composed: true, view: w, clientX: x, clientY: y, button: 0, pointerId: 1, pointerType: "mouse", isPrimary: true };
+        const fire = (Ctor, type, buttons) => target.dispatchEvent(new Ctor(type, { ...init, buttons }));
+        fire(w.PointerEvent, "pointerdown", 1);
+        fire(w.MouseEvent, "mousedown", 1);
+        if (typeof el.focus === "function")
+            el.focus({ preventScroll: true });
+        fire(w.PointerEvent, "pointerup", 0);
+        fire(w.MouseEvent, "mouseup", 0);
+        fire(w.MouseEvent, "click", 0);
+    }
+    function type(el, ref, text, enter) {
+        const field = /^(INPUT|TEXTAREA)$/.test(el.tagName) ? el : el.querySelector("input, textarea");
+        if (!field)
+            throw new ActError("not_a_field", `${ref} is not a text field. Type into a "textbox" ref.`);
+        ready(field, ref);
+        const w = field.ownerDocument.defaultView;
+        field.focus();
+        const proto = field.tagName === "TEXTAREA" ? w.HTMLTextAreaElement.prototype : w.HTMLInputElement.prototype;
+        Object.getOwnPropertyDescriptor(proto, "value").set.call(field, text);
+        field.dispatchEvent(new w.InputEvent("input", { bubbles: true, inputType: "insertText", data: text }));
+        field.dispatchEvent(new w.Event("change", { bubbles: true }));
+        if (enter)
+            for (const t of ["keydown", "keypress", "keyup"])
+                field.dispatchEvent(new w.KeyboardEvent(t, { key: "Enter", code: "Enter", keyCode: 13, which: 13, bubbles: true, cancelable: true }));
+        return field.value;
+    }
+    function scroll(ref, direction) {
+        const d = appDoc();
+        const w = d.defaultView;
+        const dx = direction === "right" ? 300 : direction === "left" ? -300 : 0;
+        const dy = direction === "down" ? 600 : direction === "up" ? -600 : 0;
+        if (!dx && !dy)
+            throw new ActError("bad_direction", "scroll needs a direction: up, down, left or right.");
+        let el = ref ? resolve(ref) : d.body;
+        // The nearest ancestor that can scroll that way. Fall back to the page.
+        for (; el && el !== d.body && el !== d.documentElement; el = el.parentElement) {
+            const cs = w.getComputedStyle(el);
+            if (dy && /auto|scroll/.test(cs.overflowY) && el.scrollHeight > el.clientHeight)
+                break;
+            if (dx && /auto|scroll/.test(cs.overflowX) && el.scrollWidth > el.clientWidth)
+                break;
+        }
+        if (!el || el === d.body || el === d.documentElement)
+            el = d.scrollingElement || d.documentElement;
+        el.scrollBy({ left: dx, top: dy, behavior: "instant" });
+    }
+    function back() {
+        const d = appDoc();
+        for (const el of d.querySelectorAll("button, [role=button], input[type=button], input[type=submit]")) {
+            const role = roleOf(el);
+            if (role === "button" && !hidden(el, win.getComputedStyle(el)) && nameOf(el, role).name === "Back") {
+                tap(el, "Back");
+                return;
+            }
+        }
+        throw new Error("no Back button on this screen");
+    }
+    // The person can see and stop the agent (slice 4 of #448). Overlay elements live in the frame page, outside the app iframe.
+    let seenAgent = false, taken = false, hideTimer;
+    const css = doc.createElement("style");
+    css.textContent = "#shipvo-agent{position:fixed;left:50%;top:40px;transform:translateX(-50%);z-index:2147483000;display:none;align-items:center;gap:8px;padding:4px 12px;border-radius:14px;background:#4f46e5;color:#fff;font:12px system-ui,sans-serif}" +
+        "#shipvo-agent.paused{background:#b45309}#shipvo-agent button{border:0;border-radius:10px;padding:2px 10px;font:inherit;cursor:pointer;display:none}#shipvo-agent.paused button{display:inline-block}" +
+        "#shipvo-capture{position:fixed;left:50%;top:84px;transform:translateX(-50%);z-index:2147483001;display:none;align-items:center;gap:8px;padding:6px 12px;border-radius:14px;background:#111;color:#fff;font:12px system-ui,sans-serif;white-space:nowrap;max-width:calc(100vw - 24px);box-sizing:border-box}#shipvo-capture button{border:0;border-radius:10px;padding:2px 10px;font:inherit;cursor:pointer}#shipvo-capture button:focus-visible{outline:2px solid #fff;outline-offset:2px}" +
+        "#shipvo-dot{position:fixed;width:22px;height:22px;margin:-11px 0 0 -11px;border-radius:50%;background:#4f46e5aa;border:2px solid #fff;box-shadow:0 0 0 2px #4f46e5;pointer-events:none;z-index:2147483000;display:none}" +
+        "#shipvo-outline{position:fixed;border:2px solid #4f46e5;border-radius:4px;pointer-events:none;z-index:2147483000;display:none}" +
+        "#shipvo-log{position:fixed;right:12px;bottom:44px;width:240px;max-width:calc(100vw - 24px);box-sizing:border-box;max-height:25%;overflow:auto;margin:0;padding:8px 8px 8px 26px;border-radius:8px;background:#111d;color:#fff;font:12px system-ui,sans-serif;z-index:2147483000;display:none}#shipvo-log li{margin:2px 0}" +
+        "#shipvo-log-toggle{position:fixed;right:12px;bottom:12px;z-index:2147483000;display:none;border:0;border-radius:10px;padding:4px 10px;background:#111d;color:#fff;font:12px system-ui,sans-serif;cursor:pointer}";
+    doc.head.appendChild(css);
+    const make = (tag, id) => { const e = doc.createElement(tag); e.id = id; doc.body.appendChild(e); return e; };
+    const banner = make("div", "shipvo-agent");
+    banner.setAttribute("role", "status");
+    const bannerText = doc.createElement("span");
+    banner.appendChild(bannerText);
+    const handBack = doc.createElement("button");
+    handBack.type = "button";
+    handBack.textContent = "Hand back";
+    banner.appendChild(handBack);
+    const dot = make("div", "shipvo-dot"), outline = make("div", "shipvo-outline"), log = make("ol", "shipvo-log");
+    log.setAttribute("aria-label", "Agent steps");
+    // The log can be hidden so it never covers what the agent is tapping.
+    let logHidden = false;
+    const logToggle = make("button", "shipvo-log-toggle");
+    logToggle.type = "button";
+    logToggle.textContent = "Hide steps";
+    logToggle.setAttribute("aria-expanded", "true");
+    const showLog = () => { log.style.display = logHidden ? "none" : "block"; logToggle.style.display = "block"; logToggle.textContent = logHidden ? "Show steps" : "Hide steps"; logToggle.setAttribute("aria-expanded", String(!logHidden)); };
+    logToggle.onclick = () => { logHidden = !logHidden; showLog(); };
+    const showBanner = () => {
+        banner.style.display = "flex";
+        banner.classList.toggle("paused", taken);
+        bannerText.textContent = taken ? "You took over. The agent is paused." : "Agent is driving";
+    };
+    const step = (text, simulated = false) => {
+        const li = doc.createElement("li");
+        li.textContent = text + (simulated ? " (simulated)" : "");
+        if (simulated)
+            li.setAttribute("data-simulated", "true");
+        log.appendChild(li);
+        showLog();
+        log.scrollTop = log.scrollHeight;
+        while (log.children.length > 50)
+            log.removeChild(log.firstChild);
+    };
+    /** Where the scaled app iframe is on the frame page. */
+    const toPage = () => { const f = doc.getElementById("app"); const r = f.getBoundingClientRect(); const k = f.offsetWidth ? r.width / f.offsetWidth : 1; return { left: r.left, top: r.top, k }; };
+    function mark(x, y, el) {
+        const o = toPage(), b = el.getBoundingClientRect();
+        dot.style.left = o.left + x * o.k + "px";
+        dot.style.top = o.top + y * o.k + "px";
+        dot.style.display = "block";
+        Object.assign(outline.style, { left: o.left + b.left * o.k + "px", top: o.top + b.top * o.k + "px", width: b.width * o.k + "px", height: b.height * o.k + "px", display: "block" });
+        win.clearTimeout(hideTimer);
+        hideTimer = win.setTimeout(() => { dot.style.display = "none"; outline.style.display = "none"; }, 1500);
+    }
+    /** Plain words for a step that worked. Taken before the action runs, so the name is the one the agent saw. */
+    const describe = (m) => {
+        const target = () => { try {
+            const e = byRef.get(String(m.ref))?.el;
+            if (!e)
+                return "an element";
+            const r = roleOf(e);
+            const n = r ? nameOf(e, r).name : norm(e.textContent);
+            return (r || "element") + (n ? " " + quote(n) : "");
+        }
+        catch {
+            return "an element";
+        } };
+        if (m.type === "tap")
+            return "Tapped " + target();
+        if (m.type === "type")
+            return "Typed " + quote(String(m.text ?? "")) + " into " + target() + (m.enter ? " and pressed Enter" : "");
+        if (m.type === "scroll")
+            return "Scrolled " + m.direction + (m.ref ? " in " + target() : "");
+        return "Pressed Back";
+    };
+    /** Lumi's caption for a step, from the same element the agent named: "Tapping Checkout". Nothing is invented; an unnamed element is "an element". */
+    const caption = (m) => {
+        const target = () => { try {
+            const e = byRef.get(String(m.ref))?.el;
+            if (!e)
+                return "an element";
+            const r = roleOf(e);
+            const n = r ? nameOf(e, r).name : norm(e.textContent);
+            return n || "an element";
+        }
+        catch {
+            return "an element";
+        } };
+        if (m.type === "tap")
+            return "Tapping " + target();
+        if (m.type === "type")
+            return "Typing into " + target();
+        if (m.type === "scroll")
+            return "Scrolling " + m.direction + (m.ref ? " in " + target() : "");
+        return "Going back";
+    };
+    /** Shows a caption on Lumi in the phone frame. Does nothing when the frame has no Lumi (it is off). */
+    const lumiSay = (text, state, hold = 0) => { try {
+        win.shipvoLumi && win.shipvoLumi.say(text, state, hold);
+    }
+    catch { } };
+    function takeOver(e) {
+        if (!e.isTrusted || taken || !seenAgent)
+            return;
+        if (e.target && e.target.closest && e.target.closest("#shipvo-agent, #shipvo-log-toggle, #shipvo-capture"))
+            return;
+        taken = true;
+        showBanner();
+        step("You took over. The agent is paused.");
+    }
+    handBack.onclick = () => { taken = false; showBanner(); step("You handed back. The agent can continue."); };
+    const watch = (d) => { if (!d)
+        return; for (const t of ["pointerdown", "touchstart", "keydown"])
+        d.addEventListener(t, takeOver, true); };
+    watch(doc);
+    const frameEl = doc.getElementById("app");
+    const watchApp = () => { try {
+        watch(frameEl.contentDocument);
+    }
+    catch { } };
+    frameEl.addEventListener("load", watchApp);
+    watchApp();
+    // Lumi (when the frame draws it) reacts to a page error in the app and to a failed action, with the real message as the reason.
+    const lumi = (ev) => { try {
+        win.shipvoLumi?.react(ev);
+    }
+    catch { } };
+    const watchErrors = () => {
+        try {
+            const w = frameEl.contentWindow;
+            w.addEventListener("error", (e) => lumi({ kind: "page-error", error: e.message || (e.error && e.error.message) }));
+            w.addEventListener("unhandledrejection", (e) => lumi({ kind: "page-error", error: e.reason && e.reason.message ? e.reason.message : e.reason }));
+        }
+        catch { }
+    };
+    frameEl.addEventListener("load", watchErrors);
+    watchErrors();
+    // Device events from the simulated phone are not the app's own: the log marks them.
+    win.addEventListener("message", (e) => {
+        const d = e.data;
+        if (d && d.type === "shipvo:system-sheet" && seenAgent)
+            step(d.open !== false ? "System permission sheet opened" : "System permission sheet closed", true);
+    });
+    const sleep = (ms) => new Promise((r) => win.setTimeout(r, ms));
+    /** Give the app a moment to react, and wait for a page load a tap started. */
+    async function settle() {
+        await sleep(250);
+        for (let i = 0; i < 30; i++) {
+            const d = appDoc();
+            if (d && d.readyState === "complete" && d.body)
+                break;
+            await sleep(100);
+        }
+        await sleep(50);
+    }
+    const screenOrUndefined = () => { try {
+        return tree();
+    }
+    catch {
+        return undefined;
+    } };
+    async function act(m) {
+        switch (m.type) {
+            case "tap":
+                tap(resolve(m.ref), m.ref);
+                break;
+            case "type": {
+                const value = type(resolve(m.ref), m.ref, String(m.text ?? ""), !!m.enter);
+                await settle();
+                return { delivery: "delivered", value, screen: tree() };
+            }
+            case "scroll":
+                scroll(m.ref, m.direction);
+                break;
+            case "back":
+                back();
+                break;
+            default: throw new Error("unknown request: " + m.type);
+        }
+        await settle();
+        return { delivery: "delivered", screen: tree() };
+    }
+    // Screenshots (slice 5 of #448). Order: tab capture if the person said yes, else the iframe drawn to an image.
+    // The permission is asked once, on the first request from the agent. getDisplayMedia needs a click, so the question is a button.
+    let capture = "ask", stream, video, asked = false;
+    const bar = make("div", "shipvo-capture");
+    bar.setAttribute("role", "group");
+    bar.setAttribute("aria-label", "Let the agent see this tab?");
+    const barText = doc.createElement("span");
+    barText.textContent = "Let the agent see this tab?";
+    bar.appendChild(barText);
+    const allow = doc.createElement("button");
+    allow.type = "button";
+    allow.textContent = "Allow";
+    bar.appendChild(allow);
+    const deny = doc.createElement("button");
+    deny.type = "button";
+    deny.textContent = "Not now";
+    bar.appendChild(deny);
+    const askCapture = () => { if (asked)
+        return; asked = true; bar.style.display = "flex"; };
+    deny.onclick = () => { capture = "refused"; bar.style.display = "none"; step("The agent cannot see this tab. Screenshots are drawn from the page."); };
+    allow.onclick = async () => {
+        bar.style.display = "none";
+        try {
+            stream = await win.navigator.mediaDevices.getDisplayMedia({ video: true, audio: false, preferCurrentTab: true });
+            video = doc.createElement("video");
+            video.muted = true;
+            video.srcObject = stream;
+            await video.play();
+            const surface = stream.getVideoTracks()[0]?.getSettings?.().displaySurface;
+            if (surface && surface !== "browser") {
+                for (const t of stream.getTracks())
+                    t.stop();
+                stream = undefined;
+                capture = "refused";
+                step("The shared surface is not this tab. Screenshots are drawn from the page.");
+                return;
+            }
+            for (const t of stream.getTracks())
+                t.addEventListener("ended", () => { capture = "ask"; asked = false; stream = undefined; step("Tab sharing stopped. Screenshots are drawn from the page; the next agent action asks again."); });
+            capture = "granted";
+            step("The agent can see this tab.");
+        }
+        catch {
+            capture = "refused";
+            step("Tab capture was refused. Screenshots are drawn from the page.");
+        }
+    };
+    // The socket drops any message over 1 MiB, so a screenshot is kept under SHOT_MAX base64 characters.
+    const SHOT_MAX = 700_000;
+    const blob64 = (c, type, q) => new Promise((ok, fail) => c.toBlob((b) => {
+        if (!b)
+            return fail(new Error("could not make an image"));
+        const r = new win.FileReader();
+        r.onload = () => ok(String(r.result).split(",")[1]);
+        r.onerror = () => fail(new Error("could not read the image"));
+        r.readAsDataURL(b);
+    }, type, q));
+    /** PNG if it fits. Otherwise a smaller PNG, then JPEG at lower scale and quality. Throws if nothing fits. */
+    async function png(c) {
+        const steps = [["image/png", 1, 1], ["image/png", 0.5, 1], ["image/jpeg", 1, 0.8], ["image/jpeg", 0.5, 0.6], ["image/jpeg", 0.3, 0.5]];
+        for (const [mime, scale, q] of steps) {
+            let src = c;
+            if (scale < 1) {
+                src = doc.createElement("canvas");
+                src.width = Math.max(1, Math.round(c.width * scale));
+                src.height = Math.max(1, Math.round(c.height * scale));
+                const g = src.getContext("2d");
+                g.fillStyle = "#fff";
+                g.fillRect(0, 0, src.width, src.height);
+                g.drawImage(c, 0, 0, src.width, src.height);
+            }
+            const image = await blob64(src, mime, q);
+            if (image.length <= SHOT_MAX)
+                return { image, mime, width: src.width, height: src.height };
+        }
+        throw new Error("the screenshot is too large to send, even scaled down");
+    }
+    /** A frame of the live tab, cropped to the phone. */
+    async function tabShot() {
+        for (let i = 0; i < 20 && !video.videoWidth; i++)
+            await sleep(50);
+        await new Promise((r) => win.requestAnimationFrame(() => r(0)));
+        await sleep(60);
+        const k = video.videoWidth / win.innerWidth, r = doc.getElementById("phone").getBoundingClientRect();
+        const c = doc.createElement("canvas");
+        c.width = Math.max(1, Math.round(r.width * k));
+        c.height = Math.max(1, Math.round(r.height * k));
+        c.getContext("2d").drawImage(video, r.left * k, r.top * k, r.width * k, r.height * k, 0, 0, c.width, c.height);
+        return await png(c);
+    }
+    /** The app iframe's DOM drawn to a canvas through an SVG foreignObject. Says what it could not draw. */
+    async function renderedShot() {
+        const d = appDoc();
+        if (!d)
+            throw new Error("the app is not loaded");
+        const w = d.defaultView, W = w.innerWidth, H = w.innerHeight, skipped = new Set();
+        const clone = d.documentElement.cloneNode(true);
+        const pair = (sel) => { const a = Array.from(d.querySelectorAll(sel)), b = Array.from(clone.querySelectorAll(sel)); return a.map((x, i) => [x, b[i]]); };
+        const placeholder = (src, dst, what) => {
+            skipped.add(what);
+            const ph = d.createElement("div");
+            const r = src.getBoundingClientRect();
+            ph.setAttribute("style", "display:inline-block;width:" + r.width + "px;height:" + r.height + "px;background:#8883");
+            dst.replaceWith(ph);
+        };
+        for (const n of Array.from(clone.querySelectorAll("script, noscript, template")))
+            n.remove();
+        for (const [a, b] of pair("video"))
+            placeholder(a, b, "video");
+        for (const [a, b] of pair("iframe"))
+            placeholder(a, b, "iframe");
+        for (const [a, b] of pair("canvas")) {
+            try {
+                b.replaceWith(Object.assign(d.createElement("img"), { src: a.toDataURL(), width: a.width, height: a.height }));
+            }
+            catch {
+                placeholder(a, b, "canvas");
+            }
+        }
+        for (const [a, b] of pair("img")) {
+            try {
+                if (!a.complete || !a.naturalWidth)
+                    throw new Error("not loaded");
+                const c = d.createElement("canvas");
+                c.width = a.naturalWidth;
+                c.height = a.naturalHeight;
+                c.getContext("2d").drawImage(a, 0, 0);
+                b.removeAttribute("srcset");
+                b.setAttribute("src", c.toDataURL());
+            }
+            catch {
+                skipped.add("cross-origin images");
+                b.removeAttribute("srcset");
+                b.removeAttribute("src");
+            }
+        }
+        for (const [a, b] of pair("link[rel~=stylesheet]")) {
+            try {
+                const st = d.createElement("style");
+                st.textContent = Array.from(a.sheet.cssRules).map((r) => r.cssText).join("\n");
+                b.replaceWith(st);
+            }
+            catch {
+                skipped.add("cross-origin stylesheets");
+                b.remove();
+            }
+        }
+        for (const [a, b] of pair("input, textarea")) {
+            if (a.tagName === "TEXTAREA")
+                b.textContent = a.value;
+            else
+                b.setAttribute("value", a.value);
+            if (a.checked)
+                b.setAttribute("checked", "");
+        }
+        const svg = '<svg xmlns="http://www.w3.org/2000/svg" width="' + W + '" height="' + H + '"><foreignObject width="100%" height="100%">' + new win.XMLSerializer().serializeToString(clone) + "</foreignObject></svg>";
+        const img = new win.Image();
+        await new Promise((ok, fail) => { img.onload = ok; img.onerror = () => fail(new Error("the page could not be drawn")); img.src = "data:image/svg+xml;charset=utf-8," + encodeURIComponent(svg); });
+        const c = doc.createElement("canvas");
+        c.width = W;
+        c.height = H;
+        const g = c.getContext("2d");
+        g.fillStyle = "#fff";
+        g.fillRect(0, 0, W, H);
+        g.drawImage(img, 0, 0);
+        return { ...(await png(c)), unsupported: Array.from(skipped) };
+    }
+    async function shot() {
+        if (capture === "granted" && stream) {
+            try {
+                return { source: "tab-capture", ...(await tabShot()) };
+            }
+            catch {
+                capture = "refused";
+            }
+        }
+        return { source: "rendered", ...(await renderedShot()) };
+    }
+    const ws = new win.WebSocket((win.location.protocol === "https:" ? "wss://" : "ws://") + win.location.host + "/__shipvo/agent?role=screen&token=" + encodeURIComponent(token));
+    ws.onmessage = (e) => {
+        let m;
+        try {
+            m = JSON.parse(String(e.data));
+        }
+        catch {
+            return;
+        }
+        if (!m || typeof m.id !== "number")
+            return;
+        // Reading the screen changes nothing, so it neither counts as the agent arriving nor is blocked by a take-over.
+        if (m.type !== "tree")
+            seenAgent = true;
+        if (m.type !== "tree" && m.type !== "screen")
+            askCapture();
+        if (m.type === "screenshot") {
+            const out = (o) => ws.send(JSON.stringify(o));
+            shot().then((r) => { const body = JSON.stringify({ id: m.id, type: "screenshot", ok: true, ...r, screen: screenOrUndefined() }); if (body.length > 1000000)
+                throw new Error("the screenshot reply is too large to send"); ws.send(body); }).catch((err) => out({ id: m.id, type: "screenshot", ok: false, error: String(err && err.message || err), screen: screenOrUndefined() }));
+            return;
+        }
+        if (taken && m.type !== "tree") {
+            ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, code: "taken_over", error: "the person took over" }));
+            return;
+        }
+        const words = /^(tap|type|scroll|back)$/.test(m.type) ? (showBanner(), describe(m)) : "";
+        const cap = /^(tap|type|scroll|back)$/.test(m.type) ? caption(m) : "";
+        if (cap)
+            lumiSay(cap, "working");
+        if (m.type === "lumi") {
+            // The MCP server names a step the page cannot see itself (opening the app). The text is cut to 80 characters in the frame.
+            const ok = typeof m.caption === "string" && ["working", "idle", "happy", "worried"].includes(m.state);
+            if (ok)
+                lumiSay(m.caption, m.state, Number(m.hold) > 0 ? Math.min(Number(m.hold), 10000) : 0);
+            ws.send(JSON.stringify({ id: m.id, type: "lumi", ok }));
+            return;
+        }
+        try {
+            if (m.type === "screen")
+                ws.send(JSON.stringify({ id: m.id, type: "screen", ok: true, screen: tree() }));
+            else if (m.type === "tree")
+                ws.send(JSON.stringify({ id: m.id, type: "tree", ok: true, tree: tree() }));
+            else if (/^(tap|type|scroll|back)$/.test(m.type)) {
+                act(m).then((r) => { step(words); lumiSay(cap, "idle", 1800); return r; }).then((r) => ws.send(JSON.stringify({ id: m.id, type: m.type, ok: true, ...r })), (err) => { lumiSay("Failed: " + cap, "worried", 2500); lumi({ kind: "action-failed", name: m.type, error: err && err.message || err }); ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, ...(err && err.code ? { code: err.code } : {}), error: String(err && err.message || err), screen: screenOrUndefined() })); });
+            }
+            else
+                ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, error: "unknown request: " + m.type }));
+        }
+        catch (err) {
+            ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, error: String(err && err.message || err) }));
+        }
+    };
+}
+/** The script text served at `/__shipvo/bridge.js`. It reads the session token from its own `data-token` attribute and then removes the attribute. */
+export function bridgeScript(rules = screenRulesSource()) {
+    return `(function(){
+var __rules=(function(){${rules}
+return {clearScreenRules:clearScreenRules,applyScreenRules:applyScreenRules,filterTree:filterTree};})();
+globalThis.__shipvoRules=__rules;
+var tag=document.currentScript;
+var tok=tag?tag.getAttribute("data-token")||"":"";
+if(tag)tag.removeAttribute("data-token");
+(${bridgeMain.toString()})(window, tok);
+})();
+`;
+}