@shipvo/cli 0.1.5 → 0.1.6

package/dist/run-bridge.js

@@ -237,6 +237,7 @@ function bridgeMain(win, token) {
     }
     function tap(el, ref) {
         const { x, y, target } = ready(el, ref);
+        mark(x, y, el);
         const w = el.ownerDocument.defaultView;
         const init = { bubbles: true, cancelable: true, composed: true, view: w, clientX: x, clientY: y, button: 0, pointerId: 1, pointerType: "mouse", isPrimary: true };
         const fire = (Ctor, type, buttons) => target.dispatchEvent(new Ctor(type, { ...init, buttons }));
@@ -295,6 +296,152 @@ function bridgeMain(win, token) {
         }
         throw new Error("no Back button on this screen");
     }
+    // The person can see and stop the agent (slice 4 of #448). Overlay elements live in the frame page, outside the app iframe.
+    let seenAgent = false, taken = false, hideTimer;
+    const css = doc.createElement("style");
+    css.textContent = "#shipvo-agent{position:fixed;left:50%;top:40px;transform:translateX(-50%);z-index:2147483000;display:none;align-items:center;gap:8px;padding:4px 12px;border-radius:14px;background:#4f46e5;color:#fff;font:12px system-ui,sans-serif}" +
+        "#shipvo-agent.paused{background:#b45309}#shipvo-agent button{border:0;border-radius:10px;padding:2px 10px;font:inherit;cursor:pointer;display:none}#shipvo-agent.paused button{display:inline-block}" +
+        "#shipvo-capture{position:fixed;left:50%;top:84px;transform:translateX(-50%);z-index:2147483001;display:none;align-items:center;gap:8px;padding:6px 12px;border-radius:14px;background:#111;color:#fff;font:12px system-ui,sans-serif;white-space:nowrap;max-width:calc(100vw - 24px);box-sizing:border-box}#shipvo-capture button{border:0;border-radius:10px;padding:2px 10px;font:inherit;cursor:pointer}#shipvo-capture button:focus-visible{outline:2px solid #fff;outline-offset:2px}" +
+        "#shipvo-dot{position:fixed;width:22px;height:22px;margin:-11px 0 0 -11px;border-radius:50%;background:#4f46e5aa;border:2px solid #fff;box-shadow:0 0 0 2px #4f46e5;pointer-events:none;z-index:2147483000;display:none}" +
+        "#shipvo-outline{position:fixed;border:2px solid #4f46e5;border-radius:4px;pointer-events:none;z-index:2147483000;display:none}" +
+        "#shipvo-log{position:fixed;right:12px;bottom:44px;width:240px;max-width:calc(100vw - 24px);box-sizing:border-box;max-height:25%;overflow:auto;margin:0;padding:8px 8px 8px 26px;border-radius:8px;background:#111d;color:#fff;font:12px system-ui,sans-serif;z-index:2147483000;display:none}#shipvo-log li{margin:2px 0}" +
+        "#shipvo-log-toggle{position:fixed;right:12px;bottom:12px;z-index:2147483000;display:none;border:0;border-radius:10px;padding:4px 10px;background:#111d;color:#fff;font:12px system-ui,sans-serif;cursor:pointer}";
+    doc.head.appendChild(css);
+    const make = (tag, id) => { const e = doc.createElement(tag); e.id = id; doc.body.appendChild(e); return e; };
+    const banner = make("div", "shipvo-agent");
+    banner.setAttribute("role", "status");
+    const bannerText = doc.createElement("span");
+    banner.appendChild(bannerText);
+    const handBack = doc.createElement("button");
+    handBack.type = "button";
+    handBack.textContent = "Hand back";
+    banner.appendChild(handBack);
+    const dot = make("div", "shipvo-dot"), outline = make("div", "shipvo-outline"), log = make("ol", "shipvo-log");
+    log.setAttribute("aria-label", "Agent steps");
+    // The log can be hidden so it never covers what the agent is tapping.
+    let logHidden = false;
+    const logToggle = make("button", "shipvo-log-toggle");
+    logToggle.type = "button";
+    logToggle.textContent = "Hide steps";
+    logToggle.setAttribute("aria-expanded", "true");
+    const showLog = () => { log.style.display = logHidden ? "none" : "block"; logToggle.style.display = "block"; logToggle.textContent = logHidden ? "Show steps" : "Hide steps"; logToggle.setAttribute("aria-expanded", String(!logHidden)); };
+    logToggle.onclick = () => { logHidden = !logHidden; showLog(); };
+    const showBanner = () => {
+        banner.style.display = "flex";
+        banner.classList.toggle("paused", taken);
+        bannerText.textContent = taken ? "You took over. The agent is paused." : "Agent is driving";
+    };
+    const step = (text, simulated = false) => {
+        const li = doc.createElement("li");
+        li.textContent = text + (simulated ? " (simulated)" : "");
+        if (simulated)
+            li.setAttribute("data-simulated", "true");
+        log.appendChild(li);
+        showLog();
+        log.scrollTop = log.scrollHeight;
+        while (log.children.length > 50)
+            log.removeChild(log.firstChild);
+    };
+    /** Where the scaled app iframe is on the frame page. */
+    const toPage = () => { const f = doc.getElementById("app"); const r = f.getBoundingClientRect(); const k = f.offsetWidth ? r.width / f.offsetWidth : 1; return { left: r.left, top: r.top, k }; };
+    function mark(x, y, el) {
+        const o = toPage(), b = el.getBoundingClientRect();
+        dot.style.left = o.left + x * o.k + "px";
+        dot.style.top = o.top + y * o.k + "px";
+        dot.style.display = "block";
+        Object.assign(outline.style, { left: o.left + b.left * o.k + "px", top: o.top + b.top * o.k + "px", width: b.width * o.k + "px", height: b.height * o.k + "px", display: "block" });
+        win.clearTimeout(hideTimer);
+        hideTimer = win.setTimeout(() => { dot.style.display = "none"; outline.style.display = "none"; }, 1500);
+    }
+    /** Plain words for a step that worked. Taken before the action runs, so the name is the one the agent saw. */
+    const describe = (m) => {
+        const target = () => { try {
+            const e = byRef.get(String(m.ref))?.el;
+            if (!e)
+                return "an element";
+            const r = roleOf(e);
+            const n = r ? nameOf(e, r).name : norm(e.textContent);
+            return (r || "element") + (n ? " " + quote(n) : "");
+        }
+        catch {
+            return "an element";
+        } };
+        if (m.type === "tap")
+            return "Tapped " + target();
+        if (m.type === "type")
+            return "Typed " + quote(String(m.text ?? "")) + " into " + target() + (m.enter ? " and pressed Enter" : "");
+        if (m.type === "scroll")
+            return "Scrolled " + m.direction + (m.ref ? " in " + target() : "");
+        return "Pressed Back";
+    };
+    /** Lumi's caption for a step, from the same element the agent named: "Tapping Checkout". Nothing is invented; an unnamed element is "an element". */
+    const caption = (m) => {
+        const target = () => { try {
+            const e = byRef.get(String(m.ref))?.el;
+            if (!e)
+                return "an element";
+            const r = roleOf(e);
+            const n = r ? nameOf(e, r).name : norm(e.textContent);
+            return n || "an element";
+        }
+        catch {
+            return "an element";
+        } };
+        if (m.type === "tap")
+            return "Tapping " + target();
+        if (m.type === "type")
+            return "Typing into " + target();
+        if (m.type === "scroll")
+            return "Scrolling " + m.direction + (m.ref ? " in " + target() : "");
+        return "Going back";
+    };
+    /** Shows a caption on Lumi in the phone frame. Does nothing when the frame has no Lumi (it is off). */
+    const lumiSay = (text, state, hold = 0) => { try {
+        win.shipvoLumi && win.shipvoLumi.say(text, state, hold);
+    }
+    catch { } };
+    function takeOver(e) {
+        if (!e.isTrusted || taken || !seenAgent)
+            return;
+        if (e.target && e.target.closest && e.target.closest("#shipvo-agent, #shipvo-log-toggle, #shipvo-capture"))
+            return;
+        taken = true;
+        showBanner();
+        step("You took over. The agent is paused.");
+    }
+    handBack.onclick = () => { taken = false; showBanner(); step("You handed back. The agent can continue."); };
+    const watch = (d) => { if (!d)
+        return; for (const t of ["pointerdown", "touchstart", "keydown"])
+        d.addEventListener(t, takeOver, true); };
+    watch(doc);
+    const frameEl = doc.getElementById("app");
+    const watchApp = () => { try {
+        watch(frameEl.contentDocument);
+    }
+    catch { } };
+    frameEl.addEventListener("load", watchApp);
+    watchApp();
+    // Lumi (when the frame draws it) reacts to a page error in the app and to a failed action, with the real message as the reason.
+    const lumi = (ev) => { try {
+        win.shipvoLumi?.react(ev);
+    }
+    catch { } };
+    const watchErrors = () => {
+        try {
+            const w = frameEl.contentWindow;
+            w.addEventListener("error", (e) => lumi({ kind: "page-error", error: e.message || (e.error && e.error.message) }));
+            w.addEventListener("unhandledrejection", (e) => lumi({ kind: "page-error", error: e.reason && e.reason.message ? e.reason.message : e.reason }));
+        }
+        catch { }
+    };
+    frameEl.addEventListener("load", watchErrors);
+    watchErrors();
+    // Device events from the simulated phone are not the app's own: the log marks them.
+    win.addEventListener("message", (e) => {
+        const d = e.data;
+        if (d && d.type === "shipvo:system-sheet" && seenAgent)
+            step(d.open !== false ? "System permission sheet opened" : "System permission sheet closed", true);
+    });
     const sleep = (ms) => new Promise((r) => win.setTimeout(r, ms));
     /** Give the app a moment to react, and wait for a page load a tap started. */
     async function settle() {
@@ -334,6 +481,184 @@ function bridgeMain(win, token) {
         await settle();
         return { delivery: "delivered", screen: tree() };
     }
+    // Screenshots (slice 5 of #448). Order: tab capture if the person said yes, else the iframe drawn to an image.
+    // The permission is asked once, on the first request from the agent. getDisplayMedia needs a click, so the question is a button.
+    let capture = "ask", stream, video, asked = false;
+    const bar = make("div", "shipvo-capture");
+    bar.setAttribute("role", "group");
+    bar.setAttribute("aria-label", "Let the agent see this tab?");
+    const barText = doc.createElement("span");
+    barText.textContent = "Let the agent see this tab?";
+    bar.appendChild(barText);
+    const allow = doc.createElement("button");
+    allow.type = "button";
+    allow.textContent = "Allow";
+    bar.appendChild(allow);
+    const deny = doc.createElement("button");
+    deny.type = "button";
+    deny.textContent = "Not now";
+    bar.appendChild(deny);
+    const askCapture = () => { if (asked)
+        return; asked = true; bar.style.display = "flex"; };
+    deny.onclick = () => { capture = "refused"; bar.style.display = "none"; step("The agent cannot see this tab. Screenshots are drawn from the page."); };
+    allow.onclick = async () => {
+        bar.style.display = "none";
+        try {
+            stream = await win.navigator.mediaDevices.getDisplayMedia({ video: true, audio: false, preferCurrentTab: true });
+            video = doc.createElement("video");
+            video.muted = true;
+            video.srcObject = stream;
+            await video.play();
+            const surface = stream.getVideoTracks()[0]?.getSettings?.().displaySurface;
+            if (surface && surface !== "browser") {
+                for (const t of stream.getTracks())
+                    t.stop();
+                stream = undefined;
+                capture = "refused";
+                step("The shared surface is not this tab. Screenshots are drawn from the page.");
+                return;
+            }
+            for (const t of stream.getTracks())
+                t.addEventListener("ended", () => { capture = "ask"; asked = false; stream = undefined; step("Tab sharing stopped. Screenshots are drawn from the page; the next agent action asks again."); });
+            capture = "granted";
+            step("The agent can see this tab.");
+        }
+        catch {
+            capture = "refused";
+            step("Tab capture was refused. Screenshots are drawn from the page.");
+        }
+    };
+    // The socket drops any message over 1 MiB, so a screenshot is kept under SHOT_MAX base64 characters.
+    const SHOT_MAX = 700_000;
+    const blob64 = (c, type, q) => new Promise((ok, fail) => c.toBlob((b) => {
+        if (!b)
+            return fail(new Error("could not make an image"));
+        const r = new win.FileReader();
+        r.onload = () => ok(String(r.result).split(",")[1]);
+        r.onerror = () => fail(new Error("could not read the image"));
+        r.readAsDataURL(b);
+    }, type, q));
+    /** PNG if it fits. Otherwise a smaller PNG, then JPEG at lower scale and quality. Throws if nothing fits. */
+    async function png(c) {
+        const steps = [["image/png", 1, 1], ["image/png", 0.5, 1], ["image/jpeg", 1, 0.8], ["image/jpeg", 0.5, 0.6], ["image/jpeg", 0.3, 0.5]];
+        for (const [mime, scale, q] of steps) {
+            let src = c;
+            if (scale < 1) {
+                src = doc.createElement("canvas");
+                src.width = Math.max(1, Math.round(c.width * scale));
+                src.height = Math.max(1, Math.round(c.height * scale));
+                const g = src.getContext("2d");
+                g.fillStyle = "#fff";
+                g.fillRect(0, 0, src.width, src.height);
+                g.drawImage(c, 0, 0, src.width, src.height);
+            }
+            const image = await blob64(src, mime, q);
+            if (image.length <= SHOT_MAX)
+                return { image, mime, width: src.width, height: src.height };
+        }
+        throw new Error("the screenshot is too large to send, even scaled down");
+    }
+    /** A frame of the live tab, cropped to the phone. */
+    async function tabShot() {
+        for (let i = 0; i < 20 && !video.videoWidth; i++)
+            await sleep(50);
+        await new Promise((r) => win.requestAnimationFrame(() => r(0)));
+        await sleep(60);
+        const k = video.videoWidth / win.innerWidth, r = doc.getElementById("phone").getBoundingClientRect();
+        const c = doc.createElement("canvas");
+        c.width = Math.max(1, Math.round(r.width * k));
+        c.height = Math.max(1, Math.round(r.height * k));
+        c.getContext("2d").drawImage(video, r.left * k, r.top * k, r.width * k, r.height * k, 0, 0, c.width, c.height);
+        return await png(c);
+    }
+    /** The app iframe's DOM drawn to a canvas through an SVG foreignObject. Says what it could not draw. */
+    async function renderedShot() {
+        const d = appDoc();
+        if (!d)
+            throw new Error("the app is not loaded");
+        const w = d.defaultView, W = w.innerWidth, H = w.innerHeight, skipped = new Set();
+        const clone = d.documentElement.cloneNode(true);
+        const pair = (sel) => { const a = Array.from(d.querySelectorAll(sel)), b = Array.from(clone.querySelectorAll(sel)); return a.map((x, i) => [x, b[i]]); };
+        const placeholder = (src, dst, what) => {
+            skipped.add(what);
+            const ph = d.createElement("div");
+            const r = src.getBoundingClientRect();
+            ph.setAttribute("style", "display:inline-block;width:" + r.width + "px;height:" + r.height + "px;background:#8883");
+            dst.replaceWith(ph);
+        };
+        for (const n of Array.from(clone.querySelectorAll("script, noscript, template")))
+            n.remove();
+        for (const [a, b] of pair("video"))
+            placeholder(a, b, "video");
+        for (const [a, b] of pair("iframe"))
+            placeholder(a, b, "iframe");
+        for (const [a, b] of pair("canvas")) {
+            try {
+                b.replaceWith(Object.assign(d.createElement("img"), { src: a.toDataURL(), width: a.width, height: a.height }));
+            }
+            catch {
+                placeholder(a, b, "canvas");
+            }
+        }
+        for (const [a, b] of pair("img")) {
+            try {
+                if (!a.complete || !a.naturalWidth)
+                    throw new Error("not loaded");
+                const c = d.createElement("canvas");
+                c.width = a.naturalWidth;
+                c.height = a.naturalHeight;
+                c.getContext("2d").drawImage(a, 0, 0);
+                b.removeAttribute("srcset");
+                b.setAttribute("src", c.toDataURL());
+            }
+            catch {
+                skipped.add("cross-origin images");
+                b.removeAttribute("srcset");
+                b.removeAttribute("src");
+            }
+        }
+        for (const [a, b] of pair("link[rel~=stylesheet]")) {
+            try {
+                const st = d.createElement("style");
+                st.textContent = Array.from(a.sheet.cssRules).map((r) => r.cssText).join("\n");
+                b.replaceWith(st);
+            }
+            catch {
+                skipped.add("cross-origin stylesheets");
+                b.remove();
+            }
+        }
+        for (const [a, b] of pair("input, textarea")) {
+            if (a.tagName === "TEXTAREA")
+                b.textContent = a.value;
+            else
+                b.setAttribute("value", a.value);
+            if (a.checked)
+                b.setAttribute("checked", "");
+        }
+        const svg = '<svg xmlns="http://www.w3.org/2000/svg" width="' + W + '" height="' + H + '"><foreignObject width="100%" height="100%">' + new win.XMLSerializer().serializeToString(clone) + "</foreignObject></svg>";
+        const img = new win.Image();
+        await new Promise((ok, fail) => { img.onload = ok; img.onerror = () => fail(new Error("the page could not be drawn")); img.src = "data:image/svg+xml;charset=utf-8," + encodeURIComponent(svg); });
+        const c = doc.createElement("canvas");
+        c.width = W;
+        c.height = H;
+        const g = c.getContext("2d");
+        g.fillStyle = "#fff";
+        g.fillRect(0, 0, W, H);
+        g.drawImage(img, 0, 0);
+        return { ...(await png(c)), unsupported: Array.from(skipped) };
+    }
+    async function shot() {
+        if (capture === "granted" && stream) {
+            try {
+                return { source: "tab-capture", ...(await tabShot()) };
+            }
+            catch {
+                capture = "refused";
+            }
+        }
+        return { source: "rendered", ...(await renderedShot()) };
+    }
     const ws = new win.WebSocket((win.location.protocol === "https:" ? "wss://" : "ws://") + win.location.host + "/__shipvo/agent?role=screen&token=" + encodeURIComponent(token));
     ws.onmessage = (e) => {
         let m;
@@ -345,11 +670,40 @@ function bridgeMain(win, token) {
         }
         if (!m || typeof m.id !== "number")
             return;
+        // Reading the screen changes nothing, so it neither counts as the agent arriving nor is blocked by a take-over.
+        if (m.type !== "tree")
+            seenAgent = true;
+        if (m.type !== "tree" && m.type !== "screen")
+            askCapture();
+        if (m.type === "screenshot") {
+            const out = (o) => ws.send(JSON.stringify(o));
+            shot().then((r) => { const body = JSON.stringify({ id: m.id, type: "screenshot", ok: true, ...r, screen: screenOrUndefined() }); if (body.length > 1000000)
+                throw new Error("the screenshot reply is too large to send"); ws.send(body); }).catch((err) => out({ id: m.id, type: "screenshot", ok: false, error: String(err && err.message || err), screen: screenOrUndefined() }));
+            return;
+        }
+        if (taken && m.type !== "tree") {
+            ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, code: "taken_over", error: "the person took over" }));
+            return;
+        }
+        const words = /^(tap|type|scroll|back)$/.test(m.type) ? (showBanner(), describe(m)) : "";
+        const cap = /^(tap|type|scroll|back)$/.test(m.type) ? caption(m) : "";
+        if (cap)
+            lumiSay(cap, "working");
+        if (m.type === "lumi") {
+            // The MCP server names a step the page cannot see itself (opening the app). The text is cut to 80 characters in the frame.
+            const ok = typeof m.caption === "string" && ["working", "idle", "happy", "worried"].includes(m.state);
+            if (ok)
+                lumiSay(m.caption, m.state, Number(m.hold) > 0 ? Math.min(Number(m.hold), 10000) : 0);
+            ws.send(JSON.stringify({ id: m.id, type: "lumi", ok }));
+            return;
+        }
         try {
-            if (m.type === "tree")
+            if (m.type === "screen")
+                ws.send(JSON.stringify({ id: m.id, type: "screen", ok: true, screen: tree() }));
+            else if (m.type === "tree")
                 ws.send(JSON.stringify({ id: m.id, type: "tree", ok: true, tree: tree() }));
             else if (/^(tap|type|scroll|back)$/.test(m.type)) {
-                act(m).then((r) => ws.send(JSON.stringify({ id: m.id, type: m.type, ok: true, ...r })), (err) => ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, ...(err && err.code ? { code: err.code } : {}), error: String(err && err.message || err), screen: screenOrUndefined() })));
+                act(m).then((r) => { step(words); lumiSay(cap, "idle", 1800); return r; }).then((r) => ws.send(JSON.stringify({ id: m.id, type: m.type, ok: true, ...r })), (err) => { lumiSay("Failed: " + cap, "worried", 2500); lumi({ kind: "action-failed", name: m.type, error: err && err.message || err }); ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, ...(err && err.code ? { code: err.code } : {}), error: String(err && err.message || err), screen: screenOrUndefined() })); });
             }
             else
                 ws.send(JSON.stringify({ id: m.id, type: m.type, ok: false, error: "unknown request: " + m.type }));

package/dist/run-net.d.ts

@@ -0,0 +1,39 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+export declare const NET_PATH = "/__shipvo/net";
+export type NetProfile = {
+    latencyMs: number;
+    downKbps: number;
+    failureRate: number;
+};
+export type NetRecord = {
+    at: number;
+    session: string;
+    method: string;
+    host: string;
+    status: number;
+    ms: number;
+    profile: string;
+    error?: string;
+};
+/** One cookie jar per phone session, like the cookies of one installed app. */
+export declare class CookieJar {
+    private list;
+    set(url: URL, header: string, now?: number): void;
+    header(url: URL, now?: number): string;
+}
+/** The profile table from @shipvo/sim (one source of truth); a built-in copy if that file cannot be loaded. */
+export declare function loadProfiles(): Promise<Record<string, NetProfile>>;
+export type NetProxy = {
+    handle: (req: IncomingMessage, res: ServerResponse) => Promise<void>;
+    log: NetRecord[];
+};
+/** The request handler for `NET_PATH`. `tokens` are the credentials it accepts; `port()` is the bound port. */
+export declare function createNetProxy(o: {
+    tokens: string[];
+    port: () => number;
+    profiles: Record<string, NetProfile>;
+    onRecord?: (r: NetRecord) => void;
+    fetchImpl?: typeof fetch;
+}): NetProxy;
+/** The script that runs inside the app's page: sends cross-origin fetch and XMLHttpRequest through `NET_PATH`. */
+export declare function netClientSource(): string;