openmausbot 0.1.96 → 0.1.98

package/dist-server/server/browser-live.js

@@ -1,12 +1,22 @@
 import { execFile } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { promisify } from "node:util";
-import { browserRuntimeEnv } from "./browser-runtime.js";
+import { BROWSER_VIEWPORT_ARGS, browserRuntimeEnv, ownsBrowserViewport } from "./browser-runtime.js";
 import { closeBrowserSession } from "./browser-engine.js";
 const execute = promisify(execFile);
 const MAX_FRAME = 3 * 1024 * 1024;
 const MAX_BUFFER = 4 * 1024 * 1024;
 const HEARTBEAT_MS = 10_000;
+const COMMAND_TIMEOUT_MS = 30_000;
+const VIEWPORT_TIMEOUT_MS = 5_000;
+// Opening a view while the bot's own command holds the session: each stream
+// probe waits up to STATUS_PROBE_MS (it may also have to launch the browser),
+// then the view shows that it is waiting and probes again with a bounded
+// backoff for as long as the viewer stays, up to a limit. A probe that is
+// waiting in line answers as soon as the bot's command finishes.
+const STATUS_PROBE_MS = 10_000;
+const BUSY_RETRY_MS = [1_000, 2_000, 5_000];
+const BUSY_LIMIT_MS = 10 * 60_000;
 // The native press resolver supplies the virtual key codes and Enter/Tab text
 // that its raw input_keyboard relay omits. Keep unknown keys literal.
 const DISCRETE_KEYS = new Set(["Backspace", "Enter", "Tab", "Escape", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown"]);
@@ -17,6 +27,20 @@ export class BrowserLiveError extends Error {
     status;
     constructor(message, status = 400) { super(message); this.name = "BrowserLiveError"; this.status = status; }
 }
+/** The engine is there but did not answer in time: another command, usually
+ * the bot's own, holds the same browser session. Not a missing engine. */
+export class BrowserBusyError extends BrowserLiveError {
+    constructor(message = "The browser is busy with another action and did not answer in time. Try again shortly.") {
+        super(message, 503);
+        this.name = "BrowserBusyError";
+    }
+}
+/** execFile's own timeout: it killed a command that had not exited, as
+ * opposed to a missing binary, a failed command, an abort or an overflow. */
+function timedOut(error) {
+    const failure = error;
+    return failure?.killed === true && failure.signal === "SIGKILL" && failure.code == null;
+}
 function object(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
 }
@@ -183,6 +207,8 @@ export class BrowserLive {
         }
         if (!viewer.res.headersSent)
             return; // A peer can change control while this viewer is still starting.
+        if (viewer.waiting && message.type !== "waiting" && message.type !== "error")
+            return;
         if (viewer.res.writableLength > MAX_BUFFER) {
             this.close(viewer);
             return;
@@ -279,6 +305,7 @@ export class BrowserLive {
         if (viewer.closed)
             return;
         viewer.closed = true;
+        viewer.abort.abort();
         this.viewers.delete(viewer.id);
         clearTimeout(viewer.heartbeat);
         clearTimeout(viewer.drainTimer);
@@ -303,13 +330,14 @@ export class BrowserLive {
         if (viewer.pressedKeys.size || viewer.pressedButtons.size)
             this.runtime.abandonHumanInput(viewer.session, viewer.id);
     }
-    async command(viewer, args) {
+    async command(viewer, args, timeout = COMMAND_TIMEOUT_MS) {
         if (!this.current(viewer))
             throw new BrowserLiveError("This browser view is no longer available.", 409);
         const env = browserRuntimeEnv({ ...viewer.spec.env, AGENT_BROWSER_SESSION: viewer.session });
         try {
             const { stdout } = await execute(viewer.spec.command, [...args, "--json", "--no-webmcp"], {
-                env, timeout: 30_000, killSignal: "SIGKILL", maxBuffer: 1024 * 1024, encoding: "utf8", windowsHide: true,
+                env, timeout, killSignal: "SIGKILL", maxBuffer: 1024 * 1024, encoding: "utf8", windowsHide: true,
+                signal: viewer.abort.signal,
             });
             if (!this.current(viewer))
                 throw new Error("stale viewer");
@@ -324,6 +352,8 @@ export class BrowserLive {
             return data;
         }
         catch (error) {
+            if (timedOut(error))
+                throw new BrowserBusyError();
             console.warn("browser-live:", error);
             if (browserLaunchFailure(error)) {
                 // The daemon is alive but holding a launch configuration Chrome
@@ -349,6 +379,73 @@ export class BrowserLive {
             throw new BrowserLiveError("The browser could not complete this action. Check that the browser engine is installed, then reconnect.", 503);
         }
     }
+    /** Find (or start) the session's stream and make sure its browser runs. */
+    async connect(viewer) {
+        // A failed launch already started this session's daemon teardown; wait
+        // for it so the status probe meets a fresh daemon, not the stuck one.
+        // Checked on every attempt: a reset can start while the view waits.
+        const resetting = this.sessionResets.get(viewer.session);
+        if (resetting)
+            await resetting;
+        let status = await this.command(viewer, ["stream", "status"], STATUS_PROBE_MS);
+        if (status.enabled !== true) {
+            try {
+                status = await this.command(viewer, ["stream", "enable"]);
+            }
+            catch (error) {
+                if (error instanceof BrowserBusyError)
+                    throw error;
+                status = await this.command(viewer, ["stream", "status"]); // Another view may have enabled the same session.
+            }
+        }
+        const port = browserStreamPort(status);
+        // `open` without a URL is an idempotent launch, never navigation away from the bot's page.
+        if (status.connected !== true)
+            await this.command(viewer, ["open"]);
+        // The status probe itself may have launched the browser. Give its page the
+        // standard size, as the bot's runtime does, so the page, the frames and
+        // the clicks share one geometry; at that size this changes nothing. Best
+        // effort and brief: a failure or a busy session keeps the current size.
+        if (ownsBrowserViewport(viewer.spec.env ?? {})) {
+            await this.command(viewer, [...BROWSER_VIEWPORT_ARGS], VIEWPORT_TIMEOUT_MS).catch(() => undefined);
+        }
+        return port;
+    }
+    /** While another command holds the session, keep the view open and waiting
+     * instead of failing it: start the event stream, say so, and retry with a
+     * bounded backoff until the browser answers, the viewer leaves, or the
+     * limit passes. */
+    async connectWhenFree(viewer) {
+        const deadline = Date.now() + BUSY_LIMIT_MS;
+        for (let attempt = 0;; attempt++) {
+            try {
+                return await this.connect(viewer);
+            }
+            catch (error) {
+                if (!(error instanceof BrowserBusyError) || !this.current(viewer))
+                    throw error;
+                if (Date.now() >= deadline)
+                    throw new BrowserBusyError("The bot is still using this browser. Reconnect when it has finished.");
+            }
+            viewer.waiting = true;
+            this.startEvents(viewer);
+            this.send(viewer, { type: "waiting", reason: "busy" });
+            await new Promise((resolve) => {
+                const done = () => { clearTimeout(timer); viewer.abort.signal.removeEventListener("abort", done); resolve(); };
+                const timer = setTimeout(done, BUSY_RETRY_MS[Math.min(attempt, BUSY_RETRY_MS.length - 1)]);
+                timer.unref?.();
+                viewer.abort.signal.addEventListener("abort", done, { once: true });
+            });
+            if (!this.current(viewer))
+                throw new BrowserLiveError("This browser view is no longer available.", 409);
+        }
+    }
+    startEvents(viewer) {
+        if (viewer.res.headersSent)
+            return;
+        viewer.res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-store", "X-Accel-Buffering": "no", Connection: "keep-alive" });
+        viewer.res.flushHeaders();
+    }
     async input(viewer, message) {
         if (!viewer.port)
             throw new BrowserLiveError("The browser stream is not connected.", 409);
@@ -415,36 +512,23 @@ export class BrowserLive {
         if (this.viewers.size >= 8 || [...this.viewers.values()].filter((v) => v.session === options.session).length >= 2) {
             throw new BrowserLiveError("Too many browser views are open. Close another browser panel first.", 429);
         }
-        const viewer = { ...options, id: randomUUID(), closed: false, blocked: false, pendingActions: 0, restarting: false, pressedKeys: new Set(), pressedButtons: new Set() };
+        const viewer = { ...options, id: randomUUID(), closed: false, blocked: false, pendingActions: 0, restarting: false, pressedKeys: new Set(), pressedButtons: new Set(), waiting: false, abort: new AbortController() };
         if (!this.current(viewer))
             throw new BrowserLiveError("This browser view is no longer available.", 409);
         this.viewers.set(viewer.id, viewer);
         options.res.once("close", () => this.close(viewer));
         options.res.once("error", () => this.close(viewer));
         try {
-            // A failed launch already started this session's daemon teardown; wait
-            // for it so the status probe meets a fresh daemon, not the stuck one.
-            const resetting = this.sessionResets.get(viewer.session);
-            if (resetting)
-                await resetting;
-            let status = await this.command(viewer, ["stream", "status"]);
-            if (status.enabled !== true) {
-                try {
-                    status = await this.command(viewer, ["stream", "enable"]);
-                }
-                catch {
-                    status = await this.command(viewer, ["stream", "status"]);
-                } // Another view may have enabled the same session.
-            }
-            const port = browserStreamPort(status);
+            const port = await this.connectWhenFree(viewer);
+            // Best-effort steps of the startup swallow their own failures, an abort
+            // by close() included; a viewer closed meanwhile gets no socket.
+            if (!this.current(viewer))
+                throw new BrowserLiveError("This browser view is no longer available.", 409);
             viewer.port = port;
-            // `open` without a URL is an idempotent launch, never navigation away from the bot's page.
-            if (status.connected !== true)
-                await this.command(viewer, ["open"]);
             const socket = new WebSocket(`ws://127.0.0.1:${port}/?pacing=ack&maxFps=15`);
             viewer.socket = socket;
-            options.res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-store", "X-Accel-Buffering": "no", Connection: "keep-alive" });
-            options.res.flushHeaders();
+            this.startEvents(viewer);
+            viewer.waiting = false;
             this.send(viewer, { type: "ready", viewerId: viewer.id });
             this.control(viewer.session);
             socket.addEventListener("message", (event) => {
@@ -511,7 +595,10 @@ export class BrowserLive {
         catch (error) {
             console.warn("browser-live:", error);
             if (options.res.headersSent) {
-                this.send(viewer, { type: "error", retryable: true, message: "The browser stream could not start." });
+                // A view that waited out a busy browser gets the reason, and no
+                // automatic retry loop on top of the wait it already did.
+                const busy = error instanceof BrowserBusyError;
+                this.send(viewer, { type: "error", retryable: !busy, message: busy ? error.message : "The browser stream could not start." });
                 this.close(viewer);
                 return;
             }

package/dist-server/server/browser-runtime.js

@@ -23,6 +23,24 @@ export function browserRuntimeEnv(overrides) {
     const env = Object.fromEntries(HOST_ENV.flatMap((key) => process.env[key] === undefined ? [] : [[key, process.env[key]]]));
     return { ...env, ...overrides };
 }
+/** The page size of a headless browser OMB launches, in CSS pixels.
+ *
+ * agent-browser sizes the Chrome *window* (`--window-size=1280,720`). Full
+ * Chrome in headless mode (Chrome for Testing, as the Docker image installs)
+ * takes its emulated browser UI out of that window, so the page itself comes
+ * out 1280×577 while the screencast metadata still reports 1280×720, and the
+ * live view aimed clicks at the wrong place. chrome-headless-shell has no such
+ * UI, so there this changes nothing. agent-browser has no launch option for
+ * the page size; `set viewport` sets it and resizes the window's content area,
+ * which the browser's later tabs inherit. */
+export const BROWSER_VIEWPORT = { width: 1280, height: 720 };
+export const BROWSER_VIEWPORT_ARGS = ["set", "viewport", String(BROWSER_VIEWPORT.width), String(BROWSER_VIEWPORT.height)];
+/** Only a headless browser this server launches is OMB's to size. A browser
+ * attached over CDP is someone's own Chrome, and a headed window is the
+ * user's to size. */
+export function ownsBrowserViewport(env) {
+    return env.AGENT_BROWSER_HEADLESS === "1" && !env.AGENT_BROWSER_CDP;
+}
 export class TransportError extends Error {
 }
 /** A server-owned JSONL client. Neither child stderr nor its environment is
@@ -215,9 +233,11 @@ export class BrowserRuntime {
     toolLists = new Map();
     options;
     closeBrowser;
-    constructor({ closeBrowser, ...options } = {}) {
+    applyViewport;
+    constructor({ closeBrowser, applyViewport, ...options } = {}) {
         this.options = { requestTimeoutMs: 120_000, takeoverTimeoutMs: 15_000, idleMs: 60_000, maxPending: 16, resultBudget: DEFAULT_BROWSER_RESULT_BUDGET, ...options };
         this.closeBrowser = closeBrowser ?? (async () => false);
+        this.applyViewport = applyViewport;
     }
     gate(session) {
         let gate = this.gates.get(session);
@@ -300,6 +320,16 @@ export class BrowserRuntime {
             beforeDispatch?.();
             if (method === "tools/call" && this.gate(session).owner !== null)
                 throw new Error(BROWSER_CONTROL_REFUSAL);
+            if (method === "tools/call" && this.applyViewport) {
+                // The bot's first call on a transport may launch the browser. Size its
+                // page first, once per transport, so the page, the bot's screenshots
+                // and the live view agree. Failure only leaves the engine's default.
+                entry.viewport ??= this.applyViewport(spec).catch(() => undefined);
+                await entry.viewport;
+                beforeDispatch?.();
+                if (this.gate(session).owner !== null)
+                    throw new Error(BROWSER_CONTROL_REFUSAL);
+            }
             try {
                 // The model sees slimmed schemas and text-only, bounded results; the
                 // launch/session parameters OMB owns never reach the engine from a call.