@melaya/runner 1.1.8 → 1.1.10

package/dist/assistantHost.py

@@ -465,6 +465,13 @@ def _build_agent():
         _budget = int(os.environ.get("MEL_LAZY_BUDGET", "25"))
         if phone_enabled:
             _budget = max(_budget, 64)
+        # Same for browser: the browser_* set (nav/click/type/scroll/screen-tree/
+        # screenshot/tabs/...) is ~24 tools and must ALL stay pinned+active, or the
+        # default 25 budget spills the tail (e.g. browser_open_tab) into the lazy
+        # pool where activate_tool then fails with "budget reached". Widen so tab
+        # management + every browser action is directly callable, no search/activate.
+        if browser_enabled:
+            _budget = max(_budget, 64)
         toolkit = build_lazy_toolkit(
             active_categories=categories,
             include_categories=categories + core_categories + connector_services,

package/dist/browserBridge.d.ts

@@ -90,6 +90,25 @@ export interface BrowserBridge {
      * grant-scoped run sessions (byRunId).
      */
     killInteractive(sessionId?: string): Promise<string[]>;
+    /**
+     * Set or clear the takeover pause flag (Fix 3).
+     *
+     * Called by the connection layer when the server emits a
+     * `browser:takeover { sessionId?, paused }` socket event.
+     *
+     * While paused, `performAct` blocks all consequential act kinds
+     * (navigate, click, tap, input_text, press_key, scroll,
+     * select_option, upload_file, submit, back, forward, batch,
+     * ask_user) and returns a `BridgeError("paused_by_user", ...)` so
+     * the agent stops trying and waits. Read-only ops (get_screen_tree,
+     * screenshot, get_text, current_target, wait,
+     * wait_for_network_idle) are not blocked.
+     *
+     * If sessionId is provided, only that interactive session is paused.
+     * If sessionId is omitted, ALL sessions on this runner are paused.
+     * The flag is cleared on teardownRun / session teardown.
+     */
+    setPaused(paused: boolean, sessionId?: string): void;
     shutdown(): Promise<void>;
 }
 export declare function startBrowserBridge(opts: {

package/dist/browserBridge.js

@@ -96,7 +96,20 @@ const KIND_MIN_EFFECT = {
     click: "read", dblclick: "read", hover: "read", scroll: "read",
     input_text: "read", press_key: "read", select_option: "read",
     wait: "read",
+    // Added kinds (Fix 1 + Fix 2):
+    tap: "read", // coordinate click; same effect ceiling as click
+    get_text: "read", // read-only page text extraction
+    wait_for_network_idle: "read", // read-only network wait
 };
+// Consequential act kinds blocked while the user has taken over the browser.
+// Read-only kinds (get_text, get_screen_tree path, screenshot, wait,
+// wait_for_network_idle) are intentionally excluded so the agent can observe.
+const CONSEQUENTIAL_KINDS = new Set([
+    "navigate", "back", "forward",
+    "click", "dblclick", "hover", "tap",
+    "input_text", "press_key", "scroll", "select_option",
+    "upload_file", "submit", "ask_user", "batch",
+]);
 // ---------------------------------------------------------------------
 // Bridge
 // ---------------------------------------------------------------------
@@ -129,6 +142,11 @@ export async function startBrowserBridge(opts) {
     // getInteractiveSession and setWatchLease can find them without iterating
     // the full grant-scoped run registry.
     const interactiveSessions = new Map();
+    // Takeover pause state (Fix 3).
+    // A sessionId entry means that specific interactive session is paused.
+    // The sentinel key "" means ALL sessions on this runner are paused.
+    // Cleared per-session on teardown, or globally on teardownAll.
+    const pausedSessions = new Set();
     let playwrightMod = null;
     async function pw() {
         if (!playwrightMod)
@@ -729,6 +747,17 @@ export async function startBrowserBridge(opts) {
         }
         throw new BridgeError("act_args_invalid", `kind '${args.kind}' needs a ref or x/y coordinates`);
     }
+    // Returns true if the given run's session is currently paused by a takeover.
+    // The global sentinel "" pauses all sessions; a specific sessionId pauses
+    // only sessions attached to that interactive session.
+    function isActPaused(reg) {
+        if (pausedSessions.has(""))
+            return true;
+        const browserSessionId = String(reg.spec.grant?.browserSession || "");
+        if (browserSessionId && pausedSessions.has(browserSessionId))
+            return true;
+        return false;
+    }
     async function performAct(reg, args) {
         const kind = String(args.kind || "");
         if (!(kind in KIND_MIN_EFFECT)) {
@@ -744,6 +773,13 @@ export async function startBrowserBridge(opts) {
             if (!d.allowed)
                 throw new BridgeError(d.code, d.message);
         }
+        // Takeover pause gate (Fix 3): block consequential acts while the user
+        // has manually taken over the browser. Read-only kinds pass through.
+        if (CONSEQUENTIAL_KINDS.has(kind) && isActPaused(reg)) {
+            throw new BridgeError("paused_by_user", "The user has taken manual control of the browser; agent actions are " +
+                "paused until they hand control back. Wait and retry, or call " +
+                "browser_get_screen_tree / browser_screenshot to observe the page.");
+        }
         const { rec, lease } = await getLease(reg);
         return sessions.runOnTarget(lease, async () => {
             if (reg.cancelled)
@@ -769,6 +805,40 @@ export async function startBrowserBridge(opts) {
                 case "click":
                 case "dblclick":
                 case "hover": {
+                    // Fix 2: if a "text" arg is present and there is no ref or x/y,
+                    // resolve the text to an element point via Playwright locators
+                    // (case-insensitive; prefers exact match, falls back to contains;
+                    // scoped to visible elements). If nothing matches, give the model
+                    // a clear directive to re-snapshot and use a @eN ref instead.
+                    if (args.text && !args.ref && args.x === undefined) {
+                        const label = String(args.text);
+                        // Try exact visible text first, then partial, then ARIA name.
+                        let locator = page.getByText(label, { exact: true }).first();
+                        let box = await locator.boundingBox({ timeout: 2_000 }).catch(() => null);
+                        if (!box) {
+                            locator = page.getByText(label, { exact: false }).first();
+                            box = await locator.boundingBox({ timeout: 2_000 }).catch(() => null);
+                        }
+                        if (!box) {
+                            locator = page.getByRole("button", { name: label, exact: false }).first();
+                            box = await locator.boundingBox({ timeout: 2_000 }).catch(() => null);
+                        }
+                        if (!box) {
+                            locator = page.getByRole("link", { name: label, exact: false }).first();
+                            box = await locator.boundingBox({ timeout: 2_000 }).catch(() => null);
+                        }
+                        if (!box) {
+                            throw new BridgeError("act_args_invalid", `click by text: no visible element matched "${label}". ` +
+                                "Call browser_get_screen_tree to get fresh @eN refs and retry " +
+                                "with a ref instead of a text label.");
+                        }
+                        const pt = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+                        if (kind === "hover")
+                            await page.mouse.move(pt.x, pt.y);
+                        else
+                            await page.mouse.click(pt.x, pt.y, { clickCount: kind === "dblclick" ? 2 : 1 });
+                        return { at: { x: Math.round(pt.x), y: Math.round(pt.y) }, matched_text: label };
+                    }
                     const pt = await resolveActionPoint(reg, rec, lease, args);
                     if (kind === "hover")
                         await page.mouse.move(pt.x, pt.y);
@@ -776,6 +846,14 @@ export async function startBrowserBridge(opts) {
                         await page.mouse.click(pt.x, pt.y, { clickCount: kind === "dblclick" ? 2 : 1 });
                     return { at: { x: Math.round(pt.x), y: Math.round(pt.y) } };
                 }
+                case "tap": {
+                    // Fix 1: tap is a coordinate click (canvas/vision fallback).
+                    // browser.py sends {x, y} as 0..1 fractions or CSS px.
+                    // resolveActionPoint already handles the fraction/px heuristic.
+                    const pt = await resolveActionPoint(reg, rec, lease, args);
+                    await page.mouse.click(pt.x, pt.y);
+                    return { at: { x: Math.round(pt.x), y: Math.round(pt.y) } };
+                }
                 case "input_text": {
                     const text = String(args.text ?? args.value ?? "");
                     if (text.length > 20_000)
@@ -801,9 +879,67 @@ export async function startBrowserBridge(opts) {
                         await cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: binding.backendNodeId }).catch(() => { });
                         return { scrolled: "into_view" };
                     }
-                    const dy = Math.max(-4000, Math.min(4000, Number(args.dy ?? 600)));
-                    await page.mouse.wheel(0, dy);
-                    return { scrolled: dy };
+                    // Fix 1 (scroll): browser.py sends {value: "small"|"medium"|"large",
+                    // text: "down"|"up"|"left"|"right"}. Map to pixel deltas.
+                    // Fall back to args.dy for callers that send raw pixel deltas directly.
+                    let dx = 0;
+                    let dy = 0;
+                    if (args.value !== undefined || args.text !== undefined) {
+                        const amountMap = { small: 200, medium: 600, large: 1400 };
+                        const amount = amountMap[String(args.value || "medium")] ?? 600;
+                        const dir = String(args.text || "down").toLowerCase();
+                        if (dir === "down")
+                            dy = amount;
+                        else if (dir === "up")
+                            dy = -amount;
+                        else if (dir === "right")
+                            dx = amount;
+                        else if (dir === "left")
+                            dx = -amount;
+                        else
+                            dy = amount;
+                    }
+                    else {
+                        dy = Math.max(-4000, Math.min(4000, Number(args.dy ?? 600)));
+                    }
+                    await page.mouse.wheel(dx, dy);
+                    return { scrolled: { dx, dy } };
+                }
+                case "get_text": {
+                    // Fix 1: read-only page-text extraction.
+                    // browser.py sends {ref?: "@eN"} (optional scoped element).
+                    // The ref arg arrives under args.ref; if absent, read full body text.
+                    const TEXT_BUDGET = 12_000;
+                    let text;
+                    if (args.ref) {
+                        // Scope to the element's innerText via CDP object resolution.
+                        const binding = sessions.resolveRef(lease, args.ref);
+                        const cdp = await getCdp(rec, page);
+                        const resolved = await cdp.send("DOM.resolveNode", { backendNodeId: binding.backendNodeId });
+                        const result = await cdp.send("Runtime.callFunctionOn", {
+                            objectId: resolved.object.objectId,
+                            functionDeclaration: `function(){ return (this.innerText || this.textContent || "").trim(); }`,
+                            returnByValue: true,
+                        });
+                        text = String(result.result.value ?? "");
+                    }
+                    else {
+                        text = await page.evaluate(() => (document.body?.innerText || document.body?.textContent || "").trim());
+                    }
+                    const truncated = text.length > TEXT_BUDGET;
+                    return {
+                        text: text.slice(0, TEXT_BUDGET) + (truncated ? `\n...[truncated; ${text.length} chars total]` : ""),
+                    };
+                }
+                case "wait_for_network_idle": {
+                    // Fix 1: wait for network idle. browser.py sends {value: seconds}.
+                    const maxMs = 30_000;
+                    const timeoutMs = Math.min(Math.max(1_000, Math.round(Number(args.value ?? 10) * 1_000)), maxMs);
+                    await page.waitForLoadState("networkidle", { timeout: timeoutMs }).catch(() => {
+                        // Timeout is not fatal: the page may have long-polling connections.
+                        // Return ok anyway; the agent will re-snapshot and observe.
+                    });
+                    return { network_idle: true, waited_ms: timeoutMs };
                 }
                 case "select_option": {
                     if (!args.ref)
@@ -1442,6 +1578,8 @@ export async function startBrowserBridge(opts) {
                     // producer) are preserved.
                     interactive.targets.delete(reg.spec.grant.target.ref);
                 }
+                // Clear any per-session pause flag so a restarted run is not stuck.
+                pausedSessions.delete(browserSessionId);
             }
             log(`browser run detached (non-owner): ${runId.slice(0, 10)} reason=${reason}`);
         }
@@ -1454,6 +1592,8 @@ export async function startBrowserBridge(opts) {
         }
     };
     const teardownAll = async (reason) => {
+        // Clear all pause flags so nothing is left paused after a full teardown.
+        pausedSessions.clear();
         // Tear down grant-scoped run sessions.
         const runIds = [...byRunId.keys()];
         // Tear down interactive sessions (those not already covered by byRunId).
@@ -1472,6 +1612,18 @@ export async function startBrowserBridge(opts) {
             }),
         ]);
     };
+    // setPaused: set or clear the takeover pause flag (Fix 3).
+    // sessionId present -> pause only that interactive session.
+    // sessionId absent  -> pause/resume all sessions (global sentinel "").
+    const setPaused = (paused, sessionId) => {
+        const key = sessionId ?? "";
+        if (paused) {
+            pausedSessions.add(key);
+        }
+        else {
+            pausedSessions.delete(key);
+        }
+    };
     // killInteractive: close one or all interactive sessions (emergency stop).
     // Does NOT touch grant-scoped run sessions in byRunId.
     const killInteractive = async (sessionId) => {
@@ -1517,6 +1669,7 @@ export async function startBrowserBridge(opts) {
         getInteractiveSession,
         setWatchLease,
         killInteractive,
+        setPaused,
         async shutdown() {
             // Stop all watch-lease producers before closing sessions.
             for (const sid of [...watchLeases.keys()])

package/dist/connection.js

@@ -360,6 +360,28 @@ export async function connect(opts) {
             console.log(chalk.yellow(`  ! browser:kill error: ${e?.message || e}`));
         }
     });
+    // ── BrowserControlPage takeover pause (browser:takeover) ─────────────
+    // The server emits browser:takeover { sessionId?, paused } when the user
+    // clicks "Take control" (paused:true) or "Hand back" (paused:false) in
+    // the BrowserControlPage UI. While paused, the bridge blocks all
+    // consequential acts (click, navigate, input_text, etc.) with a
+    // paused_by_user BridgeError so the agent waits instead of racing with
+    // the user. Read-only ops (get_screen_tree, screenshot, get_text, wait)
+    // are not blocked. If sessionId is absent the pause applies to all
+    // sessions on this runner.
+    socket.on("browser:takeover", (payload) => {
+        const sessionId = payload?.sessionId ? String(payload.sessionId) : undefined;
+        const paused = Boolean(payload?.paused);
+        if (!browserBridge) {
+            if (opts.verbose)
+                console.log(chalk.gray("  [browser-takeover] bridge not ready, ignoring"));
+            return;
+        }
+        browserBridge.setPaused(paused, sessionId);
+        if (opts.verbose) {
+            console.log(chalk.gray(`  [browser-takeover] paused=${paused} sessionId=${sessionId ? sessionId.slice(0, 16) : "(all)"}`));
+        }
+    });
     // ── Heartbeat ──────────────────────────────────────────────────────
     setInterval(() => {
         if (socket.connected)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@melaya/runner",
-  "version": "1.1.8",
+  "version": "1.1.10",
   "description": "Run Melaya AI pipelines locally with your own LM Studio or Ollama models",
   "license": "UNLICENSED",
   "private": false,