@loadmill/droid-cua 2.40.0 → 2.42.0

package/build/src/desktop/computer-use.js

@@ -3,6 +3,7 @@ import { buildTypeActionPayload, buildTypeActionText, } from "../device/sensitiv
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { getWindowsVisibleBounds } from "./windows-visible-bounds.js";
 import sharp from "sharp";
+import { capturePrimaryDisplay } from "./primary-display.js";
 const SCREENSHOT_WIDTH = 1440;
 const ACCESSIBILITY_HIT_TOLERANCE = 12;
 const MACOS_CAPTURE_INSETS = { left: 50, right: 50, top: 38, bottom: 62 };
@@ -104,7 +105,7 @@ function formatPermissionError(error, platform) {
     if (platform === "darwin" && /permission|not authorized|screencapture|screen recording|cgwindow|axerror/i.test(message)) {
         return `${message} Grant Screen & System Audio Recording and Accessibility access to Droid (or your terminal for CLI runs) in System Settings > Privacy & Security, then restart it.`;
     }
-    if (platform === "win32" && /access denied|elevat|integrity level|could not focus/i.test(message)) {
+    if (platform === "win32" && /access denied|elevat|integrity level/i.test(message)) {
         return `${message} Run Droid in an interactive desktop session and at the same integrity level as the target application; elevated Windows applications cannot be controlled by a non-elevated Droid process.`;
     }
     return message;
@@ -236,22 +237,26 @@ function normalizeShortcut(keys, platform) {
     return keys.map((key) => aliases[String(key).toUpperCase()] || String(key).toLowerCase()).join("+");
 }
 /**
- * @param {{ app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", focusOnConnect?: boolean, client?: any, accessibility?: any, visibleBoundsReader?: typeof getWindowsVisibleBounds }} options
+ * @param {{ scope?: "window" | "desktop", app?: string, windowId?: number | null, excludePids?: number[], platform?: string, inputMode?: "accessibility" | "physical", focusOnConnect?: boolean, client?: any, accessibility?: any, visibleBoundsReader?: typeof getWindowsVisibleBounds, displayCapture?: typeof capturePrimaryDisplay }} options
  */
-export async function createDesktopController({ app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", focusOnConnect = false, client: injectedClient = null, accessibility: injectedAccessibility = null, visibleBoundsReader = getWindowsVisibleBounds, } = {}) {
+export async function createDesktopController({ scope = "window", app: requestedApp, windowId: requestedWindowId = null, excludePids = [], platform = process.platform, inputMode = "accessibility", focusOnConnect = false, client: injectedClient = null, accessibility: injectedAccessibility = null, visibleBoundsReader = getWindowsVisibleBounds, displayCapture = capturePrimaryDisplay, } = {}) {
     if (!SUPPORTED_PLATFORMS.has(platform)) {
         throw new Error("Desktop control is currently supported only on macOS and Windows.");
     }
-    if (!requestedApp)
+    if (scope !== "window" && scope !== "desktop")
+        throw new Error('Desktop scope must be "window" or "desktop".');
+    if (scope === "desktop" && (requestedApp || requestedWindowId != null))
+        throw new Error("Full desktop cannot select an application or window.");
+    if (scope === "desktop" && inputMode !== "physical")
+        throw new Error("Full desktop requires physical input.");
+    if (scope === "window" && !requestedApp)
         throw new Error("--desktop-app is required with --target desktop.");
     if (!INPUT_MODES.has(inputMode))
         throw new Error('Desktop input mode must be "accessibility" or "physical".');
     const client = injectedClient || await createLibraryClient();
     try {
-        const apps = (await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid));
-        const application = chooseApplication(apps, requestedApp);
-        const windows = readWindows(await client.listWindows(application.bundleId));
-        let targetWindow = chooseWindow(windows, requestedWindowId == null ? null : Number(requestedWindowId));
+        const application = scope === "window" ? chooseApplication((await discoverApplications(client)).filter((app) => typeof app.bundleId === "string" && app.bundleId && !excludePids.includes(app.pid)), requestedApp) : null;
+        let targetWindow = application ? chooseWindow(readWindows(await client.listWindows(application.bundleId)), requestedWindowId == null ? null : Number(requestedWindowId)) : null;
         let capture = null;
         let deviceInfo = null;
         let native = injectedAccessibility;
@@ -260,19 +265,38 @@ export async function createDesktopController({ app: requestedApp, windowId: req
         async function getWindow() {
             return parseJsonResult(await client.getWindow(targetWindow.windowId), "Window lookup");
         }
-        async function focus() {
+        async function refreshWindow() {
+            const windows = readWindows(await client.listWindows(application.bundleId))
+                .filter((window) => window.isOnScreen && window.bounds?.width > 0 && window.bounds?.height > 0);
+            const selected = windows.find((window) => window.isFocused)
+                || windows.find((window) => window.windowId === targetWindow.windowId)
+                || chooseWindow(windows, null);
+            if (selected.windowId !== targetWindow.windowId)
+                capture = null;
+            targetWindow = selected;
+            if (deviceInfo) {
+                deviceInfo.desktopWindowId = selected.windowId;
+                deviceInfo.device_name = `${application.displayName} — ${selected.title || "Untitled"}`;
+            }
+        }
+        async function activateWindow() {
             const activation = parseJsonResult(await client.activateWindow(targetWindow.windowId, 2000), "Window activation");
-            const focused = await getWindow();
-            if (!activation.activated || !focused.isFocused) {
-                throw new Error("Droid could not focus the selected desktop window, so no input was sent.");
+            if (!activation.activated || !(await getWindow()).isFocused) {
+                throw new Error(`Droid could not focus the selected desktop window, so no input was sent.${activation.reason ? ` Reason: ${activation.reason}.` : ""}`);
             }
         }
+        async function focus() {
+            if (scope === "desktop")
+                return;
+            await refreshWindow();
+            await activateWindow();
+        }
         async function captureFromDesktop(window) {
             if (!window.isFocused) {
                 if (inputMode !== "physical") {
                     throw new Error("Windows desktop capture requires the target window to be visible and focused. Bring it to the foreground and retry.");
                 }
-                await focus();
+                await activateWindow();
                 window = await getWindow();
             }
             native ||= await loadNativeDriver();
@@ -293,29 +317,38 @@ export async function createDesktopController({ app: requestedApp, windowId: req
         }
         async function screenshot({ allowDesktopFallback = true } = {}) {
             try {
-                const currentWindows = readWindows(await client.listWindows(application.bundleId));
-                const focusedWindow = currentWindows.find((candidate) => candidate.isFocused && candidate.isOnScreen);
-                if (focusedWindow)
-                    targetWindow = focusedWindow;
-                const window = await getWindow();
-                if (!window.isOnScreen)
-                    throw new Error("The selected desktop window is no longer visible.");
-                const result = await client.screenshot({
-                    width: SCREENSHOT_WIDTH,
-                    quality: 0,
-                    target_window_id: targetWindow.windowId,
-                });
-                assertResult(result, "Desktop screenshot");
-                let image = { ...readScreenshot(result), bounds: { ...window.bounds }, actionBounds: { ...window.bounds } };
-                if (allowDesktopFallback && platform === "win32" && await isBlackScreenshot(image.data)) {
-                    image = await captureFromDesktop(window);
+                let image;
+                if (scope === "desktop") {
+                    native ||= await loadNativeDriver();
+                    const screen = await displayCapture(native, platform);
+                    image = { ...screen, actionBounds: { ...screen.bounds } };
+                }
+                else {
+                    await refreshWindow();
+                    const window = await getWindow();
+                    if (!window.isOnScreen)
+                        throw new Error("The selected desktop window is no longer visible.");
+                    const result = await client.screenshot({
+                        width: SCREENSHOT_WIDTH,
+                        quality: 0,
+                        target_window_id: targetWindow.windowId,
+                    });
+                    assertResult(result, "Desktop screenshot");
+                    image = { ...readScreenshot(result), bounds: { ...window.bounds }, actionBounds: { ...window.bounds } };
+                    if (allowDesktopFallback && platform === "win32" && await isBlackScreenshot(image.data)) {
+                        image = await captureFromDesktop(window);
+                    }
                 }
                 capture = {
                     bounds: image.bounds,
                     actionBounds: image.actionBounds,
                     width: image.width,
                     height: image.height,
-                    contentBounds: getScreenshotContentBounds(image.width, image.height, image.bounds, platform),
+                    windowId: targetWindow?.windowId ?? null,
+                    displayId: image.displayId,
+                    contentBounds: scope === "desktop"
+                        ? { x: 0, y: 0, width: image.width, height: image.height }
+                        : getScreenshotContentBounds(image.width, image.height, image.bounds, platform),
                 };
                 if (deviceInfo) {
                     deviceInfo.device_width = image.actionBounds.width;
@@ -323,7 +356,7 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                     deviceInfo.scaled_width = image.width;
                     deviceInfo.scaled_height = image.height;
                     deviceInfo.scale = image.width / image.actionBounds.width;
-                    deviceInfo.desktopWindowId = targetWindow.windowId;
+                    deviceInfo.desktopWindowId = targetWindow?.windowId ?? null;
                 }
                 return image.data;
             }
@@ -334,25 +367,77 @@ export async function createDesktopController({ app: requestedApp, windowId: req
         function scaledPoint(x, y) {
             if (!capture)
                 throw new Error("Capture the desktop window before executing an action.");
+            if (scope === "desktop" && (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x >= capture.width || y >= capture.height)) {
+                throw new Error("The action is outside the primary display.");
+            }
             const content = capture.contentBounds;
             const bounds = capture.actionBounds;
-            return [
+            const point = [
                 Math.round(bounds.x + ((x - content.x) * bounds.width) / content.width),
                 Math.round(bounds.y + ((y - content.y) * bounds.height) / content.height),
             ];
+            // Retina rounding at the last image pixel must stay inside the display.
+            return scope === "desktop" ? [Math.min(point[0], bounds.width - 1), Math.min(point[1], bounds.height - 1)] : point;
         }
         async function prepareAction(requireFocus) {
             if (!capture)
-                throw new Error("Capture the desktop window before executing an action.");
-            const before = await getWindow();
-            if (!sameBounds(before.bounds, capture.bounds)) {
+                throw new Error("A fresh desktop screenshot is required before acting.");
+            if (scope === "desktop") {
+                const display = native.getDisplaySize(capture.displayId);
+                if (display.width !== capture.bounds.width || display.height !== capture.bounds.height) {
+                    throw new Error("The display resized after the screenshot. A fresh screenshot is required before acting.");
+                }
+                return;
+            }
+            let window;
+            try {
+                window = parseJsonResult(await client.getWindow(capture.windowId), "Window lookup");
+            }
+            catch {
+                throw new Error("The captured desktop window is no longer available. A fresh screenshot is required before acting.");
+            }
+            if (!window.isOnScreen || window.bundleId !== application.bundleId || capture.windowId !== targetWindow.windowId) {
+                throw new Error("The captured desktop window is no longer available. A fresh screenshot is required before acting.");
+            }
+            if (!sameBounds(window.bounds, capture.bounds)) {
                 throw new Error("The desktop window moved or resized after the screenshot. A fresh screenshot is required before acting.");
             }
-            if (!requireFocus)
-                return;
-            await focus();
+            if (requireFocus) {
+                await activateWindow();
+                if (!sameBounds((await getWindow()).bounds, capture.bounds)) {
+                    throw new Error("The desktop window moved or resized during focus. A fresh screenshot is required before acting.");
+                }
+            }
         }
-        const targetOptions = () => ({ targetWindowId: targetWindow.windowId, focusStrategy: "strict" });
+        const targetOptions = () => scope === "desktop"
+            ? { focusStrategy: "none" }
+            : { targetWindowId: targetWindow.windowId, focusStrategy: "strict" };
+        // Native pointer calls avoid the library's first-monitor coordinate validation.
+        // Keyboard calls retain its text/clipboard handling, with app focus disabled.
+        const physical = scope === "window" ? client : {
+            click: (x, y) => native.mouseClick(x, y, "left", 1),
+            doubleClick: (x, y) => native.mouseClick(x, y, "left", 2),
+            moveMouse: (x, y) => native.mouseMove(x, y),
+            drag: async (to, from) => {
+                native.mouseMove(from[0], from[1]);
+                await new Promise((resolve) => setTimeout(resolve, 50));
+                native.mouseButton("press", from[0], from[1]);
+                try {
+                    await new Promise((resolve) => setTimeout(resolve, 50));
+                    native.mouseDrag(to[0], to[1]);
+                    await new Promise((resolve) => setTimeout(resolve, 50));
+                }
+                finally {
+                    native.mouseButton("release", to[0], to[1]);
+                }
+            },
+            scroll: (x, y, direction, amount) => {
+                native.mouseMove(x, y);
+                native.mouseScroll(direction === "down" ? amount : direction === "up" ? -amount : 0, direction === "right" ? amount : direction === "left" ? -amount : 0);
+            },
+            type: client.type,
+            key: client.key,
+        };
         async function performCursorFreeAction(actionType, x, y, direction = null) {
             const tree = parseJsonResult(await client.getUiTree(targetWindow.windowId, 20), "Accessibility tree");
             const selected = chooseAccessibilityElement(tree, x, y, actionType, direction);
@@ -385,8 +470,8 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                         }
                         else {
                             const result = action.type === "click"
-                                ? await client.click(x, y, application.bundleId, targetOptions())
-                                : await client.doubleClick(x, y, application.bundleId, targetOptions());
+                                ? await physical.click(x, y, application?.bundleId, targetOptions())
+                                : await physical.doubleClick(x, y, application?.bundleId, targetOptions());
                             assertResult(result, "Desktop click");
                         }
                         break;
@@ -396,7 +481,7 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                         addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...meta({ x, y, unit: "px" }) });
                         if (inputMode === "accessibility")
                             throw new Error(cursorFreeError(action.type));
-                        assertResult(await client.moveMouse(x, y, application.bundleId, targetOptions()), "Desktop mouse move");
+                        assertResult(await physical.moveMouse(x, y, application?.bundleId, targetOptions()), "Desktop mouse move");
                         break;
                     }
                     case "drag": {
@@ -408,7 +493,7 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                         addOutput({ type: "action", text: `Dragging from (${from[0]}, ${from[1]}) to (${to[0]}, ${to[1]})`, ...meta({ pathStart: { x: from[0], y: from[1] }, pathEnd: { x: to[0], y: to[1] }, unit: "px" }) });
                         if (inputMode === "accessibility")
                             throw new Error(cursorFreeError(action.type));
-                        assertResult(await client.drag(to, from, application.bundleId, targetOptions()), "Desktop drag");
+                        assertResult(await physical.drag(to, from, application?.bundleId, targetOptions()), "Desktop drag");
                         break;
                     }
                     case "scroll": {
@@ -423,20 +508,20 @@ export async function createDesktopController({ app: requestedApp, windowId: req
                             await performCursorFreeAction(action.type, x, y, direction);
                         }
                         else {
-                            assertResult(await client.scroll(x, y, direction, amount, application.bundleId, targetOptions()), "Desktop scroll");
+                            assertResult(await physical.scroll(x, y, direction, amount, application?.bundleId, targetOptions()), "Desktop scroll");
                         }
                         break;
                     }
                     case "type": {
                         const text = typeof action.text === "string" ? action.text : "";
                         addOutput({ type: "action", text: buildTypeActionText(action, text), ...meta(buildTypeActionPayload(action, text)) });
-                        assertResult(await client.type(text, application.bundleId, targetOptions()), "Desktop typing");
+                        assertResult(await physical.type(text, application?.bundleId, targetOptions()), "Desktop typing");
                         break;
                     }
                     case "keypress": {
                         const shortcut = normalizeShortcut(action.keys, platform);
                         addOutput({ type: "action", text: `Pressing key: ${shortcut}`, ...meta({ keys: action.keys, shortcut }) });
-                        assertResult(await client.key(shortcut, application.bundleId, targetOptions()), "Desktop keypress");
+                        assertResult(await physical.key(shortcut, application?.bundleId, targetOptions()), "Desktop keypress");
                         break;
                     }
                     case "wait": {
@@ -463,20 +548,21 @@ export async function createDesktopController({ app: requestedApp, windowId: req
         deviceInfo = {
             platform: "desktop",
             source: "local",
-            device_name: `${application.displayName} — ${targetWindow.title || "Untitled"}`,
+            device_name: application ? `${application.displayName} — ${targetWindow.title || "Untitled"}` : "Full desktop",
             device_width: capture.actionBounds.width,
             device_height: capture.actionBounds.height,
             scaled_width: capture.width,
             scaled_height: capture.height,
             scale: capture.width / capture.actionBounds.width,
-            desktopAppId: application.bundleId,
-            desktopWindowId: targetWindow.windowId,
+            desktopScope: scope,
+            desktopAppId: application?.bundleId ?? null,
+            desktopWindowId: targetWindow?.windowId ?? null,
             desktopInputMode: inputMode,
         };
         return {
             application,
-            window: targetWindow,
-            deviceId: `desktop:${targetWindow.windowId}`,
+            get window() { return targetWindow; },
+            deviceId: scope === "desktop" ? "desktop:primary" : `desktop:${targetWindow.windowId}`,
             deviceInfo,
             focus,
             getScreenshotAsBase64: () => screenshot(),

package/build/src/desktop/primary-display.js

@@ -0,0 +1,74 @@
+import { execFile } from "node:child_process";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import sharp from "sharp";
+const runFile = promisify(execFile);
+const WINDOWS_CAPTURE = `
+$ErrorActionPreference = 'Stop'
+Add-Type -AssemblyName System.Drawing
+Add-Type -TypeDefinition @'
+using System;
+using System.Runtime.InteropServices;
+public static class PrimaryDisplay {
+  [StructLayout(LayoutKind.Sequential)] public struct Point { public int X; public int Y; }
+  [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
+  [DllImport("user32.dll")] public static extern int GetSystemMetrics(int index);
+  [DllImport("user32.dll")] public static extern IntPtr MonitorFromPoint(Point point, uint flags);
+}
+'@
+[void][PrimaryDisplay]::SetProcessDPIAware()
+$width = [PrimaryDisplay]::GetSystemMetrics(0)
+$height = [PrimaryDisplay]::GetSystemMetrics(1)
+$bitmap = [System.Drawing.Bitmap]::new($width, $height)
+$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+try {
+  $graphics.CopyFromScreen(0, 0, 0, 0, $bitmap.Size)
+  $bitmap.Save('CAPTURE_PATH', [System.Drawing.Imaging.ImageFormat]::Png)
+  # The native driver's display lookup accepts a uint32 monitor handle.
+  [PrimaryDisplay]::MonitorFromPoint([PrimaryDisplay+Point]::new(), 1).ToInt64() -band 0xffffffff
+} finally {
+  $graphics.Dispose()
+  $bitmap.Dispose()
+}
+`;
+/** Capture only the primary display, independently of the library's cached window target. */
+export async function capturePrimaryDisplay(native, platform = process.platform, run = runFile) {
+    const directory = await mkdtemp(join(tmpdir(), "droid-desktop-"));
+    const path = join(directory, "screen.png");
+    try {
+        let displayId;
+        if (platform === "darwin") {
+            await run("/usr/sbin/screencapture", ["-x", "-m", "-t", "png", path], { timeout: 10000 });
+        }
+        else if (platform === "win32") {
+            const script = WINDOWS_CAPTURE.replace("CAPTURE_PATH", path.replaceAll("'", "''"));
+            const { stdout } = await run("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", script], {
+                windowsHide: true,
+                timeout: 10000,
+            });
+            displayId = Number(stdout.trim());
+            if (!Number.isSafeInteger(displayId) || displayId <= 0)
+                throw new Error("Primary display lookup failed.");
+        }
+        else {
+            throw new Error("Desktop control is currently supported only on macOS and Windows.");
+        }
+        // Use the native driver's coordinate units, which can differ from screenshot pixels.
+        const display = native.getDisplaySize(displayId);
+        const { data, info } = await sharp(await readFile(path))
+            .resize({ width: 1440, withoutEnlargement: true })
+            .png().toBuffer({ resolveWithObject: true });
+        return {
+            data: data.toString("base64"),
+            width: info.width,
+            height: info.height,
+            bounds: { x: 0, y: 0, width: display.width, height: display.height },
+            displayId: displayId ?? display.displayId,
+        };
+    }
+    finally {
+        await rm(directory, { recursive: true, force: true });
+    }
+}

package/build/src/device/openai.js

@@ -41,6 +41,29 @@ function normalizeMessageContent(content) {
 function buildScreenshotDataUrl(screenshotBase64) {
     return `data:image/png;base64,${screenshotBase64}`;
 }
+const queryNetworkTool = {
+    type: "function",
+    name: "query_network",
+    description: "Answer a question about captured HTTP traffic. All matching exchanges are considered in time order. Select the headers/bodies needed, or [] for metadata. Filters combine with AND; flow_ids selects any listed ID. Time filters select request start times (inclusive). from_step uses step-0001 style IDs. Null filters search the whole run. observe_ms waits the full duration before searching, maximum 30000. Narrow too_broad results; never treat errors, incomplete data or absence_supported=false as proof of absence.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            question: { type: "string", description: "Ask for the specific values or conclusion needed, rather than a full body dump." },
+            sections: { type: "array", items: { type: "string", enum: ["request_headers", "response_headers", "request_body", "response_body"] } },
+            url_contains: { type: ["string", "null"] },
+            method: { type: ["string", "null"] },
+            status: { type: ["integer", "null"] },
+            flow_ids: { type: ["array", "null"], items: { type: "string" } },
+            after: { type: ["string", "null"] },
+            before: { type: ["string", "null"] },
+            from_step: { type: ["string", "null"] },
+            observe_ms: { type: ["integer", "null"] },
+        },
+        required: ["question", "sections", "url_contains", "method", "status", "flow_ids", "after", "before", "from_step", "observe_ms"],
+        additionalProperties: false,
+    },
+};
 const previewClickTool = {
     type: "function",
     name: "preview_click",
@@ -234,6 +257,8 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
     if (toolCapabilities.closeKeyboard) {
         tools.push(deviceInfo?.platform?.toLowerCase() === "ios" ? iosCloseKeyboardTool : closeKeyboardTool);
     }
+    if (toolCapabilities.queryNetwork)
+        tools.push(queryNetworkTool);
     if (toolCapabilities.loadmill) {
         tools.push(loadmillTool);
     }

package/build/src/modes/execution-mode.js

@@ -102,6 +102,7 @@ export class ExecutionMode {
         // changed these three fields. Reset them to the values execution needs.
         this.engine.toolCapabilities = {
             assert: true,
+            queryNetwork: Boolean(this.engine.networkQuery),
             navigate: this.isBrowserSession,
             closeCurrentTab: this.isBrowserSession,
             closeKeyboard: !this.isBrowserSession,
@@ -245,6 +246,7 @@ export class ExecutionMode {
     async execute(context = {}) {
         const runContext = {
             ...context,
+            shouldStop: () => this.shouldStop,
             runId: context.runId || `run-${Date.now()}`
         };
         const addOutput = runContext.addOutput || printCliOutput;
@@ -274,6 +276,10 @@ export class ExecutionMode {
                     return { success: false, error: 'Stopped by user' };
                 }
             }
+            if (this.engine.networkQuery) {
+                this.engine.networkQuery.markStep(stepContext.stepId);
+                this.session.addToTranscript(`[Step ${stepContext.stepId}] ${instruction}`);
+            }
             this.emit(addOutput, 'user', instruction, runContext, stepContext, {
                 eventType: 'instruction_started',
                 payload: {
@@ -371,7 +377,10 @@ export class ExecutionMode {
                 captureSource: CAPTURE_SOURCES.INSTRUCTION_INPUT
             });
             const response = await this.sendCUARequest({
-                messages: this.session.messages,
+                // The chain already contains the base prompt; keep new tool-provided context.
+                messages: this.session.previousResponseId
+                    ? this.session.messages.filter((message) => message.role !== "system" || message.content !== this.session.systemPrompt)
+                    : this.session.messages,
                 screenshotBase64,
                 previousResponseId: this.session.previousResponseId,
                 deviceInfo: this.session.deviceInfo,

package/build/src/network/browser-capture.js

@@ -0,0 +1,110 @@
+import { randomUUID } from "node:crypto";
+import { BODY_CAPTURE_LIMIT } from "./network-capture.js";
+/** Observe one Playwright context without routing or changing its requests. */
+export function captureBrowserNetwork(context, record) {
+    const flows = new Map();
+    const pending = new Set();
+    function track(work) {
+        pending.add(work);
+        void work.finally(() => pending.delete(work));
+        return work;
+    }
+    function error(flowId, cause) {
+        return record({ type: "error", flowId, phase: "browser", code: null, message: cause.message });
+    }
+    function body(flowId, side, headers, buffer, { complete = true, omitted, bytes = buffer.length } = {}) {
+        return record({
+            type: `${side}_body`, flowId, headers,
+            body: buffer.subarray(0, BODY_CAPTURE_LIMIT), bytes,
+            truncated: bytes > BODY_CAPTURE_LIMIT, complete, decoded: true, omitted,
+        });
+    }
+    const onRequest = (request) => {
+        if (!/^https?:/.test(request.url()))
+            return;
+        const flowId = randomUUID();
+        const timestamp = new Date().toISOString();
+        const flow = { flowId, response: null, responseReady: null, ready: null };
+        flows.set(request, flow);
+        flow.ready = track((async () => {
+            const headers = await request.allHeaders();
+            await record({
+                type: "request", flowId, timestamp, method: request.method(), url: request.url(), headers,
+                resourceType: request.resourceType(), serviceWorker: Boolean(request.serviceWorker()),
+            });
+            await body(flowId, "request", headers, request.postDataBuffer() || Buffer.alloc(0));
+        })().catch((cause) => error(flowId, cause)));
+    };
+    const onResponse = (response) => {
+        const flow = flows.get(response.request());
+        if (!flow)
+            return; // Ignore traffic that began before this test.
+        const timestamp = new Date().toISOString();
+        flow.response = response;
+        flow.responseReady = track((async () => {
+            await flow.ready;
+            const headers = await response.allHeaders();
+            await record({
+                type: "response", flowId: flow.flowId, timestamp, status: response.status(), headers,
+                fromServiceWorker: response.fromServiceWorker(),
+            });
+            return headers;
+        })().catch((cause) => error(flow.flowId, cause)));
+    };
+    const onFinished = (request) => {
+        const flow = flows.get(request);
+        if (!flow)
+            return;
+        track((async () => {
+            await flow.ready;
+            const headers = (await flow.responseReady) || {};
+            const contentType = String(headers["content-type"] || "").toLowerCase();
+            let omitted;
+            if (!/(?:^text\/|json|xml|javascript|x-www-form-urlencoded)/.test(contentType))
+                omitted = "binary-or-unknown-content-type";
+            if (flow.response?.status() >= 300 && flow.response.status() < 400)
+                omitted = "redirect-body-unavailable";
+            // Avoid retrieving known-large resources. Unknown sizes are bounded after reading.
+            if (Number(headers["content-length"]) > BODY_CAPTURE_LIMIT)
+                omitted = "body-too-large";
+            if (omitted) {
+                await body(flow.flowId, "response", headers, Buffer.alloc(0), { omitted, bytes: Number(headers["content-length"]) || 0 });
+            }
+            else {
+                await body(flow.flowId, "response", headers, await flow.response.body());
+            }
+        })().catch(async (cause) => {
+            await error(flow.flowId, cause);
+            await body(flow.flowId, "response", {}, Buffer.alloc(0), { complete: false, omitted: "unavailable" });
+        }).finally(() => { flows.delete(request); }));
+    };
+    const onFailed = (request) => {
+        const flow = flows.get(request);
+        if (!flow)
+            return;
+        track((async () => {
+            await flow.ready;
+            await flow.responseReady;
+            await error(flow.flowId, new Error(request.failure()?.errorText || "Browser request failed."));
+            await body(flow.flowId, "response", {}, Buffer.alloc(0), { complete: false, omitted: "request-failed" });
+            flows.delete(request);
+        })());
+    };
+    const onClose = () => { track(error(undefined, new Error("Browser context closed during network capture."))); };
+    const listeners = { request: onRequest, response: onResponse, requestfinished: onFinished, requestfailed: onFailed, close: onClose };
+    for (const [name, listener] of Object.entries(listeners))
+        context.on(name, listener);
+    const flush = () => Promise.allSettled([...pending]);
+    return {
+        flush,
+        async stop() {
+            for (const [name, listener] of Object.entries(listeners))
+                context.off(name, listener);
+            await flush();
+            for (const flow of flows.values()) {
+                await body(flow.flowId, "response", {}, Buffer.alloc(0), { complete: false, omitted: "unfinished" });
+            }
+            flows.clear();
+        },
+    };
+}