@loadmill/droid-cua 2.7.0 → 2.9.0

package/build/src/modes/execution-mode.js

@@ -1,7 +1,11 @@
 import { randomUUID } from "node:crypto";
+import { CAPTURE_SOURCES } from "../core/capture-sources.js";
 import { getScreenshotAsBase64, connectToDevice, getDeviceInfo, getCurrentPlatform } from "../device/connection.js";
+import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, navigateToUrl as navigateBrowserToUrl, } from "../browser/connection.js";
+import { normalizeBrowserInfoForSession } from "../browser/normalize.js";
+import { extractNavigateTarget, isNavigateInstruction } from "../browser/navigation.js";
 import { sendCUARequest } from "../device/openai.js";
-import { buildExecutionRecoveryPrompt } from "../core/prompts.js";
+import { buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt } from "../core/prompts.js";
 import { isAssertion, extractAssertionPrompt, buildAssertionSystemPrompt, checkAssertionResult, handleAssertionFailure, handleAssertionSuccess, } from "../device/assertions.js";
 import { isLoadmillInstruction, extractLoadmillCommand, executeLoadmillInstruction, } from "../device/loadmill.js";
 import { logger } from "../utils/logger.js";
@@ -52,6 +56,16 @@ export class ExecutionMode {
             assertionsPassed: 0,
             assertionsFailed: 0,
         };
+        this.isBrowserSession = session?.targetKind === "browser" || session?.deviceInfo?.platform === "browser";
+        this.getScreenshotAsBase64 = executionEngine.getScreenshotAsBase64 || getScreenshotAsBase64;
+        this.sendCUARequest = executionEngine.sendCUARequest || sendCUARequest;
+        this.connectToDevice = executionEngine.connectToDevice || connectToDevice;
+        this.getDeviceInfo = executionEngine.getDeviceInfo || getDeviceInfo;
+        this.getCurrentPlatform =
+            executionEngine.getCurrentPlatform || (this.isBrowserSession ? getBrowserCurrentPlatform : getCurrentPlatform);
+        this.connectToBrowser = executionEngine.connectToBrowser || connectToBrowser;
+        this.getBrowserInfo = executionEngine.getBrowserInfo || getBrowserInfo;
+        this.navigateToUrl = executionEngine.navigateToUrl || navigateBrowserToUrl;
     }
     /**
      * Format duration in human-readable format (Xm Ys)
@@ -200,6 +214,94 @@ export class ExecutionMode {
     async executeInstruction(instruction, context, retryCount = 0, stepContext = null) {
         const MAX_RETRIES = 10;
         const addOutput = context.addOutput || printCliOutput;
+        if (isNavigateInstruction(instruction)) {
+            if (!this.isBrowserSession) {
+                const message = "navigate: instructions are only supported for web runs.";
+                this.emit(addOutput, "error", message, context, stepContext, {
+                    eventType: "error",
+                    payload: {
+                        instructionType: "navigate",
+                        message
+                    }
+                });
+                return { success: false, error: message };
+            }
+            const navigationTarget = extractNavigateTarget(instruction);
+            this.session.addToTranscript(`[Navigate] ${navigationTarget}`);
+            emitDesktopDebug("runner.navigate.started", "execution", {
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex
+            }, {
+                instruction,
+                target: navigationTarget
+            });
+            this.emit(addOutput, "system", `Navigating to ${navigationTarget}...`, context, stepContext, {
+                eventType: "system_message",
+                payload: {
+                    instructionType: "navigate",
+                    target: navigationTarget,
+                }
+            });
+            try {
+                const normalizedUrl = await this.navigateToUrl(this.session.browserSessionId, navigationTarget);
+                const browserInfo = normalizeBrowserInfoForSession(await this.getBrowserInfo(this.session.browserSessionId));
+                this.session.browserInfo = browserInfo;
+                this.session.deviceInfo = browserInfo;
+                this.session.currentUrl = browserInfo.currentUrl || normalizedUrl;
+                this.session.browserName = browserInfo.browserName || this.session.currentUrl;
+                this.session.deviceName = this.session.browserName;
+                const transcriptContext = this.session.getTranscriptText();
+                const navigationRecoveryPrompt = buildExecutionNavigationRecoveryPrompt({
+                    basePrompt: this.initialSystemText,
+                    transcript: transcriptContext,
+                    navigatedUrl: this.session.currentUrl,
+                });
+                this.session.installRecoverySystemPrompt(navigationRecoveryPrompt);
+                emitDesktopDebug("runner.navigate.completed", "execution", {
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }, {
+                    instruction,
+                    target: navigationTarget,
+                    url: normalizedUrl
+                });
+                this.emit(addOutput, "success", `Navigation complete: ${normalizedUrl}`, context, stepContext, {
+                    eventType: "system_message",
+                    payload: {
+                        instructionType: "navigate",
+                        url: normalizedUrl,
+                    }
+                });
+                return { success: true };
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : "Navigation failed.";
+                logger.error("Execution navigation error", {
+                    instruction,
+                    retryCount,
+                    message,
+                });
+                emitDesktopDebug("runner.navigate.failed", "execution", {
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }, {
+                    instruction,
+                    target: navigationTarget,
+                    message
+                });
+                this.emit(addOutput, "error", `Navigation failed: ${message}`, context, stepContext, {
+                    eventType: "error",
+                    payload: {
+                        instructionType: "navigate",
+                        message,
+                    }
+                });
+                return { success: false, error: message };
+            }
+        }
         // ── Check for Loadmill instruction ──
         if (isLoadmillInstruction(instruction)) {
             const loadmillCommand = extractLoadmillCommand(instruction);
@@ -230,13 +332,13 @@ export class ExecutionMode {
             this.session.addMessage("user", instruction);
         }
         try {
-            const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
             const screenshotResult = await this.engine.recordScreenshot?.(screenshotBase64, {
                 runId: context?.runId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
                 ...(assertionEventId ? { eventId: assertionEventId } : {}),
-                captureSource: isAssertionStep ? "instruction-input-assertion" : "instruction-input"
+                captureSource: isAssertionStep ? CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION : CAPTURE_SOURCES.INSTRUCTION_INPUT
             });
             const assertionScreenshotPath = isAssertionStep ? screenshotPathFromRecorderResult(screenshotResult) : null;
             const assertionScreenshotDataUrl = isAssertionStep
@@ -259,7 +361,7 @@ export class ExecutionMode {
                 messages: this.session.messages,
                 previousResponseId: this.session.previousResponseId
             });
-            const response = await sendCUARequest({
+            const response = await this.sendCUARequest({
                 messages: messagesToSend,
                 screenshotBase64,
                 previousResponseId: previousResponseIdToSend,
@@ -364,35 +466,52 @@ export class ExecutionMode {
                     attemptsExhausted: retryCount,
                     maxRetries: MAX_RETRIES
                 });
-                this.emit(addOutput, 'error', `Failed after ${MAX_RETRIES} retries. Device may be disconnected.`, context, stepContext, {
+                this.emit(addOutput, 'error', `Failed after ${MAX_RETRIES} retries. ${this.isBrowserSession ? "Browser session" : "Device"} may be disconnected.`, context, stepContext, {
                     eventType: 'error',
                     payload: {
-                        message: `Failed after ${MAX_RETRIES} retries. Device may be disconnected.`,
+                        message: `Failed after ${MAX_RETRIES} retries. ${this.isBrowserSession ? "Browser session" : "Device"} may be disconnected.`,
                         attempt: retryCount,
                         maxRetries: MAX_RETRIES
                     }
                 });
-                // Attempt to reconnect to the device
-                this.emit(addOutput, 'info', 'Attempting to reconnect to device...', context, stepContext, {
+                // Attempt to reconnect to the active target.
+                this.emit(addOutput, 'info', `Attempting to reconnect to ${this.isBrowserSession ? "browser" : "device"}...`, context, stepContext, {
                     eventType: 'system_message'
                 });
                 try {
-                    const platform = getCurrentPlatform();
-                    const deviceName = this.session.deviceName || undefined;
-                    const deviceId = await connectToDevice(deviceName, platform);
-                    const deviceInfo = await getDeviceInfo(deviceId);
-                    // Update session with new connection
-                    this.session.deviceId = deviceId;
-                    this.session.deviceInfo = deviceInfo;
+                    if (this.isBrowserSession) {
+                        const browserSessionId = await this.connectToBrowser({
+                            url: this.session.currentUrl || undefined,
+                            headless: this.isHeadlessMode,
+                            sessionMode: this.session.sessionMode || "persistent",
+                        });
+                        const browserInfo = normalizeBrowserInfoForSession(await this.getBrowserInfo(browserSessionId));
+                        this.session.browserSessionId = browserSessionId;
+                        this.session.deviceId = browserSessionId;
+                        this.session.browserInfo = browserInfo;
+                        this.session.deviceInfo = browserInfo;
+                        this.session.currentUrl = browserInfo.currentUrl || null;
+                        this.session.browserName = browserInfo.browserName || this.session.currentUrl;
+                        this.session.deviceName = this.session.browserName;
+                    }
+                    else {
+                        const platform = this.getCurrentPlatform();
+                        const deviceName = this.session.deviceName || undefined;
+                        const deviceId = await this.connectToDevice(deviceName, platform);
+                        const deviceInfo = await this.getDeviceInfo(deviceId);
+                        this.session.deviceId = deviceId;
+                        this.session.deviceInfo = deviceInfo;
+                        this.session.deviceName = deviceInfo?.device_name || this.session.deviceName;
+                    }
                     emitDesktopDebug("reconnect.attempt", "device", {
                         runId: context?.runId,
                         stepId: stepContext?.stepId,
                         instructionIndex: stepContext?.instructionIndex
                     }, {
                         stage: "success",
-                        deviceId
+                        deviceId: this.session.deviceId
                     });
-                    this.emit(addOutput, 'success', 'Reconnected to device. Resuming...', context, stepContext, {
+                    this.emit(addOutput, 'success', `Reconnected to ${this.isBrowserSession ? "browser" : "device"}. Resuming...`, context, stepContext, {
                         eventType: 'system_message'
                     });
                     // Reset retry count and try again
@@ -408,13 +527,13 @@ export class ExecutionMode {
                         message: reconnectErr.message
                     });
                     logger.error('Failed to reconnect to device', { error: reconnectErr.message });
-                    this.emit(addOutput, 'error', `Could not reconnect to device: ${reconnectErr.message}`, context, stepContext, {
+                    this.emit(addOutput, 'error', `Could not reconnect to ${this.isBrowserSession ? "browser" : "device"}: ${reconnectErr.message}`, context, stepContext, {
                         eventType: 'error',
                         payload: {
                             message: reconnectErr.message
                         }
                     });
-                    return { success: false, error: 'Device disconnected and reconnection failed' };
+                    return { success: false, error: `${this.isBrowserSession ? "Browser" : "Device"} disconnected and reconnection failed` };
                 }
             }
             this.emit(addOutput, 'info', 'Thinking...', context, stepContext, {
@@ -452,8 +571,7 @@ export class ExecutionMode {
                 transcript: transcriptContext
             });
             // Replace the system message with the enhanced one
-            this.session.messages = [{ role: "system", content: recoverySystemPrompt }];
-            this.session.updateResponseId(undefined);
+            this.session.installRecoverySystemPrompt(recoverySystemPrompt);
             // Wait a bit before retrying
             await new Promise(resolve => setTimeout(resolve, 1000));
             // Retry the same instruction with incremented counter

package/build/src/prompts/base.js

@@ -33,13 +33,20 @@ function buildStrictModeRuntimeSection(runtimeOptions = {}) {
     }
     return `STRICT MODE - Observation-First Runtime:
 - The runtime may execute only the first meaningful action from any multi-step action chain you propose.
-- After that first action, it will re-observe the device and continue from a fresh screenshot.
+- After that first action, it will re-observe the ${runtimeOptions?.platform === "browser" ? "browser" : "device"} and continue from a fresh screenshot.
 - If you propose multiple steps, assume only the first may happen before the next turn.
 - Do not assume later proposed actions succeeded unless the next screenshot confirms them.`;
 }
-function describeControlledDevice(deviceInfo = {}) {
+function isBrowserTarget(deviceInfo = {}) {
+    return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
+}
+function describeControlledTarget(deviceInfo = {}) {
     const platform = typeof deviceInfo.platform === "string" ? deviceInfo.platform.trim().toLowerCase() : "";
     const deviceName = typeof deviceInfo.device_name === "string" ? deviceInfo.device_name.trim() : "";
+    const browserName = typeof deviceInfo.browserName === "string" ? deviceInfo.browserName.trim() : "";
+    if (platform === "browser") {
+        return browserName ? `a browser session (${browserName})` : "a browser session";
+    }
     if (platform === "ios") {
         return deviceName ? `an iOS simulator (${deviceName})` : "an iOS device";
     }
@@ -48,10 +55,83 @@ function describeControlledDevice(deviceInfo = {}) {
     }
     return "a mobile device";
 }
-function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
-    const controlledDevice = describeControlledDevice(deviceInfo);
-    const prompt = `
-  You are controlling ${controlledDevice} in a sandboxed testing environment.
+function buildBrowserBaseSystemPrompt(deviceInfo) {
+    return `
+  You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
+  Follow the user's instructions to interact with the browser.
+
+  You can interact with any part of the visible browser window, including page content and browser chrome that is visible.
+
+  The screen you see is ${deviceInfo.scaled_width} x ${deviceInfo.scaled_height} pixels.
+  Pixel (0,0) is at the top-left corner.
+
+  When aiming for visual targets:
+  - Reason carefully about the approximate pixel position.
+  - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
+  - Review the returned annotated screenshot before clicking.
+  - Click if the marker is clearly inside the intended clickable target.
+  - Adjust and preview again only if the marker is outside the target, on the edge, or closer to a neighboring target than the intended one.
+  - If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
+  - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
+  - Click precisely based on your visual estimate.
+
+  Available actions: click, scroll, type, keypress, wait, screenshot.
+
+  CRITICAL - Desktop Browser Input Constraints:
+  - This is a desktop browser environment. Standard desktop keys and modifier chords are allowed when they are clearly useful.
+  - Prefer clicking visible controls over keyboard shortcuts when both are equally reliable.
+  - Use keypress chords only when they directly help the task, such as Enter to submit, Escape to dismiss a modal, or Control/Meta shortcuts for focused text fields.
+  - Do not rely on shortcuts that would switch away from the browser or operating system context.
+
+  CRITICAL - Automatic Timing:
+  - After EVERY action (click, type, keypress, scroll), there is an automatic 500ms delay
+  - This 500ms is sufficient for normal UI updates and animations
+  - DO NOT add 'wait' actions unnecessarily - trust the automatic delay
+
+  CRITICAL - Mutating Actions:
+  - Mutating actions change app state. Examples: submit, create, save, confirm, approve, reject, login, logout, send, place order, initiate transfer
+  - Before clicking a mutating action button, dismiss any open overlay or on-screen keyboard if it blocks the target and is not needed
+  - After performing a mutating action once, do NOT repeat the same mutating action unless the UI clearly shows the first attempt failed or had no effect
+  - Treat visible state change as success. Examples: form fields clear, submit button returns to normal, status changes, list refreshes, new row appears, success message appears, screen changes
+  - For form submissions specifically, if the relevant fields clear and the action button returns to its normal idle state, treat that as success even if the new row or confirmation is not obvious yet
+  - If the UI shows signs that the mutating action succeeded, stop acting for that instruction
+
+  Use explicit 'wait' action ONLY in these specific cases:
+  1. After pressing ENTER that triggers navigation (search, URL, form submit)
+  2. After clicking links or controls that trigger page navigation
+  3. After actions that trigger heavy loading, a full-page refresh, or a visible URL change
+
+  Rule of thumb: Wait for page navigation and page loads. Everything else has automatic timing.
+
+  Perform the user's requested actions within the current view.
+
+  If unsure about visual elements, take a screenshot to improve your reasoning.
+  If unsure about the user's intent, make the best decision you can based on context and continue automatically.
+
+  CRITICAL - Never Ask Questions:
+  - NEVER ask the user for confirmation, clarification, or next steps
+  - NEVER ask questions like "Should I...", "Would you like...", "Do you want me to..."
+  - NEVER wait for user guidance - make autonomous decisions
+  - If stuck, try alternative approaches (go back, try different UI element, restart app)
+  - ONLY stop when the task is complete or you've exhausted reasonable approaches
+
+  Act decisively to complete the task.
+
+  Stop acting once the task appears complete.
+  Only complete the current instruction. Do not proceed beyond the current step unless asked.
+
+  Browser-Specific Notes:
+  - You are operating a desktop browser, not a phone UI
+  - Direct URL navigation is handled by runner-owned \`navigate:\` lines when available; do not try to reach the browser address bar or fake URL entry with keyboard shortcuts
+  - ESC dismisses many overlays and dialogs
+  - Cookie or consent banners may block the main task and may need dismissal first
+  - Vertical scrolling is the default way to discover content below the fold
+  - CTRL/CMD/ALT/OPTION/SHIFT may be used when they are the most reliable path
+  `;
+}
+function buildMobileBaseSystemPrompt(deviceInfo) {
+    return `
+  You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
   Follow the user's instructions to interact with the device.
 
   The device screen has been scaled down for display.
@@ -103,18 +183,6 @@ function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
   3. After clicking links that open new apps or pages
   4. After actions that trigger heavy loading (camera, maps, etc.)
 
-  When you MUST wait:
-  - Click app icon from home → wait → Continue
-  - Type in search box → Press ENTER → wait → Continue
-  - Click link that opens new page/app → wait → Continue
-  - Open camera/maps/heavy feature → wait → Continue
-
-  When you should NOT wait (automatic 500ms handles it):
-  - Clicking UI buttons within a running app (click button - no wait needed)
-  - Typing in text fields (type text - no wait needed)
-  - Scrolling (scroll - no wait needed)
-  - Clicking tabs or menu items within an app (click - no wait needed)
-
   Rule of thumb: Wait for app launches and navigation. Everything else has automatic timing.
 
   Perform the user's requested actions within the current view.
@@ -140,6 +208,10 @@ function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
   - On iOS, ESC has no effect; use visible on-screen controls instead
   - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
   `;
-    return prompt;
 }
-export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledDevice, buildStrictModeRuntimeSection, };
+function buildBaseSystemPrompt(deviceInfo) {
+    return isBrowserTarget(deviceInfo)
+        ? buildBrowserBaseSystemPrompt(deviceInfo)
+        : buildMobileBaseSystemPrompt(deviceInfo);
+}
+export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, buildStrictModeRuntimeSection, };

package/build/src/prompts/design.js

@@ -1,17 +1,18 @@
-import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection } from "./base.js";
+import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
 export function buildDesignModePrompt(deviceInfo, customInstructions = {}, runtimeOptions = {}) {
     const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
     const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
     const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
+    const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
 DESIGN MODE:
-You are helping design a test script for an Android app.
+You are helping design a test script for ${browserTarget ? "a browser workflow" : "a mobile app"}.
 Some tests intentionally validate negative outcomes (errors, failures, rejected inputs). These are expected and should be treated as successful progress when they match the test goal.
 
 Your task:
 1. Understand what the user wants to test from their initial instruction
-2. Explore the app autonomously to understand the flows
+2. Explore the ${browserTarget ? "website" : "app"} autonomously to understand the flows
 3. Take screenshots and interact as needed to discover the UI and behavior
 4. Once you've successfully completed the user's requested flow, immediately generate the test script
 
@@ -71,16 +72,10 @@ Suggested test name: short-kebab-case-name
 \`\`\`
 
 CORRECT Example:
-Suggested test name: calculator-addition
+Suggested test name: ${browserTarget ? "docs-search" : "calculator-addition"}
 
 \`\`\`
-Open Calculator app
-assert: Calculator app is visible
-Type "2"
-Click the plus button
-Type "3"
-Click the equals button
-assert: result shows 5
+${browserTarget ? "navigate: https://docs.example.com\nType \"pricing\" in the search field\nPress Enter\nwait\nClick \"Pricing\"\nassert: Pricing page is visible" : "Open Calculator app\nassert: Calculator app is visible\nType \"2\"\nClick the plus button\nType \"3\"\nClick the equals button\nassert: result shows 5"}
 exit
 \`\`\`
 

package/build/src/prompts/execution.js

@@ -1,20 +1,21 @@
-import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection } from "./base.js";
-export function buildAppContextSection(briefing) {
-    const text = typeof briefing === "string" ? briefing.trim() : "";
-    if (!text) {
+import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
+export function buildAppContextSection(text) {
+    const trimmed = typeof text === "string" ? text.trim() : "";
+    if (!trimmed) {
         return "";
     }
-    return `APP CONTEXT BRIEFING:
-The following is a condensed description of the app you are testing, relevant to the current task.
+    return `APP CONTEXT:
+The following describes the app you are testing.
 Use this to understand screen layouts, terminology, navigation, and expected behavior.
 
-${text}`;
+${trimmed}`;
 }
-export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextBriefing = "", runtimeOptions = {}) {
+export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", runtimeOptions = {}) {
     const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
     const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
-    const appContextSection = buildAppContextSection(appContextBriefing);
+    const appContextSection = buildAppContextSection(appContextText);
     const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
+    const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
 EXECUTION MODE - Critical Behavior:
@@ -27,12 +28,12 @@ CRITICAL RULES:
 - DO NOT say "Let me know if you need help" or similar phrases
 - Just execute the action silently and stop immediately
 - Only generate text if the action FAILED or cannot be completed
-- Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses
+- ${browserTarget ? "Use browser-appropriate keyboard input only when it is the most reliable path" : "Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses"}
 - Never repeat the same mutating action with the same apparent intent unless the UI clearly shows failure or no state change
 - If a submit/create/approve/reject/login action appears to succeed, stop instead of trying to reconfirm by doing it again
 - For form submissions, cleared fields plus a reset action button are strong success signals; stop even if the created item is not yet obvious in the visible list
 - If target is not visible, perform bounded off-screen discovery first:
-  1. Scroll the screen in the likely direction to reveal hidden controls
+  1. Scroll ${browserTarget ? "vertically" : "the screen"} in the likely direction to reveal hidden controls
   2. If still missing, do one minimal fallback (e.g., close overlay or go back once), then retry
 
 Your process:
@@ -61,6 +62,27 @@ ${transcript}
 
 [IMPORTANT: Resume execution silently. Do NOT narrate or explain. Just execute the next instruction.]`;
 }
+export function buildExecutionNavigationRecoveryPrompt({ basePrompt, transcript, navigatedUrl }) {
+    const normalizedTranscript = typeof transcript === "string" ? transcript.trim() : "";
+    const normalizedUrl = typeof navigatedUrl === "string" ? navigatedUrl.trim() : "";
+    if (!normalizedTranscript && !normalizedUrl) {
+        return basePrompt;
+    }
+    const transitionLine = normalizedUrl
+        ? `[RUNNER NAVIGATION - The runtime navigated directly to ${normalizedUrl}. The page content has changed.]`
+        : "[RUNNER NAVIGATION - The runtime navigated directly. The page content has changed.]";
+    const transcriptSection = normalizedTranscript
+        ? `
+
+[PRIOR TEST CONTEXT - Steps completed before this navigation:]
+${normalizedTranscript}`
+        : "";
+    return `${basePrompt}
+
+${transitionLine}${transcriptSection}
+
+[IMPORTANT: Continue the current test from this new page. Use the prior context above, but rely on the latest screenshot for the current UI.]`;
+}
 export function buildAssertionSystemPrompt(baseSystemPrompt, assertionPrompt) {
     return `${baseSystemPrompt}
 
@@ -82,101 +104,3 @@ Remember:
 - You must include either "ASSERTION RESULT: PASS" or "ASSERTION RESULT: FAIL" in your response.
 - Be thorough but efficient. Only take the actions necessary to validate the assertion.`;
 }
-export function buildAppContextCompactionInput({ contextDocument, taskDescription, tokenBudget }) {
-    return [
-        {
-            role: "system",
-            content: [{
-                    type: "input_text",
-                    text: `You are compressing an app context document for a mobile testing agent.
-
-You will receive:
-1. A context document
-2. A test task
-
-Your job is to SELECT only the facts from the context document that are useful for the given task.
-The output will be injected into a system prompt with a strict token budget.
-
-CRITICAL:
-- Use only facts explicitly supported by the context document
-- Never invent, infer, normalize, substitute, or improve credentials, labels, screen names, button names, or numeric values
-- Preserve exact values verbatim when present in the source
-- Prefer facts that help the agent act correctly when they are not obvious from the task alone
-- Do not restate, paraphrase, summarize, or reorganize the test task
-- The output must not read like instructions or a test plan
-- Do not describe what the agent should do
-- Output only reference knowledge about the app
-- If a line could be copied from the task with minor wording changes, omit it
-- Prefer copying source facts verbatim or near-verbatim over rewriting them
-- Do not collapse multiple specific source facts into one generic summary if that removes useful distinctions
-
-Selection priority:
-1. Facts the agent would NOT know from the test script alone
-2. Facts that are hard to infer from screenshots
-3. Non-obvious navigation or interaction details
-4. Exact visible labels needed to act correctly
-5. Credentials and other exact values
-
-High-value facts:
-- exact UI labels
-- how state, mode, or account selection is performed
-- where logout is located
-- hidden or non-obvious navigation
-- which menu items are decorative or non-functional
-- screen titles and section labels used to confirm location
-- exact credentials and role labels
-
-Low-value facts:
-- restating the test steps
-- repeating literal values already present in the task
-- generic summaries like "approve the transaction"
-
-When the task involves authentication, switching state or mode, opening menus, or moving between major areas of the app, strongly prefer including:
-- how account, state, or mode selection is performed
-- exact visible labels for the relevant controls
-- where exit or sign-out actions are located
-- the screen or section labels that confirm the agent is in the right place
-
-Rules:
-- Output plain text only
-- No markdown, no bullet symbols, no numbering, no headers
-- Use terse, factual language: one fact per line, no filler words
-- Blank lines only to separate logical groups
-- Prefer exact visible UI labels over summaries
-- Do not describe step-by-step procedures
-- Do not restate the test workflow
-- State only facts about screens, elements, hidden interactions, entities, credentials, and navigation
-- If a useful fact is not explicitly stated in the context document, omit it
-- Include only information relevant to this task
-- Do not waste space repeating the task itself
-- If the task already states a value or action, include it only when the context adds non-obvious execution details
-- Return a short result or an empty string if little is relevant
-- Target: under ${tokenBudget} tokens
-
-Bad output patterns to avoid:
-- generic summaries that remove actionable details
-- lines that restate the task in generic prose
-- lines that describe obvious workflow steps instead of app knowledge
-- lines that replace exact source labels or mechanisms with broad summaries
-
-Good output characteristics:
-- preserves the exact label or mechanism from the source when it matters
-- keeps distinctions like dropdown vs tabs, drawer vs visible button, exact section titles, exact button text
-- includes hidden or non-obvious navigation details when relevant
-
-Return only the briefing text.`
-                }]
-        },
-        {
-            role: "user",
-            content: [{
-                    type: "input_text",
-                    text: `APP CONTEXT DOCUMENT:
-${contextDocument}
-
-TASK:
-${taskDescription}`
-                }]
-        }
-    ];
-}

package/build/src/utils/console-output.js

@@ -1,3 +1,8 @@
+import { classifyCaptureSource } from "../core/capture-sources.js";
+function isHiddenExecutionOutput(item) {
+    const captureSource = item?.payload?.captureSource;
+    return classifyCaptureSource(captureSource).mediaKind === "hidden-debug";
+}
 export function formatCliOutput(item) {
     if (item == null) {
         return null;
@@ -8,6 +13,9 @@ export function formatCliOutput(item) {
     if (typeof item !== "object") {
         return String(item);
     }
+    if (isHiddenExecutionOutput(item)) {
+        return null;
+    }
     const text = typeof item.text === "string" ? item.text : null;
     const type = typeof item.type === "string" ? item.type : "";
     const eventType = typeof item.eventType === "string" ? item.eventType : "";

package/build/src/utils/cua-debug-tracer.js

@@ -94,7 +94,6 @@ function redactRequestForFileLog(requestParams) {
                         ...item.output,
                         image_url: `data:image/png;base64,[${base64Data.length} chars]`
                     },
-                    current_url: item.current_url,
                     acknowledged_safety_checks: item.acknowledged_safety_checks
                 };
             }

package/build/src/utils/run-screenshot-recorder.js

@@ -126,7 +126,7 @@ async function renderAnnotatedScreenshot(image, marker = {}) {
         return null;
     }
     const placement = buildMarkerPlacement(marker, width, height);
-    const radius = Math.max(14, Math.round(Math.min(width, height) * 0.055));
+    const radius = Math.max(16, Math.min(22, Math.round(Math.min(width, height) * 0.05)));
     const overlay = Buffer.from(buildClickMarkerSvg({ width, height, x: placement.x, y: placement.y, radius, marker }));
     const outputBuffer = await image
         .composite([{ input: overlay }])

package/build/src/utils/structured-debug-log-manager.js

@@ -20,7 +20,10 @@ function compactTimestamp(iso) {
 }
 function sanitizeValue(value, pathParts = [], eventName = "") {
     if (typeof value === "string") {
-        if (eventName === "cua.request.full" || eventName === "cua.response.full" || eventName === "app_context.briefing.full") {
+        if (eventName === "cua.request.full" ||
+            eventName === "cua.response.full" ||
+            eventName === "app_context.briefing.full" ||
+            eventName === "app_context.loaded") {
             return value;
         }
         if (isPromptContentPath(pathParts)) {