@loadmill/droid-cua 2.6.0 → 2.8.0

package/LICENSE

@@ -1 +1 @@
-© 2025 Loadmill. All rights reserved.
+© 2026 Loadmill. All rights reserved.

package/README.md

@@ -155,7 +155,9 @@ Example headless config:
   },
   "appContextEnabled": true,
   "appContextBudget": 300,
-  "appContextPath": "../tests/context.md"
+  "appContextPath": "../tests/context.md",
+  "contextOptimizationEnabled": true,
+  "contextOptimizationThreshold": 30000
 }
 ```
 
@@ -177,10 +179,14 @@ Supported CLI options include:
 - `--context`
 - `--app-context-budget`
 - `--no-context`
+- `--context-optimization-threshold`
+- `--no-context-optimization`
 - `--base-prompt`
 - `--execution-prompt`
 - `--base-prompt-file`
 - `--execution-prompt-file`
+- `--strict`
+- `--no-strict`
 - `--record`
 - `--debug`
 
@@ -189,6 +195,9 @@ Config and precedence rules:
 - CLI flags override config file values.
 - `--context` overrides the config app-context path.
 - `--no-context` disables app context entirely.
+- Context Optimization is enabled by default with a `30000` input-token threshold.
+- `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
+- `--no-context-optimization` disables Context Optimization.
 - `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
 - `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
 - If both inline and file-based prompt overrides are provided, the inline prompt flags win.
@@ -216,4 +225,4 @@ Current headless behavior is documented in [docs/headless-cli-spec.md](docs/head
 
 <h2 id="license">📄 License</h2>
 
-© 2025 Loadmill. All rights reserved.
+© 2026 Loadmill. All rights reserved.

package/build/index.js

@@ -1,8 +1,11 @@
 import minimist from "minimist";
 import dotenv from "dotenv";
 import path from "path";
-import { mkdir, readFile } from "fs/promises";
+import { mkdir } from "fs/promises";
 import { connectToDevice, getDeviceInfo } from "./src/device/connection.js";
+import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "./src/browser/connection.js";
+import { handleModelAction as handleBrowserModelAction } from "./src/browser/actions.js";
+import { normalizeBrowserInfoForSession } from "./src/browser/normalize.js";
 import { Session } from "./src/core/session.js";
 import { ExecutionEngine } from "./src/core/execution-engine.js";
 import { buildBaseSystemPrompt, buildExecutionModePrompt } from "./src/core/prompts.js";
@@ -11,14 +14,18 @@ import { ExecutionMode } from "./src/modes/execution-mode.js";
 import { logger } from "./src/utils/logger.js";
 import { selectDevice } from "./src/cli/device-selector.js";
 import { buildAppContextBriefing, DEFAULT_APP_CONTEXT_BUDGET } from "./src/core/app-context.js";
+import { CONTEXT_OPTIMIZATION_ENABLED_ENV, CONTEXT_OPTIMIZATION_THRESHOLD_ENV, } from "./src/core/context-optimization.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
+import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
 import { printCliOutput } from "./src/utils/console-output.js";
 import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
 import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
+import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
 dotenv.config();
 const args = minimist(process.argv.slice(2));
-let avdName = args["avd"];
-let platform = args["platform"] || null; // 'ios' or 'android'
+const targetConfig = resolveHeadlessTarget(args);
+let avdName = targetConfig.avdName;
+let platform = targetConfig.platform || null; // 'ios' or 'android'
 const recordScreenshots = args["record"] || false;
 const instructionsFile = args.instructions || args.i || null;
 const appContextPath = typeof args.context === "string" ? args.context : null;
@@ -33,6 +40,7 @@ if (recordScreenshots)
     await mkdir(screenshotDir, { recursive: true });
 async function main() {
     const isHeadlessInstructionsRun = Boolean(instructionsFile);
+    const isBrowserTarget = targetConfig.target === "web";
     const headlessDebug = createHeadlessDebugArtifacts({
         cwd: process.cwd(),
         enabled: isHeadlessInstructionsRun && debugMode
@@ -42,19 +50,38 @@ async function main() {
     }
     // Initialize legacy plain-text debug logging only for non-headless flows.
     await logger.init(debugMode && !isHeadlessInstructionsRun);
+    if (isBrowserTarget && !isHeadlessInstructionsRun) {
+        throw new Error("--target web is only supported with --instructions headless execution.");
+    }
     // If no device specified, show interactive selection menu
-    if (!avdName && !platform) {
+    if (!isBrowserTarget && !avdName && !platform) {
         const selection = await selectDevice();
         platform = selection.platform;
         avdName = selection.deviceName;
     }
-    // Connect to device
-    const deviceId = await connectToDevice(avdName, platform);
-    const deviceInfo = await getDeviceInfo(deviceId);
-    console.log(`Using real resolution: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
-    console.log(`Model sees resolution: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+    let targetId;
+    let deviceInfo;
+    if (isBrowserTarget) {
+        targetId = await connectToBrowser({
+            url: targetConfig.url,
+            headless: true,
+            sessionMode: targetConfig.sessionMode
+        });
+        deviceInfo = normalizeBrowserInfoForSession(await getBrowserInfo(targetId));
+        console.log(`Using browser viewport: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
+        console.log(`Model sees viewport: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+        if (deviceInfo.currentUrl) {
+            console.log(`Starting URL: ${deviceInfo.currentUrl}`);
+        }
+    }
+    else {
+        targetId = await connectToDevice(avdName, platform);
+        deviceInfo = await getDeviceInfo(targetId);
+        console.log(`Using real resolution: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
+        console.log(`Model sees resolution: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+    }
     // Create session to manage state
-    const session = new Session(deviceId, deviceInfo);
+    const session = new Session(targetId, deviceInfo);
     const initialSystemText = buildBaseSystemPrompt(deviceInfo);
     session.setSystemPrompt(initialSystemText);
     // If --instructions provided, run in headless mode
@@ -74,14 +101,13 @@ async function main() {
                 console.log(`Debug logging enabled: ${currentLogFilePath}`);
             }
             console.log(`\nRunning test from: ${instructionsFile}\n`);
-            const content = await readFile(instructionsFile, "utf-8");
-            instructions = content
-                .split("\n")
-                .map(line => line.trim())
-                .filter(line => line.length > 0);
+            instructions = await loadHeadlessInstructions(instructionsFile);
             const taskText = instructions.join("\n");
             const headlessConfig = await resolveHeadlessExecutionConfig(args);
             process.env.OPENAI_CUA_MODEL = headlessConfig.cuaModel;
+            process.env[CONTEXT_OPTIMIZATION_ENABLED_ENV] = String(headlessConfig.contextOptimizationEnabled);
+            process.env[CONTEXT_OPTIMIZATION_THRESHOLD_ENV] = String(headlessConfig.contextOptimizationThreshold);
+            console.log(`Using CUA model: ${headlessConfig.cuaModel}`);
             let appContextBriefing = "";
             if (!headlessConfig.appContextEnabled) {
                 emitDesktopDebug("app_context.status", "execution", { runId }, {
@@ -136,6 +162,9 @@ async function main() {
                     budget: headlessConfig.appContextBudget,
                     status: "missing"
                 });
+                if (isBrowserTarget && deviceInfo.currentUrl) {
+                    appContextBriefing = `Starting URL: ${deviceInfo.currentUrl}`;
+                }
             }
             const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextBriefing, { strictMode: headlessConfig.strictMode });
             session.setSystemPrompt(executionPrompt);
@@ -144,17 +173,23 @@ async function main() {
                 recordScreenshots,
                 screenshotDir
             });
-            const engine = screenshotRecorder
-                ? new ExecutionEngine(session, {
+            const engineOptions = screenshotRecorder
+                ? {
                     recordScreenshots: true,
                     screenshotRecorder,
                     strictMode: headlessConfig.strictMode
-                })
-                : new ExecutionEngine(session, {
+                }
+                : {
                     recordScreenshots,
                     screenshotDir,
                     strictMode: headlessConfig.strictMode
-                });
+                };
+            if (isBrowserTarget) {
+                engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
+                engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
+                engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
+            }
+            const engine = new ExecutionEngine(session, engineOptions);
             executionMode = new ExecutionMode(session, engine, instructions, true);
             const result = await executionMode.execute({
                 runId,

package/build/src/browser/actions.js

@@ -0,0 +1,9 @@
+import { getBrowserBackend, getCurrentPlatform } from "./factory.js";
+export async function handleModelAction(browserSessionId, action, scale = 1.0, context = null) {
+    const platform = getCurrentPlatform();
+    if (!platform) {
+        throw new Error("No platform set. Call connectToBrowser first.");
+    }
+    const backend = getBrowserBackend(platform);
+    return backend.handleModelAction(browserSessionId, action, scale, context);
+}

package/build/src/browser/connection.js

@@ -0,0 +1,37 @@
+import { getBrowserBackend, detectPlatform, setCurrentPlatform, getCurrentPlatform } from "./factory.js";
+let currentBackend = null;
+export async function connectToBrowser(targetOrOptions, platform = null) {
+    if (platform && platform !== "browser") {
+        throw new Error(`Unsupported platform: ${platform}. Only browser is supported.`);
+    }
+    const detectedPlatform = platform || detectPlatform(targetOrOptions);
+    setCurrentPlatform(detectedPlatform);
+    currentBackend = getBrowserBackend(detectedPlatform);
+    console.log(`Platform: ${detectedPlatform}`);
+    return currentBackend.connectToBrowser(targetOrOptions);
+}
+export async function getBrowserInfo(browserSessionId) {
+    if (!currentBackend) {
+        throw new Error("Not connected to a browser session. Call connectToBrowser first.");
+    }
+    return currentBackend.getBrowserInfo(browserSessionId);
+}
+export async function getScreenshotAsBase64(browserSessionId, browserInfo) {
+    if (!currentBackend) {
+        throw new Error("Not connected to a browser session. Call connectToBrowser first.");
+    }
+    return currentBackend.getScreenshotAsBase64(browserSessionId, browserInfo);
+}
+export async function navigateToUrl(browserSessionId, url) {
+    if (!currentBackend?.navigateToUrl) {
+        throw new Error("Active browser backend does not support direct navigation.");
+    }
+    return currentBackend.navigateToUrl(browserSessionId, url);
+}
+export { getCurrentPlatform };
+export async function disconnect() {
+    if (currentBackend?.disconnect) {
+        await currentBackend.disconnect();
+    }
+    currentBackend = null;
+}

package/build/src/browser/factory.js

@@ -0,0 +1,22 @@
+import * as browserConnection from "./playwright/connection.js";
+import * as browserActions from "./playwright/actions.js";
+let currentPlatform = null;
+export function detectPlatform() {
+    return "browser";
+}
+export function getBrowserBackend(_platform = "browser") {
+    return {
+        connectToBrowser: browserConnection.connectToBrowser,
+        getBrowserInfo: browserConnection.getBrowserInfo,
+        getScreenshotAsBase64: browserConnection.getScreenshotAsBase64,
+        navigateToUrl: browserConnection.navigateToUrl,
+        handleModelAction: browserActions.handleModelAction,
+        disconnect: browserConnection.disconnect,
+    };
+}
+export function setCurrentPlatform(platform) {
+    currentPlatform = platform;
+}
+export function getCurrentPlatform() {
+    return currentPlatform;
+}

package/build/src/browser/interface.js

@@ -0,0 +1,14 @@
+/**
+ * Browser interface - contract the browser backend must implement.
+ */
+export const SUPPORTED_ACTIONS = [
+    "click",
+    "double_click",
+    "move",
+    "type",
+    "scroll",
+    "drag",
+    "keypress",
+    "wait",
+    "screenshot",
+];

package/build/src/browser/navigation.js

@@ -0,0 +1,11 @@
+export function isNavigateInstruction(userInput) {
+    const trimmed = typeof userInput === "string" ? userInput.trim() : "";
+    return trimmed.toLowerCase().startsWith("navigate:");
+}
+export function extractNavigateTarget(userInput) {
+    const trimmed = typeof userInput === "string" ? userInput.trim() : "";
+    if (!trimmed.toLowerCase().startsWith("navigate:")) {
+        return trimmed;
+    }
+    return trimmed.slice("navigate:".length).trim();
+}

package/build/src/browser/normalize.js

@@ -0,0 +1,36 @@
+const DEFAULT_BROWSER_VIEWPORT = {
+    width: 1440,
+    height: 900,
+};
+function finiteNumber(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+export function normalizeBrowserInfoForSession(browserInfo = {}) {
+    const viewportWidth = finiteNumber(browserInfo.viewportWidth, DEFAULT_BROWSER_VIEWPORT.width);
+    const viewportHeight = finiteNumber(browserInfo.viewportHeight, DEFAULT_BROWSER_VIEWPORT.height);
+    const scaledWidth = finiteNumber(browserInfo.scaledWidth, viewportWidth);
+    const scaledHeight = finiteNumber(browserInfo.scaledHeight, viewportHeight);
+    const scale = finiteNumber(browserInfo.scale, 1);
+    const browserName = typeof browserInfo.browserName === "string" && browserInfo.browserName.trim()
+        ? browserInfo.browserName.trim()
+        : "Browser";
+    const browserFamily = browserInfo.browserFamily === "chrome" || browserInfo.browserFamily === "edge"
+        ? browserInfo.browserFamily
+        : null;
+    const currentUrl = typeof browserInfo.currentUrl === "string" && browserInfo.currentUrl.trim()
+        ? browserInfo.currentUrl.trim()
+        : null;
+    return {
+        ...browserInfo,
+        platform: "browser",
+        device_width: viewportWidth,
+        device_height: viewportHeight,
+        scaled_width: scaledWidth,
+        scaled_height: scaledHeight,
+        scale,
+        device_name: browserName,
+        browserName,
+        browserFamily,
+        currentUrl,
+    };
+}

package/build/src/browser/playwright/actions.js

@@ -0,0 +1,199 @@
+import { logger } from "../../utils/logger.js";
+import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
+import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
+import { buildActionEventMeta } from "../../device/action-event-meta.js";
+import { getActiveBrowserPage } from "./session-state.js";
+const KEY_ALIASES = {
+    CTRL: "Control",
+    CONTROL: "Control",
+    CMD: "Meta",
+    COMMAND: "Meta",
+    META: "Meta",
+    ALT: "Alt",
+    OPTION: "Alt",
+    SHIFT: "Shift",
+    ENTER: "Enter",
+    RETURN: "Enter",
+    ESC: "Escape",
+    ESCAPE: "Escape",
+    BACKSPACE: "Backspace",
+    DELETE: "Delete",
+    TAB: "Tab",
+    SPACE: "Space",
+    HOME: "Home",
+    END: "End",
+    PAGEUP: "PageUp",
+    PAGEDOWN: "PageDown",
+    ARROWUP: "ArrowUp",
+    ARROWDOWN: "ArrowDown",
+    ARROWLEFT: "ArrowLeft",
+    ARROWRIGHT: "ArrowRight",
+};
+function normalizeKeyEntry(key) {
+    const raw = String(key ?? "").trim();
+    if (!raw) {
+        throw new Error("Keypress action is missing keys.");
+    }
+    const normalized = KEY_ALIASES[raw.toUpperCase()];
+    if (normalized) {
+        return normalized;
+    }
+    if (raw.length === 1) {
+        return raw;
+    }
+    return raw[0].toUpperCase() + raw.slice(1);
+}
+function normalizeShortcut(keys = []) {
+    if (!Array.isArray(keys) || keys.length === 0) {
+        throw new Error("Keypress action is missing keys.");
+    }
+    return keys.map(normalizeKeyEntry).join("+");
+}
+function buildMeta(context, action, payload = {}) {
+    return buildActionEventMeta({
+        context,
+        action,
+        platform: "browser",
+        payload,
+    });
+}
+export async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const page = getActiveBrowserPage(browserSessionId);
+    try {
+        emitDesktopDebug("browser.action.execute", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex,
+        }, {
+            platform: "browser",
+            browserSessionId,
+            actionType: action?.type,
+            scale,
+            x: action?.x,
+            y: action?.y,
+            x1: action?.x1,
+            y1: action?.y1,
+            x2: action?.x2,
+            y2: action?.y2,
+            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
+            pathPoints: Array.isArray(action?.path) ? action.path.length : 0,
+        });
+        switch (action.type) {
+            case "click": {
+                const x = Math.round(action.x / scale);
+                const y = Math.round(action.y / scale);
+                addOutput({ type: "action", text: "Click", ...buildMeta(context, action, { x, y, unit: "px" }) });
+                await page.mouse.click(x, y);
+                break;
+            }
+            case "double_click": {
+                const x = Math.round(action.x / scale);
+                const y = Math.round(action.y / scale);
+                addOutput({ type: "action", text: `Double-clicking at (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
+                await page.mouse.dblclick(x, y);
+                break;
+            }
+            case "move": {
+                const x = Math.round(action.x / scale);
+                const y = Math.round(action.y / scale);
+                addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
+                await page.mouse.move(x, y);
+                break;
+            }
+            case "scroll": {
+                const scrollX = Math.round((action.scroll_x ?? 0) / scale);
+                const scrollY = Math.round((action.scroll_y ?? 0) / scale);
+                addOutput({
+                    type: "action",
+                    text: `Scrolling by (${scrollX}, ${scrollY})`,
+                    ...buildMeta(context, action, { scrollX, scrollY, unit: "px" }),
+                });
+                await page.mouse.wheel(scrollX, scrollY);
+                break;
+            }
+            case "drag": {
+                const dragPath = Array.isArray(action.path) ? action.path : [];
+                if (dragPath.length < 2) {
+                    throw new Error("Drag action requires a path with at least two points.");
+                }
+                const [start] = dragPath;
+                const end = dragPath[dragPath.length - 1];
+                const scaledStart = { x: Math.round(start.x / scale), y: Math.round(start.y / scale) };
+                const scaledEnd = { x: Math.round(end.x / scale), y: Math.round(end.y / scale) };
+                addOutput({
+                    type: "action",
+                    text: `Dragging from (${scaledStart.x}, ${scaledStart.y}) to (${scaledEnd.x}, ${scaledEnd.y})`,
+                    ...buildMeta(context, action, { pathStart: scaledStart, pathEnd: scaledEnd, unit: "px" }),
+                });
+                await page.mouse.move(scaledStart.x, scaledStart.y);
+                await page.mouse.down();
+                for (const point of dragPath.slice(1)) {
+                    await page.mouse.move(Math.round(point.x / scale), Math.round(point.y / scale));
+                }
+                await page.mouse.up();
+                break;
+            }
+            case "type": {
+                const text = typeof action.text === "string" ? action.text : "";
+                addOutput({ type: "action", text: `Typing text: ${text}`, ...buildMeta(context, action, { text }) });
+                await page.keyboard.type(text);
+                break;
+            }
+            case "keypress": {
+                const shortcut = normalizeShortcut(action.keys);
+                addOutput({
+                    type: "action",
+                    text: `Pressing key: ${shortcut}`,
+                    ...buildMeta(context, action, { keys: action.keys, shortcut }),
+                });
+                await page.keyboard.press(shortcut);
+                break;
+            }
+            case "wait": {
+                addOutput({ type: "action", text: "Waiting...", ...buildMeta(context, action) });
+                const timeoutMs = typeof action.ms === "number" && Number.isFinite(action.ms) && action.ms >= 0
+                    ? Math.round(action.ms)
+                    : getConfiguredStepDelayMs();
+                await page.waitForTimeout(timeoutMs);
+                break;
+            }
+            default:
+                addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+        }
+    }
+    catch (error) {
+        logger.error("Action execution error", {
+            action,
+            message: error.message,
+            stack: error.stack,
+        });
+        emitDesktopDebug("browser.error", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex,
+        }, {
+            platform: "browser",
+            operation: "action.execute",
+            actionType: action?.type,
+            message: error.message,
+        });
+        addOutput({
+            type: "error",
+            text: `Error executing action: ${error.message}`,
+            eventType: "error",
+            actionType: action?.type,
+            runId: context?.runId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex,
+            payload: {
+                message: error.message,
+                platform: "browser",
+            },
+        });
+        addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+    }
+}