@loadmill/droid-cua 2.7.0 → 2.9.0

package/README.md

@@ -40,6 +40,7 @@ Under the hood, `droid-cua` uses an AI agent to explore your app, execute action
 
 - Mac: [Download](https://github.com/loadmill/droid-cua-release/releases/latest/download/Loadmill-Droid-CUA.dmg)
 - Mac (Intel): [Download](https://github.com/loadmill/droid-cua-release/releases/latest/download/Loadmill-Droid-CUA-intel.dmg)
+- Windows: [Download](https://github.com/loadmill/droid-cua-release/releases/latest/download/Loadmill-Droid-CUA-Setup.exe)
 
 **2. Launch the app**
 
@@ -67,6 +68,10 @@ You can also keep project run history in a results folder and review past runs f
 - **Android** - Physical devices and emulators
 - **iOS** - Simulators on macOS
 
+Desktop app support:
+- macOS: Android and iOS simulator workflows
+- Windows: Android workflows only
+
 ## Requirements
 
 **All platforms:**
@@ -154,10 +159,9 @@ Example headless config:
     "executionModeInstructions": ""
   },
   "appContextEnabled": true,
-  "appContextBudget": 300,
   "appContextPath": "../tests/context.md",
   "contextOptimizationEnabled": true,
-  "contextOptimizationThreshold": 30000
+  "contextOptimizationThreshold": 20000
 }
 ```
 
@@ -177,7 +181,6 @@ Supported CLI options include:
 - `--config`
 - `--cua-model`
 - `--context`
-- `--app-context-budget`
 - `--no-context`
 - `--context-optimization-threshold`
 - `--no-context-optimization`
@@ -195,7 +198,7 @@ Config and precedence rules:
 - CLI flags override config file values.
 - `--context` overrides the config app-context path.
 - `--no-context` disables app context entirely.
-- Context Optimization is enabled by default with a `30000` input-token threshold.
+- Context Optimization is enabled by default with a `20000` input-token threshold.
 - `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
 - `--no-context-optimization` disables Context Optimization.
 - `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.

package/build/index.js

@@ -3,24 +3,31 @@ import dotenv from "dotenv";
 import path from "path";
 import { mkdir } from "fs/promises";
 import { connectToDevice, getDeviceInfo } from "./src/device/connection.js";
+import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "./src/browser/connection.js";
+import { handleModelAction as handleBrowserModelAction } from "./src/browser/actions.js";
+import { normalizeBrowserInfoForSession } from "./src/browser/normalize.js";
 import { Session } from "./src/core/session.js";
 import { ExecutionEngine } from "./src/core/execution-engine.js";
 import { buildBaseSystemPrompt, buildExecutionModePrompt } from "./src/core/prompts.js";
 import { startInkShell } from "./src/cli/ink-shell.js";
 import { ExecutionMode } from "./src/modes/execution-mode.js";
+import { createOpenAiExecutor } from "./src/device/openai.js";
 import { logger } from "./src/utils/logger.js";
 import { selectDevice } from "./src/cli/device-selector.js";
-import { buildAppContextBriefing, DEFAULT_APP_CONTEXT_BUDGET } from "./src/core/app-context.js";
-import { CONTEXT_OPTIMIZATION_ENABLED_ENV, CONTEXT_OPTIMIZATION_THRESHOLD_ENV, } from "./src/core/context-optimization.js";
+import { resolveExecutionAppContext } from "./src/core/app-context.js";
+import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
+import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
 import { printCliOutput } from "./src/utils/console-output.js";
 import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
 import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
 import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
+import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
 dotenv.config();
 const args = minimist(process.argv.slice(2));
-let avdName = args["avd"];
-let platform = args["platform"] || null; // 'ios' or 'android'
+const targetConfig = resolveHeadlessTarget(args);
+let avdName = targetConfig.avdName;
+let platform = targetConfig.platform || null; // 'ios' or 'android'
 const recordScreenshots = args["record"] || false;
 const instructionsFile = args.instructions || args.i || null;
 const appContextPath = typeof args.context === "string" ? args.context : null;
@@ -33,8 +40,31 @@ if (strictFlag && noStrictFlag) {
 const screenshotDir = path.join("droid-cua-recording-" + Date.now());
 if (recordScreenshots)
     await mkdir(screenshotDir, { recursive: true });
+function createCliRuntimeContext(configOverrides = {}) {
+    // CLI env/config is bootstrap input only. After this boundary the shared
+    // runtime uses explicit run-scoped config and credentials objects.
+    const context = createRunContext({
+        config: resolveRuntimeConfigFromEnv(process.env, configOverrides),
+        credentials: resolveRuntimeCredentialsFromEnv(process.env),
+    });
+    const openAi = createOpenAiExecutor({
+        client: context.clients.openai,
+        config: context.config,
+    });
+    const loadmillClient = createLoadmillApiClient({
+        token: context.credentials.loadmillApiToken,
+        baseUrl: context.credentials.loadmillBaseUrl,
+    });
+    return {
+        ...context,
+        openAi,
+        openAiClient: context.clients.openai,
+        loadmillClient,
+    };
+}
 async function main() {
     const isHeadlessInstructionsRun = Boolean(instructionsFile);
+    const isBrowserTarget = targetConfig.target === "web";
     const headlessDebug = createHeadlessDebugArtifacts({
         cwd: process.cwd(),
         enabled: isHeadlessInstructionsRun && debugMode
@@ -44,19 +74,38 @@ async function main() {
     }
     // Initialize legacy plain-text debug logging only for non-headless flows.
     await logger.init(debugMode && !isHeadlessInstructionsRun);
+    if (isBrowserTarget && !isHeadlessInstructionsRun) {
+        throw new Error("--target web is only supported with --instructions headless execution.");
+    }
     // If no device specified, show interactive selection menu
-    if (!avdName && !platform) {
+    if (!isBrowserTarget && !avdName && !platform) {
         const selection = await selectDevice();
         platform = selection.platform;
         avdName = selection.deviceName;
     }
-    // Connect to device
-    const deviceId = await connectToDevice(avdName, platform);
-    const deviceInfo = await getDeviceInfo(deviceId);
-    console.log(`Using real resolution: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
-    console.log(`Model sees resolution: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+    let targetId;
+    let deviceInfo;
+    if (isBrowserTarget) {
+        targetId = await connectToBrowser({
+            url: targetConfig.url,
+            headless: true,
+            sessionMode: targetConfig.sessionMode
+        });
+        deviceInfo = normalizeBrowserInfoForSession(await getBrowserInfo(targetId));
+        console.log(`Using browser viewport: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
+        console.log(`Model sees viewport: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+        if (deviceInfo.currentUrl) {
+            console.log(`Starting URL: ${deviceInfo.currentUrl}`);
+        }
+    }
+    else {
+        targetId = await connectToDevice(avdName, platform);
+        deviceInfo = await getDeviceInfo(targetId);
+        console.log(`Using real resolution: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
+        console.log(`Model sees resolution: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+    }
     // Create session to manage state
-    const session = new Session(deviceId, deviceInfo);
+    const session = new Session(targetId, deviceInfo);
     const initialSystemText = buildBaseSystemPrompt(deviceInfo);
     session.setSystemPrompt(initialSystemText);
     // If --instructions provided, run in headless mode
@@ -77,89 +126,91 @@ async function main() {
             }
             console.log(`\nRunning test from: ${instructionsFile}\n`);
             instructions = await loadHeadlessInstructions(instructionsFile);
-            const taskText = instructions.join("\n");
             const headlessConfig = await resolveHeadlessExecutionConfig(args);
-            process.env.OPENAI_CUA_MODEL = headlessConfig.cuaModel;
-            process.env[CONTEXT_OPTIMIZATION_ENABLED_ENV] = String(headlessConfig.contextOptimizationEnabled);
-            process.env[CONTEXT_OPTIMIZATION_THRESHOLD_ENV] = String(headlessConfig.contextOptimizationThreshold);
-            console.log(`Using CUA model: ${headlessConfig.cuaModel}`);
-            let appContextBriefing = "";
-            if (!headlessConfig.appContextEnabled) {
-                emitDesktopDebug("app_context.status", "execution", { runId }, {
+            const appContext = await resolveExecutionAppContext({
+                enabled: headlessConfig.appContextEnabled,
+                contextPath: headlessConfig.appContextPath,
+            });
+            const appContextText = appContext.text;
+            const runtimeContext = createCliRuntimeContext({
+                cuaModel: headlessConfig.cuaModel,
+                strictMode: headlessConfig.strictMode,
+                promptCustomizations: headlessConfig.promptCustomizations,
+                contextOptimizationEnabled: headlessConfig.contextOptimizationEnabled,
+                contextOptimizationThreshold: headlessConfig.contextOptimizationThreshold,
+                appContext: {
+                    enabled: headlessConfig.appContextEnabled,
+                    path: appContext.contextPath,
+                    status: appContext.status,
+                    text: appContextText,
+                    message: appContext.message,
+                },
+            });
+            console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
+            if (appContext.status === "loaded" || appContext.status === "empty") {
+                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
                     source: "cli_context_flag",
-                    contextPath: null,
-                    budget: headlessConfig.appContextBudget,
-                    status: "disabled"
+                    contextPath: appContext.contextPath,
+                    status: appContext.status,
+                    text: appContextText
                 });
-            }
-            else if (headlessConfig.appContextPath) {
-                try {
-                    const result = await buildAppContextBriefing({
-                        contextPath: headlessConfig.appContextPath,
-                        taskText,
-                        budget: headlessConfig.appContextBudget,
-                    });
-                    appContextBriefing = result.briefing;
-                    emitDesktopDebug("app_context.status", "execution", { runId }, {
-                        source: "cli_context_flag",
-                        contextPath: result.contextPath,
-                        budget: headlessConfig.appContextBudget,
-                        outputTokens: result.outputTokens,
-                        status: appContextBriefing.trim().length > 0 ? "loaded" : "empty"
-                    });
-                    if (appContextBriefing) {
-                        emitDesktopDebug("app_context.briefing.full", "execution", { runId }, {
-                            source: "cli_context_flag",
-                            contextPath: result.contextPath,
-                            budget: headlessConfig.appContextBudget,
-                            outputTokens: result.outputTokens,
-                            briefing: appContextBriefing
-                        });
-                        console.log(`Using app context briefing from: ${result.contextPath}`);
-                    }
-                }
-                catch (error) {
-                    const message = error instanceof Error ? error.message : "Unknown app context error";
-                    emitDesktopDebug("app_context.status", "execution", { runId }, {
-                        source: "cli_context_flag",
-                        contextPath: headlessConfig.appContextPath,
-                        budget: headlessConfig.appContextBudget,
-                        status: "failed",
-                        message
-                    });
-                    console.warn(`Warning: could not load app context from ${headlessConfig.appContextPath}. Running without briefing.`);
+                if (appContextText) {
+                    console.log(`Using app context from: ${appContext.contextPath}`);
                 }
             }
+            else if (appContext.status === "failed") {
+                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
+                    source: "cli_context_flag",
+                    contextPath: appContext.contextPath,
+                    status: "failed",
+                    message: appContext.message
+                });
+                console.warn(`Warning: could not load app context from ${appContext.contextPath}. Running without it.`);
+            }
             else {
-                emitDesktopDebug("app_context.status", "execution", { runId }, {
+                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
                     source: "cli_context_flag",
-                    contextPath: null,
-                    budget: headlessConfig.appContextBudget,
-                    status: "missing"
+                    contextPath: appContext.contextPath,
+                    status: appContext.status
                 });
             }
-            const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextBriefing, { strictMode: headlessConfig.strictMode });
+            const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { strictMode: headlessConfig.strictMode });
             session.setSystemPrompt(executionPrompt);
             const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
                 runId,
                 recordScreenshots,
                 screenshotDir
             });
-            const engine = screenshotRecorder
-                ? new ExecutionEngine(session, {
+            const engineOptions = screenshotRecorder
+                ? {
                     recordScreenshots: true,
                     screenshotRecorder,
-                    strictMode: headlessConfig.strictMode
-                })
-                : new ExecutionEngine(session, {
+                    strictMode: runtimeContext.config.strictMode,
+                    stepDelayMs: runtimeContext.config.stepDelayMs,
+                    sendCUARequest: runtimeContext.openAi.sendCUARequest,
+                    reviseTestScript: runtimeContext.openAi.reviseTestScript,
+                }
+                : {
                     recordScreenshots,
                     screenshotDir,
-                    strictMode: headlessConfig.strictMode
-                });
+                    strictMode: runtimeContext.config.strictMode,
+                    stepDelayMs: runtimeContext.config.stepDelayMs,
+                    sendCUARequest: runtimeContext.openAi.sendCUARequest,
+                    reviseTestScript: runtimeContext.openAi.reviseTestScript,
+                };
+            if (isBrowserTarget) {
+                engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
+                engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
+                engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
+            }
+            const engine = new ExecutionEngine(session, engineOptions);
             executionMode = new ExecutionMode(session, engine, instructions, true);
             const result = await executionMode.execute({
                 runId,
-                addOutput: printCliOutput
+                addOutput: printCliOutput,
+                openAiClient: runtimeContext.openAiClient,
+                loadmillClient: runtimeContext.loadmillClient,
+                runtimeCredentials: runtimeContext.credentials,
             });
             const stats = executionMode.stats || {};
             const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
@@ -204,15 +255,21 @@ async function main() {
             throw error;
         }
     }
+    const runtimeContext = createCliRuntimeContext({
+        strictMode: strictFlag,
+    });
     const engine = new ExecutionEngine(session, {
         recordScreenshots,
         screenshotDir,
-        strictMode: strictFlag,
+        strictMode: runtimeContext.config.strictMode,
+        stepDelayMs: runtimeContext.config.stepDelayMs,
+        sendCUARequest: runtimeContext.openAi.sendCUARequest,
+        reviseTestScript: runtimeContext.openAi.reviseTestScript,
     });
     // Otherwise, start interactive Ink shell
     await startInkShell(session, engine, {
         appContextPath: appContextPath ? path.resolve(appContextPath) : null,
-        appContextBudget: DEFAULT_APP_CONTEXT_BUDGET,
+        runtimeContext,
     });
 }
 main().catch((error) => {

package/build/src/browser/actions.js

@@ -0,0 +1,9 @@
+import { getBrowserBackend, getCurrentPlatform } from "./factory.js";
+export async function handleModelAction(browserSessionId, action, scale = 1.0, context = null) {
+    const platform = getCurrentPlatform();
+    if (!platform) {
+        throw new Error("No platform set. Call connectToBrowser first.");
+    }
+    const backend = getBrowserBackend(platform);
+    return backend.handleModelAction(browserSessionId, action, scale, context);
+}

package/build/src/browser/connection.js

@@ -0,0 +1,37 @@
+import { getBrowserBackend, detectPlatform, setCurrentPlatform, getCurrentPlatform } from "./factory.js";
+let currentBackend = null;
+export async function connectToBrowser(targetOrOptions, platform = null) {
+    if (platform && platform !== "browser") {
+        throw new Error(`Unsupported platform: ${platform}. Only browser is supported.`);
+    }
+    const detectedPlatform = platform || detectPlatform(targetOrOptions);
+    setCurrentPlatform(detectedPlatform);
+    currentBackend = getBrowserBackend(detectedPlatform);
+    console.log(`Platform: ${detectedPlatform}`);
+    return currentBackend.connectToBrowser(targetOrOptions);
+}
+export async function getBrowserInfo(browserSessionId) {
+    if (!currentBackend) {
+        throw new Error("Not connected to a browser session. Call connectToBrowser first.");
+    }
+    return currentBackend.getBrowserInfo(browserSessionId);
+}
+export async function getScreenshotAsBase64(browserSessionId, browserInfo) {
+    if (!currentBackend) {
+        throw new Error("Not connected to a browser session. Call connectToBrowser first.");
+    }
+    return currentBackend.getScreenshotAsBase64(browserSessionId, browserInfo);
+}
+export async function navigateToUrl(browserSessionId, url) {
+    if (!currentBackend?.navigateToUrl) {
+        throw new Error("Active browser backend does not support direct navigation.");
+    }
+    return currentBackend.navigateToUrl(browserSessionId, url);
+}
+export { getCurrentPlatform };
+export async function disconnect() {
+    if (currentBackend?.disconnect) {
+        await currentBackend.disconnect();
+    }
+    currentBackend = null;
+}

package/build/src/browser/factory.js

@@ -0,0 +1,22 @@
+import * as browserConnection from "./playwright/connection.js";
+import * as browserActions from "./playwright/actions.js";
+let currentPlatform = null;
+export function detectPlatform() {
+    return "browser";
+}
+export function getBrowserBackend(_platform = "browser") {
+    return {
+        connectToBrowser: browserConnection.connectToBrowser,
+        getBrowserInfo: browserConnection.getBrowserInfo,
+        getScreenshotAsBase64: browserConnection.getScreenshotAsBase64,
+        navigateToUrl: browserConnection.navigateToUrl,
+        handleModelAction: browserActions.handleModelAction,
+        disconnect: browserConnection.disconnect,
+    };
+}
+export function setCurrentPlatform(platform) {
+    currentPlatform = platform;
+}
+export function getCurrentPlatform() {
+    return currentPlatform;
+}

package/build/src/browser/interface.js

@@ -0,0 +1,14 @@
+/**
+ * Browser interface - contract the browser backend must implement.
+ */
+export const SUPPORTED_ACTIONS = [
+    "click",
+    "double_click",
+    "move",
+    "type",
+    "scroll",
+    "drag",
+    "keypress",
+    "wait",
+    "screenshot",
+];

package/build/src/browser/navigation.js

@@ -0,0 +1,11 @@
+export function isNavigateInstruction(userInput) {
+    const trimmed = typeof userInput === "string" ? userInput.trim() : "";
+    return trimmed.toLowerCase().startsWith("navigate:");
+}
+export function extractNavigateTarget(userInput) {
+    const trimmed = typeof userInput === "string" ? userInput.trim() : "";
+    if (!trimmed.toLowerCase().startsWith("navigate:")) {
+        return trimmed;
+    }
+    return trimmed.slice("navigate:".length).trim();
+}

package/build/src/browser/normalize.js

@@ -0,0 +1,36 @@
+const DEFAULT_BROWSER_VIEWPORT = {
+    width: 1440,
+    height: 900,
+};
+function finiteNumber(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+export function normalizeBrowserInfoForSession(browserInfo = {}) {
+    const viewportWidth = finiteNumber(browserInfo.viewportWidth, DEFAULT_BROWSER_VIEWPORT.width);
+    const viewportHeight = finiteNumber(browserInfo.viewportHeight, DEFAULT_BROWSER_VIEWPORT.height);
+    const scaledWidth = finiteNumber(browserInfo.scaledWidth, viewportWidth);
+    const scaledHeight = finiteNumber(browserInfo.scaledHeight, viewportHeight);
+    const scale = finiteNumber(browserInfo.scale, 1);
+    const browserName = typeof browserInfo.browserName === "string" && browserInfo.browserName.trim()
+        ? browserInfo.browserName.trim()
+        : "Browser";
+    const browserFamily = browserInfo.browserFamily === "chrome" || browserInfo.browserFamily === "edge"
+        ? browserInfo.browserFamily
+        : null;
+    const currentUrl = typeof browserInfo.currentUrl === "string" && browserInfo.currentUrl.trim()
+        ? browserInfo.currentUrl.trim()
+        : null;
+    return {
+        ...browserInfo,
+        platform: "browser",
+        device_width: viewportWidth,
+        device_height: viewportHeight,
+        scaled_width: scaledWidth,
+        scaled_height: scaledHeight,
+        scale,
+        device_name: browserName,
+        browserName,
+        browserFamily,
+        currentUrl,
+    };
+}