@loadmill/droid-cua 2.7.0 → 2.9.0

package/build/src/cli/headless-execution-config.js

@@ -1,9 +1,7 @@
 import path from "path";
 import { readFile } from "fs/promises";
-import { DEFAULT_APP_CONTEXT_BUDGET, MAX_APP_CONTEXT_BUDGET, MIN_APP_CONTEXT_BUDGET, } from "../core/app-context.js";
 import { DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD, MAX_CONTEXT_OPTIMIZATION_THRESHOLD, MIN_CONTEXT_OPTIMIZATION_THRESHOLD, } from "../core/context-optimization.js";
-const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
-const VALID_CUA_MODEL_LIST = ["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"];
+import { CUA_MODELS, isValidCuaModel, resolveCuaModel } from "../core/cua-models.js";
 function createEmptyPromptCustomizations() {
     return {
         basePromptInstructions: "",
@@ -20,17 +18,13 @@ function assertNonEmptyString(value, label) {
     }
 }
 function validateCuaModel(value, label) {
-    if (typeof value !== "string" || !VALID_CUA_MODELS.has(value)) {
-        throw new Error(`${label} must be one of: ${VALID_CUA_MODEL_LIST.join(", ")}.`);
+    if (typeof value !== "string" || !isValidCuaModel(value)) {
+        throw new Error(`${label} must be one of: ${CUA_MODELS.join(", ")}.`);
     }
     return value;
 }
 function readEnvDefaultCuaModel() {
-    const rawValue = process.env.OPENAI_CUA_MODEL;
-    if (typeof rawValue !== "string" || rawValue.trim().length === 0) {
-        return "gpt-5.4";
-    }
-    return VALID_CUA_MODELS.has(rawValue) ? rawValue : "gpt-5.4";
+    return resolveCuaModel(process.env.OPENAI_CUA_MODEL);
 }
 function parseBooleanFlagState(args, key) {
     return {
@@ -38,24 +32,6 @@ function parseBooleanFlagState(args, key) {
         disabled: args[`no-${key}`] === true || args[key] === false,
     };
 }
-function parseBudgetValue(rawValue, label) {
-    const numericValue = (() => {
-        if (typeof rawValue === "number") {
-            return rawValue;
-        }
-        if (typeof rawValue === "string" && /^-?\d+$/.test(rawValue.trim())) {
-            return Number.parseInt(rawValue, 10);
-        }
-        return Number.NaN;
-    })();
-    if (!Number.isInteger(numericValue)) {
-        throw new Error(`${label} must be an integer between ${MIN_APP_CONTEXT_BUDGET} and ${MAX_APP_CONTEXT_BUDGET}.`);
-    }
-    if (numericValue < MIN_APP_CONTEXT_BUDGET || numericValue > MAX_APP_CONTEXT_BUDGET) {
-        throw new Error(`${label} must be between ${MIN_APP_CONTEXT_BUDGET} and ${MAX_APP_CONTEXT_BUDGET}.`);
-    }
-    return numericValue;
-}
 function parseContextOptimizationThreshold(rawValue, label) {
     const numericValue = (() => {
         if (typeof rawValue === "number") {
@@ -133,7 +109,6 @@ async function loadConfigFromFile(configPath) {
         cuaModel: undefined,
         promptCustomizations: createEmptyPromptCustomizations(),
         appContextEnabled: undefined,
-        appContextBudget: undefined,
         appContextPath: undefined,
         contextOptimizationEnabled: undefined,
         contextOptimizationThreshold: undefined,
@@ -151,9 +126,6 @@ async function loadConfigFromFile(configPath) {
         }
         normalized.appContextEnabled = rawConfig.appContextEnabled;
     }
-    if ("appContextBudget" in rawConfig) {
-        normalized.appContextBudget = parseBudgetValue(rawConfig.appContextBudget, "config.appContextBudget");
-    }
     if ("appContextPath" in rawConfig && rawConfig.appContextPath != null) {
         assertNonEmptyString(rawConfig.appContextPath, "config.appContextPath");
         normalized.appContextPath = path.resolve(configDir, rawConfig.appContextPath);
@@ -210,7 +182,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
         cuaModel: fileConfig?.cuaModel || readEnvDefaultCuaModel(),
         promptCustomizations,
         appContextEnabled: fileConfig?.appContextEnabled ?? true,
-        appContextBudget: fileConfig?.appContextBudget ?? DEFAULT_APP_CONTEXT_BUDGET,
         appContextPath: fileConfig?.appContextPath || null,
         contextOptimizationEnabled: fileConfig?.contextOptimizationEnabled ?? true,
         contextOptimizationThreshold: fileConfig?.contextOptimizationThreshold ?? DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD,
@@ -219,9 +190,6 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
     if (typeof args["cua-model"] === "string") {
         resolved.cuaModel = validateCuaModel(args["cua-model"], "--cua-model");
     }
-    if (args["app-context-budget"] != null) {
-        resolved.appContextBudget = parseBudgetValue(args["app-context-budget"], "--app-context-budget");
-    }
     if (hasContextOptimizationThresholdFlag) {
         resolved.contextOptimizationEnabled = true;
         resolved.contextOptimizationThreshold = parseContextOptimizationThreshold(args["context-optimization-threshold"], "--context-optimization-threshold");

package/build/src/cli/headless-target.js

@@ -0,0 +1,64 @@
+function normalizeTarget(value) {
+    if (value == null || value === "") {
+        return "mobile";
+    }
+    if (typeof value !== "string") {
+        throw new Error("--target must be one of: mobile, web.");
+    }
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "mobile" || normalized === "web") {
+        return normalized;
+    }
+    throw new Error("--target must be one of: mobile, web.");
+}
+function normalizeSessionMode(value) {
+    if (value == null || value === "") {
+        return "persistent";
+    }
+    if (typeof value !== "string") {
+        throw new Error("--session-mode must be one of: persistent, fresh.");
+    }
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "persistent" || normalized === "fresh") {
+        return normalized;
+    }
+    throw new Error("--session-mode must be one of: persistent, fresh.");
+}
+function hasNonEmptyString(value) {
+    return typeof value === "string" && value.trim().length > 0;
+}
+export function resolveHeadlessTarget(args = {}) {
+    const target = normalizeTarget(args.target);
+    const sessionMode = normalizeSessionMode(args["session-mode"]);
+    const url = hasNonEmptyString(args.url) ? args.url.trim() : null;
+    const avdName = hasNonEmptyString(args.avd) ? args.avd.trim() : null;
+    const platform = hasNonEmptyString(args.platform) ? args.platform.trim() : null;
+    if (target === "web") {
+        if (!url) {
+            throw new Error("--target web requires --url.");
+        }
+        if (avdName || platform) {
+            throw new Error("--avd and --platform cannot be used with --target web.");
+        }
+        return {
+            target,
+            url,
+            sessionMode,
+            avdName: null,
+            platform: null,
+        };
+    }
+    if (hasNonEmptyString(args.url)) {
+        throw new Error("--url can only be used with --target web.");
+    }
+    if (args["session-mode"] != null) {
+        throw new Error("--session-mode can only be used with --target web.");
+    }
+    return {
+        target,
+        url: null,
+        sessionMode: null,
+        avdName,
+        platform,
+    };
+}

package/build/src/cli/ink-shell.js

@@ -7,7 +7,7 @@ import { routeCommand } from '../commands/index.js';
  * Start the Ink-based conversational shell
  * @param {Object} session - Session object with device info
  * @param {Object} executionEngine - Execution engine instance
- * @param {{ appContextPath?: string | null, appContextBudget?: number }} [options]
+ * @param {{ appContextPath?: string | null, runtimeContext?: Object | null }} [options]
  * @returns {Promise<void>}
  */
 export async function startInkShell(session, executionEngine, options = {}) {
@@ -34,7 +34,10 @@ export async function startInkShell(session, executionEngine, options = {}) {
                 ...context,
                 engine: executionEngine,
                 appContextPath: options.appContextPath ?? null,
-                appContextBudget: options.appContextBudget
+                runtimeContext: options.runtimeContext ?? null,
+                openAiClient: options.runtimeContext?.openAiClient ?? null,
+                loadmillClient: options.runtimeContext?.loadmillClient ?? null,
+                runtimeCredentials: options.runtimeContext?.credentials ?? null
             });
             if (!shouldContinue) {
                 shouldExit = true;

package/build/src/commands/help.js

@@ -10,19 +10,22 @@
  */
 export async function handleHelp(args, session, context) {
     const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
-    addOutput({ type: 'system', text: 'droid-cua - AI-powered mobile device testing CLI' });
+    addOutput({ type: 'system', text: 'droid-cua - AI-powered mobile and web testing CLI' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Usage:' });
     addOutput({ type: 'info', text: '  droid-cua --avd <device-name> [options]' });
+    addOutput({ type: 'info', text: '  droid-cua --target web --url <starting-url> --instructions <file> [options]' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Options:' });
+    addOutput({ type: 'info', text: '  --target <kind>        Target kind: mobile (default) or web' });
     addOutput({ type: 'info', text: '  --avd <name>           Device name (Android device ID/serial or iOS Simulator)' });
     addOutput({ type: 'info', text: '  --platform <platform>  Force platform: android or ios' });
     addOutput({ type: 'info', text: '  --instructions <file>  Run test file in headless mode' });
+    addOutput({ type: 'info', text: '  --url <value>          Required starting URL for --target web headless runs' });
+    addOutput({ type: 'info', text: '  --session-mode <mode>  Browser session mode for web runs: persistent or fresh' });
     addOutput({ type: 'info', text: '  --config <file>        Headless execution JSON config for prompt parity' });
-    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4, gpt-5.4-mini, or computer-use-preview' });
-    addOutput({ type: 'info', text: '  --context <file>       Optional app context file used to brief execution runs' });
-    addOutput({ type: 'info', text: '  --app-context-budget   Headless app context token budget override' });
+    addOutput({ type: 'info', text: '  --cua-model <model>    Headless CUA model override: gpt-5.4 or gpt-5.4-mini' });
+    addOutput({ type: 'info', text: '  --context <file>       Optional app context file injected into execution runs' });
     addOutput({ type: 'info', text: '  --no-context           Disable app context for headless execution' });
     addOutput({ type: 'info', text: '  --context-optimization-threshold <tokens> Headless Context Optimization compaction threshold' });
     addOutput({ type: 'info', text: '  --no-context-optimization Disable Context Optimization for headless execution' });
@@ -52,6 +55,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: 'Platform Support:' });
     addOutput({ type: 'info', text: '  Android: Uses ADB to communicate with Android devices (physical or emulator)' });
     addOutput({ type: 'info', text: '  iOS:     Uses Appium + XCUITest for iOS Simulator automation' });
+    addOutput({ type: 'info', text: '  Web:     Uses Playwright with installed Google Chrome or Microsoft Edge for browser execution' });
     addOutput({ type: 'info', text: '' });
     addOutput({ type: 'info', text: 'Platform Detection:' });
     addOutput({ type: 'info', text: '  - Use --platform flag to force a specific platform' });
@@ -71,6 +75,7 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  droid-cua --avd "iPhone 16"             (iOS Simulator, auto-detected)' });
     addOutput({ type: 'info', text: '  droid-cua --platform ios --avd MySim    (Force iOS platform)' });
     addOutput({ type: 'info', text: '  droid-cua --instructions tests/login.dcua --context app/context.md' });
+    addOutput({ type: 'info', text: '  droid-cua --target web --url docs.example.com --instructions tests/search.dcua' });
     addOutput({ type: 'info', text: '  /create login-test                      (design a new test)' });
     addOutput({ type: 'info', text: '  /list                                   (see all tests)' });
     addOutput({ type: 'info', text: '  /view login-test                        (view test contents)' });

package/build/src/commands/loadmill.js

@@ -1,7 +1,8 @@
 /**
  * Loadmill command handler
  */
-import { executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
+import { createLoadmillApiClient, executeLoadmillCommand, getApiToken } from "../integrations/loadmill/index.js";
+import { createOpenAiClient, resolveRuntimeCredentialsFromEnv } from "../core/runtime-config.js";
 /**
  * Handle /loadmill command
  * @param {string} args - Command arguments (natural language command)
@@ -47,8 +48,16 @@ export async function handleLoadmill(args, session, context) {
     if (context.setAgentWorking) {
         context.setAgentWorking(true, 'Running Loadmill flow...');
     }
+    const runtimeCredentials = context.runtimeCredentials || resolveRuntimeCredentialsFromEnv();
+    const loadmillClient = context.loadmillClient || createLoadmillApiClient({
+        token: runtimeCredentials.loadmillApiToken,
+        baseUrl: runtimeCredentials.loadmillBaseUrl,
+    });
+    const openAiClient = context.openAiClient || createOpenAiClient(runtimeCredentials.openAiApiKey);
     // Execute the command
     const result = await executeLoadmillCommand(command, {
+        loadmillClient,
+        openAiClient,
         onProgress: ({ message }) => {
             addOutput({ type: 'info', text: `[Loadmill] ${message}` });
         }

package/build/src/commands/run.js

@@ -4,7 +4,7 @@
 import { loadTest, listTests, testExists } from "../test-store/test-manager.js";
 import { ExecutionMode } from "../modes/execution-mode.js";
 import { buildExecutionModePrompt } from "../core/prompts.js";
-import { buildAppContextBriefing } from "../core/app-context.js";
+import { resolveExecutionAppContext } from "../core/app-context.js";
 import { logger } from "../utils/logger.js";
 /**
  * Handle /run command
@@ -51,7 +51,6 @@ export async function handleRun(args, session, context) {
     // Load test instructions
     addOutput({ type: 'system', text: `Loading test: ${testName}` });
     const instructions = await loadTest(testName);
-    const taskText = instructions.join("\n");
     addOutput({ type: 'info', text: `Loaded ${instructions.length} instructions` });
     addOutput({ type: 'info', text: '' });
     // Disable free-form input during execution (only allow commands like /exit)
@@ -72,34 +71,24 @@ export async function handleRun(args, session, context) {
     // Each test instruction should execute in isolation
     session.updateResponseId(undefined);
     session.clearMessages();
-    let appContextBriefing = '';
-    if (context.appContextPath) {
-        try {
-            const result = await buildAppContextBriefing({
-                contextPath: context.appContextPath,
-                taskText,
-                budget: context.appContextBudget,
-            });
-            appContextBriefing = result.briefing;
-            if (appContextBriefing) {
-                addOutput({ type: 'info', text: `Loaded app context briefing from: ${result.contextPath}` });
-            }
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : 'Unknown app context error';
-            logger.error('CLI app context compaction failed', {
-                contextPath: context.appContextPath,
-                testName,
-                message,
-            });
-            addOutput({
-                type: 'warning',
-                text: `Warning: could not load app context from ${context.appContextPath}. Running without briefing.`,
-            });
-        }
+    const appContext = await resolveExecutionAppContext({ contextPath: context.appContextPath });
+    const appContextText = appContext.text;
+    if (appContext.status === 'loaded' && appContext.contextPath) {
+        addOutput({ type: 'info', text: `Loaded app context from: ${appContext.contextPath}` });
+    }
+    else if (appContext.status === 'failed') {
+        logger.error('CLI app context load failed', {
+            contextPath: appContext.contextPath,
+            testName,
+            message: appContext.message,
+        });
+        addOutput({
+            type: 'warning',
+            text: `Warning: could not load app context from ${appContext.contextPath}. Running without it.`,
+        });
     }
     // Set execution mode system prompt (replaces any design mode prompt)
-    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextBriefing, { strictMode: Boolean(context.engine?.strictMode) });
+    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextText, { strictMode: Boolean(context.engine?.strictMode) });
     session.setSystemPrompt(executionPrompt);
     // Create execution mode
     const executionMode = new ExecutionMode(session, context.engine, instructions);

package/build/src/core/app-context.js

@@ -1,24 +1,7 @@
 import path from "path";
 import { access, readFile } from "fs/promises";
 import { constants as fsConstants } from "fs";
-import { compactAppContext } from "../device/openai.js";
 export const APP_CONTEXT_FILENAME = "context.md";
-export const DEFAULT_APP_CONTEXT_BUDGET = 300;
-export const MIN_APP_CONTEXT_BUDGET = 100;
-export const MAX_APP_CONTEXT_BUDGET = 2000;
-export function normalizeAppContextBudget(value) {
-    if (typeof value !== "number" || !Number.isFinite(value)) {
-        return DEFAULT_APP_CONTEXT_BUDGET;
-    }
-    const normalized = Math.round(value);
-    if (normalized < MIN_APP_CONTEXT_BUDGET) {
-        return MIN_APP_CONTEXT_BUDGET;
-    }
-    if (normalized > MAX_APP_CONTEXT_BUDGET) {
-        return MAX_APP_CONTEXT_BUDGET;
-    }
-    return normalized;
-}
 export function getDefaultProjectContextPath(projectPath) {
     return path.join(projectPath, APP_CONTEXT_FILENAME);
 }
@@ -35,23 +18,64 @@ export async function appContextFileExists(filePath) {
         return false;
     }
 }
-export async function buildAppContextBriefing({ contextPath, taskText, budget = DEFAULT_APP_CONTEXT_BUDGET, }) {
+/**
+ * @param {{ contextPath?: string | null }} [options]
+ */
+export async function loadAppContext({ contextPath }) {
+    if (!contextPath) {
+        return { text: "", contextPath: null };
+    }
+    const text = await readAppContextFile(contextPath);
+    return { text, contextPath };
+}
+/**
+ * @param {{
+ *   enabled?: boolean,
+ *   contextPath?: string | null,
+ *   missingStatus?: string,
+ *   missingFileStatus?: string | null,
+ * }} [options]
+ */
+export async function resolveExecutionAppContext({ enabled = true, contextPath = null, missingStatus = "missing", missingFileStatus = null, } = {}) {
+    if (!enabled) {
+        return {
+            status: "disabled",
+            text: "",
+            contextPath: null,
+            message: null,
+        };
+    }
     if (!contextPath) {
-        return { briefing: "", contextPath: null };
+        return {
+            status: missingStatus,
+            text: "",
+            contextPath: null,
+            message: null,
+        };
+    }
+    try {
+        const result = await loadAppContext({ contextPath });
+        return {
+            status: result.text.trim().length > 0 ? "loaded" : "empty",
+            text: result.text,
+            contextPath: result.contextPath,
+            message: null,
+        };
     }
-    const normalizedBudget = normalizeAppContextBudget(budget);
-    if (normalizedBudget === 0) {
-        return { briefing: "", contextPath };
+    catch (error) {
+        if (missingFileStatus && error && typeof error === "object" && error.code === "ENOENT") {
+            return {
+                status: missingFileStatus,
+                text: "",
+                contextPath,
+                message: null,
+            };
+        }
+        return {
+            status: "failed",
+            text: "",
+            contextPath,
+            message: error instanceof Error ? error.message : "Unknown app context error",
+        };
     }
-    const rawContext = await readAppContextFile(contextPath);
-    const result = await compactAppContext({
-        contextDocument: rawContext,
-        taskDescription: taskText,
-        tokenBudget: normalizedBudget,
-    });
-    return {
-        briefing: result.briefing,
-        outputTokens: result.outputTokens,
-        contextPath,
-    };
 }

package/build/src/core/capture-sources.js

@@ -0,0 +1,76 @@
+// Single source of truth for screenshot/click-annotation provenance. Producers
+// stamp `captureSource` on every event; consumers (renderer log filter, main
+// result-artifact router, debug recorder) read it through classifyCaptureSource.
+//
+// classifyCaptureSource returns three orthogonal axes — each consumer reads only
+// what it needs:
+//   - mediaKind: how the renderer should display the event ('hidden-debug'
+//     hides it from the timeline, 'screenshot' / 'click-annotation' put it in
+//     the gallery, null leaves it as plain text).
+//   - persistsToResults: whether the main process copies the artifact into the
+//     run's user-visible result directory (vs. debug-only).
+//   - visibility: whether the event represents user-facing action ('user'),
+//     internal preview/debug noise ('debug'), or has no associated media
+//     ('none'). Distinct from mediaKind because INSTRUCTION_INPUT and
+//     DESIGN_INPUT are real user inputs but don't appear in the gallery.
+export const CAPTURE_SOURCES = Object.freeze({
+    PREVIEW_CLICK: "preview-click",
+    ACTION_CLICK_ANNOTATION: "action-click-annotation",
+    CALL_OUTPUT_POST_ACTION: "call-output-post-action",
+    CALL_OUTPUT_EXPLICIT_ACTION: "call-output-explicit-action",
+    INSTRUCTION_INPUT: "instruction-input",
+    INSTRUCTION_INPUT_ASSERTION: "instruction-input-assertion",
+    DESIGN_INPUT: "design-input",
+});
+const HIDDEN_DEBUG_MEDIA_KINDS = new Set([
+    CAPTURE_SOURCES.PREVIEW_CLICK,
+]);
+const CLICK_ANNOTATION_MEDIA_KINDS = new Set([
+    CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
+]);
+const SCREENSHOT_MEDIA_KINDS = new Set([
+    CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
+    CAPTURE_SOURCES.CALL_OUTPUT_EXPLICIT_ACTION,
+    CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
+]);
+const USER_VISIBLE_RESULT_CAPTURE_SOURCES = new Set([
+    CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION,
+    CAPTURE_SOURCES.CALL_OUTPUT_EXPLICIT_ACTION,
+    CAPTURE_SOURCES.INSTRUCTION_INPUT_ASSERTION,
+    CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
+]);
+export function classifyCaptureSource(source) {
+    if (typeof source !== "string" || !source.trim()) {
+        return {
+            mediaKind: null,
+            persistsToResults: false,
+            visibility: "none",
+        };
+    }
+    if (HIDDEN_DEBUG_MEDIA_KINDS.has(source)) {
+        return {
+            mediaKind: "hidden-debug",
+            persistsToResults: false,
+            visibility: "debug",
+        };
+    }
+    if (CLICK_ANNOTATION_MEDIA_KINDS.has(source)) {
+        return {
+            mediaKind: "click-annotation",
+            persistsToResults: true,
+            visibility: "user",
+        };
+    }
+    if (SCREENSHOT_MEDIA_KINDS.has(source)) {
+        return {
+            mediaKind: "screenshot",
+            persistsToResults: true,
+            visibility: "user",
+        };
+    }
+    return {
+        mediaKind: null,
+        persistsToResults: USER_VISIBLE_RESULT_CAPTURE_SOURCES.has(source),
+        visibility: source === CAPTURE_SOURCES.INSTRUCTION_INPUT || source === CAPTURE_SOURCES.DESIGN_INPUT ? "user" : "none",
+    };
+}

package/build/src/core/context-optimization.js

@@ -1,5 +1,5 @@
 export const DEFAULT_CONTEXT_OPTIMIZATION_ENABLED = true;
-export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 30000;
+export const DEFAULT_CONTEXT_OPTIMIZATION_THRESHOLD = 20000;
 export const MIN_CONTEXT_OPTIMIZATION_THRESHOLD = 10000;
 export const MAX_CONTEXT_OPTIMIZATION_THRESHOLD = 900000;
 export const CONTEXT_OPTIMIZATION_ENABLED_ENV = "DROID_CUA_CONTEXT_OPTIMIZATION_ENABLED";

package/build/src/core/cua-models.js

@@ -0,0 +1,9 @@
+export const CUA_MODELS = ["gpt-5.4", "gpt-5.4-mini"];
+export const DEFAULT_CUA_MODEL = "gpt-5.4";
+const VALID_CUA_MODELS = new Set(CUA_MODELS);
+export function isValidCuaModel(value) {
+    return VALID_CUA_MODELS.has(value);
+}
+export function resolveCuaModel(value) {
+    return isValidCuaModel(value) ? value : DEFAULT_CUA_MODEL;
+}

package/build/src/core/execution-engine.js

@@ -8,7 +8,7 @@ import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { printCliOutput } from "../utils/console-output.js";
 import { annotateScreenshotArtifact, createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
-const VALID_CUA_MODELS = new Set(["gpt-5.4", "gpt-5.4-mini", "computer-use-preview"]);
+import { CAPTURE_SOURCES } from "./capture-sources.js";
 function extractActionableOutputItems(items) {
     const entries = [];
     for (const item of items) {
@@ -188,13 +188,14 @@ export class ExecutionEngine {
         this.screenshotDir = options.screenshotDir || null;
         this.screenshotRecorder = options.screenshotRecorder || null;
         this.strictMode = options.strictMode === true;
-        this.stepDelayMs = getConfiguredStepDelayMs();
+        this.stepDelayMs = options.stepDelayMs ?? getConfiguredStepDelayMs();
         this.reportedScreenshotWriteError = false;
         this.reportedScreenshotAnnotateError = false;
         this.lastRecordedScreenshotPaths = [];
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
+        this.reviseTestScript = options.reviseTestScript || null;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
     }
     async saveScreenshotArtifact(screenshotBase64, metadata = {}, options = {}) {
@@ -319,23 +320,51 @@ export class ExecutionEngine {
                 }));
             }
             const previewEventId = randomUUID();
-            await this.recordPreviewScreenshot(previewResult.screenshotBase64, {
+            const previewScreenshotResult = await this.recordPreviewScreenshot(previewResult.screenshotBase64, {
                 runId: context?.runId,
                 sessionId: context?.sessionId,
                 stepId: stepContext?.stepId,
                 instructionIndex: stepContext?.instructionIndex,
                 callId,
                 eventId: previewEventId,
-                captureSource: "preview-click",
+                captureSource: CAPTURE_SOURCES.PREVIEW_CLICK,
             });
+            const previewText = buildPreviewOutputText({
+                requestedX: previewResult.marker.requestedX,
+                requestedY: previewResult.marker.requestedY,
+                renderedX: previewResult.marker.x,
+                renderedY: previewResult.marker.y,
+                wasClamped: previewResult.marker.wasClamped,
+            });
+            const addOutput = context?.addOutput || null;
+            if (addOutput) {
+                const previewScreenshotPath = screenshotPathFromRecorderResult(previewScreenshotResult);
+                const previewScreenshotDataUrl = context?.includeScreenshotDataUrl
+                    ? `data:image/png;base64,${previewResult.screenshotBase64}`
+                    : null;
+                addOutput({
+                    type: "action",
+                    text: "Click",
+                    eventType: "tool_call",
+                    runId: context?.runId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex,
+                    payload: {
+                        platform: this.session.deviceInfo?.platform,
+                        toolName: "preview_click",
+                        captureSource: CAPTURE_SOURCES.PREVIEW_CLICK,
+                        requestedX: previewResult.marker.requestedX,
+                        requestedY: previewResult.marker.requestedY,
+                        renderedX: previewResult.marker.x,
+                        renderedY: previewResult.marker.y,
+                        wasClamped: previewResult.marker.wasClamped,
+                        ...(previewScreenshotPath ? { screenshotPath: previewScreenshotPath } : {}),
+                        ...(previewScreenshotDataUrl ? { screenshotDataUrl: previewScreenshotDataUrl } : {})
+                    }
+                });
+            }
             return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
-                text: buildPreviewOutputText({
-                    requestedX: previewResult.marker.requestedX,
-                    requestedY: previewResult.marker.requestedY,
-                    renderedX: previewResult.marker.x,
-                    renderedY: previewResult.marker.y,
-                    wasClamped: previewResult.marker.wasClamped,
-                }),
+                text: previewText,
                 annotatedScreenshotBase64: previewResult.screenshotBase64,
             }));
         }
@@ -465,7 +494,7 @@ export class ExecutionEngine {
                         stepId: stepContext?.stepId,
                         instructionIndex: stepContext?.instructionIndex,
                         eventId: actionEventId,
-                        captureSource: "action-click-annotation"
+                        captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION
                     });
                     const annotatedScreenshotPath = screenshotPathFromRecorderResult(annotatedScreenshotResult);
                     const annotatedScreenshotDataUrl = screenshotDataUrlFromRecorderResult(annotatedScreenshotResult);
@@ -475,6 +504,7 @@ export class ExecutionEngine {
                         stepId: stepContext?.stepId,
                         instructionIndex: stepContext?.instructionIndex,
                         eventId: actionEventId,
+                        captureSource: CAPTURE_SOURCES.ACTION_CLICK_ANNOTATION,
                         ...(annotatedScreenshotPath ? { screenshotPath: annotatedScreenshotPath } : {}),
                         ...(context?.includeScreenshotDataUrl && annotatedScreenshotDataUrl ? { screenshotDataUrl: annotatedScreenshotDataUrl } : {})
                     });
@@ -505,7 +535,7 @@ export class ExecutionEngine {
                 base64Length: screenshotBase64.length
             });
             const screenshotEventId = randomUUID();
-            const captureSource = sawExplicitScreenshotAction ? "call-output-explicit-action" : "call-output-post-action";
+            const captureSource = sawExplicitScreenshotAction ? CAPTURE_SOURCES.CALL_OUTPUT_EXPLICIT_ACTION : CAPTURE_SOURCES.CALL_OUTPUT_POST_ACTION;
             const screenshotResult = await this.recordScreenshot(screenshotBase64, {
                 runId: context?.runId,
                 sessionId: context?.sessionId,
@@ -549,9 +579,6 @@ export class ExecutionEngine {
                     note: runtimeNote
                 });
             }
-            const selectedCuaModel = VALID_CUA_MODELS.has(process.env.OPENAI_CUA_MODEL)
-                ? process.env.OPENAI_CUA_MODEL
-                : "gpt-5.4";
             const input = [{
                     type: "computer_call_output",
                     call_id,
@@ -559,9 +586,6 @@ export class ExecutionEngine {
                         type: "computer_screenshot",
                         image_url: `data:image/png;base64,${screenshotBase64}`,
                     },
-                    ...(selectedCuaModel === "computer-use-preview"
-                        ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
-                        : {}),
                     ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
                 }];
             if (runtimeNote) {

package/build/src/core/prompts.js

@@ -1,3 +1,3 @@
 export { buildBaseSystemPrompt } from "../prompts/base.js";
 export { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../prompts/design.js";
-export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";
+export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionNavigationRecoveryPrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";