@loadmill/droid-cua 2.3.0 → 2.5.0

package/README.md

@@ -177,6 +177,8 @@ Supported CLI options include:
 - `--context`
 - `--app-context-budget`
 - `--no-context`
+- `--base-prompt`
+- `--execution-prompt`
 - `--base-prompt-file`
 - `--execution-prompt-file`
 - `--record`
@@ -187,7 +189,18 @@ Config and precedence rules:
 - CLI flags override config file values.
 - `--context` overrides the config app-context path.
 - `--no-context` disables app context entirely.
+- `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
 - `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
+- If both inline and file-based prompt overrides are provided, the inline prompt flags win.
+
+Example without a config file:
+```sh
+droid-cua \
+  --avd adb:emulator-5554 \
+  --instructions tests/login.dcua \
+  --context tests/context.md \
+  --base-prompt "stop and look at the screen after every action you take."
+```
 
 Headless debug artifacts:
 - `--debug` writes desktop-style structured JSONL artifacts under `logs/`.

package/build/index.js

@@ -23,6 +23,11 @@ const recordScreenshots = args["record"] || false;
 const instructionsFile = args.instructions || args.i || null;
 const appContextPath = typeof args.context === "string" ? args.context : null;
 const debugMode = args["debug"] || false;
+const strictFlag = args.strict === true;
+const noStrictFlag = args["no-strict"] === true || args.strict === false;
+if (strictFlag && noStrictFlag) {
+    throw new Error("--strict and --no-strict cannot be used together.");
+}
 const screenshotDir = path.join("droid-cua-recording-" + Date.now());
 if (recordScreenshots)
     await mkdir(screenshotDir, { recursive: true });
@@ -132,7 +137,7 @@ async function main() {
                     status: "missing"
                 });
             }
-            const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextBriefing);
+            const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextBriefing, { strictMode: headlessConfig.strictMode });
             session.setSystemPrompt(executionPrompt);
             const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
                 runId,
@@ -142,11 +147,13 @@ async function main() {
             const engine = screenshotRecorder
                 ? new ExecutionEngine(session, {
                     recordScreenshots: true,
-                    screenshotRecorder
+                    screenshotRecorder,
+                    strictMode: headlessConfig.strictMode
                 })
                 : new ExecutionEngine(session, {
                     recordScreenshots,
                     screenshotDir,
+                    strictMode: headlessConfig.strictMode
                 });
             executionMode = new ExecutionMode(session, engine, instructions, true);
             const result = await executionMode.execute({
@@ -199,6 +206,7 @@ async function main() {
     const engine = new ExecutionEngine(session, {
         recordScreenshots,
         screenshotDir,
+        strictMode: strictFlag,
     });
     // Otherwise, start interactive Ink shell
     await startInkShell(session, engine, {

package/build/src/cli/headless-execution-config.js

@@ -23,6 +23,12 @@ function validateCuaModel(value, label) {
     }
     return value;
 }
+function parseBooleanFlagState(args, key) {
+    return {
+        enabled: args[key] === true,
+        disabled: args[`no-${key}`] === true || args[key] === false,
+    };
+}
 function parseBudgetValue(rawValue, label) {
     const numericValue = (() => {
         if (typeof rawValue === "number") {
@@ -102,6 +108,7 @@ async function loadConfigFromFile(configPath) {
         appContextEnabled: undefined,
         appContextBudget: undefined,
         appContextPath: undefined,
+        strictMode: undefined,
     };
     if ("cuaModel" in rawConfig) {
         normalized.cuaModel = validateCuaModel(rawConfig.cuaModel, "config.cuaModel");
@@ -122,18 +129,30 @@ async function loadConfigFromFile(configPath) {
         assertNonEmptyString(rawConfig.appContextPath, "config.appContextPath");
         normalized.appContextPath = path.resolve(configDir, rawConfig.appContextPath);
     }
+    if ("strictMode" in rawConfig) {
+        if (typeof rawConfig.strictMode !== "boolean") {
+            throw new Error("config.strictMode must be a boolean.");
+        }
+        normalized.strictMode = rawConfig.strictMode;
+    }
     return normalized;
 }
 export async function resolveHeadlessExecutionConfig(args, options = {}) {
     const cwd = typeof options.cwd === "string" ? options.cwd : process.cwd();
     const configPath = typeof args.config === "string" ? args.config : null;
     const explicitContextPath = typeof args.context === "string" ? path.resolve(cwd, args.context) : null;
-    const noContext = args["no-context"] === true;
+    const noContext = args["no-context"] === true || args.context === false;
+    const { enabled: strictEnabledByFlag, disabled: strictDisabledByFlag } = parseBooleanFlagState(args, "strict");
+    const basePromptText = typeof args["base-prompt"] === "string" ? args["base-prompt"] : null;
+    const executionPromptText = typeof args["execution-prompt"] === "string" ? args["execution-prompt"] : null;
     const basePromptFilePath = typeof args["base-prompt-file"] === "string" ? path.resolve(cwd, args["base-prompt-file"]) : null;
     const executionPromptFilePath = typeof args["execution-prompt-file"] === "string" ? path.resolve(cwd, args["execution-prompt-file"]) : null;
     if (explicitContextPath && noContext) {
         throw new Error("--context and --no-context cannot be used together.");
     }
+    if (strictEnabledByFlag && strictDisabledByFlag) {
+        throw new Error("--strict and --no-strict cannot be used together.");
+    }
     const fileConfig = configPath ? await loadConfigFromFile(configPath) : null;
     const promptCustomizations = {
         ...createEmptyPromptCustomizations(),
@@ -146,6 +165,7 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
         appContextEnabled: fileConfig?.appContextEnabled ?? true,
         appContextBudget: fileConfig?.appContextBudget ?? DEFAULT_APP_CONTEXT_BUDGET,
         appContextPath: fileConfig?.appContextPath || null,
+        strictMode: fileConfig?.strictMode ?? false,
     };
     if (typeof args["cua-model"] === "string") {
         resolved.cuaModel = validateCuaModel(args["cua-model"], "--cua-model");
@@ -159,6 +179,12 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
     if (executionPromptFilePath) {
         resolved.promptCustomizations.executionModeInstructions = await readTextFile(executionPromptFilePath, "--execution-prompt-file");
     }
+    if (basePromptText !== null) {
+        resolved.promptCustomizations.basePromptInstructions = basePromptText;
+    }
+    if (executionPromptText !== null) {
+        resolved.promptCustomizations.executionModeInstructions = executionPromptText;
+    }
     if (explicitContextPath) {
         resolved.appContextEnabled = true;
         resolved.appContextPath = explicitContextPath;
@@ -167,5 +193,11 @@ export async function resolveHeadlessExecutionConfig(args, options = {}) {
         resolved.appContextEnabled = false;
         resolved.appContextPath = null;
     }
+    if (strictEnabledByFlag) {
+        resolved.strictMode = true;
+    }
+    else if (strictDisabledByFlag) {
+        resolved.strictMode = false;
+    }
     return resolved;
 }

package/build/src/commands/help.js

@@ -24,8 +24,12 @@ export async function handleHelp(args, session, context) {
     addOutput({ type: 'info', text: '  --context <file>       Optional app context file used to brief execution runs' });
     addOutput({ type: 'info', text: '  --app-context-budget   Headless app context token budget override' });
     addOutput({ type: 'info', text: '  --no-context           Disable app context for headless execution' });
+    addOutput({ type: 'info', text: '  --base-prompt <text>   Headless base prompt customization text' });
+    addOutput({ type: 'info', text: '  --execution-prompt <text> Headless execution prompt customization text' });
     addOutput({ type: 'info', text: '  --base-prompt-file     Headless base prompt customization file' });
     addOutput({ type: 'info', text: '  --execution-prompt-file Headless execution prompt customization file' });
+    addOutput({ type: 'info', text: '  --strict               Strict Mode: re-observe after the first action in a chain' });
+    addOutput({ type: 'info', text: '  --no-strict            Disable Strict Mode for headless config-driven runs' });
     addOutput({ type: 'info', text: '  --record               Record screenshots during execution' });
     addOutput({ type: 'info', text: '  --debug                Enable structured JSONL debug artifacts' });
     addOutput({ type: 'info', text: '' });

package/build/src/commands/run.js

@@ -99,7 +99,7 @@ export async function handleRun(args, session, context) {
         }
     }
     // Set execution mode system prompt (replaces any design mode prompt)
-    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextBriefing);
+    const executionPrompt = buildExecutionModePrompt(session.deviceInfo, {}, appContextBriefing, { strictMode: Boolean(context.engine?.strictMode) });
     session.setSystemPrompt(executionPrompt);
     // Create execution mode
     const executionMode = new ExecutionMode(session, context.engine, instructions);

package/build/src/core/execution-engine.js

@@ -31,14 +31,68 @@ function extractComputerCalls(items) {
     }
     return entries;
 }
+function getScopeAndIds(context = null, stepContext = null) {
+    const scope = context?.sessionId ? "design" : "execution";
+    const ids = scope === "design"
+        ? {
+            sessionId: context?.sessionId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex
+        }
+        : {
+            runId: context?.runId,
+            stepId: stepContext?.stepId,
+            instructionIndex: stepContext?.instructionIndex
+        };
+    return { scope, ids };
+}
+function buildStrictModePlan(actions = [], strictMode = false) {
+    if (!strictMode || actions.length === 0) {
+        return {
+            actionsToExecute: actions,
+            droppedActions: [],
+            truncationReason: null,
+            runtimeNote: null
+        };
+    }
+    const actionsToExecute = [actions[0]];
+    const droppedActions = actions.slice(1);
+    if (droppedActions.length === 0) {
+        return {
+            actionsToExecute,
+            droppedActions,
+            truncationReason: null,
+            runtimeNote: null
+        };
+    }
+    const leadingActionType = actionsToExecute[0]?.type;
+    const truncationReason = leadingActionType === "screenshot"
+        ? "leading_screenshot_reobserve"
+        : "post_first_action_reobserve";
+    const droppedActionTypes = droppedActions
+        .map((action) => action?.type)
+        .filter(Boolean);
+    const runtimeNote = `Strict Mode: I executed only the first ${leadingActionType === "screenshot" ? "screenshot request" : "action"} from your previous chain and intentionally skipped the remaining ${droppedActions.length} action${droppedActions.length === 1 ? "" : "s"} (${droppedActionTypes.join(", ")}) so I could re-observe the device before continuing. Base your next step only on what is visible now.`;
+    return {
+        actionsToExecute,
+        droppedActions,
+        truncationReason,
+        runtimeNote
+    };
+}
 export class ExecutionEngine {
     constructor(session, options = {}) {
         this.session = session;
         this.recordScreenshots = options.recordScreenshots || false;
         this.screenshotDir = options.screenshotDir || null;
         this.screenshotRecorder = options.screenshotRecorder || null;
+        this.strictMode = options.strictMode === true;
         this.stepDelayMs = getConfiguredStepDelayMs();
         this.reportedScreenshotWriteError = false;
+        this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
+        this.handleModelAction = options.handleModelAction || handleModelAction;
+        this.sendCUARequest = options.sendCUARequest || sendCUARequest;
+        this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
     }
     async recordScreenshot(screenshotBase64, metadata = {}) {
         if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
@@ -89,6 +143,7 @@ export class ExecutionEngine {
         const addOutput = context?.addOutput || printCliOutput;
         let newResponseId = response.id;
         const shouldStop = () => Boolean(trackAction?.());
+        const { scope, ids } = getScopeAndIds(context, stepContext);
         const eventMeta = (extra = {}) => ({
             runId: context?.runId,
             stepId: stepContext?.stepId,
@@ -156,8 +211,9 @@ export class ExecutionEngine {
             for (const { call_id, actions } of computerCalls) {
                 if (!call_id)
                     continue;
+                const { actionsToExecute, droppedActions, truncationReason, runtimeNote } = buildStrictModePlan(actions, this.strictMode);
                 let sawExplicitScreenshotAction = false;
-                for (const action of actions) {
+                for (const action of actionsToExecute) {
                     if (shouldStop()) {
                         return newResponseId;
                     }
@@ -177,7 +233,7 @@ export class ExecutionEngine {
                         });
                     }
                     else {
-                        await handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
+                        await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
                             ...context,
                             shouldStop,
                             stepId: stepContext?.stepId,
@@ -201,7 +257,7 @@ export class ExecutionEngine {
                 if (shouldStop()) {
                     return newResponseId;
                 }
-                const screenshotBase64 = await getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+                const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
                 emitDesktopDebug("device.screenshot", "device", {
                     runId: context?.runId,
                     stepId: stepContext?.stepId,
@@ -221,6 +277,23 @@ export class ExecutionEngine {
                     callId: call_id,
                     captureSource: sawExplicitScreenshotAction ? "call-output-explicit-action" : "call-output-post-action"
                 });
+                if (runtimeNote) {
+                    const executedActionTypes = actionsToExecute
+                        .map((action) => action?.type)
+                        .filter(Boolean);
+                    const droppedActionTypes = droppedActions
+                        .map((action) => action?.type)
+                        .filter(Boolean);
+                    emitDesktopDebug("cua.strict_mode.truncation", scope, ids, {
+                        callId: call_id,
+                        executedCount: actionsToExecute.length,
+                        droppedCount: droppedActions.length,
+                        executedActionTypes,
+                        droppedActionTypes,
+                        reason: truncationReason,
+                        note: runtimeNote
+                    });
+                }
                 // Build next input: screenshot + any carryover reasoning
                 const selectedCuaModel = process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
                 const input = [{
@@ -231,19 +304,25 @@ export class ExecutionEngine {
                             image_url: `data:image/png;base64,${screenshotBase64}`,
                         },
                         ...(selectedCuaModel === "computer-use-preview"
-                            ? { current_url: getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
+                            ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
                             : {}),
                         ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
                     }];
+                if (runtimeNote) {
+                    input.push({
+                        role: "user",
+                        content: runtimeNote
+                    });
+                }
                 if (shouldStop()) {
                     return newResponseId;
                 }
-                response = await sendCUARequest({
+                response = await this.sendCUARequest({
                     messages: input,
                     previousResponseId: newResponseId,
                     deviceInfo: this.session.deviceInfo,
                     debugContext: {
-                        scope: context?.sessionId ? "design" : "execution",
+                        scope,
                         runId: context?.runId,
                         sessionId: context?.sessionId,
                         stepId: stepContext?.stepId,

package/build/src/core/prompts.js

@@ -1,279 +1,3 @@
-/**
- * System prompt templates for different modes
- */
-function buildCustomInstructionsSection(sections = []) {
-    const nonEmptySections = sections
-        .map((section) => ({
-        title: section?.title,
-        text: typeof section?.text === "string" ? section.text.trim() : ""
-    }))
-        .filter((section) => section.title && section.text);
-    if (nonEmptySections.length === 0) {
-        return "";
-    }
-    const renderedSections = nonEmptySections
-        .map((section) => `${section.title}:\n${section.text}`)
-        .join("\n\n");
-    return `USER CUSTOM INSTRUCTIONS:
-Follow these user-configured instructions in addition to the default behavior below.
-Prefer these custom instructions when deciding how to behave.
-
-${renderedSections}`;
-}
-function appendCustomSections(prompt, sections = []) {
-    const customSection = buildCustomInstructionsSection(sections);
-    if (!customSection) {
-        return prompt;
-    }
-    return `${prompt}
-
-${customSection}
-`;
-}
-function describeControlledDevice(deviceInfo = {}) {
-    const platform = typeof deviceInfo.platform === "string" ? deviceInfo.platform.trim().toLowerCase() : "";
-    const deviceName = typeof deviceInfo.device_name === "string" ? deviceInfo.device_name.trim() : "";
-    if (platform === "ios") {
-        return deviceName ? `an iOS simulator (${deviceName})` : "an iOS device";
-    }
-    if (platform === "android") {
-        return deviceName ? `an Android device (${deviceName})` : "an Android device";
-    }
-    return "a mobile device";
-}
-function buildAppContextSection(briefing) {
-    const text = typeof briefing === "string" ? briefing.trim() : "";
-    if (!text) {
-        return "";
-    }
-    return `APP CONTEXT BRIEFING:
-The following is a condensed description of the app you are testing, relevant to the current task.
-Use this to understand screen layouts, terminology, navigation, and expected behavior.
-
-${text}`;
-}
-export function buildBaseSystemPrompt(deviceInfo, customInstructions = {}) {
-    const controlledDevice = describeControlledDevice(deviceInfo);
-    const prompt = `
-  You are controlling ${controlledDevice} in a sandboxed testing environment.
-  Follow the user's instructions to interact with the device.
-
-  The device screen has been scaled down for display.
-  You can interact with any part of the visible phone screen, including system UI, browser UI, and app content.
-
-  The screen you see is ${deviceInfo.scaled_width} x ${deviceInfo.scaled_height} pixels.
-  Pixel (0,0) is at the top-left corner.
-
-  When aiming for visual targets:
-  - Reason carefully about the approximate pixel position.
-  - Click precisely based on your visual estimate.
-
-  Available actions: click, scroll, type, keypress, wait, screenshot.
-
-  CRITICAL - Mobile Input Constraints:
-  - This is a mobile device, not a desktop. Do NOT use desktop keyboard shortcuts or modifier chords.
-  - NEVER emit key combinations such as CTRL+A, CMD+A, CTRL+C, CTRL+V, ALT+TAB, SHIFT+ENTER, or similar shortcuts.
-  - Use 'keypress' only for a single mobile-safe key when absolutely necessary.
-  - To replace text, tap into the field and type the desired value. If correction is needed, use mobile-safe deletion only.
-  - Prefer tapping visible controls over hardware key events.
-  - Prefer on-screen navigation controls such as menus, tabs, drawer items, back arrows, close buttons, and explicit logout buttons over keypress actions.
-  - Do NOT use Back or ESC for normal app navigation when a reliable on-screen control is visible.
-  - Avoid using Back or ESC from a main or root screen, because it may leave the app.
-  - Exception: if the software keyboard is open and blocking the next needed control, Back or ESC may be used to dismiss the keyboard before continuing.
-  - Treat keypress actions as a fallback for limited cases only, such as a clearly needed single mobile-safe key or dismissing transient UI when no better visible control exists.
-
-  CRITICAL - Automatic Timing:
-  - After EVERY action (click, type, keypress, scroll), there is an automatic 500ms delay
-  - This 500ms is sufficient for normal UI updates and animations
-  - DO NOT add 'wait' actions unnecessarily - trust the automatic delay
-
-  CRITICAL - Mutating Actions:
-  - Mutating actions change app state. Examples: submit, create, save, confirm, approve, reject, login, logout, send, place order, initiate transfer
-  - Before tapping a mutating action button, dismiss the software keyboard first when it is open and not required for the tap
-  - After performing a mutating action once, do NOT repeat the same mutating action unless the UI clearly shows the first attempt failed or had no effect
-  - Treat visible state change as success. Examples: form fields clear, submit button returns to normal, status changes, list refreshes, new row appears, success message appears, screen changes
-  - For form submissions specifically, if the relevant fields clear and the action button returns to its normal idle state, treat that as success even if the new row or confirmation is not obvious yet
-  - If the UI shows signs that the mutating action succeeded, stop acting for that instruction
-
-  Use explicit 'wait' action ONLY in these specific cases:
-  1. After launching apps from home screen or app drawer
-  2. After pressing ENTER that triggers navigation (search, URL, form submit)
-  3. After clicking links that open new apps or pages
-  4. After actions that trigger heavy loading (camera, maps, etc.)
-
-  When you MUST wait:
-  - Click app icon from home → wait → Continue
-  - Type in search box → Press ENTER → wait → Continue
-  - Click link that opens new page/app → wait → Continue
-  - Open camera/maps/heavy feature → wait → Continue
-
-  When you should NOT wait (automatic 500ms handles it):
-  - Clicking UI buttons within a running app (click button - no wait needed)
-  - Typing in text fields (type text - no wait needed)
-  - Scrolling (scroll - no wait needed)
-  - Clicking tabs or menu items within an app (click - no wait needed)
-
-  Rule of thumb: Wait for app launches and navigation. Everything else has automatic timing.
-
-  Perform the user's requested actions within the current view.
-
-  If unsure about visual elements, take a screenshot to improve your reasoning.
-  If unsure about the user's intent, make the best decision you can based on context and continue automatically.
-
-  CRITICAL - Never Ask Questions:
-  - NEVER ask the user for confirmation, clarification, or next steps
-  - NEVER ask questions like "Should I...", "Would you like...", "Do you want me to..."
-  - NEVER wait for user guidance - make autonomous decisions
-  - If stuck, try alternative approaches (go back, try different UI element, restart app)
-  - ONLY stop when the task is complete or you've exhausted reasonable approaches
-
-  Act decisively to complete the task.
-
-  Stop acting once the task appears complete.
-  Only complete the current instruction. Do not proceed beyond the current step unless asked.
-
-  Mobile-Specific Notes:
-  - HOME key returns to the home screen
-  - On Android, ESC key maps to Back
-  - On iOS, ESC has no effect; use visible on-screen controls instead
-  - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
-  `;
-    return prompt;
-}
-export function buildDesignModePrompt(deviceInfo, customInstructions = {}) {
-    const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
-    const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
-    const prompt = `${basePrompt}
-
-DESIGN MODE:
-You are helping design a test script for an Android app.
-Some tests intentionally validate negative outcomes (errors, failures, rejected inputs). These are expected and should be treated as successful progress when they match the test goal.
-
-Your task:
-1. Understand what the user wants to test from their initial instruction
-2. Explore the app autonomously to understand the flows
-3. Take screenshots and interact as needed to discover the UI and behavior
-4. Once you've successfully completed the user's requested flow, immediately generate the test script
-
-CRITICAL - After Completing the Task:
-- DO NOT navigate back or away from the final screen
-- The final screen state is what matters for verification
-- Generate the test script immediately showing the current state
-- Use assertions to verify state, not navigation
-- "Check that it changed" means verify the current visual state, not navigate elsewhere
-- If the target validation state is visible (including expected error states), STOP actions and immediately output the final test script
-
-CRITICAL - Recognizing When You Are Stuck:
-If you find yourself:
-- Repeating similar actions multiple times (e.g., opening/closing the same app repeatedly)
-- Not reaching a new screen or state after several attempts
-- Unsure about a higher-level decision (which tab to use, which mode to enter, where to start)
-- Unable to find the UI element or feature the user mentioned
-
-THEN STOP ACTING IMMEDIATELY and ask the user for guidance:
-1. Briefly describe what you see on screen now
-2. Explain what you were trying to do and why you're stuck
-3. Ask a single, concrete question to unblock the next step
-
-Example:
-"Chrome is open but I don't see a search bar or new tab button. Should I open a new tab, or is there a specific way you'd like me to navigate?"
-
-DO NOT continue brute-forcing the UI when stuck. The user prefers being asked over watching repeated failed attempts.
-DO NOT ask if the user wants a script after successfully completing the flow - just generate it automatically.
-
-CRITICAL - Off-Screen Element Discovery:
-- If a required element is not visible, assume it may be off-screen before changing strategy
-- Humans naturally scroll when UI appears cropped; do the same
-- Use this discovery sequence before retries or fallback navigation:
-  1. Scroll the screen in the likely direction to reveal hidden content
-  2. If still missing, do one minimal fallback (e.g., close overlay or go back once), then retry discovery
-- Do not repeat already-successful actions while searching for an off-screen target
-
-CRITICAL - Test Script Format Rules:
-- One simple instruction per line (NO numbers, NO bullets)
-- Use imperative commands: "Open X", "Click Y", "Type Z"
-- Include "assert: <condition>" lines to validate expected behavior
-- Normalize validation wording into assertions:
-  - Convert "check", "verify", "ensure", "fetch", and "compare" intent into explicit "assert: ..." lines
-  - Do not leave standalone "Check ..." or "Verify ..." lines in the final script
-- Merge duplicate or near-duplicate validation lines into one clear assertion
-- End with "exit"
-- Keep it simple and executable
-- When you generate the final result, include a suggested test name before the script
-- The suggested test name must be very short: prefer 2 to 4 words
-- Focus on the main user goal, not every assertion or detail
-- The suggested test name must be lowercase, kebab-case, and filename-safe
-- Use this exact final format:
-Suggested test name: short-kebab-case-name
-
-\`\`\`
-<test script here>
-\`\`\`
-
-CORRECT Example:
-Suggested test name: calculator-addition
-
-\`\`\`
-Open Calculator app
-assert: Calculator app is visible
-Type "2"
-Click the plus button
-Type "3"
-Click the equals button
-assert: result shows 5
-exit
-\`\`\`
-
-WRONG Example (DON'T DO THIS):
-\`\`\`
-1. Open Calculator app
-2. Verify the app opened
-3. etc...
-\`\`\`
-
-Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.
-`;
-    return appendCustomSections(prompt, [
-        { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
-        { title: "Design Mode Instructions", text: designCustomText }
-    ]);
-}
-export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextBriefing = "") {
-    const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
-    const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
-    const appContextSection = buildAppContextSection(appContextBriefing);
-    const prompt = `${basePrompt}
-
-EXECUTION MODE - Critical Behavior:
-You are executing test script commands one at a time. This is NOT a conversation.
-
-CRITICAL RULES:
-- DO NOT generate conversational text or narration
-- DO NOT ask questions like "What should I do next?", "Would you like...", "Can I assist...?"
-- DO NOT describe what you see on screen
-- DO NOT say "Let me know if you need help" or similar phrases
-- Just execute the action silently and stop immediately
-- Only generate text if the action FAILED or cannot be completed
-- Never emit desktop keyboard shortcuts or modifier combos; mobile execution only supports mobile-safe single-key presses
-- Never repeat the same mutating action with the same apparent intent unless the UI clearly shows failure or no state change
-- If a submit/create/approve/reject/login action appears to succeed, stop instead of trying to reconfirm by doing it again
-- For form submissions, cleared fields plus a reset action button are strong success signals; stop even if the created item is not yet obvious in the visible list
-- If target is not visible, perform bounded off-screen discovery first:
-  1. Scroll the screen in the likely direction to reveal hidden controls
-  2. If still missing, do one minimal fallback (e.g., close overlay or go back once), then retry
-
-Your process:
-1. Read the instruction
-2. Execute the required actions
-3. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
-4. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
-5. Stop as soon as success is visible
-6. Stop immediately - no commentary, no questions
-
-Each instruction is independent. Do not reference previous instructions or ask about next steps.
-${appContextSection ? `\n\n${appContextSection}` : ""}`;
-    return appendCustomSections(prompt, [
-        { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
-        { title: "Execution Mode Instructions", text: executionCustomText }
-    ]);
-}
+export { buildBaseSystemPrompt } from "../prompts/base.js";
+export { buildDesignModePrompt, buildDesignRecoveryPrompt } from "../prompts/design.js";
+export { buildAppContextSection, buildAssertionSystemPrompt, buildExecutionModePrompt, buildExecutionRecoveryPrompt, } from "../prompts/execution.js";

package/build/src/device/android/actions.js

@@ -1,6 +1,7 @@
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 import { execAdb } from "./tools.js";
 function adbShell(deviceId, command) {
     return execAdb(["-s", deviceId, "shell", command]);
@@ -74,13 +75,16 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 await adbShell(deviceId, `input tap ${realX} ${realY}`);
                 break;
             case "scroll":
-                const scrollX = Math.round(action.scroll_x / scale);
-                const scrollY = Math.round(action.scroll_y / scale);
-                addOutput({ type: 'action', text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY, unit: 'px' }) });
-                const startX = 500;
-                const startY = 500;
-                const endX = startX + scrollX;
-                const endY = startY - scrollY; // <--- INVERT Y
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    fallbackStartX: 500,
+                    fallbackStartY: 500
+                });
+                addOutput({
+                    type: 'action',
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY})`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? 'action' : 'fallback', unit: 'px' })
+                });
                 await adbShell(deviceId, `input swipe ${startX} ${startY} ${endX} ${endY} 500`);
                 break;
             case "drag":