@loadmill/droid-cua 2.9.0 → 2.10.1-rc.8

package/build/src/prompts/base.js

@@ -27,16 +27,6 @@ function appendCustomSections(prompt, sections = []) {
 ${customSection}
 `;
 }
-function buildStrictModeRuntimeSection(runtimeOptions = {}) {
-    if (runtimeOptions?.strictMode !== true) {
-        return "";
-    }
-    return `STRICT MODE - Observation-First Runtime:
-- The runtime may execute only the first meaningful action from any multi-step action chain you propose.
-- After that first action, it will re-observe the ${runtimeOptions?.platform === "browser" ? "browser" : "device"} and continue from a fresh screenshot.
-- If you propose multiple steps, assume only the first may happen before the next turn.
-- Do not assume later proposed actions succeeded unless the next screenshot confirms them.`;
-}
 function isBrowserTarget(deviceInfo = {}) {
     return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
 }
@@ -55,7 +45,41 @@ function describeControlledTarget(deviceInfo = {}) {
     }
     return "a mobile device";
 }
-function buildBrowserBaseSystemPrompt(deviceInfo) {
+function buildAimAssistGuidance(targetKind, aimAssistMode = "always") {
+    if (aimAssistMode === "off") {
+        return [
+            "- Click directly based on your best visual estimate.",
+            "- If a click has no effect, treat it as a likely misclick and try a meaningfully different coordinate.",
+            "- Click precisely based on your visual estimate.",
+        ].join("\n  ");
+    }
+    if (aimAssistMode === "smart") {
+        return [
+            "- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
+            "- Review the returned annotated screenshot before clicking.",
+            `- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable target" : "tappable control"}.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
+            `- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
+            "- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
+            "- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
+            "- Click precisely based on your visual estimate.",
+            "- You may skip preview only when the target is broad, isolated, stable, and unambiguous, and you are confident the click point is clearly inside the intended target.",
+            "- Good skip examples are large empty text fields, large search fields, large CTA buttons with generous padding, launcher icons, and wide rows with generous spacing and no nearby competing actions.",
+            "- Always use preview for dropdowns, expanded option lists, popovers, select menus, dense lists or search results, bottom sheets or cards, modals, overlays, date or time pickers, calendar cells, wheels, segmented selectors, and small inline controls.",
+            "- Do not skip preview when the target is one of many similar options, close to neighboring controls, in a keyboard-driven flow, after a recent screen change, or whenever you are not confident.",
+        ].join("\n  ");
+    }
+    return [
+        "- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
+        "- Review the returned annotated screenshot before clicking.",
+        `- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable target" : "tappable control"}.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
+        `- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
+        "- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
+        "- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
+        "- Click precisely based on your visual estimate.",
+    ].join("\n  ");
+}
+function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
+    const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
     return `
   You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
   Follow the user's instructions to interact with the browser.
@@ -67,13 +91,7 @@ function buildBrowserBaseSystemPrompt(deviceInfo) {
 
   When aiming for visual targets:
   - Reason carefully about the approximate pixel position.
-  - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
-  - Review the returned annotated screenshot before clicking.
-  - Click if the marker is clearly inside the intended clickable target.
-  - Adjust and preview again only if the marker is outside the target, on the edge, or closer to a neighboring target than the intended one.
-  - If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
-  - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
-  - Click precisely based on your visual estimate.
+  ${buildAimAssistGuidance("browser", aimAssistMode)}
 
   Available actions: click, scroll, type, keypress, wait, screenshot.
 
@@ -122,14 +140,15 @@ function buildBrowserBaseSystemPrompt(deviceInfo) {
 
   Browser-Specific Notes:
   - You are operating a desktop browser, not a phone UI
-  - Direct URL navigation is handled by runner-owned \`navigate:\` lines when available; do not try to reach the browser address bar or fake URL entry with keyboard shortcuts
+  - Use the \`navigate\` tool for direct URL changes when available; do not try to reach the browser address bar or fake URL entry with keyboard shortcuts
   - ESC dismisses many overlays and dialogs
   - Cookie or consent banners may block the main task and may need dismissal first
   - Vertical scrolling is the default way to discover content below the fold
   - CTRL/CMD/ALT/OPTION/SHIFT may be used when they are the most reliable path
   `;
 }
-function buildMobileBaseSystemPrompt(deviceInfo) {
+function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
+    const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
     return `
   You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
   Follow the user's instructions to interact with the device.
@@ -142,13 +161,7 @@ function buildMobileBaseSystemPrompt(deviceInfo) {
 
   When aiming for visual targets:
   - Reason carefully about the approximate pixel position.
-  - Before every click action, call \`preview_click\` with the exact coordinates you intend to click.
-  - Review the returned annotated screenshot before clicking.
-  - Click if the marker is clearly inside the intended tappable control. Text inside a button, field, row, or bottom bar is okay.
-  - Adjust and preview again only if the marker is outside the target, on the target border/edge, or closer to a neighboring target than the intended one.
-  - If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.
-  - If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.
-  - Click precisely based on your visual estimate.
+  ${buildAimAssistGuidance("mobile", aimAssistMode)}
 
   Available actions: click, scroll, type, keypress, wait, screenshot.
 
@@ -209,9 +222,9 @@ function buildMobileBaseSystemPrompt(deviceInfo) {
   - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
   `;
 }
-function buildBaseSystemPrompt(deviceInfo) {
+function buildBaseSystemPrompt(deviceInfo, options = {}) {
     return isBrowserTarget(deviceInfo)
-        ? buildBrowserBaseSystemPrompt(deviceInfo)
-        : buildMobileBaseSystemPrompt(deviceInfo);
+        ? buildBrowserBaseSystemPrompt(deviceInfo, options)
+        : buildMobileBaseSystemPrompt(deviceInfo, options);
 }
-export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, buildStrictModeRuntimeSection, };
+export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, };

package/build/src/prompts/design.js

@@ -1,8 +1,7 @@
-import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
-export function buildDesignModePrompt(deviceInfo, customInstructions = {}, runtimeOptions = {}) {
+import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
+export function buildDesignModePrompt(deviceInfo, customInstructions = {}) {
     const designCustomText = typeof customInstructions.designModeInstructions === "string" ? customInstructions.designModeInstructions.trim() : "";
     const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
-    const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -14,15 +13,21 @@ Your task:
 1. Understand what the user wants to test from their initial instruction
 2. Explore the ${browserTarget ? "website" : "app"} autonomously to understand the flows
 3. Take screenshots and interact as needed to discover the UI and behavior
-4. Once you've successfully completed the user's requested flow, immediately generate the test script
+4. Once you've successfully completed the user's requested flow, call \`test_design_complete\` with a structured summary of what was proven
 
 CRITICAL - After Completing the Task:
 - DO NOT navigate back or away from the final screen
 - The final screen state is what matters for verification
-- Generate the test script immediately showing the current state
+- Call \`test_design_complete\` immediately once the requested flow has been proven
 - Use assertions to verify state, not navigation
 - "Check that it changed" means verify the current visual state, not navigate elsewhere
-- If the target validation state is visible (including expected error states), STOP actions and immediately output the final test script
+- If the target validation state is visible (including expected error states), STOP actions and call \`test_design_complete\`
+- If an assertion or validation PASS confirms the user goal, treat the task as complete
+- After a passing assertion confirms the goal, your very next action should be calling \`test_design_complete\`
+- Do NOT repeat the same assertion or validation on an unchanged screen after it has already passed
+- Do NOT keep browsing, screenshotting, or re-checking once the goal has been confirmed
+- Do NOT output the final test script before calling \`test_design_complete\`
+- After calling \`test_design_complete\`, stop exploring and wait for the final script-generation turn
 
 CRITICAL - Recognizing When You Are Stuck:
 If you find yourself:
@@ -40,7 +45,7 @@ Example:
 "Chrome is open but I don't see a search bar or new tab button. Should I open a new tab, or is there a specific way you'd like me to navigate?"
 
 DO NOT continue brute-forcing the UI when stuck. The user prefers being asked over watching repeated failed attempts.
-DO NOT ask if the user wants a script after successfully completing the flow - just generate it automatically.
+DO NOT ask if the user wants a script after successfully completing the flow - complete the handoff automatically.
 
 CRITICAL - Off-Screen Element Discovery:
 - If a required element is not visible, assume it may be off-screen before changing strategy
@@ -53,11 +58,11 @@ CRITICAL - Off-Screen Element Discovery:
 CRITICAL - Test Script Format Rules:
 - One simple instruction per line (NO numbers, NO bullets)
 - Use imperative commands: "Open X", "Click Y", "Type Z"
-- Include "assert: <condition>" lines to validate expected behavior
-- Normalize validation wording into assertions:
-  - Convert "check", "verify", "ensure", "fetch", and "compare" intent into explicit "assert: ..." lines
-  - Do not leave standalone "Check ..." or "Verify ..." lines in the final script
-- Merge duplicate or near-duplicate validation lines into one clear assertion
+- Use plain-language verification lines such as "Verify ..." or "Validate ..."
+- Normalize validation wording into plain-language verification:
+  - Convert "check", "assert", "ensure", "fetch", and "compare" intent into clear "Verify ..." or "Validate ..." lines
+  - Do not leave vague standalone "Check ..." lines in the final script
+- Merge duplicate or near-duplicate verification lines into one clear validation step
 - End with "exit"
 - Keep it simple and executable
 - When you generate the final result, include a suggested test name before the script
@@ -75,7 +80,7 @@ CORRECT Example:
 Suggested test name: ${browserTarget ? "docs-search" : "calculator-addition"}
 
 \`\`\`
-${browserTarget ? "navigate: https://docs.example.com\nType \"pricing\" in the search field\nPress Enter\nwait\nClick \"Pricing\"\nassert: Pricing page is visible" : "Open Calculator app\nassert: Calculator app is visible\nType \"2\"\nClick the plus button\nType \"3\"\nClick the equals button\nassert: result shows 5"}
+${browserTarget ? "Go to https://docs.example.com\nType \"pricing\" in the search field\nPress Enter\nwait\nClick \"Pricing\"\nVerify the Pricing page is visible" : "Open Calculator app\nVerify the Calculator app is visible\nType \"2\"\nClick the plus button\nType \"3\"\nClick the equals button\nVerify the result shows 5"}
 exit
 \`\`\`
 
@@ -87,7 +92,7 @@ WRONG Example (DON'T DO THIS):
 \`\`\`
 
 Remember: You are autonomous. Explore confidently. Generate simple, executable test scripts.
-` + (strictModeSection ? `\n\n${strictModeSection}` : "");
+`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
         { title: "Design Mode Instructions", text: designCustomText }
@@ -106,5 +111,40 @@ ${transcript}
 Original objective:
 ${objective ?? "(not provided)"}
 
-If the objective is already completed, generate the final test script now.`;
+If the objective is already completed, or a recent passing validation already confirmed the goal, call \`test_design_complete\` now.
+Do not repeat the same assertion on an unchanged screen.`;
+}
+function formatStringList(items = []) {
+    const normalized = Array.isArray(items)
+        ? items
+            .map((item) => (typeof item === "string" ? item.trim() : ""))
+            .filter(Boolean)
+        : [];
+    if (normalized.length === 0) {
+        return "- (none provided)";
+    }
+    return normalized.map((item) => `- ${item}`).join("\n");
+}
+export function buildDesignScriptGenerationPrompt(summary = {}) {
+    const suggestedTestName = typeof summary.suggestedTestName === "string" ? summary.suggestedTestName.trim() : "";
+    const goalSummary = typeof summary.goalSummary === "string" ? summary.goalSummary.trim() : "";
+    const finalState = typeof summary.finalState === "string" ? summary.finalState.trim() : "";
+    const notesForScript = typeof summary.notesForScript === "string" ? summary.notesForScript.trim() : "";
+    return `AUTHORING MODE:
+Test design exploration is complete. Use the validated completion summary below and the conversation so far to produce the final test script now.
+
+Do not call tools.
+Do not continue exploring.
+Do not add explanation.
+Output only the final result in the required design-mode format.
+
+Completion summary:
+Suggested test name: ${suggestedTestName || "(choose a short kebab-case name)"}
+Goal summary: ${goalSummary || "(not provided)"}
+Final state: ${finalState || "(not provided)"}
+Important steps:
+${formatStringList(summary.importantSteps)}
+Verification points:
+${formatStringList(summary.verificationPoints)}
+${notesForScript ? `Notes for script:\n${notesForScript}\n` : ""}`;
 }

package/build/src/prompts/editor.js

@@ -12,7 +12,7 @@ Apply the user's changes and output the revised test script.
 FORMAT RULES:
 - One simple instruction per line (NO numbers, NO bullets)
 - Use imperative commands: "Open X", "Click Y", "Type Z"
-- Include "assert: <condition>" lines to validate expected behavior
+- Use plain-language verification lines such as "Verify ..." or "Validate ..." for expected behavior
 - End with "exit"
 
 Output only the revised test script, nothing else.`;

package/build/src/prompts/execution.js

@@ -1,4 +1,5 @@
-import { appendCustomSections, buildBaseSystemPrompt, buildStrictModeRuntimeSection, isBrowserTarget } from "./base.js";
+import { DEFAULT_AIM_ASSIST_MODE } from "../core/aim-assist.js";
+import { appendCustomSections, buildBaseSystemPrompt, isBrowserTarget } from "./base.js";
 export function buildAppContextSection(text) {
     const trimmed = typeof text === "string" ? text.trim() : "";
     if (!trimmed) {
@@ -10,11 +11,11 @@ Use this to understand screen layouts, terminology, navigation, and expected beh
 
 ${trimmed}`;
 }
-export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", runtimeOptions = {}) {
+export function buildExecutionModePrompt(deviceInfo, customInstructions = {}, appContextText = "", options = {}) {
     const executionCustomText = typeof customInstructions.executionModeInstructions === "string" ? customInstructions.executionModeInstructions.trim() : "";
-    const basePrompt = buildBaseSystemPrompt(deviceInfo, customInstructions);
+    const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : DEFAULT_AIM_ASSIST_MODE;
+    const basePrompt = buildBaseSystemPrompt(deviceInfo, { aimAssistMode });
     const appContextSection = buildAppContextSection(appContextText);
-    const strictModeSection = buildStrictModeRuntimeSection(runtimeOptions);
     const browserTarget = isBrowserTarget(deviceInfo);
     const prompt = `${basePrompt}
 
@@ -38,14 +39,18 @@ CRITICAL RULES:
 
 Your process:
 1. Read the instruction
-2. Execute the required actions
-3. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
-4. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
-5. Stop as soon as success is visible
-6. Stop immediately - no commentary, no questions
+2. Choose the right tool when the instruction is really a higher-level capability:
+   - Use \`assert\` for assertions and verification intent, including legacy \`assert:\` lines and natural language like verify, validate, ensure, or confirm
+   - Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
+   - Use \`loadmill\` only when the instruction explicitly references the word "loadmill"
+3. Execute the required actions
+4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
+5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
+6. Stop as soon as success is visible
+7. Stop immediately - no commentary, no questions
 
 Each instruction is independent. Do not reference previous instructions or ask about next steps.
-${appContextSection ? `\n\n${appContextSection}` : ""}` + (strictModeSection ? `\n\n${strictModeSection}` : "");
+${appContextSection ? `\n\n${appContextSection}` : ""}`;
     return appendCustomSections(prompt, [
         { title: "Base Prompt Instructions", text: customInstructions.basePromptInstructions },
         { title: "Execution Mode Instructions", text: executionCustomText }

package/build/src/utils/cua-debug-tracer.js

@@ -191,7 +191,9 @@ export class CuaDebugTracer {
             ...(Number.isInteger(debugContext?.instructionIndex) ? { instructionIndex: debugContext.instructionIndex } : {})
         };
         const chainId = scope === "design" ? ids.sessionId : ids.runId;
-        const chainKey = `${scope}:${chainId || "unknown"}`;
+        const chainKey = typeof debugContext?.chainId === "string" && debugContext.chainId.trim()
+            ? debugContext.chainId.trim()
+            : `${scope}:${chainId || "unknown"}`;
         const previousPendingSafetyChecks = Array.isArray(this.lastPendingSafetyChecksByChain.get(chainKey))
             ? this.lastPendingSafetyChecksByChain.get(chainKey)
             : [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.9.0",
+  "version": "2.10.1-rc.8",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -23,9 +23,8 @@
     "test": "node --test unit-tests/*.test.js",
     "build": "tsc",
     "clean": "rm -rf build",
-    "prerelease": "npm run build",
-    "release:patch": "npm version patch && npm publish",
-    "postversion": "git push && git push --tags"
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run build"
   },
   "repository": {
     "type": "git",

package/build/src/device/screenshot-resolution.js

@@ -1,33 +0,0 @@
-export const TARGET_SCALED_WIDTH = 400;
-export const SCREENSHOT_RESOLUTION_MODE_DOWNSCALED = "downscaled";
-export const SCREENSHOT_RESOLUTION_MODE_NATIVE = "native";
-export function normalizeScreenshotResolutionMode(value) {
-    return value === SCREENSHOT_RESOLUTION_MODE_NATIVE
-        ? SCREENSHOT_RESOLUTION_MODE_NATIVE
-        : SCREENSHOT_RESOLUTION_MODE_DOWNSCALED;
-}
-export function validateScreenshotResolutionMode(value, label) {
-    if (typeof value !== "string") {
-        throw new Error(`${label} must be one of: downscaled, native.`);
-    }
-    const normalized = normalizeScreenshotResolutionMode(value);
-    if (normalized !== value) {
-        throw new Error(`${label} must be one of: downscaled, native.`);
-    }
-    return normalized;
-}
-export function buildResolutionAwareDeviceInfo({ width, height, screenshotResolutionMode, }) {
-    const normalizedMode = normalizeScreenshotResolutionMode(screenshotResolutionMode);
-    const scale = normalizedMode === SCREENSHOT_RESOLUTION_MODE_NATIVE || width <= TARGET_SCALED_WIDTH
-        ? 1.0
-        : TARGET_SCALED_WIDTH / width;
-    return {
-        scaled_width: Math.round(width * scale),
-        scaled_height: Math.round(height * scale),
-        scale,
-        screenshot_resolution_mode: normalizedMode,
-    };
-}
-export function readScreenshotResolutionModeFromEnv() {
-    return normalizeScreenshotResolutionMode(process.env.DROID_CUA_SCREENSHOT_RESOLUTION_MODE);
-}