@loadmill/droid-cua 2.10.0 → 2.10.1-rc.8

package/build/src/prompts/base.js

@@ -45,17 +45,6 @@ function describeControlledTarget(deviceInfo = {}) {
     }
     return "a mobile device";
 }
-function buildAlwaysAimAssistGuidance(targetKind) {
-    return [
-        "- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
-        "- Review the returned annotated screenshot before clicking.",
-        `- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable" : "tappable"} control.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
-        `- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
-        "- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
-        "- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
-        "- Click precisely based on your visual estimate.",
-    ];
-}
 function buildAimAssistGuidance(targetKind, aimAssistMode = "always") {
     if (aimAssistMode === "off") {
         return [
@@ -65,19 +54,29 @@ function buildAimAssistGuidance(targetKind, aimAssistMode = "always") {
         ].join("\n  ");
     }
     if (aimAssistMode === "smart") {
-        const targetWord = targetKind === "browser" ? "clickable" : "tappable";
         return [
-            ...buildAlwaysAimAssistGuidance(targetKind),
-            "- In Smart mode, you may skip preview only when the target is broad, isolated, stable, and unambiguous, and you are confident the click point is clearly inside the intended target.",
+            "- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
+            "- Review the returned annotated screenshot before clicking.",
+            `- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable target" : "tappable control"}.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
+            `- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
+            "- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
+            "- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
+            "- Click precisely based on your visual estimate.",
+            "- You may skip preview only when the target is broad, isolated, stable, and unambiguous, and you are confident the click point is clearly inside the intended target.",
             "- Good skip examples are large empty text fields, large search fields, large CTA buttons with generous padding, launcher icons, and wide rows with generous spacing and no nearby competing actions.",
-            `- Do not skip preview when the target is small, dense, visually ambiguous, close to neighboring controls, or you are not confident the marker will land clearly inside the intended ${targetWord} target.`,
-            "- If a dropdown, expanded option list, popover, or select menu is visible, always use preview before selecting an option.",
-            "- Do not skip preview in search results, bottom sheets, bottom cards, modals, overlays, keyboard-driven flows, or after recent screen changes.",
-            "- Do not skip preview for known tricky UI such as dense lists, date pickers, time pickers, calendar cells, wheels, segmented selectors, and small inline controls inside larger rows or cards.",
-            "- If the target is one option among many visually similar options, use preview before clicking.",
+            "- Always use preview for dropdowns, expanded option lists, popovers, select menus, dense lists or search results, bottom sheets or cards, modals, overlays, date or time pickers, calendar cells, wheels, segmented selectors, and small inline controls.",
+            "- Do not skip preview when the target is one of many similar options, close to neighboring controls, in a keyboard-driven flow, after a recent screen change, or whenever you are not confident.",
         ].join("\n  ");
     }
-    return buildAlwaysAimAssistGuidance(targetKind).join("\n  ");
+    return [
+        "- Before every click action, call `preview_click` with the exact coordinates you intend to click.",
+        "- Review the returned annotated screenshot before clicking.",
+        `- Click if the marker is clearly inside the intended ${targetKind === "browser" ? "clickable target" : "tappable control"}.${targetKind === "browser" ? "" : " Text inside a button, field, row, or bottom bar is okay."}`,
+        `- Adjust and preview again only if the marker is outside the target, on the ${targetKind === "browser" ? "edge" : "target border/edge"}, or closer to a neighboring target than the intended one.`,
+        "- If you preview more than once, the next click should use the most recent accepted preview coordinates unless the screen changed.",
+        "- If the click does not cause the expected UI change, treat it as a likely misclick and try a meaningfully different coordinate.",
+        "- Click precisely based on your visual estimate.",
+    ].join("\n  ");
 }
 function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
@@ -141,7 +140,7 @@ function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
 
   Browser-Specific Notes:
   - You are operating a desktop browser, not a phone UI
-  - Direct URL navigation is handled by runner-owned \`navigate:\` lines when available; do not try to reach the browser address bar or fake URL entry with keyboard shortcuts
+  - Use the \`navigate\` tool for direct URL changes when available; do not try to reach the browser address bar or fake URL entry with keyboard shortcuts
   - ESC dismisses many overlays and dialogs
   - Cookie or consent banners may block the main task and may need dismissal first
   - Vertical scrolling is the default way to discover content below the fold

package/build/src/prompts/design.js

@@ -13,15 +13,21 @@ Your task:
 1. Understand what the user wants to test from their initial instruction
 2. Explore the ${browserTarget ? "website" : "app"} autonomously to understand the flows
 3. Take screenshots and interact as needed to discover the UI and behavior
-4. Once you've successfully completed the user's requested flow, immediately generate the test script
+4. Once you've successfully completed the user's requested flow, call \`test_design_complete\` with a structured summary of what was proven
 
 CRITICAL - After Completing the Task:
 - DO NOT navigate back or away from the final screen
 - The final screen state is what matters for verification
-- Generate the test script immediately showing the current state
+- Call \`test_design_complete\` immediately once the requested flow has been proven
 - Use assertions to verify state, not navigation
 - "Check that it changed" means verify the current visual state, not navigate elsewhere
-- If the target validation state is visible (including expected error states), STOP actions and immediately output the final test script
+- If the target validation state is visible (including expected error states), STOP actions and call \`test_design_complete\`
+- If an assertion or validation PASS confirms the user goal, treat the task as complete
+- After a passing assertion confirms the goal, your very next action should be calling \`test_design_complete\`
+- Do NOT repeat the same assertion or validation on an unchanged screen after it has already passed
+- Do NOT keep browsing, screenshotting, or re-checking once the goal has been confirmed
+- Do NOT output the final test script before calling \`test_design_complete\`
+- After calling \`test_design_complete\`, stop exploring and wait for the final script-generation turn
 
 CRITICAL - Recognizing When You Are Stuck:
 If you find yourself:
@@ -39,7 +45,7 @@ Example:
 "Chrome is open but I don't see a search bar or new tab button. Should I open a new tab, or is there a specific way you'd like me to navigate?"
 
 DO NOT continue brute-forcing the UI when stuck. The user prefers being asked over watching repeated failed attempts.
-DO NOT ask if the user wants a script after successfully completing the flow - just generate it automatically.
+DO NOT ask if the user wants a script after successfully completing the flow - complete the handoff automatically.
 
 CRITICAL - Off-Screen Element Discovery:
 - If a required element is not visible, assume it may be off-screen before changing strategy
@@ -52,11 +58,11 @@ CRITICAL - Off-Screen Element Discovery:
 CRITICAL - Test Script Format Rules:
 - One simple instruction per line (NO numbers, NO bullets)
 - Use imperative commands: "Open X", "Click Y", "Type Z"
-- Include "assert: <condition>" lines to validate expected behavior
-- Normalize validation wording into assertions:
-  - Convert "check", "verify", "ensure", "fetch", and "compare" intent into explicit "assert: ..." lines
-  - Do not leave standalone "Check ..." or "Verify ..." lines in the final script
-- Merge duplicate or near-duplicate validation lines into one clear assertion
+- Use plain-language verification lines such as "Verify ..." or "Validate ..."
+- Normalize validation wording into plain-language verification:
+  - Convert "check", "assert", "ensure", "fetch", and "compare" intent into clear "Verify ..." or "Validate ..." lines
+  - Do not leave vague standalone "Check ..." lines in the final script
+- Merge duplicate or near-duplicate verification lines into one clear validation step
 - End with "exit"
 - Keep it simple and executable
 - When you generate the final result, include a suggested test name before the script
@@ -74,7 +80,7 @@ CORRECT Example:
 Suggested test name: ${browserTarget ? "docs-search" : "calculator-addition"}
 
 \`\`\`
-${browserTarget ? "navigate: https://docs.example.com\nType \"pricing\" in the search field\nPress Enter\nwait\nClick \"Pricing\"\nassert: Pricing page is visible" : "Open Calculator app\nassert: Calculator app is visible\nType \"2\"\nClick the plus button\nType \"3\"\nClick the equals button\nassert: result shows 5"}
+${browserTarget ? "Go to https://docs.example.com\nType \"pricing\" in the search field\nPress Enter\nwait\nClick \"Pricing\"\nVerify the Pricing page is visible" : "Open Calculator app\nVerify the Calculator app is visible\nType \"2\"\nClick the plus button\nType \"3\"\nClick the equals button\nVerify the result shows 5"}
 exit
 \`\`\`
 
@@ -105,5 +111,40 @@ ${transcript}
 Original objective:
 ${objective ?? "(not provided)"}
 
-If the objective is already completed, generate the final test script now.`;
+If the objective is already completed, or a recent passing validation already confirmed the goal, call \`test_design_complete\` now.
+Do not repeat the same assertion on an unchanged screen.`;
+}
+function formatStringList(items = []) {
+    const normalized = Array.isArray(items)
+        ? items
+            .map((item) => (typeof item === "string" ? item.trim() : ""))
+            .filter(Boolean)
+        : [];
+    if (normalized.length === 0) {
+        return "- (none provided)";
+    }
+    return normalized.map((item) => `- ${item}`).join("\n");
+}
+export function buildDesignScriptGenerationPrompt(summary = {}) {
+    const suggestedTestName = typeof summary.suggestedTestName === "string" ? summary.suggestedTestName.trim() : "";
+    const goalSummary = typeof summary.goalSummary === "string" ? summary.goalSummary.trim() : "";
+    const finalState = typeof summary.finalState === "string" ? summary.finalState.trim() : "";
+    const notesForScript = typeof summary.notesForScript === "string" ? summary.notesForScript.trim() : "";
+    return `AUTHORING MODE:
+Test design exploration is complete. Use the validated completion summary below and the conversation so far to produce the final test script now.
+
+Do not call tools.
+Do not continue exploring.
+Do not add explanation.
+Output only the final result in the required design-mode format.
+
+Completion summary:
+Suggested test name: ${suggestedTestName || "(choose a short kebab-case name)"}
+Goal summary: ${goalSummary || "(not provided)"}
+Final state: ${finalState || "(not provided)"}
+Important steps:
+${formatStringList(summary.importantSteps)}
+Verification points:
+${formatStringList(summary.verificationPoints)}
+${notesForScript ? `Notes for script:\n${notesForScript}\n` : ""}`;
 }

package/build/src/prompts/editor.js

@@ -12,7 +12,7 @@ Apply the user's changes and output the revised test script.
 FORMAT RULES:
 - One simple instruction per line (NO numbers, NO bullets)
 - Use imperative commands: "Open X", "Click Y", "Type Z"
-- Include "assert: <condition>" lines to validate expected behavior
+- Use plain-language verification lines such as "Verify ..." or "Validate ..." for expected behavior
 - End with "exit"
 
 Output only the revised test script, nothing else.`;

package/build/src/prompts/execution.js

@@ -39,11 +39,15 @@ CRITICAL RULES:
 
 Your process:
 1. Read the instruction
-2. Execute the required actions
-3. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
-4. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
-5. Stop as soon as success is visible
-6. Stop immediately - no commentary, no questions
+2. Choose the right tool when the instruction is really a higher-level capability:
+   - Use \`assert\` for assertions and verification intent, including legacy \`assert:\` lines and natural language like verify, validate, ensure, or confirm
+   - Use \`navigate\` for direct browser navigation instead of trying to manipulate the address bar
+   - Use \`loadmill\` only when the instruction explicitly references the word "loadmill"
+3. Execute the required actions
+4. Before tapping a mutating action, dismiss the keyboard if it is open and not needed
+5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation
+6. Stop as soon as success is visible
+7. Stop immediately - no commentary, no questions
 
 Each instruction is independent. Do not reference previous instructions or ask about next steps.
 ${appContextSection ? `\n\n${appContextSection}` : ""}`;

package/build/src/utils/cua-debug-tracer.js

@@ -191,7 +191,9 @@ export class CuaDebugTracer {
             ...(Number.isInteger(debugContext?.instructionIndex) ? { instructionIndex: debugContext.instructionIndex } : {})
         };
         const chainId = scope === "design" ? ids.sessionId : ids.runId;
-        const chainKey = `${scope}:${chainId || "unknown"}`;
+        const chainKey = typeof debugContext?.chainId === "string" && debugContext.chainId.trim()
+            ? debugContext.chainId.trim()
+            : `${scope}:${chainId || "unknown"}`;
         const previousPendingSafetyChecks = Array.isArray(this.lastPendingSafetyChecksByChain.get(chainKey))
             ? this.lastPendingSafetyChecksByChain.get(chainKey)
             : [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.10.0",
+  "version": "2.10.1-rc.8",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -23,9 +23,8 @@
     "test": "node --test unit-tests/*.test.js",
     "build": "tsc",
     "clean": "rm -rf build",
-    "prerelease": "npm run build",
-    "release:patch": "npm version patch && npm publish",
-    "postversion": "git push && git push --tags"
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run build"
   },
   "repository": {
     "type": "git",

package/build/src/device/screenshot-resolution.js

@@ -1,33 +0,0 @@
-export const TARGET_SCALED_WIDTH = 400;
-export const SCREENSHOT_RESOLUTION_MODE_DOWNSCALED = "downscaled";
-export const SCREENSHOT_RESOLUTION_MODE_NATIVE = "native";
-export function normalizeScreenshotResolutionMode(value) {
-    return value === SCREENSHOT_RESOLUTION_MODE_NATIVE
-        ? SCREENSHOT_RESOLUTION_MODE_NATIVE
-        : SCREENSHOT_RESOLUTION_MODE_DOWNSCALED;
-}
-export function validateScreenshotResolutionMode(value, label) {
-    if (typeof value !== "string") {
-        throw new Error(`${label} must be one of: downscaled, native.`);
-    }
-    const normalized = normalizeScreenshotResolutionMode(value);
-    if (normalized !== value) {
-        throw new Error(`${label} must be one of: downscaled, native.`);
-    }
-    return normalized;
-}
-export function buildResolutionAwareDeviceInfo({ width, height, screenshotResolutionMode, }) {
-    const normalizedMode = normalizeScreenshotResolutionMode(screenshotResolutionMode);
-    const scale = normalizedMode === SCREENSHOT_RESOLUTION_MODE_NATIVE || width <= TARGET_SCALED_WIDTH
-        ? 1.0
-        : TARGET_SCALED_WIDTH / width;
-    return {
-        scaled_width: Math.round(width * scale),
-        scaled_height: Math.round(height * scale),
-        scale,
-        screenshot_resolution_mode: normalizedMode,
-    };
-}
-export function readScreenshotResolutionModeFromEnv() {
-    return normalizeScreenshotResolutionMode(process.env.DROID_CUA_SCREENSHOT_RESOLUTION_MODE);
-}