@loadmill/droid-cua 2.33.0 → 2.35.0

package/build/src/device/openai.js

@@ -103,7 +103,7 @@ const closeCurrentTabTool = {
 const closeKeyboardTool = {
     type: "function",
     name: "close_keyboard",
-    description: "Close the mobile software keyboard after text entry. Call this when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden. Do not call it for a slim hardware-keyboard toolbar or suggestions alone. On cloud iOS, optionally provide x and y for a dismissal control or clearly noninteractive background that you trust; never target Return, Go, Search, Next, Continue, Send, or an app action.",
+    description: "Close the mobile software keyboard after text entry. Call this when a full software keyboard is visible. Do not call it for a slim hardware-keyboard toolbar or suggestions alone.",
     strict: false,
     parameters: {
         type: "object",
@@ -115,6 +115,18 @@ const closeKeyboardTool = {
         additionalProperties: false,
     },
 };
+const iosCloseKeyboardTool = {
+    ...closeKeyboardTool,
+    description: "Close the iOS software keyboard after text entry by tapping a safe dismissal point from the current screenshot. Call this when a full software keyboard is visible, or after typing into a secure field even when its keyboard is hidden. Prefer a dedicated keyboard dismissal control or clearly noninteractive background. Never target Return, Go, Search, Next, Continue, Send, or an app action.",
+    parameters: {
+        ...closeKeyboardTool.parameters,
+        properties: {
+            x: { type: "number", description: "Horizontal coordinate of the dismissal point in the current screenshot." },
+            y: { type: "number", description: "Vertical coordinate of the dismissal point in the current screenshot." },
+        },
+        required: ["x", "y"],
+    },
+};
 const loadmillTool = {
     type: "function",
     name: "loadmill",
@@ -143,6 +155,34 @@ const typeSensitiveTextTool = {
         additionalProperties: false,
     },
 };
+const typeTextSlowlyTool = {
+    type: "function",
+    name: "type_text_slowly",
+    description: "Type literal text into the currently focused iOS field using the slower XCTest compatibility input method. Use this when app context says normal typing may drop characters, or after a screenshot shows incomplete input. This method can still drop characters, so inspect the returned screenshot. It does not clear existing text; clear the field first when retrying.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            text: { type: "string" },
+        },
+        required: ["text"],
+        additionalProperties: false,
+    },
+};
+const typeSecretSlowlyTool = {
+    type: "function",
+    name: "type_secret_slowly",
+    description: "Type a listed secret into the currently focused iOS field using the slower XCTest compatibility input method. Use this when app context says normal typing may drop characters, or when separate evidence shows normal secret entry failed. This method can still drop characters and does not clear existing text; clear the field first when retrying.",
+    strict: true,
+    parameters: {
+        type: "object",
+        properties: {
+            secretKey: { type: "string" },
+        },
+        required: ["secretKey"],
+        additionalProperties: false,
+    },
+};
 // Design completion is the handoff into script generation; it is separate from
 // execution step-completion markers used by model adapters.
 const testDesignCompleteTool = {
@@ -177,7 +217,7 @@ const testDesignCompleteTool = {
         additionalProperties: false,
     },
 };
-export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, toolCapabilities = {}, } = {}) {
+export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, toolCapabilities = {}, deviceInfo = null, } = {}) {
     const tools = [];
     if (aimAssistMode !== "off" && toolCapabilities.previewClick !== false) {
         tools.push(previewClickTool);
@@ -192,7 +232,7 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
         tools.push(closeCurrentTabTool);
     }
     if (toolCapabilities.closeKeyboard) {
-        tools.push(closeKeyboardTool);
+        tools.push(deviceInfo?.platform?.toLowerCase() === "ios" ? iosCloseKeyboardTool : closeKeyboardTool);
     }
     if (toolCapabilities.loadmill) {
         tools.push(loadmillTool);
@@ -200,6 +240,12 @@ export function buildFunctionTools({ aimAssistMode = DEFAULT_AIM_ASSIST_MODE, to
     if (toolCapabilities.typeSensitiveText) {
         tools.push(typeSensitiveTextTool);
     }
+    if (toolCapabilities.typeTextSlowly) {
+        tools.push(typeTextSlowlyTool);
+    }
+    if (toolCapabilities.typeTextSlowly && toolCapabilities.typeSensitiveText) {
+        tools.push(typeSecretSlowlyTool);
+    }
     if (toolCapabilities.testDesignComplete) {
         tools.push(testDesignCompleteTool);
     }
@@ -333,7 +379,7 @@ export function buildCuaRequestParams({ cuaModel, aimAssistMode = DEFAULT_AIM_AS
         truncation: "auto",
         tools: [
             ...(includeComputerTool ? [buildComputerTool({ cuaModel, deviceInfo })] : []),
-            ...buildFunctionTools({ aimAssistMode, toolCapabilities }),
+            ...buildFunctionTools({ aimAssistMode, toolCapabilities, deviceInfo }),
         ],
     };
 }

package/build/src/modes/design-mode-ink.js

@@ -42,6 +42,7 @@ export class DesignModeInk {
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
             closeKeyboard: this.session?.deviceInfo?.platform !== "browser",
+            typeTextSlowly: this.session?.deviceInfo?.platform === "ios",
             loadmill: false,
             typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,

package/build/src/modes/design-mode.js

@@ -39,6 +39,7 @@ export class DesignMode {
             navigate: this.session?.deviceInfo?.platform === "browser",
             closeCurrentTab: this.session?.deviceInfo?.platform === "browser",
             closeKeyboard: this.session?.deviceInfo?.platform !== "browser",
+            typeTextSlowly: this.session?.deviceInfo?.platform === "ios",
             loadmill: false,
             typeSensitiveText: this.engine.hasSecrets(),
             testDesignComplete: true,

package/build/src/modes/execution-mode.js

@@ -96,6 +96,7 @@ export class ExecutionMode {
             navigate: this.isBrowserSession,
             closeCurrentTab: this.isBrowserSession,
             closeKeyboard: !this.isBrowserSession,
+            typeTextSlowly: this.session?.deviceInfo?.platform === "ios",
             loadmill: true,
             typeSensitiveText: this.engine.hasSecrets(),
         };

package/build/src/prompts/base.js

@@ -30,6 +30,9 @@ ${customSection}
 function isBrowserTarget(deviceInfo = {}) {
     return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "browser";
 }
+function isDesktopTarget(deviceInfo = {}) {
+    return typeof deviceInfo.platform === "string" && deviceInfo.platform.trim().toLowerCase() === "desktop";
+}
 function isCloudIosTarget(deviceInfo = {}) {
     const platform = deviceInfo.platform?.trim().toLowerCase();
     const source = deviceInfo.source?.trim().toLowerCase() || "local";
@@ -42,6 +45,9 @@ function describeControlledTarget(deviceInfo = {}) {
     if (platform === "browser") {
         return browserName ? `a browser session (${browserName})` : "a browser session";
     }
+    if (platform === "desktop") {
+        return deviceName ? `a desktop application (${deviceName})` : "a desktop application";
+    }
     if (platform === "ios") {
         const target = isCloudIosTarget(deviceInfo) ? "an iOS device" : "an iOS simulator";
         return deviceName ? `${target} (${deviceName})` : target;
@@ -156,11 +162,10 @@ function buildBrowserBaseSystemPrompt(deviceInfo, options = {}) {
 }
 function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
     const aimAssistMode = typeof options.aimAssistMode === "string" ? options.aimAssistMode : "always";
-    const cloudIosKeyboardGuidance = isCloudIosTarget(deviceInfo) ? `
-  - On this cloud iOS device, \`close_keyboard\` can optionally tap a point you choose from the current screenshot.
-  - If you can identify a dedicated keyboard dismissal control or clearly noninteractive background that you trust, pass its \`x\` and \`y\` coordinates to \`close_keyboard\`.
+    const iosKeyboardGuidance = deviceInfo.platform?.trim().toLowerCase() === "ios" ? `
+  - On iOS, \`close_keyboard\` requires \`x\` and \`y\` coordinates from the current screenshot and taps that point.
+  - Prefer a dedicated keyboard dismissal control. Otherwise choose a clearly noninteractive background that you trust.
   - Never use Return, Go, Search, Next, Continue, Send, or an app action as the dismissal point because it may submit or mutate the app.
-  - If dismissal is required but no safe point is identifiable, call \`close_keyboard\` without coordinates. Do not guess.
   - Choose coordinates directly from the post-type screenshot; do not call \`preview_click\` between \`type\` and \`close_keyboard\`.` : "";
     return `
   You are controlling ${describeControlledTarget(deviceInfo)} in a sandboxed testing environment.
@@ -188,14 +193,14 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
   - If a full software keyboard is visibly covering part of the screen, your next action must be \`close_keyboard\`.
   - Do not finish the current instruction while a required \`close_keyboard\` action is still pending.
   - A slim hardware-keyboard toolbar, autocomplete suggestions, or a focused field without a full software keyboard does not require \`close_keyboard\`.
-  - Treat \`type_sensitive_text\` as text entry too and apply the same dismissal decision after it returns.
+  - Treat \`type_sensitive_text\`, \`type_text_slowly\`, and \`type_secret_slowly\` as text entry too and apply the same dismissal decision after any of them returns.
   - Whenever dismissal is required, do not click, scroll, inspect, submit, type again, issue a keypress, or call another tool before \`close_keyboard\`.
   - Consecutive DELETE or BACKSPACE keypresses may be used for correction without closing the keyboard between them. After typing the replacement, call \`close_keyboard\`.
   - Password and other secure fields may appear empty in screenshots even after text was entered successfully.
   - On iOS, a secure field may also hide an open keyboard from Appium screenshots. After typing into an iOS secure field, call \`close_keyboard\` even when the keyboard is not visible.
   - Do not type into a secure field again solely because its characters or bullets are not visible.
   - Retry secure-field input only when there is separate evidence of failure, such as another field changing or the app rejecting the value.
-  ${cloudIosKeyboardGuidance}
+  ${iosKeyboardGuidance}
   - Prefer tapping visible controls over hardware key events.
   - Prefer on-screen navigation controls such as menus, tabs, drawer items, back arrows, close buttons, and explicit logout buttons over keypress actions.
   - Do NOT use Back or ESC for normal app navigation when a reliable on-screen control is visible.
@@ -249,9 +254,37 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
   - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action
   `;
 }
+function buildDesktopBaseSystemPrompt(deviceInfo, options = {}) {
+    let browserPrompt = buildBrowserBaseSystemPrompt(deviceInfo, options)
+        .replace("Follow the user's instructions to interact with the browser.", "Follow the user's instructions to interact with the selected application.")
+        .replace("visible browser window", "selected application window")
+        .replace("CRITICAL - Desktop Browser Input Constraints:", "CRITICAL - Desktop Input Constraints:")
+        .replace("This is a desktop browser environment.", "This is a desktop application environment.");
+    if (deviceInfo.desktopInputMode !== "physical") {
+        browserPrompt = browserPrompt
+            .replace("Available actions: click, scroll, type, keypress, wait, screenshot.", "Available actions: click, scroll, wait, screenshot.")
+            .replace("Standard desktop keys and modifier chords are allowed when they are clearly useful.", "Keyboard input is unavailable because it would focus the application.")
+            .replace("Prefer clicking visible controls over keyboard shortcuts when both are equally reliable.", "Use visible application controls for every interaction.")
+            .replace("Use keypress chords only when they directly help the task, such as Enter to submit, Escape to dismiss a modal, or Control/Meta shortcuts for focused text fields.", "Use visible application controls for submit, dismiss, and correction actions.");
+        browserPrompt = browserPrompt.replace("After EVERY action (click, type, keypress, scroll)", "After EVERY action (click, scroll)");
+    }
+    const notesIndex = browserPrompt.indexOf("  Browser-Specific Notes:");
+    const withoutBrowserNotes = notesIndex === -1 ? browserPrompt : browserPrompt.slice(0, notesIndex);
+    const inputNote = deviceInfo.desktopInputMode === "physical"
+        ? "Droid may use the physical OS pointer for this run"
+        : "Cursor-free mode never focuses the application: use only clicks and scrolling on controls with matching accessibility actions; never emit type, keypress, move, hover, or drag";
+    return `${withoutBrowserNotes}  Desktop-Specific Notes:
+  - Droid is controlling only the selected application window, not the whole display
+  - ${inputNote}
+  - Do not use shortcuts that switch applications, virtual desktops, or operating-system context
+  - Dialogs owned by the selected application may be controlled when they become its active window
+  `;
+}
 function buildBaseSystemPrompt(deviceInfo, options = {}) {
-    return isBrowserTarget(deviceInfo)
-        ? buildBrowserBaseSystemPrompt(deviceInfo, options)
-        : buildMobileBaseSystemPrompt(deviceInfo, options);
+    if (isBrowserTarget(deviceInfo))
+        return buildBrowserBaseSystemPrompt(deviceInfo, options);
+    if (isDesktopTarget(deviceInfo))
+        return buildDesktopBaseSystemPrompt(deviceInfo, options);
+    return buildMobileBaseSystemPrompt(deviceInfo, options);
 }
-export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, };
+export { appendCustomSections, buildBaseSystemPrompt, buildCustomInstructionsSection, describeControlledTarget, isBrowserTarget, isDesktopTarget, };

package/build/src/prompts/execution.js

@@ -32,6 +32,7 @@ export function buildSecretKeysContextSection(keys = []) {
     return `SECRET KEYS FOR THIS RUN:
 The following secret keys are available. You can see only the keys, not the values.
 Use the \`type_sensitive_text\` tool when you need to enter passwords, OTPs, tokens, API keys, or other hidden credentials.
+When the iOS \`type_secret_slowly\` tool is available and slow typing is needed, pass it the matching \`secretKey\`.
 Never ask the user to reveal these values and never type placeholder text when a matching secret key is available.
 
 ${normalized.map((key) => `- ${key}`).join("\n")}`;
@@ -80,7 +81,7 @@ ${browserTarget ? "" : `   - Every \`type\` action must end its computer-action
    - Call \`close_keyboard\` next when a full software keyboard is visible, or after typing into an iOS secure field even when its keyboard is hidden
    - Do not complete the current instruction while that required \`close_keyboard\` action is pending
    - Otherwise, including when only a slim hardware-keyboard toolbar or suggestions are visible, continue without \`close_keyboard\`
-   - Apply the same dismissal decision after \`type_sensitive_text\`
+   - Apply the same dismissal decision after \`type_sensitive_text\`, \`type_text_slowly\`, and \`type_secret_slowly\`
 `}
    - Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
 3. Execute the required actions

package/build/src/prompts/test-data-import.js

@@ -17,8 +17,8 @@ export function buildTestDataImportMessages({ currentContent = "", importedFileN
                 "- Do not invent values that are not present in the current file, imported file, or user instructions.",
                 "- If currentContent is template-only, replace it with real imported data and do not keep commented template examples.",
                 "- If currentContent is not template-only, preserve its useful structure and add or update entities within it.",
-                "- For matrix-like CSV rows, prefer splitting repeated dimensions into separate entities when the structure is clear.",
-                "- If rows are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
+                "- For tabular or nested data, preserve meaningful groups and relationships. Split repeated dimensions into separate entities when the structure is clear.",
+                "- If records are indivisible scenarios or examples, keep them grouped as Test Data Entries or Use Case Data.",
                 "- User instructions can guide structure and naming, but content must remain concrete data, not test instructions.",
                 "- Output markdown only inside the JSON content string; do not use code fences.",
             ].join("\n")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.33.0",
+  "version": "2.35.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -12,6 +12,7 @@
     "bin/**/*",
     "skill/**/*",
     "README.md",
+    "THIRD_PARTY_NOTICES.md",
     "LICENSE"
   ],
   "scripts": {
@@ -63,6 +64,7 @@
   "license": "UNLICENSED",
   "dependencies": {
     "@aws-sdk/client-device-farm": "^3.1065.0",
+    "@zavora-ai/computer-use-mcp": "7.1.0",
     "dotenv": "^16.4.7",
     "ink": "^4.4.1",
     "ink-spinner": "^5.0.0",
@@ -78,6 +80,6 @@
     "typescript": "^5.7.3"
   },
   "engines": {
-    "node": ">=18.17.0"
+    "node": ">=20.0.0"
   }
 }