@loadmill/droid-cua 2.37.0 → 2.38.0

package/build/src/device/cloud/actions.js

@@ -3,7 +3,7 @@ import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { buildActionEventMeta } from "../action-event-meta.js";
 import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
-import { buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
+import { buildIosAppSwitcherGesture, buildMobileKeypressPayload, getCloudAndroidTransport, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { resolveScrollGesture } from "../scroll-gesture.js";
 import { closeAppiumKeyboard, closeAppiumKeyboardWithTap } from "../keyboard.js";
@@ -149,6 +149,15 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                         });
                         await session.client.pressIosButton(session.sessionId, transport.transportValue);
                     }
+                    else if (transport.transportType === "ios-gesture") {
+                        const gesture = buildIosAppSwitcherGesture(await session.client.getWindowRect(session.sessionId));
+                        addOutput({
+                            type: "action",
+                            text: normalized.logText,
+                            ...meta(buildMobileKeypressPayload(normalized, transport, { gesture }))
+                        });
+                        await session.client.dragFromToForDuration(session.sessionId, gesture);
+                    }
                     else {
                         addOutput({
                             type: "action",

package/build/src/device/cloud/cloud-appium-client.js

@@ -175,6 +175,12 @@ export class CloudAppiumClient {
             ]
         });
     }
+    async dragFromToForDuration(sessionId, gesture) {
+        await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: dragFromToForDuration",
+            args: [gesture]
+        });
+    }
     async pressIosButton(sessionId, buttonName) {
         await this.request("POST", `/session/${sessionId}/execute/sync`, {
             script: "mobile: pressButton",

package/build/src/device/ios/actions.js

@@ -5,7 +5,7 @@
  */
 import * as appium from "./appium-client.js";
 import { buildActionEventMeta } from "../action-event-meta.js";
-import { buildMobileKeypressPayload, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
+import { buildIosAppSwitcherGesture, buildMobileKeypressPayload, getIosTransport, normalizeMobileKeypress } from "../mobile-keypress.js";
 import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../sensitive-action.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { logger } from "../../utils/logger.js";
@@ -151,6 +151,15 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
                     });
                     await appium.pressButton(session.sessionId, transport.transportValue);
                 }
+                else if (transport.transportType === "ios-gesture") {
+                    const gesture = buildIosAppSwitcherGesture(await appium.getWindowSize(session.sessionId));
+                    addOutput({
+                        type: "action",
+                        text: normalized.logText,
+                        ...meta(buildMobileKeypressPayload(normalized, transport, { gesture }))
+                    });
+                    await appium.dragFromToForDuration(session.sessionId, gesture);
+                }
                 else {
                     addOutput({
                         type: "action",

package/build/src/device/ios/appium-client.js

@@ -206,6 +206,12 @@ export async function drag(sessionId, startX, startY, endX, endY, duration = 500
         ],
     });
 }
+export async function dragFromToForDuration(sessionId, gesture) {
+    await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: dragFromToForDuration",
+        args: [gesture],
+    });
+}
 /**
  * Press a device button (home, volumeUp, volumeDown)
  * @param {string} sessionId

package/build/src/device/mobile-keypress.js

@@ -15,6 +15,7 @@ export function normalizeMobileKeypress(platform, keys = []) {
             ESC: { semanticKey: "IGNORE_ESC", logText: "Ignoring Escape on iOS" },
             ESCAPE: { semanticKey: "IGNORE_ESC", logText: "Ignoring Escape on iOS" },
             HOME: { semanticKey: "HOME", logText: "Pressing Home" },
+            APP_SWITCH: { semanticKey: "APP_SWITCH", logText: "Opening App Switcher" },
             ENTER: { semanticKey: "ENTER", logText: "Pressing Enter" },
             RETURN: { semanticKey: "ENTER", logText: "Pressing Enter" },
             BACKSPACE: { semanticKey: "BACKSPACE", logText: "Pressing Backspace" },
@@ -98,6 +99,8 @@ export function getIosTransport(normalized) {
             return { transportType: "noop", transportValue: null, ignored: true };
         case "HOME":
             return { transportType: "ios-button", transportValue: "home" };
+        case "APP_SWITCH":
+            return { transportType: "ios-gesture", transportValue: "mobile: dragFromToForDuration" };
         case "ENTER":
             return { transportType: "text", transportValue: "\n" };
         case "BACKSPACE":
@@ -108,6 +111,16 @@ export function getIosTransport(normalized) {
             throw new Error(`Unsupported iOS semantic key: ${normalized.semanticKey}`);
     }
 }
+export function buildIosAppSwitcherGesture({ width, height }) {
+    const centerX = Math.round(width / 2);
+    return {
+        duration: 1,
+        fromX: centerX,
+        fromY: Math.round(height) - 5,
+        toX: centerX,
+        toY: Math.round(height / 2)
+    };
+}
 export function buildMobileKeypressPayload(normalized, transport, extras = {}) {
     return {
         keys: [normalized.requestedKey],

package/build/src/device/openai.js

@@ -130,12 +130,12 @@ const iosCloseKeyboardTool = {
 const loadmillTool = {
     type: "function",
     name: "loadmill",
-    description: "Search Loadmill for the requested flow, run the best match, and wait for completion. Use this when the instruction explicitly mentions running or using a Loadmill flow.",
+    description: "Run a saved Loadmill API flow. Use this tool only when the current instruction contains the word 'Loadmill' (case-insensitive) or starts with the explicit 'loadmill:' directive. Never call it based only on generic words such as 'run', 'execute', 'flow', 'test', or 'script'. Without an explicit Loadmill marker, treat the instruction as a normal UI or browser test step.",
     strict: true,
     parameters: {
         type: "object",
         properties: {
-            command: { type: "string" },
+            command: { type: "string", description: "The explicitly requested Loadmill flow name or intent and any parameters." },
         },
         required: ["command"],
         additionalProperties: false,

package/build/src/prompts/base.js

@@ -248,7 +248,7 @@ function buildMobileBaseSystemPrompt(deviceInfo, options = {}) {
 
   Mobile-Specific Notes:
   - HOME key returns to the home screen
-  - On Android, APP_SWITCH opens Recent Apps
+  - APP_SWITCH opens Recent Apps on Android and the App Switcher on iOS
   - On Android, ESC key maps to Back
   - On iOS, ESC has no effect; use visible on-screen controls instead
   - Never use CTRL, CMD, ALT, OPTION, or SHIFT in a keypress action

package/build/src/prompts/execution.js

@@ -83,7 +83,7 @@ ${browserTarget ? "" : `   - Every \`type\` action must end its computer-action
    - Otherwise, including when only a slim hardware-keyboard toolbar or suggestions are visible, continue without \`close_keyboard\`
    - Apply the same dismissal decision after \`type_sensitive_text\`, \`type_text_slowly\`, and \`type_secret_slowly\`
 `}
-   - Use the \`loadmill\` tool whenever the instruction explicitly references running or using a Loadmill flow. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Do not handle Loadmill by typing in the app UI.
+   - Use the \`loadmill\` tool only when the current instruction contains the word "Loadmill" (case-insensitive) or starts with the explicit \`loadmill:\` directive. Pass the requested flow name/intent and parameters as the command; the tool will search Loadmill, select the best matching flow, run it, and wait for completion. Never infer Loadmill intent from generic words such as "run", "execute", "flow", "test", or "script". Without an explicit Loadmill marker, handle the instruction as a normal UI or browser test step. Do not handle an explicitly requested Loadmill flow by typing in the app UI.
 3. Execute the required actions
 4. ${browserTarget ? "Before tapping a mutating action, dismiss the keyboard if it is open and not needed" : "Use `close_keyboard` before any later interaction when the post-type dismissal rules require it"}
 5. After a mutating action, inspect the resulting screen for success cues such as cleared fields, reset buttons, changed status, refreshed content, or navigation

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.37.0",
+  "version": "2.38.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",