@loadmill/droid-cua 2.4.0 → 2.5.0

package/build/src/device/android/actions.js

@@ -1,6 +1,7 @@
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 import { execAdb } from "./tools.js";
 function adbShell(deviceId, command) {
     return execAdb(["-s", deviceId, "shell", command]);
@@ -74,13 +75,16 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 await adbShell(deviceId, `input tap ${realX} ${realY}`);
                 break;
             case "scroll":
-                const scrollX = Math.round(action.scroll_x / scale);
-                const scrollY = Math.round(action.scroll_y / scale);
-                addOutput({ type: 'action', text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY, unit: 'px' }) });
-                const startX = 500;
-                const startY = 500;
-                const endX = startX + scrollX;
-                const endY = startY - scrollY; // <--- INVERT Y
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    fallbackStartX: 500,
+                    fallbackStartY: 500
+                });
+                addOutput({
+                    type: 'action',
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY})`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? 'action' : 'fallback', unit: 'px' })
+                });
                 await adbShell(deviceId, `input swipe ${startX} ${startY} ${endX} ${endY} 500`);
                 break;
             case "drag":

package/build/src/device/cloud/actions.js

@@ -2,6 +2,7 @@ import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
 import { getActiveSession, getDevicePixelRatio } from "./connection.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 function normalizeMobileKeypress(platform, keys = []) {
     if (!Array.isArray(keys) || keys.length === 0) {
         throw new Error("Keypress action is missing keys");
@@ -93,14 +94,18 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
                 break;
             }
             case "scroll": {
-                const scrollX = Math.round((action.scroll_x / scale) / dpr);
-                const scrollY = Math.round((action.scroll_y / scale) / dpr);
-                const centerX = 200;
-                const centerY = 400;
-                const endX = centerX + scrollX;
-                const endY = centerY - scrollY;
-                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY }) });
-                await session.client.scroll(session.sessionId, centerX, centerY, endX, endY);
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    dpr,
+                    fallbackStartX: 200,
+                    fallbackStartY: 400
+                });
+                addOutput({
+                    type: "action",
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY})`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? "action" : "fallback" })
+                });
+                await session.client.scroll(session.sessionId, startX, startY, endX, endY);
                 break;
             }
             case "drag": {

package/build/src/device/ios/actions.js

@@ -8,6 +8,7 @@ import { getActiveSession, getDevicePixelRatio } from "./connection.js";
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 function normalizeMobileKeypress(keys = []) {
     if (!Array.isArray(keys) || keys.length === 0) {
         throw new Error("Keypress action is missing keys");
@@ -92,15 +93,18 @@ export async function handleModelAction(simulatorId, action, scale = 1.0, contex
             }
             case "scroll": {
                 const dpr = getDevicePixelRatio();
-                const scrollX = Math.round((action.scroll_x / scale) / dpr);
-                const scrollY = Math.round((action.scroll_y / scale) / dpr);
-                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY}) points`, ...meta({ scrollX, scrollY, unit: "points" }) });
-                // Start from center of screen (in logical points)
-                const centerX = 197; // Center of iPhone 16 (393/2)
-                const centerY = 426; // Center of iPhone 16 (852/2)
-                const endX = centerX + scrollX;
-                const endY = centerY - scrollY; // Invert Y for natural scrolling
-                await appium.scroll(session.sessionId, centerX, centerY, endX, endY);
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    dpr,
+                    fallbackStartX: 197,
+                    fallbackStartY: 426
+                });
+                addOutput({
+                    type: "action",
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY}) points`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? "action" : "fallback", unit: "points" })
+                });
+                await appium.scroll(session.sessionId, startX, startY, endX, endY);
                 break;
             }
             case "drag": {

package/build/src/device/screenshot-resolution.js

@@ -0,0 +1,33 @@
+export const TARGET_SCALED_WIDTH = 400;
+export const SCREENSHOT_RESOLUTION_MODE_DOWNSCALED = "downscaled";
+export const SCREENSHOT_RESOLUTION_MODE_NATIVE = "native";
+export function normalizeScreenshotResolutionMode(value) {
+    return value === SCREENSHOT_RESOLUTION_MODE_NATIVE
+        ? SCREENSHOT_RESOLUTION_MODE_NATIVE
+        : SCREENSHOT_RESOLUTION_MODE_DOWNSCALED;
+}
+export function validateScreenshotResolutionMode(value, label) {
+    if (typeof value !== "string") {
+        throw new Error(`${label} must be one of: downscaled, native.`);
+    }
+    const normalized = normalizeScreenshotResolutionMode(value);
+    if (normalized !== value) {
+        throw new Error(`${label} must be one of: downscaled, native.`);
+    }
+    return normalized;
+}
+export function buildResolutionAwareDeviceInfo({ width, height, screenshotResolutionMode, }) {
+    const normalizedMode = normalizeScreenshotResolutionMode(screenshotResolutionMode);
+    const scale = normalizedMode === SCREENSHOT_RESOLUTION_MODE_NATIVE || width <= TARGET_SCALED_WIDTH
+        ? 1.0
+        : TARGET_SCALED_WIDTH / width;
+    return {
+        scaled_width: Math.round(width * scale),
+        scaled_height: Math.round(height * scale),
+        scale,
+        screenshot_resolution_mode: normalizedMode,
+    };
+}
+export function readScreenshotResolutionModeFromEnv() {
+    return normalizeScreenshotResolutionMode(process.env.DROID_CUA_SCREENSHOT_RESOLUTION_MODE);
+}

package/build/src/device/scroll-gesture.js

@@ -0,0 +1,20 @@
+// The model returns scroll actions with `scroll_x` / `scroll_y` plus optional
+// anchor coordinates `x` / `y`. This helper converts those model coordinates
+// into backend gesture coordinates: startX/startY and endX/endY.
+export function resolveScrollGesture(action, { scale = 1.0, dpr = 1.0, fallbackStartX = 0, fallbackStartY = 0 } = {}) {
+    const divisor = scale * dpr;
+    const scrollX = Math.round((action?.scroll_x ?? 0) / divisor);
+    const scrollY = Math.round((action?.scroll_y ?? 0) / divisor);
+    const hasAnchor = Number.isFinite(action?.x) && Number.isFinite(action?.y);
+    const startX = hasAnchor ? Math.round(action.x / divisor) : Math.round(fallbackStartX);
+    const startY = hasAnchor ? Math.round(action.y / divisor) : Math.round(fallbackStartY);
+    return {
+        scrollX,
+        scrollY,
+        startX,
+        startY,
+        endX: startX + scrollX,
+        endY: startY - scrollY,
+        hasAnchor
+    };
+}

package/build/src/test-store/test-manager.js

@@ -1,5 +1,6 @@
 import { readdir, readFile, writeFile, unlink, stat, mkdir } from "fs/promises";
 import path from "path";
+import { countExecutableInstructions, parseTestScript } from "./test-script.js";
 // Tests directory is relative to current working directory
 const TESTS_DIR = path.join(process.cwd(), "tests");
 /**
@@ -27,10 +28,7 @@ export async function loadTest(name) {
     const filename = name.endsWith(".dcua") ? name : `${name}.dcua`;
     const filepath = path.join(TESTS_DIR, filename);
     const content = await readFile(filepath, "utf-8");
-    return content
-        .split("\n")
-        .map(line => line.trim())
-        .filter(line => line.length > 0);
+    return parseTestScript(content);
 }
 /**
  * Get the raw content of a test file
@@ -60,7 +58,7 @@ export async function listTests() {
         const filepath = path.join(TESTS_DIR, filename);
         const stats = await stat(filepath);
         const content = await readFile(filepath, "utf-8");
-        const lines = content.split("\n").filter(line => line.trim().length > 0).length;
+        const lines = countExecutableInstructions(content);
         return {
             name: filename.replace(".dcua", ""),
             filename: filename,

package/build/src/test-store/test-script.js

@@ -0,0 +1,50 @@
+/**
+ * Find the first comment marker in a test script line.
+ * `//` starts a comment only at line start or when preceded by whitespace.
+ *
+ * @param {string} line
+ * @returns {number}
+ */
+export function findCommentStart(line) {
+    for (let i = 0; i < line.length - 1; i++) {
+        if (line[i] !== "/" || line[i + 1] !== "/") {
+            continue;
+        }
+        if (i === 0 || /\s/.test(line[i - 1])) {
+            return i;
+        }
+    }
+    return -1;
+}
+/**
+ * Remove inline comments and surrounding instruction whitespace from a line.
+ *
+ * @param {string} line
+ * @returns {string}
+ */
+export function stripInstructionComment(line) {
+    const commentStart = findCommentStart(line);
+    const instruction = commentStart >= 0 ? line.slice(0, commentStart) : line;
+    return instruction.trim();
+}
+/**
+ * Parse executable instructions from raw .dcua content.
+ *
+ * @param {string} content
+ * @returns {string[]}
+ */
+export function parseTestScript(content) {
+    return content
+        .split("\n")
+        .map(stripInstructionComment)
+        .filter((line) => line.length > 0);
+}
+/**
+ * Count executable instructions in raw .dcua content.
+ *
+ * @param {string} content
+ * @returns {number}
+ */
+export function countExecutableInstructions(content) {
+    return parseTestScript(content).length;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",