@loadmill/droid-cua 1.0.0 → 1.1.1

package/build/src/device/ios/actions.js

@@ -0,0 +1,117 @@
+/**
+ * iOS Action Execution Module
+ *
+ * Executes CUA model actions on iOS Simulator via Appium.
+ */
+import * as appium from "./appium-client.js";
+import { getActiveSession, getDevicePixelRatio } from "./connection.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Handle an action from the CUA model
+ * @param {string} simulatorId - The simulator UDID
+ * @param {object} action - The action to execute
+ * @param {number} scale - Scale factor for coordinates
+ * @param {object} context - Context with addOutput function
+ */
+export async function handleModelAction(simulatorId, action, scale = 1.0, context = null) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const session = getActiveSession();
+    if (!session) {
+        throw new Error("No active iOS session");
+    }
+    try {
+        switch (action.type) {
+            case "click": {
+                // Convert scaled coordinates to pixels, then to logical points for Appium
+                const dpr = getDevicePixelRatio();
+                const pixelX = Math.round(action.x / scale);
+                const pixelY = Math.round(action.y / scale);
+                const pointX = Math.round(pixelX / dpr);
+                const pointY = Math.round(pixelY / dpr);
+                addOutput({ type: "action", text: `Tapping at (${pointX}, ${pointY}) points` });
+                await appium.tap(session.sessionId, pointX, pointY);
+                break;
+            }
+            case "type": {
+                addOutput({ type: "action", text: `Typing text: ${action.text}` });
+                await appium.type(session.sessionId, action.text);
+                break;
+            }
+            case "scroll": {
+                const dpr = getDevicePixelRatio();
+                const scrollX = Math.round((action.scroll_x / scale) / dpr);
+                const scrollY = Math.round((action.scroll_y / scale) / dpr);
+                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY}) points` });
+                // Start from center of screen (in logical points)
+                const centerX = 197; // Center of iPhone 16 (393/2)
+                const centerY = 426; // Center of iPhone 16 (852/2)
+                const endX = centerX + scrollX;
+                const endY = centerY - scrollY; // Invert Y for natural scrolling
+                await appium.scroll(session.sessionId, centerX, centerY, endX, endY);
+                break;
+            }
+            case "drag": {
+                const { path } = action;
+                if (path && path.length >= 2) {
+                    const dpr = getDevicePixelRatio();
+                    const start = path[0];
+                    const end = path[path.length - 1];
+                    // Convert to pixels then to logical points
+                    const startX = Math.round((start.x / scale) / dpr);
+                    const startY = Math.round((start.y / scale) / dpr);
+                    const endX = Math.round((end.x / scale) / dpr);
+                    const endY = Math.round((end.y / scale) / dpr);
+                    addOutput({
+                        type: "action",
+                        text: `Dragging from (${startX}, ${startY}) to (${endX}, ${endY}) points`,
+                    });
+                    await appium.drag(session.sessionId, startX, startY, endX, endY);
+                }
+                else {
+                    addOutput({ type: "info", text: `Drag action missing valid path: ${JSON.stringify(action)}` });
+                }
+                break;
+            }
+            case "keypress": {
+                const { keys } = action;
+                for (const key of keys) {
+                    const upperKey = key.toUpperCase();
+                    if (upperKey === "ESC" || upperKey === "ESCAPE") {
+                        // Map ESC to home button on iOS
+                        addOutput({ type: "action", text: "Pressing Home button" });
+                        await appium.pressButton(session.sessionId, "home");
+                    }
+                    else if (upperKey === "ENTER" || upperKey === "RETURN") {
+                        addOutput({ type: "action", text: "Pressing Return key" });
+                        await appium.type(session.sessionId, "\n");
+                    }
+                    else if (upperKey === "BACKSPACE" || upperKey === "DELETE") {
+                        addOutput({ type: "action", text: "Pressing Delete key" });
+                        await appium.type(session.sessionId, "\b");
+                    }
+                    else {
+                        addOutput({ type: "action", text: `Pressing key: ${key}` });
+                        await appium.type(session.sessionId, key);
+                    }
+                }
+                break;
+            }
+            case "wait": {
+                addOutput({ type: "action", text: "Waiting..." });
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+                break;
+            }
+            default:
+                addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+        }
+    }
+    catch (error) {
+        logger.error("iOS action execution error", {
+            action,
+            message: error.message,
+            stack: error.stack,
+        });
+        addOutput({ type: "error", text: `Error executing action: ${error.message}` });
+        addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+    }
+}

package/build/src/device/ios/appium-client.js

@@ -0,0 +1,207 @@
+/**
+ * Appium WebDriver Client
+ *
+ * Low-level HTTP client for Appium/WebDriver protocol.
+ * Implements W3C WebDriver and Appium-specific endpoints.
+ */
+import { getAppiumUrl } from "./appium-server.js";
+/**
+ * Make a request to the Appium server
+ * @param {string} method - HTTP method
+ * @param {string} path - API path
+ * @param {object} body - Request body (optional)
+ * @returns {Promise<object>}
+ */
+async function appiumRequest(method, path, body = null) {
+    const url = `${getAppiumUrl()}${path}`;
+    const options = {
+        method,
+        headers: {
+            "Content-Type": "application/json",
+        },
+    };
+    if (body) {
+        options.body = JSON.stringify(body);
+    }
+    const response = await fetch(url, options);
+    const data = await response.json();
+    if (!response.ok) {
+        const errorMessage = data.value?.message || data.message || JSON.stringify(data);
+        throw new Error(`Appium request failed: ${errorMessage}`);
+    }
+    return data;
+}
+/**
+ * Create a new Appium session
+ * @param {object} capabilities - Desired capabilities
+ * @returns {Promise<{sessionId: string, value: object}>}
+ */
+export async function createSession(capabilities) {
+    const response = await appiumRequest("POST", "/session", {
+        capabilities: {
+            alwaysMatch: capabilities,
+        },
+    });
+    return {
+        sessionId: response.value.sessionId,
+        value: response.value,
+    };
+}
+/**
+ * Delete an Appium session
+ * @param {string} sessionId
+ * @returns {Promise<void>}
+ */
+export async function deleteSession(sessionId) {
+    await appiumRequest("DELETE", `/session/${sessionId}`);
+}
+/**
+ * Get a screenshot
+ * @param {string} sessionId
+ * @returns {Promise<string>} Base64-encoded PNG
+ */
+export async function getScreenshot(sessionId) {
+    const response = await appiumRequest("GET", `/session/${sessionId}/screenshot`);
+    return response.value;
+}
+/**
+ * Get window size
+ * @param {string} sessionId
+ * @returns {Promise<{width: number, height: number}>}
+ */
+export async function getWindowSize(sessionId) {
+    const response = await appiumRequest("GET", `/session/${sessionId}/window/rect`);
+    return {
+        width: response.value.width,
+        height: response.value.height,
+    };
+}
+/**
+ * Perform a tap action using W3C Actions
+ * @param {string} sessionId
+ * @param {number} x
+ * @param {number} y
+ * @returns {Promise<void>}
+ */
+export async function tap(sessionId, x, y) {
+    await appiumRequest("POST", `/session/${sessionId}/actions`, {
+        actions: [
+            {
+                type: "pointer",
+                id: "finger1",
+                parameters: { pointerType: "touch" },
+                actions: [
+                    { type: "pointerMove", duration: 0, x: Math.round(x), y: Math.round(y) },
+                    { type: "pointerDown", button: 0 },
+                    { type: "pause", duration: 100 },
+                    { type: "pointerUp", button: 0 },
+                ],
+            },
+        ],
+    });
+}
+/**
+ * Type text
+ * @param {string} sessionId
+ * @param {string} text
+ * @returns {Promise<void>}
+ */
+export async function type(sessionId, text) {
+    // Use W3C key actions
+    const keyActions = [];
+    for (const char of text) {
+        keyActions.push({ type: "keyDown", value: char });
+        keyActions.push({ type: "keyUp", value: char });
+    }
+    await appiumRequest("POST", `/session/${sessionId}/actions`, {
+        actions: [
+            {
+                type: "key",
+                id: "keyboard",
+                actions: keyActions,
+            },
+        ],
+    });
+}
+/**
+ * Perform a scroll action using W3C Actions
+ * @param {string} sessionId
+ * @param {number} startX
+ * @param {number} startY
+ * @param {number} endX
+ * @param {number} endY
+ * @param {number} duration - Duration in ms
+ * @returns {Promise<void>}
+ */
+export async function scroll(sessionId, startX, startY, endX, endY, duration = 500) {
+    await appiumRequest("POST", `/session/${sessionId}/actions`, {
+        actions: [
+            {
+                type: "pointer",
+                id: "finger1",
+                parameters: { pointerType: "touch" },
+                actions: [
+                    { type: "pointerMove", duration: 0, x: Math.round(startX), y: Math.round(startY) },
+                    { type: "pointerDown", button: 0 },
+                    { type: "pointerMove", duration, x: Math.round(endX), y: Math.round(endY) },
+                    { type: "pointerUp", button: 0 },
+                ],
+            },
+        ],
+    });
+}
+/**
+ * Perform a drag action using W3C Actions
+ * @param {string} sessionId
+ * @param {number} startX
+ * @param {number} startY
+ * @param {number} endX
+ * @param {number} endY
+ * @param {number} duration - Duration in ms
+ * @returns {Promise<void>}
+ */
+export async function drag(sessionId, startX, startY, endX, endY, duration = 500) {
+    // Drag is similar to scroll but with longer press at start
+    await appiumRequest("POST", `/session/${sessionId}/actions`, {
+        actions: [
+            {
+                type: "pointer",
+                id: "finger1",
+                parameters: { pointerType: "touch" },
+                actions: [
+                    { type: "pointerMove", duration: 0, x: Math.round(startX), y: Math.round(startY) },
+                    { type: "pointerDown", button: 0 },
+                    { type: "pause", duration: 200 }, // Hold before drag
+                    { type: "pointerMove", duration, x: Math.round(endX), y: Math.round(endY) },
+                    { type: "pointerUp", button: 0 },
+                ],
+            },
+        ],
+    });
+}
+/**
+ * Press a device button (home, volumeUp, volumeDown)
+ * @param {string} sessionId
+ * @param {string} buttonName - 'home', 'volumeUp', or 'volumeDown'
+ * @returns {Promise<void>}
+ */
+export async function pressButton(sessionId, buttonName) {
+    await appiumRequest("POST", `/session/${sessionId}/execute/sync`, {
+        script: "mobile: pressButton",
+        args: [{ name: buttonName }],
+    });
+}
+/**
+ * Get session status
+ * @param {string} sessionId
+ * @returns {Promise<object>}
+ */
+export async function getSessionStatus(sessionId) {
+    try {
+        const response = await appiumRequest("GET", `/session/${sessionId}`);
+        return response.value;
+    }
+    catch {
+        return null;
+    }
+}

package/build/src/device/ios/appium-server.js

@@ -0,0 +1,101 @@
+/**
+ * Appium Server Manager
+ *
+ * Manages Appium server lifecycle for iOS Simulator automation.
+ * Auto-starts if not running, cleans up on process exit.
+ */
+import { spawn } from "child_process";
+const APPIUM_URL = process.env.APPIUM_URL || "http://localhost:4723";
+let appiumProcess = null;
+let cleanupRegistered = false;
+/**
+ * Check if Appium server is running
+ * @returns {Promise<boolean>}
+ */
+export async function isAppiumRunning() {
+    try {
+        const response = await fetch(`${APPIUM_URL}/status`);
+        return response.ok;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Wait for Appium to be ready
+ * @param {number} timeoutMs - Maximum time to wait
+ * @returns {Promise<boolean>}
+ */
+async function waitForAppiumReady(timeoutMs = 30000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        if (await isAppiumRunning()) {
+            return true;
+        }
+        await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+    return false;
+}
+/**
+ * Start Appium server if not already running
+ * @returns {Promise<void>}
+ */
+export async function startAppium() {
+    if (await isAppiumRunning()) {
+        console.log("Appium server is already running");
+        return;
+    }
+    console.log("Starting Appium server...");
+    appiumProcess = spawn("appium", [], {
+        detached: true,
+        stdio: "ignore"
+    });
+    appiumProcess.unref();
+    const ready = await waitForAppiumReady(30000);
+    if (!ready) {
+        throw new Error("Appium server failed to start within 30 seconds");
+    }
+    console.log("Appium server is ready");
+}
+/**
+ * Stop the Appium server if we started it
+ */
+export function stopAppium() {
+    if (appiumProcess) {
+        try {
+            // Kill the process group
+            process.kill(-appiumProcess.pid, "SIGTERM");
+        }
+        catch {
+            // Process may already be dead
+        }
+        appiumProcess = null;
+    }
+}
+/**
+ * Register cleanup handlers for process exit
+ */
+export function setupAppiumCleanup() {
+    if (cleanupRegistered)
+        return;
+    cleanupRegistered = true;
+    const cleanup = () => {
+        stopAppium();
+    };
+    process.on("exit", cleanup);
+    process.on("SIGINT", () => {
+        cleanup();
+        process.exit(0);
+    });
+    process.on("SIGTERM", () => {
+        cleanup();
+        process.exit(0);
+    });
+}
+/**
+ * Get the Appium server URL
+ * @returns {string}
+ */
+export function getAppiumUrl() {
+    return APPIUM_URL;
+}

package/build/src/device/ios/connection.js

@@ -0,0 +1,280 @@
+/**
+ * iOS Simulator Connection Module
+ *
+ * Manages connection to iOS Simulator via Appium.
+ */
+import { exec } from "child_process";
+import { promisify } from "util";
+import sharp from "sharp";
+import { startAppium, setupAppiumCleanup } from "./appium-server.js";
+import * as appium from "./appium-client.js";
+import { logger } from "../../utils/logger.js";
+const execAsync = promisify(exec);
+// Active session state
+let activeSession = null;
+/**
+ * Get simulator UDID by name (prefers already-booted simulator)
+ * @param {string} simulatorName
+ * @returns {Promise<string|null>}
+ */
+async function getSimulatorUDID(simulatorName) {
+    try {
+        const { stdout } = await execAsync("xcrun simctl list devices --json");
+        const data = JSON.parse(stdout);
+        let firstMatch = null;
+        let bootedMatch = null;
+        for (const runtime of Object.values(data.devices)) {
+            for (const device of runtime) {
+                if (device.name === simulatorName && device.isAvailable) {
+                    if (!firstMatch) {
+                        firstMatch = device.udid;
+                    }
+                    // Prefer the already-booted one
+                    if (device.state === "Booted") {
+                        bootedMatch = device.udid;
+                    }
+                }
+            }
+        }
+        if (bootedMatch) {
+            logger.debug(`Found booted simulator: ${bootedMatch}`);
+            return bootedMatch;
+        }
+        return firstMatch;
+    }
+    catch (error) {
+        logger.error("Failed to get simulator UDID", { error: error.message });
+    }
+    return null;
+}
+/**
+ * Find any available simulator (prefers booted, then first available)
+ * @returns {Promise<{name: string, udid: string}|null>}
+ */
+async function findAnySimulator() {
+    try {
+        const { stdout } = await execAsync("xcrun simctl list devices --json");
+        const data = JSON.parse(stdout);
+        let firstAvailable = null;
+        let bootedDevice = null;
+        for (const runtime of Object.values(data.devices)) {
+            for (const device of runtime) {
+                if (device.isAvailable) {
+                    if (!firstAvailable) {
+                        firstAvailable = { name: device.name, udid: device.udid };
+                    }
+                    if (device.state === "Booted") {
+                        bootedDevice = { name: device.name, udid: device.udid };
+                    }
+                }
+            }
+        }
+        return bootedDevice || firstAvailable;
+    }
+    catch (error) {
+        logger.error("Failed to find simulators", { error: error.message });
+    }
+    return null;
+}
+/**
+ * Boot simulator if not already booted
+ * @param {string} simulatorName
+ * @returns {Promise<string>} UDID of the simulator
+ */
+async function bootSimulator(simulatorName) {
+    const udid = await getSimulatorUDID(simulatorName);
+    if (!udid) {
+        throw new Error(`Simulator "${simulatorName}" not found. Run "xcrun simctl list devices" to see available simulators.`);
+    }
+    console.log(`Found simulator "${simulatorName}" with UDID: ${udid}`);
+    // Check if already booted
+    const { stdout } = await execAsync("xcrun simctl list devices --json");
+    const data = JSON.parse(stdout);
+    for (const runtime of Object.values(data.devices)) {
+        for (const device of runtime) {
+            if (device.udid === udid) {
+                if (device.state === "Booted") {
+                    console.log(`Simulator "${simulatorName}" is already booted`);
+                    return udid;
+                }
+                break;
+            }
+        }
+    }
+    console.log(`Booting simulator "${simulatorName}"...`);
+    await execAsync(`xcrun simctl boot ${udid}`);
+    // Open Simulator app to show the device
+    await execAsync("open -a Simulator");
+    // Wait for boot to complete
+    await waitForSimulatorBoot(udid);
+    console.log(`Simulator "${simulatorName}" is fully booted`);
+    return udid;
+}
+/**
+ * Wait for simulator to finish booting
+ * @param {string} udid
+ * @param {number} timeoutMs
+ */
+async function waitForSimulatorBoot(udid, timeoutMs = 60000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        try {
+            const { stdout } = await execAsync("xcrun simctl list devices --json");
+            const data = JSON.parse(stdout);
+            for (const runtime of Object.values(data.devices)) {
+                for (const device of runtime) {
+                    if (device.udid === udid && device.state === "Booted") {
+                        // Additional check: wait for springboard
+                        await new Promise(resolve => setTimeout(resolve, 3000));
+                        return;
+                    }
+                }
+            }
+        }
+        catch { }
+        await new Promise(resolve => setTimeout(resolve, 2000));
+    }
+    throw new Error("Simulator did not boot in time");
+}
+/**
+ * Connect to iOS Simulator
+ * @param {string} simulatorName - Name of the simulator (e.g., "iPhone 16"), or null to auto-detect
+ * @returns {Promise<string>} Simulator ID for use with other functions
+ */
+export async function connectToDevice(simulatorName) {
+    // Setup cleanup handlers
+    setupAppiumCleanup();
+    // Start Appium if not running
+    await startAppium();
+    // If no simulator specified, auto-detect
+    if (!simulatorName) {
+        const found = await findAnySimulator();
+        if (!found) {
+            console.error("No iOS Simulators found. Create one with Xcode or run:");
+            console.error("  xcrun simctl create 'iPhone 16' 'com.apple.CoreSimulator.SimDeviceType.iPhone-16'");
+            process.exit(1);
+        }
+        simulatorName = found.name;
+        console.log(`No simulator specified, using: ${simulatorName}`);
+    }
+    // Boot simulator
+    const udid = await bootSimulator(simulatorName);
+    // Create Appium session
+    console.log("Creating Appium session...");
+    const session = await appium.createSession({
+        platformName: "iOS",
+        "appium:automationName": "XCUITest",
+        "appium:deviceName": simulatorName,
+        "appium:udid": udid,
+        "appium:noReset": true,
+        "appium:shouldTerminateApp": false,
+    });
+    activeSession = {
+        sessionId: session.sessionId,
+        udid,
+        simulatorName,
+    };
+    console.log(`Connected to simulator "${simulatorName}" (${udid})`);
+    return udid;
+}
+/**
+ * Get device info (screen dimensions and scale)
+ * @param {string} simulatorId - The UDID returned from connectToDevice
+ * @returns {Promise<object>}
+ */
+export async function getDeviceInfo(simulatorId) {
+    await ensureSessionAlive();
+    // Get logical window size (points)
+    const windowSize = await appium.getWindowSize(activeSession.sessionId);
+    // Take a test screenshot to get actual pixel dimensions
+    // iOS screenshots are at Retina resolution (2x or 3x)
+    const testScreenshot = await appium.getScreenshot(activeSession.sessionId);
+    const testBuffer = Buffer.from(testScreenshot, "base64");
+    const metadata = await sharp(testBuffer).metadata();
+    const pixelWidth = metadata.width;
+    const pixelHeight = metadata.height;
+    // Calculate the iOS device scale factor (typically 2 or 3)
+    const devicePixelRatio = Math.round(pixelWidth / windowSize.width);
+    console.log(`Device pixel ratio: ${devicePixelRatio}x (${pixelWidth}x${pixelHeight} pixels, ${windowSize.width}x${windowSize.height} points)`);
+    // Store the pixel ratio for coordinate conversion
+    activeSession.devicePixelRatio = devicePixelRatio;
+    // Use pixel dimensions as the "real" resolution (like Android does)
+    const targetWidth = 400;
+    const scale = pixelWidth > targetWidth ? targetWidth / pixelWidth : 1.0;
+    const scaledWidth = Math.round(pixelWidth * scale);
+    const scaledHeight = Math.round(pixelHeight * scale);
+    return {
+        device_width: pixelWidth,
+        device_height: pixelHeight,
+        scaled_width: scaledWidth,
+        scaled_height: scaledHeight,
+        scale,
+    };
+}
+/**
+ * Get screenshot as base64
+ * @param {string} simulatorId
+ * @param {object} deviceInfo
+ * @returns {Promise<string>}
+ */
+export async function getScreenshotAsBase64(simulatorId, deviceInfo) {
+    await ensureSessionAlive();
+    const base64 = await appium.getScreenshot(activeSession.sessionId);
+    let buffer = Buffer.from(base64, "base64");
+    logger.debug(`iOS screenshot captured: ${buffer.length} bytes before scaling`);
+    if (deviceInfo.scale < 1.0) {
+        buffer = await sharp(buffer)
+            .resize({ width: deviceInfo.scaled_width, height: deviceInfo.scaled_height })
+            .png()
+            .toBuffer();
+        logger.debug(`iOS screenshot scaled: ${buffer.length} bytes after scaling`);
+    }
+    return buffer.toString("base64");
+}
+/**
+ * Ensure the Appium session is still alive, recreate if dead
+ */
+async function ensureSessionAlive() {
+    if (!activeSession) {
+        throw new Error("No active iOS session. Call connectToDevice first.");
+    }
+    const status = await appium.getSessionStatus(activeSession.sessionId);
+    if (!status) {
+        console.log("Session died, recreating...");
+        const session = await appium.createSession({
+            platformName: "iOS",
+            "appium:automationName": "XCUITest",
+            "appium:deviceName": activeSession.simulatorName,
+            "appium:udid": activeSession.udid,
+            "appium:noReset": true,
+            "appium:shouldTerminateApp": false,
+        });
+        activeSession.sessionId = session.sessionId;
+    }
+}
+/**
+ * Get the active session (for use by actions module)
+ * @returns {object|null}
+ */
+export function getActiveSession() {
+    return activeSession;
+}
+/**
+ * Get the device pixel ratio (for coordinate conversion)
+ * @returns {number}
+ */
+export function getDevicePixelRatio() {
+    return activeSession?.devicePixelRatio || 3;
+}
+/**
+ * Disconnect and cleanup
+ */
+export async function disconnect() {
+    if (activeSession) {
+        try {
+            await appium.deleteSession(activeSession.sessionId);
+        }
+        catch { }
+        activeSession = null;
+    }
+}