@loadmill/droid-cua 2.0.5 → 2.1.0

package/build/src/device/cloud/actions.js

@@ -0,0 +1,185 @@
+import { logger } from "../../utils/logger.js";
+import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
+import { getActiveSession, getDevicePixelRatio } from "./connection.js";
+function normalizeMobileKeypress(platform, keys = []) {
+    if (!Array.isArray(keys) || keys.length === 0) {
+        throw new Error("Keypress action is missing keys");
+    }
+    if (keys.length > 1) {
+        throw new Error(`Unsupported mobile key chord: ${keys.join(", ")}. Use taps and text entry instead.`);
+    }
+    const key = String(keys[0]).trim().toUpperCase();
+    if (platform === "ios") {
+        if (key === "ESC" || key === "ESCAPE" || key === "HOME") {
+            return { kind: "ios-button", originalKey: keys[0], mapped: "home" };
+        }
+        if (key === "ENTER" || key === "RETURN") {
+            return { kind: "text", originalKey: keys[0], mapped: "\n", label: "Return key" };
+        }
+        if (key === "BACKSPACE" || key === "DELETE") {
+            return { kind: "text", originalKey: keys[0], mapped: "\b", label: "Delete key" };
+        }
+        if (key === "SPACE") {
+            return { kind: "text", originalKey: keys[0], mapped: " ", label: "Space key" };
+        }
+        throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
+    }
+    const androidKeyMap = {
+        ESC: 3,
+        ESCAPE: 3,
+        HOME: 3,
+        BACK: 4,
+        ENTER: 66,
+        RETURN: 66,
+        BACKSPACE: 67,
+        DELETE: 67,
+        SPACE: 62
+    };
+    const keycode = androidKeyMap[key];
+    if (!keycode) {
+        throw new Error(`Unsupported mobile keypress: ${keys[0]}. Only single mobile-safe keys are allowed.`);
+    }
+    return { kind: "android-keycode", originalKey: keys[0], keycode };
+}
+export async function handleModelAction(deviceId, action, scale = 1.0, context = null) {
+    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+    const session = getActiveSession();
+    const platform = session?.platform;
+    const meta = (payload = {}) => ({
+        eventType: "tool_call",
+        actionType: action?.type,
+        runId: context?.runId,
+        stepId: context?.stepId,
+        instructionIndex: context?.instructionIndex,
+        payload: {
+            platform,
+            source: session?.providerId,
+            ...payload
+        }
+    });
+    if (!session || !platform) {
+        throw new Error("No active cloud session");
+    }
+    try {
+        emitDesktopDebug("device.action.execute", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex
+        }, {
+            platform,
+            source: session.providerId,
+            deviceId,
+            actionType: action?.type,
+            scale,
+            text: typeof action?.text === "string" ? truncateForDebug(action.text, 300) : undefined,
+            keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
+            pathPoints: Array.isArray(action?.path) ? action.path.length : 0
+        });
+        const dpr = getDevicePixelRatio();
+        switch (action.type) {
+            case "click":
+            case "double_click": {
+                const targetX = Math.round((action.x / scale) / dpr);
+                const targetY = Math.round((action.y / scale) / dpr);
+                addOutput({ type: "action", text: `Tapping at (${targetX}, ${targetY})`, ...meta({ x: targetX, y: targetY }) });
+                await session.client.tap(session.sessionId, targetX, targetY);
+                break;
+            }
+            case "type": {
+                addOutput({ type: "action", text: `Typing text: ${action.text}`, ...meta({ text: action.text }) });
+                await session.client.type(session.sessionId, action.text);
+                break;
+            }
+            case "scroll": {
+                const scrollX = Math.round((action.scroll_x / scale) / dpr);
+                const scrollY = Math.round((action.scroll_y / scale) / dpr);
+                const centerX = 200;
+                const centerY = 400;
+                const endX = centerX + scrollX;
+                const endY = centerY - scrollY;
+                addOutput({ type: "action", text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY }) });
+                await session.client.scroll(session.sessionId, centerX, centerY, endX, endY);
+                break;
+            }
+            case "drag": {
+                const { path } = action;
+                if (!path || path.length < 2) {
+                    addOutput({ type: "info", text: `Drag action missing valid path: ${JSON.stringify(action)}` });
+                    break;
+                }
+                const start = path[0];
+                const end = path[path.length - 1];
+                const startX = Math.round((start.x / scale) / dpr);
+                const startY = Math.round((start.y / scale) / dpr);
+                const endX = Math.round((end.x / scale) / dpr);
+                const endY = Math.round((end.y / scale) / dpr);
+                addOutput({
+                    type: "action",
+                    text: `Dragging from (${startX}, ${startY}) to (${endX}, ${endY})`,
+                    ...meta({ pathStart: { x: startX, y: startY }, pathEnd: { x: endX, y: endY } })
+                });
+                await session.client.drag(session.sessionId, startX, startY, endX, endY);
+                break;
+            }
+            case "keypress": {
+                const normalized = normalizeMobileKeypress(platform, action.keys);
+                if (normalized.kind === "ios-button") {
+                    addOutput({ type: "action", text: "Pressing Home button", ...meta({ keys: [normalized.originalKey], mapped: normalized.mapped }) });
+                    await session.client.pressIosButton(session.sessionId, normalized.mapped);
+                }
+                else if (normalized.kind === "android-keycode") {
+                    addOutput({ type: "action", text: `Pressing keycode ${normalized.keycode}`, ...meta({ keys: [normalized.originalKey], keycode: normalized.keycode }) });
+                    await session.client.pressAndroidKeyCode(session.sessionId, normalized.keycode);
+                }
+                else {
+                    addOutput({ type: "action", text: `Pressing ${normalized.label}`, ...meta({ keys: [normalized.originalKey], mapped: normalized.label }) });
+                    await session.client.type(session.sessionId, normalized.mapped);
+                }
+                break;
+            }
+            case "wait":
+                addOutput({ type: "action", text: "Waiting...", ...meta({}) });
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+                break;
+            default:
+                addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+        }
+    }
+    catch (error) {
+        logger.error("Cloud action execution error", {
+            action,
+            message: error.message,
+            stack: error.stack,
+            providerId: session.providerId,
+            platform
+        });
+        emitDesktopDebug("device.error", "device", {
+            runId: context?.runId,
+            sessionId: context?.sessionId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex
+        }, {
+            platform,
+            source: session.providerId,
+            operation: "action.execute",
+            actionType: action?.type,
+            message: error.message
+        });
+        addOutput({
+            type: "error",
+            text: `Error executing action: ${error.message}`,
+            eventType: "error",
+            actionType: action?.type,
+            runId: context?.runId,
+            stepId: context?.stepId,
+            instructionIndex: context?.instructionIndex,
+            payload: {
+                message: error.message,
+                platform,
+                source: session.providerId
+            }
+        });
+        addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+    }
+}

package/build/src/device/cloud/adapter.js

@@ -0,0 +1 @@
+export {};

package/build/src/device/cloud/browserstack/adapter.js

@@ -0,0 +1,334 @@
+import path from "node:path";
+import { readFile } from "node:fs/promises";
+const HUB_URL = "https://hub-cloud.browserstack.com/wd/hub";
+const PLAN_URL = "https://api-cloud.browserstack.com/app-automate/plan.json";
+const DEVICES_URL = "https://api-cloud.browserstack.com/app-automate/devices.json";
+const UPLOAD_URL = "https://api-cloud.browserstack.com/app-automate/upload";
+const RECENT_APPS_URL = "https://api-cloud.browserstack.com/app-automate/recent_apps";
+function notImplemented(methodName) {
+    throw new Error(`BrowserStack adapter stub: ${methodName} is not implemented yet.`);
+}
+function sanitizeCustomId(value) {
+    const sanitized = value.replace(/[^A-Za-z0-9._-]/g, "");
+    return sanitized.slice(0, 100) || "droid-cua-app";
+}
+function buildCustomId(localPath) {
+    const baseName = path.basename(localPath, path.extname(localPath));
+    return sanitizeCustomId(`droid-cua.${baseName}`);
+}
+function normalizePlanLabel(value) {
+    if (typeof value !== "string") {
+        if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+            return normalizePlanLabel(value.name) ?? normalizePlanLabel(value.label) ?? normalizePlanLabel(value.type);
+        }
+        return undefined;
+    }
+    const trimmed = value.trim();
+    return trimmed ? trimmed : undefined;
+}
+function readPlanLabel(payload) {
+    const candidates = [
+        payload.plan,
+        payload.plan_name,
+        payload.account_plan,
+        payload.automate_plan,
+        payload.automate_plan_name,
+        payload.billing_plan,
+        payload.plan_type,
+        payload.account_type
+    ];
+    for (const candidate of candidates) {
+        const normalized = normalizePlanLabel(candidate);
+        if (normalized) {
+            return normalized;
+        }
+    }
+    return undefined;
+}
+function normalizeParallelLimit(payload) {
+    const candidates = [
+        payload.parallel_sessions_max_allowed,
+        payload.parallel_sessions,
+        payload.parallelSessions,
+        payload.max_allowed_parallel_sessions
+    ];
+    for (const candidate of candidates) {
+        if (typeof candidate === "number" && Number.isFinite(candidate)) {
+            return candidate;
+        }
+        if (typeof candidate === "string" && candidate.trim().length > 0) {
+            const parsed = Number(candidate);
+            if (Number.isFinite(parsed)) {
+                return parsed;
+            }
+        }
+    }
+    return undefined;
+}
+async function requestPlan(creds) {
+    const response = await fetch(PLAN_URL, {
+        method: "GET",
+        headers: {
+            Authorization: browserStackAdapter.getAuthHeader(creds)
+        }
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new Error("BrowserStack rejected these credentials. Check the username and access key and try again.");
+    }
+    if (!response.ok) {
+        throw new Error(`BrowserStack validation failed with status ${response.status}.`);
+    }
+    const payload = await response.json();
+    if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {
+        throw new Error("BrowserStack returned an unexpected validation response.");
+    }
+    const plan = readPlanLabel(payload);
+    const parallelLimit = normalizeParallelLimit(payload);
+    const summary = typeof payload.message === "string" && payload.message.trim().length > 0 ? payload.message.trim() : undefined;
+    return {
+        plan,
+        parallelLimit,
+        summary
+    };
+}
+function normalizePlatform(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const normalized = value.trim().toLowerCase();
+    if (normalized === "android") {
+        return "android";
+    }
+    if (normalized === "ios") {
+        return "ios";
+    }
+    return null;
+}
+function sortOsVersionsDescending(values) {
+    return [...values].sort((left, right) => right.localeCompare(left, undefined, { numeric: true, sensitivity: "base" }));
+}
+async function requestDevices(creds) {
+    const response = await fetch(DEVICES_URL, {
+        method: "GET",
+        headers: {
+            Authorization: browserStackAdapter.getAuthHeader(creds)
+        }
+    });
+    if (response.status === 401 || response.status === 403) {
+        throw new Error("BrowserStack rejected these credentials. Reconnect BrowserStack and try refreshing devices again.");
+    }
+    if (!response.ok) {
+        throw new Error(`BrowserStack device catalog failed with status ${response.status}.`);
+    }
+    const payload = await response.json();
+    if (!Array.isArray(payload)) {
+        throw new Error("BrowserStack returned an unexpected device catalog response.");
+    }
+    const deduped = new Map();
+    for (const item of payload) {
+        if (typeof item !== "object" || item === null || Array.isArray(item)) {
+            continue;
+        }
+        const platform = normalizePlatform(item.os);
+        const deviceName = typeof item.device === "string" && item.device.trim().length > 0
+            ? item.device.trim()
+            : typeof item.deviceName === "string" && item.deviceName.trim().length > 0
+                ? item.deviceName.trim()
+                : null;
+        const osVersion = typeof item.os_version === "string" && item.os_version.trim().length > 0
+            ? item.os_version.trim()
+            : typeof item.osVersion === "string" && item.osVersion.trim().length > 0
+                ? item.osVersion.trim()
+                : null;
+        if (!platform || !deviceName || !osVersion) {
+            continue;
+        }
+        const key = `${platform}::${deviceName}::${osVersion}`;
+        if (!deduped.has(key)) {
+            deduped.set(key, {
+                id: key,
+                name: deviceName,
+                deviceName,
+                platform,
+                osVersion
+            });
+        }
+    }
+    return [...deduped.values()].sort((left, right) => {
+        if (left.platform !== right.platform) {
+            return left.platform.localeCompare(right.platform);
+        }
+        if (left.name !== right.name) {
+            return left.name.localeCompare(right.name);
+        }
+        return sortOsVersionsDescending([left.osVersion ?? "", right.osVersion ?? ""])[0] === (left.osVersion ?? "") ? -1 : 1;
+    });
+}
+function readAppReference(payload) {
+    if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {
+        throw new Error("BrowserStack returned an unexpected app upload response.");
+    }
+    const remotePath = typeof payload.app_url === "string" ? payload.app_url.trim() : "";
+    if (!remotePath.startsWith("bs://")) {
+        throw new Error("BrowserStack did not return a valid app reference.");
+    }
+    const id = typeof payload.custom_id === "string" && payload.custom_id.trim().length > 0
+        ? payload.custom_id.trim()
+        : remotePath.slice("bs://".length);
+    return { id, remotePath };
+}
+function readAppStatusEntry(payload) {
+    if (!Array.isArray(payload)) {
+        throw new Error("BrowserStack returned an unexpected uploaded-apps response.");
+    }
+    const entries = [];
+    for (const item of payload) {
+        if (typeof item !== "object" || item === null || Array.isArray(item)) {
+            continue;
+        }
+        const appUrl = typeof item.app_url === "string" ? item.app_url.trim() : "";
+        if (!appUrl.startsWith("bs://")) {
+            continue;
+        }
+        const customId = typeof item.custom_id === "string" && item.custom_id.trim().length > 0 ? item.custom_id.trim() : undefined;
+        entries.push({ appUrl, customId });
+    }
+    return entries;
+}
+export const browserStackAdapter = {
+    id: "browserstack",
+    displayName: "BrowserStack",
+    async validateCredentials(creds) {
+        try {
+            const account = await requestPlan(creds);
+            return {
+                ok: true,
+                message: account.summary ?? "BrowserStack credentials validated successfully.",
+                account
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : "Failed to validate BrowserStack credentials.";
+            if (/ENOTFOUND|fetch failed|network|timed out|ECONN/i.test(message)) {
+                return {
+                    ok: false,
+                    message: "Could not reach BrowserStack. Check your network connection and try again."
+                };
+            }
+            return {
+                ok: false,
+                message
+            };
+        }
+    },
+    async getAccountInfo(creds) {
+        return requestPlan(creds);
+    },
+    async getAvailableDevices(creds) {
+        return requestDevices(creds);
+    },
+    async uploadApp(creds, localPath) {
+        const fileContents = await readFile(localPath);
+        const form = new FormData();
+        form.append("file", new Blob([new Uint8Array(fileContents)]), path.basename(localPath));
+        form.append("custom_id", buildCustomId(localPath));
+        const response = await fetch(UPLOAD_URL, {
+            method: "POST",
+            headers: {
+                Authorization: browserStackAdapter.getAuthHeader(creds)
+            },
+            body: form
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new Error("BrowserStack rejected these credentials. Reconnect BrowserStack and try again.");
+        }
+        if (!response.ok) {
+            throw new Error(`BrowserStack app upload failed with status ${response.status}.`);
+        }
+        return readAppReference(await response.json());
+    },
+    async getAppStatus(creds, ref) {
+        const lookupId = sanitizeCustomId(ref.id);
+        const response = await fetch(`${RECENT_APPS_URL}/${encodeURIComponent(lookupId)}`, {
+            method: "GET",
+            headers: {
+                Authorization: browserStackAdapter.getAuthHeader(creds)
+            }
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new Error("BrowserStack rejected these credentials. Reconnect BrowserStack and try again.");
+        }
+        if (!response.ok) {
+            throw new Error(`BrowserStack uploaded-app lookup failed with status ${response.status}.`);
+        }
+        const apps = readAppStatusEntry(await response.json());
+        const matches = apps.some((entry) => entry.appUrl === ref.remotePath);
+        return {
+            status: matches ? "uploaded" : "missing",
+            message: matches
+                ? "Uploaded app reference is still available on BrowserStack."
+                : "Uploaded app reference is missing or has expired on BrowserStack."
+        };
+    },
+    async deleteApp(_creds, _ref) {
+        return notImplemented("deleteApp");
+    },
+    buildCapabilities(opts) {
+        const bstackOptions = {
+            projectName: opts.projectName,
+            buildName: opts.buildName,
+            sessionName: opts.sessionName,
+            debug: true,
+            video: true
+        };
+        if (typeof opts.username === "string" && opts.username.trim().length > 0) {
+            bstackOptions.userName = opts.username.trim();
+        }
+        if (typeof opts.accessKey === "string" && opts.accessKey.trim().length > 0) {
+            bstackOptions.accessKey = opts.accessKey.trim();
+        }
+        return {
+            platformName: opts.platform,
+            "appium:deviceName": opts.deviceName,
+            "appium:platformVersion": opts.osVersion,
+            "appium:app": opts.app,
+            "appium:automationName": opts.platform === "ios" ? "XCUITest" : "UiAutomator2",
+            "bstack:options": {
+                ...bstackOptions
+            }
+        };
+    },
+    getHubUrl() {
+        return HUB_URL;
+    },
+    getAuthHeader(creds) {
+        const username = typeof creds.username === "string" ? creds.username : "";
+        const accessKey = typeof creds.accessKey === "string" ? creds.accessKey : "";
+        return `Basic ${Buffer.from(`${username}:${accessKey}`).toString("base64")}`;
+    },
+    async getSessionArtifacts(creds, sessionId) {
+        const response = await fetch(`https://api-cloud.browserstack.com/app-automate/sessions/${encodeURIComponent(sessionId)}.json`, {
+            method: "GET",
+            headers: {
+                Authorization: browserStackAdapter.getAuthHeader(creds)
+            }
+        });
+        if (response.status === 401 || response.status === 403) {
+            throw new Error("BrowserStack rejected these credentials. Reconnect BrowserStack and try again.");
+        }
+        if (!response.ok) {
+            throw new Error(`BrowserStack session lookup failed with status ${response.status}.`);
+        }
+        const payload = await response.json();
+        const session = payload?.automation_session;
+        const logsUrl = typeof session?.logs === "string" && session.logs.trim().length > 0 ? session.logs.trim() : undefined;
+        const dashboardUrl = logsUrl ? logsUrl.replace(/\/logs\/?$/, "") : undefined;
+        return {
+            dashboardUrl,
+            logsUrl
+        };
+    },
+    async setSessionStatus(_creds, _sessionId, _status) {
+        return notImplemented("setSessionStatus");
+    }
+};

package/build/src/device/cloud/cloud-appium-client.js

@@ -0,0 +1,147 @@
+/**
+ * Shared Appium WebDriver client for cloud providers.
+ */
+export class CloudAppiumClient {
+    constructor(platform, provider, creds) {
+        this.platform = platform;
+        this.provider = provider;
+        this.creds = creds;
+    }
+    async request(method, path, body = null) {
+        const response = await fetch(`${this.provider.getHubUrl()}${path}`, {
+            method,
+            headers: {
+                Authorization: this.provider.getAuthHeader(this.creds),
+                ...(body ? { "Content-Type": "application/json" } : {})
+            },
+            body: body ? JSON.stringify(body) : undefined
+        });
+        const text = await response.text();
+        const payload = text ? JSON.parse(text) : {};
+        if (!response.ok) {
+            const message = payload?.value?.message ||
+                payload?.message ||
+                `Cloud Appium request failed with status ${response.status}.`;
+            throw new Error(message);
+        }
+        return payload;
+    }
+    async createSession(options) {
+        const capabilities = {
+            ...this.provider.buildCapabilities({
+                ...options,
+                platform: this.platform
+            })
+        };
+        const response = await this.request("POST", "/session", {
+            capabilities: {
+                alwaysMatch: capabilities
+            }
+        });
+        const sessionId = response.value?.sessionId ?? response.sessionId;
+        if (!sessionId) {
+            throw new Error("Cloud Appium session was created without a session ID.");
+        }
+        return {
+            sessionId,
+            capabilities
+        };
+    }
+    async deleteSession(sessionId) {
+        await this.request("DELETE", `/session/${sessionId}`);
+    }
+    async getSessionStatus(sessionId) {
+        const response = await this.request("GET", `/session/${sessionId}`);
+        return response.value;
+    }
+    async getScreenshot(sessionId) {
+        const response = await this.request("GET", `/session/${sessionId}/screenshot`);
+        return response.value;
+    }
+    async getWindowRect(sessionId) {
+        const response = await this.request("GET", `/session/${sessionId}/window/rect`);
+        return {
+            width: Number(response.value?.width),
+            height: Number(response.value?.height)
+        };
+    }
+    async tap(sessionId, x, y) {
+        await this.request("POST", `/session/${sessionId}/actions`, {
+            actions: [
+                {
+                    type: "pointer",
+                    id: "finger1",
+                    parameters: { pointerType: "touch" },
+                    actions: [
+                        { type: "pointerMove", duration: 0, x: Math.round(x), y: Math.round(y) },
+                        { type: "pointerDown", button: 0 },
+                        { type: "pause", duration: 100 },
+                        { type: "pointerUp", button: 0 }
+                    ]
+                }
+            ]
+        });
+    }
+    async type(sessionId, text) {
+        const keyActions = [];
+        for (const char of text) {
+            keyActions.push({ type: "keyDown", value: char });
+            keyActions.push({ type: "keyUp", value: char });
+        }
+        await this.request("POST", `/session/${sessionId}/actions`, {
+            actions: [
+                {
+                    type: "key",
+                    id: "keyboard",
+                    actions: keyActions
+                }
+            ]
+        });
+    }
+    async scroll(sessionId, startX, startY, endX, endY, duration = 500) {
+        await this.request("POST", `/session/${sessionId}/actions`, {
+            actions: [
+                {
+                    type: "pointer",
+                    id: "finger1",
+                    parameters: { pointerType: "touch" },
+                    actions: [
+                        { type: "pointerMove", duration: 0, x: Math.round(startX), y: Math.round(startY) },
+                        { type: "pointerDown", button: 0 },
+                        { type: "pointerMove", duration, x: Math.round(endX), y: Math.round(endY) },
+                        { type: "pointerUp", button: 0 }
+                    ]
+                }
+            ]
+        });
+    }
+    async drag(sessionId, startX, startY, endX, endY, duration = 500) {
+        await this.request("POST", `/session/${sessionId}/actions`, {
+            actions: [
+                {
+                    type: "pointer",
+                    id: "finger1",
+                    parameters: { pointerType: "touch" },
+                    actions: [
+                        { type: "pointerMove", duration: 0, x: Math.round(startX), y: Math.round(startY) },
+                        { type: "pointerDown", button: 0 },
+                        { type: "pause", duration: 200 },
+                        { type: "pointerMove", duration, x: Math.round(endX), y: Math.round(endY) },
+                        { type: "pointerUp", button: 0 }
+                    ]
+                }
+            ]
+        });
+    }
+    async pressIosButton(sessionId, buttonName) {
+        await this.request("POST", `/session/${sessionId}/execute/sync`, {
+            script: "mobile: pressButton",
+            args: [{ name: buttonName }]
+        });
+    }
+    async pressAndroidKeyCode(sessionId, keycode) {
+        await this.request("POST", `/session/${sessionId}/appium/device/press_keycode`, {
+            keycode
+        });
+    }
+}

package/build/src/device/cloud/connection.js

@@ -0,0 +1,116 @@
+import sharp from "sharp";
+import { getCloudProviderAdapter } from "./registry.js";
+import { CloudAppiumClient } from "./cloud-appium-client.js";
+let activeSession = null;
+function buildCloudCapabilities(options) {
+    const { providerId, platform, deviceName, osVersion, appRef, credentials } = options;
+    const adapter = getCloudProviderAdapter(providerId);
+    if (!adapter) {
+        throw new Error(`Cloud provider "${providerId}" is not registered.`);
+    }
+    return {
+        adapter,
+        capabilities: {
+            platform,
+            deviceName,
+            osVersion,
+            app: appRef,
+            username: credentials?.username,
+            accessKey: credentials?.accessKey,
+            projectName: "droid-cua",
+            buildName: `desktop-${new Date().toISOString().slice(0, 10)}`,
+            sessionName: `${deviceName} (${platform})`
+        }
+    };
+}
+export async function connectToDevice(options) {
+    if (!options || typeof options !== "object") {
+        throw new Error("Cloud device connection requires a provider payload.");
+    }
+    const { providerId, platform, deviceName, osVersion, appRef, credentials } = options;
+    if (!providerId || !platform || !deviceName || !osVersion || !appRef) {
+        throw new Error("Cloud device connection is missing required session details.");
+    }
+    if (!credentials?.username || !credentials?.accessKey) {
+        throw new Error("Cloud device connection is missing provider credentials.");
+    }
+    const { adapter, capabilities } = buildCloudCapabilities({
+        providerId,
+        platform,
+        deviceName,
+        osVersion,
+        appRef,
+        credentials
+    });
+    const client = new CloudAppiumClient(platform, adapter, credentials);
+    console.log(`Creating ${adapter.displayName} session on ${deviceName} (${osVersion})...`);
+    const session = await client.createSession(capabilities);
+    activeSession = {
+        client,
+        providerId,
+        platform,
+        deviceName,
+        osVersion,
+        sessionId: session.sessionId
+    };
+    console.log(`Connected to ${adapter.displayName} device "${deviceName}" (${session.sessionId})`);
+    return session.sessionId;
+}
+export async function getDeviceInfo(sessionId) {
+    if (!activeSession || activeSession.sessionId !== sessionId) {
+        throw new Error("No active cloud session. Call connectToDevice first.");
+    }
+    const windowRect = await activeSession.client.getWindowRect(sessionId);
+    const screenshot = await activeSession.client.getScreenshot(sessionId);
+    const screenshotBuffer = Buffer.from(screenshot, "base64");
+    const metadata = await sharp(screenshotBuffer).metadata();
+    const pixelWidth = metadata.width;
+    const pixelHeight = metadata.height;
+    const devicePixelRatio = windowRect.width > 0 && pixelWidth ? Math.max(1, Math.round(pixelWidth / windowRect.width)) : 1;
+    activeSession.devicePixelRatio = devicePixelRatio;
+    const targetWidth = 400;
+    const scale = pixelWidth > targetWidth ? targetWidth / pixelWidth : 1.0;
+    const scaledWidth = Math.round(pixelWidth * scale);
+    const scaledHeight = Math.round(pixelHeight * scale);
+    return {
+        platform: activeSession.platform,
+        device_name: activeSession.deviceName,
+        model: activeSession.deviceName,
+        device_width: pixelWidth,
+        device_height: pixelHeight,
+        scaled_width: scaledWidth,
+        scaled_height: scaledHeight,
+        scale
+    };
+}
+export async function getScreenshotAsBase64(sessionId, deviceInfo) {
+    if (!activeSession || activeSession.sessionId !== sessionId) {
+        throw new Error("No active cloud session. Call connectToDevice first.");
+    }
+    const rawBase64 = await activeSession.client.getScreenshot(sessionId);
+    let buffer = Buffer.from(rawBase64, "base64");
+    if (deviceInfo.scale < 1.0) {
+        buffer = await sharp(buffer)
+            .resize({ width: deviceInfo.scaled_width, height: deviceInfo.scaled_height })
+            .png()
+            .toBuffer();
+    }
+    return buffer.toString("base64");
+}
+export function getActiveSession() {
+    return activeSession;
+}
+export function getDevicePixelRatio() {
+    return activeSession?.devicePixelRatio || 1;
+}
+export async function disconnect() {
+    if (!activeSession) {
+        return;
+    }
+    try {
+        await activeSession.client.deleteSession(activeSession.sessionId);
+    }
+    finally {
+        activeSession = null;
+    }
+}

package/build/src/device/cloud/registry.js

@@ -0,0 +1,8 @@
+import { browserStackAdapter } from "./browserstack/adapter.js";
+const availableAdapters = [browserStackAdapter];
+export function listCloudProviderAdapters() {
+    return availableAdapters;
+}
+export function getCloudProviderAdapter(providerId) {
+    return availableAdapters.find((adapter) => adapter.id === providerId) ?? null;
+}

package/build/src/device/connection.js

@@ -4,7 +4,7 @@
  * Thin wrapper that delegates to the appropriate platform backend.
  * Maintains backwards compatibility with existing code.
  */
-import { getDeviceBackend, detectPlatform, setCurrentPlatform, getCurrentPlatform } from "./factory.js";
+import { getDeviceBackend, detectPlatform, setCurrentPlatform, setCurrentSource, getCurrentPlatform } from "./factory.js";
 let currentBackend = null;
 /**
  * Connect to a device (Android target ID/serial or iOS simulator name)
@@ -12,12 +12,22 @@ let currentBackend = null;
  * @param {string} platform - Optional platform override ('android' or 'ios')
  * @returns {Promise<string>} Device ID
  */
-export async function connectToDevice(deviceName, platform = null) {
-    const detectedPlatform = platform || detectPlatform(deviceName);
+export async function connectToDevice(deviceNameOrOptions, platform = null) {
+    if (deviceNameOrOptions && typeof deviceNameOrOptions === "object" && !Array.isArray(deviceNameOrOptions)) {
+        const cloudOptions = deviceNameOrOptions;
+        const detectedPlatform = cloudOptions.platform || detectPlatform(cloudOptions.deviceName);
+        setCurrentSource(cloudOptions.source || "local");
+        setCurrentPlatform(detectedPlatform);
+        currentBackend = getDeviceBackend(detectedPlatform);
+        console.log(`Platform: ${detectedPlatform}`);
+        return currentBackend.connectToDevice(cloudOptions);
+    }
+    const detectedPlatform = platform || detectPlatform(deviceNameOrOptions);
+    setCurrentSource("local");
     setCurrentPlatform(detectedPlatform);
     currentBackend = getDeviceBackend(detectedPlatform);
     console.log(`Platform: ${detectedPlatform}`);
-    return currentBackend.connectToDevice(deviceName);
+    return currentBackend.connectToDevice(deviceNameOrOptions);
 }
 /**
  * Get device info (screen dimensions and scale factor)
@@ -55,4 +65,5 @@ export async function disconnect() {
         await currentBackend.disconnect();
     }
     currentBackend = null;
+    setCurrentSource("local");
 }

package/build/src/device/factory.js

@@ -5,10 +5,13 @@
  */
 import * as androidConnection from "./android/connection.js";
 import * as androidActions from "./android/actions.js";
+import * as cloudConnection from "./cloud/connection.js";
+import * as cloudActions from "./cloud/actions.js";
 import * as iosConnection from "./ios/connection.js";
 import * as iosActions from "./ios/actions.js";
 // Current platform state
 let currentPlatform = null;
+let currentSource = "local";
 /**
  * Detect platform from device name or environment variable
  * @param {string} deviceName - The device ID/name/simulator name
@@ -38,6 +41,15 @@ export function detectPlatform(deviceName) {
  * @returns {object} Backend with connection and action functions
  */
 export function getDeviceBackend(platform) {
+    if (currentSource !== "local") {
+        return {
+            connectToDevice: cloudConnection.connectToDevice,
+            getDeviceInfo: cloudConnection.getDeviceInfo,
+            getScreenshotAsBase64: cloudConnection.getScreenshotAsBase64,
+            handleModelAction: cloudActions.handleModelAction,
+            disconnect: cloudConnection.disconnect,
+        };
+    }
     if (platform === "ios") {
         return {
             connectToDevice: iosConnection.connectToDevice,
@@ -63,6 +75,9 @@ export function getDeviceBackend(platform) {
 export function setCurrentPlatform(platform) {
     currentPlatform = platform;
 }
+export function setCurrentSource(source) {
+    currentSource = source || "local";
+}
 /**
  * Get the current platform
  * @returns {string|null}
@@ -70,3 +85,6 @@ export function setCurrentPlatform(platform) {
 export function getCurrentPlatform() {
     return currentPlatform;
 }
+export function getCurrentSource() {
+    return currentSource;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.0.5",
+  "version": "2.1.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",