@loadmill/droid-cua 2.28.0 → 2.30.0

package/build/src/browser/normalize.js

@@ -1,7 +1,9 @@
+import { CLOUD_BROWSER_FAMILIES, LOCAL_BROWSER_FAMILIES } from "./cloud-browser-catalog.js";
 const DEFAULT_BROWSER_VIEWPORT = {
     width: 1440,
     height: 900,
 };
+const SUPPORTED_BROWSER_FAMILIES = new Set([...LOCAL_BROWSER_FAMILIES, ...CLOUD_BROWSER_FAMILIES]);
 function finiteNumber(value, fallback) {
     return typeof value === "number" && Number.isFinite(value) ? value : fallback;
 }
@@ -14,7 +16,7 @@ export function normalizeBrowserInfoForSession(browserInfo = {}) {
     const browserName = typeof browserInfo.browserName === "string" && browserInfo.browserName.trim()
         ? browserInfo.browserName.trim()
         : "Browser";
-    const browserFamily = browserInfo.browserFamily === "chrome" || browserInfo.browserFamily === "edge"
+    const browserFamily = SUPPORTED_BROWSER_FAMILIES.has(browserInfo.browserFamily)
         ? browserInfo.browserFamily
         : null;
     const currentUrl = typeof browserInfo.currentUrl === "string" && browserInfo.currentUrl.trim()

package/build/src/browser/playwright/actions.js

@@ -1,272 +1,6 @@
-import { logger } from "../../utils/logger.js";
-import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
-import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
-import { buildActionEventMeta } from "../../device/action-event-meta.js";
-import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../../device/sensitive-action.js";
-import { normalizeAutomationUrl } from "./url.js";
+import { createGestureDispatch } from "../gesture-dispatch.js";
 import { getActiveBrowserSession, getActiveBrowserPage } from "./session-state.js";
-const KEY_ALIASES = {
-    CTRL: "Control",
-    CONTROL: "Control",
-    CMD: "Meta",
-    COMMAND: "Meta",
-    META: "Meta",
-    ALT: "Alt",
-    OPTION: "Alt",
-    SHIFT: "Shift",
-    ENTER: "Enter",
-    RETURN: "Enter",
-    ESC: "Escape",
-    ESCAPE: "Escape",
-    BACKSPACE: "Backspace",
-    DELETE: "Delete",
-    TAB: "Tab",
-    SPACE: "Space",
-    HOME: "Home",
-    END: "End",
-    PAGEUP: "PageUp",
-    PAGEDOWN: "PageDown",
-    ARROWUP: "ArrowUp",
-    ARROWDOWN: "ArrowDown",
-    ARROWLEFT: "ArrowLeft",
-    ARROWRIGHT: "ArrowRight",
-};
-function normalizeKeyEntry(key) {
-    const raw = String(key ?? "").trim();
-    if (!raw) {
-        throw new Error("Keypress action is missing keys.");
-    }
-    const normalized = KEY_ALIASES[raw.toUpperCase()];
-    if (normalized) {
-        return normalized;
-    }
-    if (raw.length === 1) {
-        return raw;
-    }
-    return raw[0].toUpperCase() + raw.slice(1);
-}
-function normalizeShortcut(keys = []) {
-    if (!Array.isArray(keys) || keys.length === 0) {
-        throw new Error("Keypress action is missing keys.");
-    }
-    return keys.map(normalizeKeyEntry).join("+");
-}
-function buildMeta(context, action, payload = {}) {
-    return buildActionEventMeta({
-        context,
-        action,
-        platform: "browser",
-        payload,
-    });
-}
-function handleNavigationTimeout(error, description) {
-    if (!(error instanceof Error) || error.name !== "TimeoutError") {
-        throw error;
-    }
-    console.warn(`Page load did not finish within the timeout for ${description}. Continuing with the current page state.`);
-}
-async function gotoWithTimeoutTolerance(page, url) {
-    try {
-        await page.goto(url, { waitUntil: "load" });
-    }
-    catch (error) {
-        handleNavigationTimeout(error, url);
-    }
-}
-async function goBackWithTimeoutTolerance(page) {
-    try {
-        await page.goBack({ waitUntil: "load" });
-    }
-    catch (error) {
-        handleNavigationTimeout(error, "browser history back");
-    }
-}
-async function goForwardWithTimeoutTolerance(page) {
-    try {
-        await page.goForward({ waitUntil: "load" });
-    }
-    catch (error) {
-        handleNavigationTimeout(error, "browser history forward");
-    }
-}
-function updateSessionUrlFromPage(session, page) {
-    const currentUrl = typeof page.url === "function" ? page.url() : null;
-    if (!currentUrl) {
-        return;
-    }
-    session.pageUrls?.set?.(page, currentUrl);
-    session.url = currentUrl;
-}
-export async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
-    const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
-    const session = getActiveBrowserSession(browserSessionId);
-    const page = getActiveBrowserPage(browserSessionId);
-    try {
-        emitDesktopDebug("browser.action.execute", "device", {
-            runId: context?.runId,
-            sessionId: context?.sessionId,
-            stepId: context?.stepId,
-            instructionIndex: context?.instructionIndex,
-        }, {
-            platform: "browser",
-            browserSessionId,
-            actionType: action?.type,
-            scale,
-            x: action?.x,
-            y: action?.y,
-            x1: action?.x1,
-            y1: action?.y1,
-            x2: action?.x2,
-            y2: action?.y2,
-            text: buildDebugTextField(action, truncateForDebug),
-            ...buildSensitiveDebugFields(action),
-            keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
-            pathPoints: Array.isArray(action?.path) ? action.path.length : 0,
-        });
-        switch (action.type) {
-            case "click": {
-                const x = Math.round(action.x / scale);
-                const y = Math.round(action.y / scale);
-                addOutput({ type: "action", text: "Click", ...buildMeta(context, action, { x, y, unit: "px" }) });
-                await page.mouse.click(x, y);
-                break;
-            }
-            case "double_click": {
-                const x = Math.round(action.x / scale);
-                const y = Math.round(action.y / scale);
-                addOutput({ type: "action", text: `Double-clicking at (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
-                await page.mouse.dblclick(x, y);
-                break;
-            }
-            case "move": {
-                const x = Math.round(action.x / scale);
-                const y = Math.round(action.y / scale);
-                addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
-                await page.mouse.move(x, y);
-                break;
-            }
-            case "scroll": {
-                const scrollX = Math.round((action.scroll_x ?? 0) / scale);
-                const scrollY = Math.round((action.scroll_y ?? 0) / scale);
-                addOutput({
-                    type: "action",
-                    text: `Scrolling by (${scrollX}, ${scrollY})`,
-                    ...buildMeta(context, action, { scrollX, scrollY, unit: "px" }),
-                });
-                await page.mouse.wheel(scrollX, scrollY);
-                break;
-            }
-            case "drag": {
-                const dragPath = Array.isArray(action.path) ? action.path : [];
-                if (dragPath.length < 2) {
-                    throw new Error("Drag action requires a path with at least two points.");
-                }
-                const [start] = dragPath;
-                const end = dragPath[dragPath.length - 1];
-                const scaledStart = { x: Math.round(start.x / scale), y: Math.round(start.y / scale) };
-                const scaledEnd = { x: Math.round(end.x / scale), y: Math.round(end.y / scale) };
-                addOutput({
-                    type: "action",
-                    text: `Dragging from (${scaledStart.x}, ${scaledStart.y}) to (${scaledEnd.x}, ${scaledEnd.y})`,
-                    ...buildMeta(context, action, { pathStart: scaledStart, pathEnd: scaledEnd, unit: "px" }),
-                });
-                await page.mouse.move(scaledStart.x, scaledStart.y);
-                await page.mouse.down();
-                for (const point of dragPath.slice(1)) {
-                    await page.mouse.move(Math.round(point.x / scale), Math.round(point.y / scale));
-                }
-                await page.mouse.up();
-                break;
-            }
-            case "type": {
-                const text = typeof action.text === "string" ? action.text : "";
-                addOutput({
-                    type: "action",
-                    text: buildTypeActionText(action, text),
-                    ...buildMeta(context, action, buildTypeActionPayload(action, text))
-                });
-                await page.keyboard.type(text);
-                break;
-            }
-            case "keypress": {
-                const shortcut = normalizeShortcut(action.keys);
-                addOutput({
-                    type: "action",
-                    text: `Pressing key: ${shortcut}`,
-                    ...buildMeta(context, action, { keys: action.keys, shortcut }),
-                });
-                await page.keyboard.press(shortcut);
-                break;
-            }
-            case "wait": {
-                addOutput({ type: "action", text: "Waiting...", ...buildMeta(context, action) });
-                const timeoutMs = typeof action.ms === "number" && Number.isFinite(action.ms) && action.ms >= 0
-                    ? Math.round(action.ms)
-                    : getConfiguredStepDelayMs();
-                await page.waitForTimeout(timeoutMs);
-                break;
-            }
-            case "goto": {
-                const normalizedUrl = normalizeAutomationUrl(action.url);
-                addOutput({
-                    type: "action",
-                    text: `Navigating to ${normalizedUrl}`,
-                    ...buildMeta(context, action, { url: normalizedUrl }),
-                });
-                await gotoWithTimeoutTolerance(page, normalizedUrl);
-                session.pageUrls?.set?.(page, normalizedUrl);
-                session.url = normalizedUrl;
-                break;
-            }
-            case "back": {
-                addOutput({ type: "action", text: "Going back", ...buildMeta(context, action) });
-                await goBackWithTimeoutTolerance(page);
-                updateSessionUrlFromPage(session, page);
-                break;
-            }
-            case "forward": {
-                addOutput({ type: "action", text: "Going forward", ...buildMeta(context, action) });
-                await goForwardWithTimeoutTolerance(page);
-                updateSessionUrlFromPage(session, page);
-                break;
-            }
-            default:
-                addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
-        }
-        return { success: true };
-    }
-    catch (error) {
-        const outputError = buildActionErrorOutput(action, error);
-        logger.error("Action execution error", {
-            action: actionForErrorLog(action),
-            message: outputError.message,
-            stack: isSensitiveAction(action) ? undefined : error.stack,
-        });
-        emitDesktopDebug("browser.error", "device", {
-            runId: context?.runId,
-            sessionId: context?.sessionId,
-            stepId: context?.stepId,
-            instructionIndex: context?.instructionIndex,
-        }, {
-            platform: "browser",
-            operation: "action.execute",
-            actionType: action?.type,
-            message: outputError.message,
-        });
-        addOutput({
-            type: "error",
-            text: outputError.text,
-            eventType: "error",
-            actionType: action?.type,
-            runId: context?.runId,
-            stepId: context?.stepId,
-            instructionIndex: context?.instructionIndex,
-            payload: {
-                message: outputError.message,
-                platform: "browser",
-            },
-        });
-        addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
-        return { success: false, error: outputError.message };
-    }
-}
+export const handleModelAction = createGestureDispatch({
+    getSession: getActiveBrowserSession,
+    getPage: getActiveBrowserPage,
+});

package/build/src/browser/webdriver/actions.js

@@ -0,0 +1,10 @@
+import { createGestureDispatch } from "../gesture-dispatch.js";
+import { getActiveWebDriverPage, getActiveWebDriverSession, reportRunningOnFirstUse, } from "./session-state.js";
+const dispatchModelAction = createGestureDispatch({
+    getSession: getActiveWebDriverSession,
+    getPage: getActiveWebDriverPage,
+});
+export async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
+    reportRunningOnFirstUse(browserSessionId);
+    return dispatchModelAction(browserSessionId, action, scale, context);
+}

package/build/src/browser/webdriver/client.js

@@ -0,0 +1,83 @@
+/**
+ * Thin fetch wrapper over an already-attached W3C WebDriver session. The
+ * backend built on this client never issues `POST /session` or
+ * `DELETE /session/:id` — it only ever drives a session the Loadmill grid
+ * already created.
+ */
+const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+export class WebDriverClient {
+    constructor({ url, sessionId, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS }) {
+        this.url = url;
+        this.sessionId = sessionId;
+        this.requestTimeoutMs = requestTimeoutMs;
+    }
+    async request(method, path, body = null) {
+        let response;
+        try {
+            response = await fetch(`${this.url}${path}`, {
+                method,
+                headers: body !== null ? { "Content-Type": "application/json" } : {},
+                body: body !== null ? JSON.stringify(body) : undefined,
+                signal: AbortSignal.timeout(this.requestTimeoutMs),
+            });
+        }
+        catch (error) {
+            if (error?.name === "AbortError" || error?.name === "TimeoutError") {
+                throw new Error(`WebDriver request to session ${this.sessionId} timed out after ${this.requestTimeoutMs}ms; the grid is not responding.`);
+            }
+            throw error;
+        }
+        const text = await response.text();
+        const payload = text ? JSON.parse(text) : {};
+        if (!response.ok) {
+            const message = payload?.value?.message ||
+                payload?.message ||
+                `WebDriver request failed with status ${response.status}.`;
+            throw new Error(message);
+        }
+        return payload;
+    }
+    async navigateTo(url) {
+        await this.request("POST", `/session/${this.sessionId}/url`, { url });
+    }
+    async goBack() {
+        await this.request("POST", `/session/${this.sessionId}/back`, {});
+    }
+    async goForward() {
+        await this.request("POST", `/session/${this.sessionId}/forward`, {});
+    }
+    async getScreenshot() {
+        const response = await this.request("GET", `/session/${this.sessionId}/screenshot`);
+        return response.value;
+    }
+    async setWindowRect({ width, height }) {
+        await this.request("POST", `/session/${this.sessionId}/window/rect`, {
+            x: 0,
+            y: 0,
+            width,
+            height,
+        });
+    }
+    async getWindowHandle() {
+        const response = await this.request("GET", `/session/${this.sessionId}/window`);
+        return response.value;
+    }
+    async getWindowHandles() {
+        const response = await this.request("GET", `/session/${this.sessionId}/window/handles`);
+        return response.value;
+    }
+    async switchToWindow(handle) {
+        await this.request("POST", `/session/${this.sessionId}/window`, { handle });
+    }
+    async closeWindow() {
+        const response = await this.request("DELETE", `/session/${this.sessionId}/window`);
+        return response.value;
+    }
+    async performActions(actions) {
+        await this.request("POST", `/session/${this.sessionId}/actions`, { actions });
+    }
+    async executeScript(script, args = []) {
+        const response = await this.request("POST", `/session/${this.sessionId}/execute/sync`, { script, args });
+        return response.value;
+    }
+}