@loadmill/droid-cua 2.28.0 → 2.30.0

package/README.md

@@ -222,7 +222,7 @@ Example headless config:
   "appContextEnabled": true,
   "appContextPath": "../tests/context.md",
   "contextOptimizationEnabled": true,
-  "contextOptimizationThreshold": 30000
+  "contextOptimizationThreshold": 60000
 }
 ```
 
@@ -271,11 +271,11 @@ Config and precedence rules:
 - CLI flags override config file values.
 - `--context` overrides the config app-context path.
 - `--no-context` disables app context entirely.
-- Context Optimization is enabled by default with a `30000` input-token threshold.
+- Context Optimization is enabled by default with a `60000` input-token threshold.
 - `--context-optimization-threshold <tokens>` sets the threshold and enables Context Optimization.
 - `--no-context-optimization` disables Context Optimization.
-- `--explain` enables Explain Mode for the run. Explain Mode asks the agent for concise explanations of non-obvious decisions, skipped actions, and completion evidence; explanations appear in terminal output and reports.
-- Set `"explainMode": true` in the JSON config to enable the same behavior without a flag. Explain Mode is off by default, and `--explain` enables it when the config leaves it off.
+- Explain Mode is enabled by default and asks the agent for concise explanations of non-obvious decisions, skipped actions, and completion evidence; explanations appear in terminal output and reports.
+- Set `"explainMode": false` in the JSON config to disable it. The `--explain` flag remains available to explicitly enable it for a run.
 - `--base-prompt` and `--execution-prompt` let you pass prompt customizations inline on the command line.
 - `--base-prompt-file` and `--execution-prompt-file` override the corresponding prompt customizations from config.
 - If both inline and file-based prompt overrides are provided, the inline prompt flags win.
@@ -291,7 +291,7 @@ HTML report (`--report`):
 - Writes a self-contained HTML report of the run — the same Run Details view the desktop app shows, with screenshots inlined — that you can share and open in any browser.
 - With no value it saves `<test>--<date>--<time>.html` (e.g. `login--2026-06-03--16-09-20.html`) in the current directory; pass `--report <file>` to pick a filename or `--report <dir>` to pick a folder.
 - In the desktop app the same report is available from a run's **Share → Download HTML report** button.
-- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 browser replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename.
+- Runs with captured screenshots also expose **Share → Generate video**. The desktop app renders a square mobile replay or 16:9 browser replay locally, then prompts for a save location using `<test>--<date>--<time>--test-replay.mp4` as the suggested filename. While rendering, a compact **Background tasks** card shows progress for **Generating test replay**; it can be minimized to a task-count card or canceled without opening the save dialog.
 - Authenticated headless runs upload their HTML report and screenshots to Loadmill by default. Use `--no-cloud-report` to disable that artifact upload. The basic Droid run row is still reported to Loadmill.
 
 ```sh

package/build/index.js

@@ -4,7 +4,8 @@ import dotenv from "dotenv";
 import path from "path";
 import { mkdir } from "fs/promises";
 import { connectToDevice, disconnect as disconnectDevice, getDeviceInfo } from "./src/device/connection.js";
-import { connectToBrowser, getBrowserInfo, } from "./src/browser/connection.js";
+import { connectToBrowser, disconnect as disconnectBrowser, getBrowserInfo, } from "./src/browser/connection.js";
+import { setBrowserBackendId } from "./src/browser/factory.js";
 import { normalizeBrowserInfoForSession } from "./src/browser/normalize.js";
 import { Session } from "./src/core/session.js";
 import { ExecutionEngine } from "./src/core/execution-engine.js";
@@ -18,11 +19,12 @@ import { resolveExecutionTestData } from "./src/core/test-data.js";
 import { getDefaultSecretsPath, resolveExecutionSecrets } from "./src/core/secrets.js";
 import { assertRuntimeCredentialsForProvider, createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
-import { resolveHeadlessTarget, shouldWaitForLoadmillCloudCompletion } from "./src/cli/headless-target.js";
+import { artifactKindsForPool, resolveHeadlessTarget, scopeArtifactKindsToPool, shouldWaitForLoadmillCloudCompletion, } from "./src/cli/headless-target.js";
 import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
 import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
 import { listHeadlessInstructionFiles, resolveHeadlessInstructionsTarget } from "./src/cli/headless-instruction-target.js";
 import { runHeadlessInstructionRuns } from "./src/cli/headless-runner.js";
+import { buildLoadmillCloudDeviceConnectSummaryLines } from "./src/cli/loadmill-cloud-device-connect-summary.js";
 import { planScenarioRuns } from "./src/cli/scenario-planner.js";
 import { resolveLambdaTestCredentials, uploadLambdaTestCliApp } from "./src/cli/lambdatest-app-upload.js";
 import { resolveAwsDeviceFarmCredentials, uploadAwsDeviceFarmCliApp } from "./src/cli/aws-device-farm-app-upload.js";
@@ -35,9 +37,12 @@ import { droidReportUploadManager } from "./src/integrations/loadmill/report-upl
 import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
 import { setCloudProgressListener } from "./src/device/cloud/progress.js";
 import { runSkillCommand } from "./src/cli/skill-command.js";
-import { isLoadmillCloudUsageLimitError } from "./src/device/cloud/loadmill-cloud/errors.js";
-import { loadmillCloudAdapter } from "./src/device/cloud/loadmill-cloud/adapter.js";
+import { formatLoadmillCloudNotEntitledMessage, isLoadmillCloudNotEntitledError, isLoadmillCloudUsageLimitError, } from "./src/device/cloud/loadmill-cloud/errors.js";
+import { fetchAndDownloadArtifacts, loadmillCloudAdapter } from "./src/device/cloud/loadmill-cloud/adapter.js";
+import { resolveRunPool } from "./src/device/cloud/loadmill-cloud/run-pool.js";
+import { ARTIFACT_READY_STATUSES } from "./src/device/cloud/loadmill-cloud/run-status.js";
 import { formatDuration } from "./src/core/report/run-log-model.js";
+import { createBrowserSessionTeardown, withLoadmillCloudSession } from "./src/cli/loadmill-cloud-session.js";
 dotenv.config();
 process.once("SIGTERM", async () => {
     process.exitCode = 143;
@@ -45,7 +50,7 @@ process.once("SIGTERM", async () => {
     // explicitly; await the /stop because a bare fire-and-forget fetch may not
     // dispatch before exit. Capped so a slow server can't hang the exit.
     await Promise.race([
-        loadmillCloudAdapter.releaseActiveSession(),
+        Promise.all([disconnectBrowser().catch(() => { }), loadmillCloudAdapter.releaseActiveSession()]),
         new Promise((resolve) => setTimeout(resolve, 1_500)),
     ]);
     process.exit(143);
@@ -243,6 +248,26 @@ async function main() {
         });
         try {
             retroactiveStatusLine.start("Waiting for Loadmill Cloud artifacts");
+            const { pool, client, state } = await resolveRunPool({
+                credentials,
+                sessionId: targetConfig.cloudSessionId,
+            });
+            const kinds = scopeArtifactKindsToPool(targetConfig.artifacts, pool);
+            if (kinds.length === 0) {
+                throw new Error(`A Loadmill Cloud ${pool} run has none of the requested artifact kinds (${targetConfig.artifacts.join(", ")}); it produces: ${artifactKindsForPool(pool).join(", ")}.`);
+            }
+            if (pool === "web") {
+                if (!ARTIFACT_READY_STATUSES.has(state.status)) {
+                    retroactiveStatusLine.stop();
+                    console.error(`Loadmill Cloud session ${targetConfig.cloudSessionId} is not terminal yet; add --instructions to drive it, or wait until it finishes and re-run this same command to fetch artifacts.`);
+                    process.exit(1);
+                }
+                await fetchAndDownloadArtifacts({ client, sessionId: targetConfig.cloudSessionId, kinds });
+                retroactiveStatusLine.stop();
+                console.log("Loadmill Cloud artifact fetch complete.");
+                console.log(`Done in ${formatDuration(Date.now() - processStartTime)}.`);
+                process.exit(0);
+            }
             await connectToDevice({
                 source: "loadmill-cloud",
                 providerId: "loadmill-cloud",
@@ -252,7 +277,7 @@ async function main() {
                 appRef: targetConfig.appPath,
                 credentials,
                 cloudSessionId: targetConfig.cloudSessionId,
-                artifacts: targetConfig.artifacts,
+                artifacts: kinds,
             }, null, { log: () => { } });
             retroactiveStatusLine.stop();
             // A resumable (non-terminal) session reconnected successfully instead of
@@ -290,6 +315,7 @@ async function main() {
     const isLambdaTestTarget = targetConfig.deviceSource === "lambdatest";
     const isAwsDeviceFarmTarget = targetConfig.deviceSource === "aws-device-farm";
     const isLoadmillCloudTarget = targetConfig.deviceSource === "loadmill-cloud";
+    const isLoadmillCloudBrowserTarget = isBrowserTarget && isLoadmillCloudTarget;
     const isCloudTarget = isLambdaTestTarget || isAwsDeviceFarmTarget || isLoadmillCloudTarget;
     let lambdaTestSessionConnected = false;
     let lambdaTestSessionUrl = null;
@@ -297,6 +323,8 @@ async function main() {
     let awsSessionConnected = false;
     let awsDeviceFarmStatusLine = null;
     let loadmillCloudSessionConnected = false;
+    let loadmillCloudBrowserSessionConnected = false;
+    let loadmillCloudBrowserTeardown = null;
     let loadmillCloudStatusLine = null;
     const headlessDebug = createHeadlessDebugArtifacts({
         cwd: process.cwd(),
@@ -377,7 +405,94 @@ async function main() {
     }
     let targetId;
     let deviceInfo;
-    if (isBrowserTarget) {
+    if (isLoadmillCloudBrowserTarget) {
+        const loadmillCloudCredentials = resolveLoadmillCloudCredentials(process.env);
+        loadmillCloudStatusLine = createStatusLine();
+        const statusLine = loadmillCloudStatusLine;
+        setCloudProgressListener((event) => statusLine.handleEvent(event));
+        setBrowserBackendId("webdriver");
+        loadmillCloudBrowserTeardown = createBrowserSessionTeardown({
+            disconnect: () => disconnectBrowser(),
+            releaseSession: () => loadmillCloudAdapter.releaseSession(loadmillCloudCredentials, {
+                waitForCompletion: shouldWaitForLoadmillCloudCompletion(targetConfig),
+            }),
+        });
+        const provisioningAbortController = new AbortController();
+        let loadmillCloudBrowserSigintHandled = false;
+        let loadmillCloudBrowserInterruptRequested = false;
+        process.once("SIGINT", async () => {
+            if (loadmillCloudBrowserSigintHandled)
+                return;
+            loadmillCloudBrowserSigintHandled = true;
+            loadmillCloudBrowserInterruptRequested = true;
+            console.log("\nStopping Loadmill Cloud session...");
+            provisioningAbortController.abort(new Error("Stopped by user"));
+            if (!loadmillCloudBrowserSessionConnected) {
+                process.exitCode = 130;
+                return;
+            }
+            await loadmillCloudBrowserTeardown();
+            process.exit(130);
+        });
+        try {
+            statusLine.start("Starting Loadmill Cloud browser session");
+            targetId = await withLoadmillCloudSession({
+                credentials: loadmillCloudCredentials,
+                connect: () => connectToBrowser({
+                    source: "loadmill-cloud",
+                    credentials: loadmillCloudCredentials,
+                    browserFamily: targetConfig.browserFamily,
+                    browserVersion: targetConfig.browserVersion,
+                    cloudSessionId: targetConfig.cloudSessionId,
+                    artifacts: targetConfig.artifacts,
+                    signal: provisioningAbortController.signal,
+                }),
+                teardown: loadmillCloudBrowserTeardown,
+            });
+            loadmillCloudBrowserSessionConnected = true;
+            if (loadmillCloudBrowserInterruptRequested) {
+                try {
+                    await loadmillCloudBrowserTeardown();
+                }
+                finally {
+                    statusLine.stop();
+                    loadmillCloudBrowserSessionConnected = false;
+                    process.exitCode = 130;
+                }
+                return;
+            }
+            deviceInfo = normalizeBrowserInfoForSession(await getBrowserInfo(targetId));
+            statusLine.stop();
+        }
+        catch (error) {
+            statusLine.stop();
+            if (error?.artifactsOnlyExit) {
+                console.log(error.message);
+                process.exitCode = 0;
+                return;
+            }
+            if (loadmillCloudBrowserInterruptRequested) {
+                process.exitCode = 130;
+                return;
+            }
+            if (isLoadmillCloudNotEntitledError(error)) {
+                console.error(`\n${formatLoadmillCloudNotEntitledMessage(error)}`);
+                process.exitCode = 1;
+                return;
+            }
+            if (isLoadmillCloudUsageLimitError(error)) {
+                const suffix = error.code ? ` (${error.code})` : "";
+                console.error(`\n${error.message}${suffix}`);
+                process.exitCode = 1;
+                return;
+            }
+            throw error;
+        }
+        console.log(`Connected to Loadmill Cloud browser "${targetConfig.browserFamily}" (${targetId})`);
+        console.log(`Using browser viewport: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
+        console.log(`Model sees viewport: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+    }
+    else if (isBrowserTarget) {
         targetId = await connectToBrowser({
             url: startupUrl,
             headless: true,
@@ -587,6 +702,11 @@ async function main() {
                 process.exitCode = 130;
                 return;
             }
+            if (isLoadmillCloudNotEntitledError(error)) {
+                console.error(`\n${formatLoadmillCloudNotEntitledMessage(error)}`);
+                process.exitCode = 1;
+                return;
+            }
             if (isLoadmillCloudUsageLimitError(error)) {
                 const suffix = error.code ? ` (${error.code})` : "";
                 console.error(`\n${error.message}${suffix}`);
@@ -595,9 +715,9 @@ async function main() {
             }
             throw error;
         }
-        console.log(`Connected to Loadmill Cloud device "${targetConfig.deviceName}" (${targetId})`);
-        console.log(`Using real resolution: ${deviceInfo.device_width}x${deviceInfo.device_height}`);
-        console.log(`Model sees resolution: ${deviceInfo.scaled_width}x${deviceInfo.scaled_height}`);
+        for (const line of buildLoadmillCloudDeviceConnectSummaryLines({ deviceInfo })) {
+            console.log(line);
+        }
     }
     else {
         targetId = await connectToDevice(avdName, platform);
@@ -668,6 +788,7 @@ async function main() {
                 screenshotDir,
                 isBrowserTarget,
                 isCloudTarget,
+                runTarget: { isBrowserTarget, isCloudTarget },
                 report: reportConfig
             });
             exitCode = result.exitCode;
@@ -713,6 +834,17 @@ async function main() {
                     console.log(`Done in ${formatDuration(Date.now() - processStartTime)}.`);
                 }
             }
+            if (loadmillCloudBrowserSessionConnected) {
+                console.log("Ending Loadmill Cloud session...");
+                loadmillCloudStatusLine.start("Ending Loadmill Cloud session");
+                await loadmillCloudBrowserTeardown();
+                loadmillCloudStatusLine.stop();
+                loadmillCloudBrowserSessionConnected = false;
+                console.log("Loadmill Cloud session ended.");
+                if (succeeded) {
+                    console.log(`Done in ${formatDuration(Date.now() - processStartTime)}.`);
+                }
+            }
         }
         if (exitCode !== null) {
             process.exit(exitCode);

package/build/src/browser/cloud-browser-catalog.js

@@ -0,0 +1,27 @@
+export const LOCAL_BROWSER_FAMILIES = ["chrome", "edge"];
+export const CLOUD_BROWSER_FAMILIES = ["chrome", "firefox"];
+export const CLOUD_BROWSER_VERSION_ALIASES = ["latest", "latest-1", "latest-2"];
+export const DEFAULT_CLOUD_BROWSER_FAMILY = "chrome";
+export const DEFAULT_CLOUD_BROWSER_VERSION = "latest";
+const CLOUD_BROWSER_DISPLAY_NAMES = {
+    chrome: "Chrome",
+    firefox: "Firefox",
+};
+export function isSupportedCloudBrowserFamily(value) {
+    return typeof value === "string" && CLOUD_BROWSER_FAMILIES.includes(value.trim().toLowerCase());
+}
+export function isSupportedCloudBrowserVersion(value) {
+    return typeof value === "string" && CLOUD_BROWSER_VERSION_ALIASES.includes(value.trim().toLowerCase());
+}
+export function assertSupportedCloudBrowser(browserFamily, browserVersion) {
+    if (!isSupportedCloudBrowserFamily(browserFamily)) {
+        throw new Error(`Loadmill Cloud browser must be one of: ${CLOUD_BROWSER_FAMILIES.join(", ")}.`);
+    }
+    if (!isSupportedCloudBrowserVersion(browserVersion)) {
+        throw new Error(`Loadmill Cloud browser version must be one of: ${CLOUD_BROWSER_VERSION_ALIASES.join(", ")}.`);
+    }
+}
+export function formatCloudBrowserName(browserFamily, browserVersion) {
+    const label = CLOUD_BROWSER_DISPLAY_NAMES[browserFamily] ?? browserFamily;
+    return browserVersion ? `${label} (${browserVersion})` : label;
+}

package/build/src/browser/factory.js

@@ -1,20 +1,43 @@
-import * as browserConnection from "./playwright/connection.js";
-import * as browserActions from "./playwright/actions.js";
+import * as playwrightConnection from "./playwright/connection.js";
+import * as playwrightActions from "./playwright/actions.js";
+import * as webdriverConnection from "./webdriver/connection.js";
+import * as webdriverActions from "./webdriver/actions.js";
 let currentPlatform = null;
+let currentBackendId = "playwright";
 export function detectPlatform() {
     return "browser";
 }
-export function getBrowserBackend(_platform = "browser") {
+function buildPlaywrightBackend() {
+    return {
+        connectToBrowser: playwrightConnection.connectToBrowser,
+        getBrowserInfo: playwrightConnection.getBrowserInfo,
+        getScreenshotAsBase64: playwrightConnection.getScreenshotAsBase64,
+        navigateToUrl: playwrightConnection.navigateToUrl,
+        closeCurrentTab: playwrightConnection.closeCurrentTab,
+        handleModelAction: playwrightActions.handleModelAction,
+        disconnect: playwrightConnection.disconnect,
+    };
+}
+function buildWebDriverBackend() {
     return {
-        connectToBrowser: browserConnection.connectToBrowser,
-        getBrowserInfo: browserConnection.getBrowserInfo,
-        getScreenshotAsBase64: browserConnection.getScreenshotAsBase64,
-        navigateToUrl: browserConnection.navigateToUrl,
-        closeCurrentTab: browserConnection.closeCurrentTab,
-        handleModelAction: browserActions.handleModelAction,
-        disconnect: browserConnection.disconnect,
+        connectToBrowser: webdriverConnection.connectToBrowser,
+        getBrowserInfo: webdriverConnection.getBrowserInfo,
+        getScreenshotAsBase64: webdriverConnection.getScreenshotAsBase64,
+        navigateToUrl: webdriverConnection.navigateToUrl,
+        closeCurrentTab: webdriverConnection.closeCurrentTab,
+        handleModelAction: webdriverActions.handleModelAction,
+        disconnect: webdriverConnection.disconnect,
     };
 }
+export function getBrowserBackend(_platform = "browser") {
+    return currentBackendId === "webdriver" ? buildWebDriverBackend() : buildPlaywrightBackend();
+}
+export function setBrowserBackendId(backendId) {
+    currentBackendId = backendId === "webdriver" ? "webdriver" : "playwright";
+}
+export function getBrowserBackendId() {
+    return currentBackendId;
+}
 export function setCurrentPlatform(platform) {
     currentPlatform = platform;
 }

package/build/src/browser/gesture-dispatch.js

@@ -0,0 +1,281 @@
+import { logger } from "../utils/logger.js";
+import { emitDesktopDebug, truncateForDebug } from "../utils/desktop-debug.js";
+import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
+import { buildActionEventMeta } from "../device/action-event-meta.js";
+import { actionForErrorLog, buildActionErrorOutput, buildDebugTextField, buildSensitiveDebugFields, buildTypeActionPayload, buildTypeActionText, isSensitiveAction, } from "../device/sensitive-action.js";
+import { normalizeAutomationUrl } from "./playwright/url.js";
+const KEY_ALIASES = {
+    CTRL: "Control",
+    CONTROL: "Control",
+    CMD: "Meta",
+    COMMAND: "Meta",
+    META: "Meta",
+    ALT: "Alt",
+    OPTION: "Alt",
+    SHIFT: "Shift",
+    ENTER: "Enter",
+    RETURN: "Enter",
+    ESC: "Escape",
+    ESCAPE: "Escape",
+    BACKSPACE: "Backspace",
+    DELETE: "Delete",
+    TAB: "Tab",
+    SPACE: "Space",
+    HOME: "Home",
+    END: "End",
+    PAGEUP: "PageUp",
+    PAGEDOWN: "PageDown",
+    ARROWUP: "ArrowUp",
+    ARROWDOWN: "ArrowDown",
+    ARROWLEFT: "ArrowLeft",
+    ARROWRIGHT: "ArrowRight",
+};
+function normalizeKeyEntry(key) {
+    const raw = String(key ?? "").trim();
+    if (!raw) {
+        throw new Error("Keypress action is missing keys.");
+    }
+    const normalized = KEY_ALIASES[raw.toUpperCase()];
+    if (normalized) {
+        return normalized;
+    }
+    if (raw.length === 1) {
+        return raw;
+    }
+    return raw[0].toUpperCase() + raw.slice(1);
+}
+function normalizeShortcut(keys = []) {
+    if (!Array.isArray(keys) || keys.length === 0) {
+        throw new Error("Keypress action is missing keys.");
+    }
+    return keys.map(normalizeKeyEntry).join("+");
+}
+function buildMeta(context, action, payload = {}) {
+    return buildActionEventMeta({
+        context,
+        action,
+        platform: "browser",
+        payload,
+    });
+}
+function handleNavigationTimeout(error, description) {
+    if (!(error instanceof Error) || error.name !== "TimeoutError") {
+        throw error;
+    }
+    console.warn(`Page load did not finish within the timeout for ${description}. Continuing with the current page state.`);
+}
+async function gotoWithTimeoutTolerance(page, url) {
+    try {
+        await page.goto(url, { waitUntil: "load" });
+    }
+    catch (error) {
+        handleNavigationTimeout(error, url);
+    }
+}
+async function goBackWithTimeoutTolerance(page) {
+    try {
+        await page.goBack({ waitUntil: "load" });
+    }
+    catch (error) {
+        handleNavigationTimeout(error, "browser history back");
+    }
+}
+async function goForwardWithTimeoutTolerance(page) {
+    try {
+        await page.goForward({ waitUntil: "load" });
+    }
+    catch (error) {
+        handleNavigationTimeout(error, "browser history forward");
+    }
+}
+async function updateSessionUrlFromPage(session, page) {
+    const currentUrl = typeof page.url === "function" ? await page.url() : null;
+    if (!currentUrl) {
+        return;
+    }
+    session.pageUrls?.set?.(page, currentUrl);
+    session.url = currentUrl;
+}
+/**
+ * Builds the shared 11-case model-action dispatcher over a page-shaped facade.
+ * `playwright/actions.js` and `webdriver/actions.js` bind their own
+ * `{ getSession, getPage }` accessors to this factory.
+ *
+ * @param {{ getSession: (browserSessionId?: string) => object, getPage: (browserSessionId?: string) => object }} accessors
+ * @returns {(browserSessionId: string, action: object, scale?: number, context?: object|null) => Promise<{ success: boolean, error?: string }>}
+ */
+export function createGestureDispatch({ getSession, getPage }) {
+    return async function handleModelAction(browserSessionId, action, scale = 1, context = null) {
+        const addOutput = context?.addOutput || ((item) => console.log(item.text || item));
+        const session = getSession(browserSessionId);
+        const page = getPage(browserSessionId);
+        try {
+            emitDesktopDebug("browser.action.execute", "device", {
+                runId: context?.runId,
+                sessionId: context?.sessionId,
+                stepId: context?.stepId,
+                instructionIndex: context?.instructionIndex,
+            }, {
+                platform: "browser",
+                browserSessionId,
+                actionType: action?.type,
+                scale,
+                x: action?.x,
+                y: action?.y,
+                x1: action?.x1,
+                y1: action?.y1,
+                x2: action?.x2,
+                y2: action?.y2,
+                text: buildDebugTextField(action, truncateForDebug),
+                ...buildSensitiveDebugFields(action),
+                keyCount: Array.isArray(action?.keys) ? action.keys.length : 0,
+                pathPoints: Array.isArray(action?.path) ? action.path.length : 0,
+            });
+            switch (action.type) {
+                case "click": {
+                    const x = Math.round(action.x / scale);
+                    const y = Math.round(action.y / scale);
+                    addOutput({ type: "action", text: "Click", ...buildMeta(context, action, { x, y, unit: "px" }) });
+                    await page.mouse.click(x, y);
+                    break;
+                }
+                case "double_click": {
+                    const x = Math.round(action.x / scale);
+                    const y = Math.round(action.y / scale);
+                    addOutput({ type: "action", text: `Double-clicking at (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
+                    await page.mouse.dblclick(x, y);
+                    break;
+                }
+                case "move": {
+                    const x = Math.round(action.x / scale);
+                    const y = Math.round(action.y / scale);
+                    addOutput({ type: "action", text: `Moving mouse to (${x}, ${y})`, ...buildMeta(context, action, { x, y, unit: "px" }) });
+                    await page.mouse.move(x, y);
+                    break;
+                }
+                case "scroll": {
+                    const scrollX = Math.round((action.scroll_x ?? 0) / scale);
+                    const scrollY = Math.round((action.scroll_y ?? 0) / scale);
+                    addOutput({
+                        type: "action",
+                        text: `Scrolling by (${scrollX}, ${scrollY})`,
+                        ...buildMeta(context, action, { scrollX, scrollY, unit: "px" }),
+                    });
+                    await page.mouse.wheel(scrollX, scrollY);
+                    break;
+                }
+                case "drag": {
+                    const dragPath = Array.isArray(action.path) ? action.path : [];
+                    if (dragPath.length < 2) {
+                        throw new Error("Drag action requires a path with at least two points.");
+                    }
+                    const [start] = dragPath;
+                    const end = dragPath[dragPath.length - 1];
+                    const scaledStart = { x: Math.round(start.x / scale), y: Math.round(start.y / scale) };
+                    const scaledEnd = { x: Math.round(end.x / scale), y: Math.round(end.y / scale) };
+                    addOutput({
+                        type: "action",
+                        text: `Dragging from (${scaledStart.x}, ${scaledStart.y}) to (${scaledEnd.x}, ${scaledEnd.y})`,
+                        ...buildMeta(context, action, { pathStart: scaledStart, pathEnd: scaledEnd, unit: "px" }),
+                    });
+                    await page.mouse.move(scaledStart.x, scaledStart.y);
+                    await page.mouse.down();
+                    for (const point of dragPath.slice(1)) {
+                        await page.mouse.move(Math.round(point.x / scale), Math.round(point.y / scale));
+                    }
+                    await page.mouse.up();
+                    break;
+                }
+                case "type": {
+                    const text = typeof action.text === "string" ? action.text : "";
+                    addOutput({
+                        type: "action",
+                        text: buildTypeActionText(action, text),
+                        ...buildMeta(context, action, buildTypeActionPayload(action, text))
+                    });
+                    await page.keyboard.type(text);
+                    break;
+                }
+                case "keypress": {
+                    const shortcut = normalizeShortcut(action.keys);
+                    addOutput({
+                        type: "action",
+                        text: `Pressing key: ${shortcut}`,
+                        ...buildMeta(context, action, { keys: action.keys, shortcut }),
+                    });
+                    await page.keyboard.press(shortcut);
+                    break;
+                }
+                case "wait": {
+                    addOutput({ type: "action", text: "Waiting...", ...buildMeta(context, action) });
+                    const timeoutMs = typeof action.ms === "number" && Number.isFinite(action.ms) && action.ms >= 0
+                        ? Math.round(action.ms)
+                        : getConfiguredStepDelayMs();
+                    await page.waitForTimeout(timeoutMs);
+                    break;
+                }
+                case "goto": {
+                    const normalizedUrl = normalizeAutomationUrl(action.url);
+                    addOutput({
+                        type: "action",
+                        text: `Navigating to ${normalizedUrl}`,
+                        ...buildMeta(context, action, { url: normalizedUrl }),
+                    });
+                    await gotoWithTimeoutTolerance(page, normalizedUrl);
+                    session.pageUrls?.set?.(page, normalizedUrl);
+                    session.url = normalizedUrl;
+                    break;
+                }
+                case "back": {
+                    addOutput({ type: "action", text: "Going back", ...buildMeta(context, action) });
+                    await goBackWithTimeoutTolerance(page);
+                    await updateSessionUrlFromPage(session, page);
+                    break;
+                }
+                case "forward": {
+                    addOutput({ type: "action", text: "Going forward", ...buildMeta(context, action) });
+                    await goForwardWithTimeoutTolerance(page);
+                    await updateSessionUrlFromPage(session, page);
+                    break;
+                }
+                default:
+                    addOutput({ type: "info", text: `Unknown action: ${JSON.stringify(action)}` });
+            }
+            return { success: true };
+        }
+        catch (error) {
+            const outputError = buildActionErrorOutput(action, error);
+            logger.error("Action execution error", {
+                action: actionForErrorLog(action),
+                message: outputError.message,
+                stack: isSensitiveAction(action) ? undefined : error.stack,
+            });
+            emitDesktopDebug("browser.error", "device", {
+                runId: context?.runId,
+                sessionId: context?.sessionId,
+                stepId: context?.stepId,
+                instructionIndex: context?.instructionIndex,
+            }, {
+                platform: "browser",
+                operation: "action.execute",
+                actionType: action?.type,
+                message: outputError.message,
+            });
+            addOutput({
+                type: "error",
+                text: outputError.text,
+                eventType: "error",
+                actionType: action?.type,
+                runId: context?.runId,
+                stepId: context?.stepId,
+                instructionIndex: context?.instructionIndex,
+                payload: {
+                    message: outputError.message,
+                    platform: "browser",
+                },
+            });
+            addOutput({ type: "info", text: "Full error details have been logged to the debug log." });
+            return { success: false, error: outputError.message };
+        }
+    };
+}

package/build/src/browser/interface.js

@@ -11,4 +11,7 @@ export const SUPPORTED_ACTIONS = [
     "keypress",
     "wait",
     "screenshot",
+    "goto",
+    "back",
+    "forward",
 ];