@loadmill/droid-cua 2.13.0 → 2.15.1

package/README.md

@@ -142,6 +142,9 @@ droid-cua
 # Headless Android run
 droid-cua --avd adb:emulator-5554 --instructions tests/login.dcua
 
+# Headless Android folder run
+droid-cua --avd adb:emulator-5554 --instructions tests
+
 # Headless iOS simulator run
 droid-cua --platform ios --avd "iPhone 16" --instructions tests/login.dcua
 

package/build/index.js

@@ -3,14 +3,12 @@ import dotenv from "dotenv";
 import path from "path";
 import { mkdir } from "fs/promises";
 import { connectToDevice, disconnect as disconnectDevice, getDeviceInfo } from "./src/device/connection.js";
-import { connectToBrowser, getBrowserInfo, getCurrentPlatform as getBrowserCurrentPlatform, getScreenshotAsBase64 as getBrowserScreenshotAsBase64, } from "./src/browser/connection.js";
-import { handleModelAction as handleBrowserModelAction } from "./src/browser/actions.js";
+import { connectToBrowser, getBrowserInfo, } from "./src/browser/connection.js";
 import { normalizeBrowserInfoForSession } from "./src/browser/normalize.js";
 import { Session } from "./src/core/session.js";
 import { ExecutionEngine } from "./src/core/execution-engine.js";
-import { buildBaseSystemPrompt, buildExecutionModePrompt } from "./src/core/prompts.js";
+import { buildBaseSystemPrompt } from "./src/core/prompts.js";
 import { startInkShell } from "./src/cli/ink-shell.js";
-import { ExecutionMode } from "./src/modes/execution-mode.js";
 import { createOpenAiExecutor } from "./src/device/openai.js";
 import { logger } from "./src/utils/logger.js";
 import { selectDevice } from "./src/cli/device-selector.js";
@@ -18,10 +16,10 @@ import { resolveExecutionAppContext } from "./src/core/app-context.js";
 import { createRunContext, resolveRuntimeConfigFromEnv, resolveRuntimeCredentialsFromEnv, } from "./src/core/runtime-config.js";
 import { resolveHeadlessExecutionConfig } from "./src/cli/headless-execution-config.js";
 import { resolveHeadlessTarget } from "./src/cli/headless-target.js";
-import { printCliOutput } from "./src/utils/console-output.js";
-import { emitDesktopDebug } from "./src/utils/desktop-debug.js";
 import { createHeadlessDebugArtifacts } from "./src/cli/headless-debug.js";
-import { loadHeadlessInstructions, resolveHeadlessInstructionsPath } from "./src/cli/headless-instructions.js";
+import { loadHeadlessInstructions } from "./src/cli/headless-instructions.js";
+import { listHeadlessInstructionFiles, resolveHeadlessInstructionsTarget } from "./src/cli/headless-instruction-target.js";
+import { runHeadlessInstructionRuns } from "./src/cli/headless-runner.js";
 import { resolveLambdaTestCredentials, uploadLambdaTestCliApp } from "./src/cli/lambdatest-app-upload.js";
 import { createLoadmillApiClient } from "./src/integrations/loadmill/client.js";
 import { getCloudProviderAdapter } from "./src/device/cloud/registry.js";
@@ -109,13 +107,30 @@ async function main() {
         throw new Error("--device-source lambdatest is only supported with --instructions headless execution.");
     }
     let preloadedHeadlessConfig = null;
-    let preloadedHeadlessInstructions = null;
-    let resolvedInstructionsFile = instructionsFile;
+    let preloadedHeadlessTarget = null;
+    let headlessInstructionRuns = [];
     let preloadedAppContext = null;
     if (isHeadlessInstructionsRun) {
         preloadedHeadlessConfig = await resolveHeadlessExecutionConfig(args);
-        resolvedInstructionsFile = await resolveHeadlessInstructionsPath(instructionsFile);
-        preloadedHeadlessInstructions = await loadHeadlessInstructions(resolvedInstructionsFile);
+        preloadedHeadlessTarget = await resolveHeadlessInstructionsTarget(instructionsFile);
+        if (preloadedHeadlessTarget.type === "directory") {
+            const files = await listHeadlessInstructionFiles(preloadedHeadlessTarget.path);
+            if (files.length === 0) {
+                throw new Error(`No .dcua tests found in instructions directory ${preloadedHeadlessTarget.path}.`);
+            }
+            headlessInstructionRuns = files.map((file) => ({
+                testPath: file.path,
+                testName: file.filename,
+                instructions: null
+            }));
+        }
+        else {
+            headlessInstructionRuns = [{
+                    testPath: preloadedHeadlessTarget.path,
+                    testName: path.basename(preloadedHeadlessTarget.path),
+                    instructions: await loadHeadlessInstructions(preloadedHeadlessTarget.path)
+                }];
+        }
         preloadedAppContext = await resolveExecutionAppContext({
             enabled: preloadedHeadlessConfig.appContextEnabled,
             contextPath: preloadedHeadlessConfig.appContextPath,
@@ -198,25 +213,9 @@ async function main() {
     session.setSystemPrompt(initialSystemText);
     // If --instructions provided, run in headless mode
     if (instructionsFile) {
-        const runId = `run-${Date.now()}`;
-        const testName = path.basename(resolvedInstructionsFile);
-        let instructions = [];
-        let executionMode = null;
         let exitCode = null;
         try {
             const headlessConfig = preloadedHeadlessConfig;
-            await headlessDebug.startExecutionSession(runId, {
-                testName,
-                platform: deviceInfo.platform,
-                deviceName: deviceInfo.device_name,
-                aimAssistMode: headlessConfig.aimAssistMode
-            });
-            const currentLogFilePath = await headlessDebug.getCurrentLogFilePath();
-            if (currentLogFilePath) {
-                console.log(`Debug logging enabled: ${currentLogFilePath}`);
-            }
-            console.log(`\nRunning test from: ${resolvedInstructionsFile}\n`);
-            instructions = preloadedHeadlessInstructions;
             const appContext = preloadedAppContext;
             const appContextText = appContext.text;
             const runtimeContext = createCliRuntimeContext({
@@ -232,130 +231,21 @@ async function main() {
                     message: appContext.message,
                 },
             });
-            console.log(`Using CUA model: ${runtimeContext.config.cuaModel}`);
-            if (appContext.status === "loaded" || appContext.status === "empty") {
-                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
-                    source: "cli_context_flag",
-                    contextPath: appContext.contextPath,
-                    status: appContext.status,
-                    text: appContextText
-                });
-                if (appContextText) {
-                    console.log(`Using app context from: ${appContext.contextPath}`);
-                }
-            }
-            else if (appContext.status === "failed") {
-                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
-                    source: "cli_context_flag",
-                    contextPath: appContext.contextPath,
-                    status: "failed",
-                    message: appContext.message
-                });
-                console.warn(`Warning: could not load app context from ${appContext.contextPath}. Running without it.`);
-            }
-            else {
-                emitDesktopDebug("app_context.loaded", "execution", { runId }, {
-                    source: "cli_context_flag",
-                    contextPath: appContext.contextPath,
-                    status: appContext.status
-                });
-            }
-            const executionPrompt = buildExecutionModePrompt(deviceInfo, headlessConfig.promptCustomizations, appContextText, { aimAssistMode: headlessConfig.aimAssistMode });
-            session.setSystemPrompt(executionPrompt);
-            const screenshotRecorder = headlessDebug.createExecutionScreenshotRecorder({
-                runId,
+            const result = await runHeadlessInstructionRuns({
+                runs: headlessInstructionRuns,
+                instructionsTarget: preloadedHeadlessTarget,
+                session,
+                deviceInfo,
+                headlessConfig,
+                appContext,
+                runtimeContext,
+                headlessDebug,
                 recordScreenshots,
-                screenshotDir
-            });
-            const executionEngineOptions = {
-                toolCapabilities: {
-                    assert: true,
-                    navigate: isBrowserTarget,
-                    loadmill: true,
-                },
-                baseSystemPrompt: executionPrompt,
-                isHeadlessMode: true,
-            };
-            const engineOptions = screenshotRecorder
-                ? {
-                    recordScreenshots: true,
-                    screenshotRecorder,
-                    stepDelayMs: runtimeContext.config.stepDelayMs,
-                    sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
-                        ...requestArgs,
-                        aimAssistMode: headlessConfig.aimAssistMode,
-                    }),
-                    reviseTestScript: runtimeContext.openAi.reviseTestScript,
-                    ...executionEngineOptions,
-                }
-                : {
-                    recordScreenshots,
-                    screenshotDir,
-                    stepDelayMs: runtimeContext.config.stepDelayMs,
-                    sendCUARequest: (requestArgs) => runtimeContext.openAi.sendCUARequest({
-                        ...requestArgs,
-                        aimAssistMode: headlessConfig.aimAssistMode,
-                    }),
-                    reviseTestScript: runtimeContext.openAi.reviseTestScript,
-                    ...executionEngineOptions,
-                };
-            if (isBrowserTarget) {
-                engineOptions.getScreenshotAsBase64 = (browserSessionId, browserInfo) => getBrowserScreenshotAsBase64(browserSessionId, browserInfo);
-                engineOptions.handleModelAction = (browserSessionId, action, scale, actionContext) => handleBrowserModelAction(browserSessionId, action, scale, actionContext);
-                engineOptions.getCurrentPlatform = getBrowserCurrentPlatform;
-            }
-            else if (isLambdaTestTarget) {
-                engineOptions.connectToDevice = async () => {
-                    throw new Error("LambdaTest cloud session disconnected; automatic CLI reconnect is not supported yet.");
-                };
-            }
-            const engine = new ExecutionEngine(session, engineOptions);
-            executionMode = new ExecutionMode(session, engine, instructions, true);
-            const result = await executionMode.execute({
-                runId,
-                addOutput: printCliOutput,
-                openAiClient: runtimeContext.openAiClient,
-                loadmillClient: runtimeContext.loadmillClient,
-                runtimeCredentials: runtimeContext.credentials,
-            });
-            const stats = executionMode.stats || {};
-            const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
-            await headlessDebug.endExecutionSession(runId, {
-                success: Boolean(result.success),
-                error: result.error ?? null,
-                durationMs,
-                instructionsTotal: instructions.length,
-                instructionsCompleted: stats.instructionsCompleted ?? 0,
-                actionsTotal: stats.actionCount ?? 0,
-                assertionsPassed: stats.assertionsPassed ?? 0,
-                assertionsFailed: stats.assertionsFailed ?? 0,
-                retries: stats.retryCount ?? 0
+                screenshotDir,
+                isBrowserTarget,
+                isLambdaTestTarget
             });
-            if (!result.success) {
-                console.error(`\nTest failed: ${result.error}`);
-            }
-            exitCode = result.success ? 0 : 1;
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : "Failed to start execution.";
-            const stats = executionMode?.stats || {};
-            const durationMs = stats.startTime ? Math.max(0, Date.now() - stats.startTime) : 0;
-            await headlessDebug.endExecutionSession(runId, {
-                success: false,
-                error: message,
-                ...(stats.startTime
-                    ? {
-                        durationMs,
-                        instructionsTotal: instructions.length,
-                        instructionsCompleted: stats.instructionsCompleted ?? 0,
-                        actionsTotal: stats.actionCount ?? 0,
-                        assertionsPassed: stats.assertionsPassed ?? 0,
-                        assertionsFailed: stats.assertionsFailed ?? 0,
-                        retries: stats.retryCount ?? 0
-                    }
-                    : { reason: "start_failed" })
-            });
-            throw error;
+            exitCode = result.exitCode;
         }
         finally {
             if (lambdaTestSessionConnected) {

package/build/src/browser/connection.js

@@ -28,6 +28,12 @@ export async function navigateToUrl(browserSessionId, url) {
     }
     return currentBackend.navigateToUrl(browserSessionId, url);
 }
+export async function closeCurrentTab(browserSessionId) {
+    if (!currentBackend?.closeCurrentTab) {
+        throw new Error("Active browser backend does not support closing browser tabs.");
+    }
+    return currentBackend.closeCurrentTab(browserSessionId);
+}
 export { getCurrentPlatform };
 export async function disconnect() {
     if (currentBackend?.disconnect) {

package/build/src/browser/factory.js

@@ -10,6 +10,7 @@ export function getBrowserBackend(_platform = "browser") {
         getBrowserInfo: browserConnection.getBrowserInfo,
         getScreenshotAsBase64: browserConnection.getScreenshotAsBase64,
         navigateToUrl: browserConnection.navigateToUrl,
+        closeCurrentTab: browserConnection.closeCurrentTab,
         handleModelAction: browserActions.handleModelAction,
         disconnect: browserConnection.disconnect,
     };

package/build/src/browser/playwright/connection.js

@@ -4,7 +4,7 @@ import { mkdtemp, rm } from "node:fs/promises";
 import { randomUUID } from "node:crypto";
 import { emitDesktopDebug } from "../../utils/desktop-debug.js";
 import { requireInstalledBrowser } from "./installed-browser.js";
-import { clearActiveBrowserSession, getActiveBrowserSession, setActiveBrowserSession } from "./session-state.js";
+import { clearActiveBrowserSession, getActiveBrowserSession, getSessionActiveBrowserPage, setActiveBrowserSession } from "./session-state.js";
 import { normalizeAutomationUrl, normalizeOptionalAutomationUrl } from "./url.js";
 import { getWelcomePageUrl, isWelcomePageUrl } from "./welcome-page.js";
 const DEFAULT_VIEWPORT = { width: 1440, height: 900 };
@@ -77,6 +77,62 @@ function isCurrentSession(session) {
         return false;
     }
 }
+function pageIsClosed(page) {
+    try {
+        return typeof page?.isClosed === "function" && page.isClosed();
+    }
+    catch {
+        return false;
+    }
+}
+function getActivePage(session) {
+    return getSessionActiveBrowserPage(session);
+}
+async function bringPageToFront(page) {
+    if (typeof page?.bringToFront !== "function") {
+        return;
+    }
+    try {
+        await page.bringToFront();
+    }
+    catch {
+        // Best-effort only. The logical active tab is still tracked in pageStack.
+    }
+}
+function pruneClosedPages(session) {
+    session.pageStack = session.pageStack.filter((page) => !pageIsClosed(page));
+}
+function trackPage(session, page) {
+    if (!page || session.pageStack.includes(page)) {
+        return;
+    }
+    page.on?.("close", () => {
+        handleTrackedPageClosed(session, page);
+    });
+    session.pageStack.push(page);
+    void bringPageToFront(page);
+}
+function handleTrackedPageClosed(session, page) {
+    if (session.disconnecting) {
+        return;
+    }
+    if (page === session._rootPage) {
+        handleExternalSessionClosed(session, "page_closed");
+        return;
+    }
+    const wasActive = getActivePage(session) === page;
+    session.pageStack = session.pageStack.filter((stackedPage) => stackedPage !== page);
+    pruneClosedPages(session);
+    if (wasActive) {
+        const restoredPage = getActivePage(session);
+        void bringPageToFront(restoredPage);
+        emitDesktopDebug("browser.popup", "device", { sessionId: session.sessionId }, {
+            stage: "closed",
+            restoredUrl: restoredPage?.url?.() || null,
+            stackDepth: session.pageStack.length,
+        });
+    }
+}
 async function waitForPendingBrowserSessionShutdown() {
     if (!pendingBrowserSessionShutdown) {
         return;
@@ -267,21 +323,15 @@ export async function connectToBrowser(targetOrOptions) {
     const welcomePageUrl = await getWelcomePageUrl();
     const initialUrl = normalizedUrl || welcomePageUrl;
     await closeExtraPages(context, primaryPage);
-    context.on("page", (page) => {
-        if (page === primaryPage) {
-            return;
-        }
-        console.warn("New tab or popup opened. Automation will remain attached to the original page only.");
-        emitDesktopDebug("browser.popup", "device", { sessionId }, {
-            stage: "opened",
-            url: page.url() || null,
-        });
-    });
-    await gotoWithTimeoutTolerance(primaryPage, initialUrl);
     const session = {
         sessionId,
         context,
-        page: primaryPage,
+        _rootPage: primaryPage,
+        get page() {
+            return getActivePage(this);
+        },
+        pageStack: [primaryPage],
+        pageUrls: new WeakMap([[primaryPage, normalizedUrl]]),
         url: normalizedUrl,
         welcomePageUrl,
         sessionMode: options.sessionMode,
@@ -293,10 +343,22 @@ export async function connectToBrowser(targetOrOptions) {
         externalDisconnectNotified: false,
         shutdownPromise: null,
     };
-    setActiveBrowserSession(session);
     primaryPage.on?.("close", () => {
-        handleExternalSessionClosed(session, "page_closed");
+        handleTrackedPageClosed(session, primaryPage);
+    });
+    context.on("page", (page) => {
+        if (page === primaryPage) {
+            return;
+        }
+        trackPage(session, page);
+        emitDesktopDebug("browser.popup", "device", { sessionId }, {
+            stage: "opened",
+            url: page.url() || null,
+            stackDepth: session.pageStack.length,
+        });
     });
+    await gotoWithTimeoutTolerance(primaryPage, initialUrl);
+    setActiveBrowserSession(session);
     context.on?.("close", () => {
         handleExternalSessionClosed(session, "context_closed");
     });
@@ -313,18 +375,21 @@ export async function connectToBrowser(targetOrOptions) {
 }
 export async function getBrowserInfo(browserSessionId) {
     const session = getActiveBrowserSession(browserSessionId);
-    const currentUrl = normalizeKnownPageUrl(session.page.url()) || session.url || null;
+    const activePage = getActivePage(session);
+    const currentUrl = normalizeKnownPageUrl(activePage.url()) || session.pageUrls.get(activePage) || null;
     return buildSessionBrowserInfo(currentUrl, session.installedBrowser);
 }
 export async function getScreenshotAsBase64(browserSessionId) {
     const session = getActiveBrowserSession(browserSessionId);
-    const imageBuffer = await session.page.screenshot({ type: "png" });
+    const imageBuffer = await getActivePage(session).screenshot({ type: "png" });
     return imageBuffer.toString("base64");
 }
 export async function navigateToUrl(browserSessionId, url) {
     const session = getActiveBrowserSession(browserSessionId);
     const normalizedUrl = normalizeAutomationUrl(url);
-    await gotoWithTimeoutTolerance(session.page, normalizedUrl);
+    const activePage = getActivePage(session);
+    await gotoWithTimeoutTolerance(activePage, normalizedUrl);
+    session.pageUrls.set(activePage, normalizedUrl);
     session.url = normalizedUrl;
     emitDesktopDebug("browser.navigate", "device", { browserSessionId }, {
         platform: "browser",
@@ -332,6 +397,40 @@ export async function navigateToUrl(browserSessionId, url) {
     });
     return normalizedUrl;
 }
+export async function closeCurrentTab(browserSessionId) {
+    const session = getActiveBrowserSession(browserSessionId);
+    pruneClosedPages(session);
+    const activePage = getActivePage(session);
+    const previousPage = session.pageStack.length > 1
+        ? session.pageStack[session.pageStack.length - 2]
+        : null;
+    if (!activePage || activePage === session._rootPage || !previousPage) {
+        return {
+            closed: false,
+            restoredUrl: normalizeKnownPageUrl(activePage?.url?.()) || session.pageUrls.get(activePage) || null,
+            message: "No previous browser tab is available. The current tab was left open.",
+        };
+    }
+    await activePage.close();
+    if (session.pageStack.includes(activePage)) {
+        handleTrackedPageClosed(session, activePage);
+    }
+    const restoredPage = getActivePage(session);
+    const restoredUrl = normalizeKnownPageUrl(restoredPage?.url?.()) || session.pageUrls.get(restoredPage) || null;
+    emitDesktopDebug("browser.tab.close_current", "device", { browserSessionId }, {
+        platform: "browser",
+        closed: true,
+        restoredUrl,
+        stackDepth: session.pageStack.length,
+    });
+    return {
+        closed: true,
+        restoredUrl,
+        message: restoredUrl
+            ? `Closed current tab. Returned to ${restoredUrl}.`
+            : "Closed current tab. Returned to the previous browser tab.",
+    };
+}
 export async function disconnect() {
     let session;
     try {

package/build/src/browser/playwright/session-state.js

@@ -14,6 +14,13 @@ export function getActiveBrowserSession(browserSessionId = null) {
     }
     return activeSession;
 }
+export function getSessionActiveBrowserPage(session) {
+    const stack = Array.isArray(session?.pageStack) ? session.pageStack : [];
+    if (stack.length > 0) {
+        return stack[stack.length - 1];
+    }
+    return session?._rootPage || session?.page;
+}
 export function getActiveBrowserPage(browserSessionId = null) {
-    return getActiveBrowserSession(browserSessionId).page;
+    return getSessionActiveBrowserPage(getActiveBrowserSession(browserSessionId));
 }

package/build/src/cli/headless-instruction-target.js

@@ -0,0 +1,64 @@
+import path from "path";
+import { readdir, stat } from "fs/promises";
+import { resolveHeadlessInstructionsPath } from "./headless-instructions.js";
+const testNameCollator = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" });
+const filenameCollator = new Intl.Collator(undefined, { sensitivity: "base" });
+function compareInstructionFiles(left, right) {
+    const byName = testNameCollator.compare(left.name, right.name);
+    if (byName !== 0) {
+        return byName;
+    }
+    const byModified = right.modified.localeCompare(left.modified);
+    if (byModified !== 0) {
+        return byModified;
+    }
+    return filenameCollator.compare(left.filename, right.filename);
+}
+/**
+ * Resolve a CLI instructions target as either a single .dcua file or a directory
+ * containing .dcua files.
+ *
+ * @param {string} instructionsPath
+ * @returns {Promise<{type: 'file'|'directory', path: string}>}
+ */
+export async function resolveHeadlessInstructionsTarget(instructionsPath) {
+    const resolvedPath = await resolveHeadlessInstructionsPath(instructionsPath);
+    try {
+        const pathStat = await stat(resolvedPath);
+        if (pathStat.isDirectory()) {
+            return { type: "directory", path: resolvedPath };
+        }
+    }
+    catch {
+        // Missing paths stay as file targets so the existing read error remains clear.
+    }
+    return { type: "file", path: resolvedPath };
+}
+/**
+ * List runnable .dcua files from a headless instructions directory.
+ *
+ * @param {string} instructionsDirectory
+ * @returns {Promise<Array<{name: string, filename: string, path: string, modified: string}>>}
+ */
+export async function listHeadlessInstructionFiles(instructionsDirectory) {
+    let entries;
+    try {
+        entries = await readdir(instructionsDirectory);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : "Unknown directory read error.";
+        throw new Error(`Could not read instructions directory ${instructionsDirectory}: ${message}`);
+    }
+    const dcuaFiles = entries.filter((entry) => entry.toLowerCase().endsWith(".dcua"));
+    const tests = await Promise.all(dcuaFiles.map(async (filename) => {
+        const fullPath = path.join(instructionsDirectory, filename);
+        const fileStat = await stat(fullPath);
+        return {
+            name: filename.replace(/\.dcua$/i, ""),
+            filename,
+            path: fullPath,
+            modified: fileStat.mtime.toISOString()
+        };
+    }));
+    return tests.sort(compareInstructionFiles);
+}