@loadmill/droid-cua 2.18.0 → 2.19.0

package/build/src/core/execution-engine.js

@@ -185,6 +185,19 @@ export class ExecutionEngine {
             if (shouldStop()) {
                 return { stopped: true, sawExplicitScreenshotAction };
             }
+            if (context?.waitIfPaused) {
+                const pauseResult = await context.waitIfPaused({
+                    boundary: "before_action",
+                    refreshScreenOnResume: true,
+                    stepContext,
+                });
+                if (pauseResult?.status === "stopped" || shouldStop()) {
+                    return { stopped: true, sawExplicitScreenshotAction };
+                }
+                if (pauseResult?.refreshScreen) {
+                    return { refreshModelState: true, sawExplicitScreenshotAction };
+                }
+            }
             if (action.type === "screenshot") {
                 sawExplicitScreenshotAction = true;
                 continue;
@@ -223,6 +236,17 @@ export class ExecutionEngine {
         }
         return { stopped: false, sawExplicitScreenshotAction };
     }
+    async waitIfPaused(context = null, stepContext = null, boundary = "turn_boundary", refreshScreenOnResume = true) {
+        if (!context?.waitIfPaused) {
+            return { status: "running", refreshScreen: false };
+        }
+        const result = await context.waitIfPaused({
+            boundary,
+            refreshScreenOnResume,
+            stepContext,
+        });
+        return result ?? { status: "running", refreshScreen: false };
+    }
     async buildActionFollowUp(callId, pendingSafetyChecks, sawExplicitScreenshotAction, addOutput, eventMeta, context = null, stepContext = null) {
         const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
         emitDesktopDebug("device.screenshot", "device", {
@@ -358,11 +382,21 @@ export class ExecutionEngine {
             instructionIndex: stepContext?.instructionIndex,
             ...extra
         });
+        let refreshBeforeNextComputerAction = false;
         while (true) {
             // Check for interruption before processing next batch of actions
             if (shouldStop()) {
+                summary.stopped = true;
+                return finalize();
+            }
+            const pauseResult = await this.waitIfPaused(context, stepContext, "before_model_output", true);
+            if (pauseResult.status === "stopped" || shouldStop()) {
+                summary.stopped = true;
                 return finalize();
             }
+            if (pauseResult.refreshScreen) {
+                refreshBeforeNextComputerAction = true;
+            }
             const items = response.output || [];
             const actionableItems = extractActionableOutputItems(items);
             const pendingSafetyChecks = collectPendingSafetyChecks(items);
@@ -385,8 +419,17 @@ export class ExecutionEngine {
                     currentScreenshot = toolResult.currentScreenshot;
                 }
                 if (shouldStop()) {
+                    summary.stopped = true;
+                    return finalize();
+                }
+                const toolPauseResult = await this.waitIfPaused(context, stepContext, "after_tool_call", true);
+                if (toolPauseResult.status === "stopped" || shouldStop()) {
+                    summary.stopped = true;
                     return finalize();
                 }
+                if (toolPauseResult.refreshScreen) {
+                    refreshBeforeNextComputerAction = true;
+                }
                 if (toolResult.kind === "fail") {
                     summary.failed = true;
                     summary.error = toolResult.error;
@@ -419,14 +462,39 @@ export class ExecutionEngine {
                 continue;
             }
             const { call_id, actions } = actionableItem;
+            if (refreshBeforeNextComputerAction) {
+                const followUp = await this.buildActionFollowUp(call_id, pendingSafetyChecks, true, addOutput, eventMeta, context, stepContext);
+                currentScreenshot = followUp.currentScreenshot ?? currentScreenshot;
+                response = await this.requestNextTurn(followUp.input, newResponseId, context, stepContext);
+                newResponseId = response.id;
+                refreshBeforeNextComputerAction = false;
+                continue;
+            }
             const actionBatchResult = await this.executeActionBatch(actions, currentScreenshot, shouldStop, context, stepContext, trackAction);
             if (actionBatchResult.stopped) {
+                summary.stopped = true;
                 return finalize();
             }
+            if (actionBatchResult.refreshModelState) {
+                const followUp = await this.buildActionFollowUp(call_id, pendingSafetyChecks, true, addOutput, eventMeta, context, stepContext);
+                currentScreenshot = followUp.currentScreenshot ?? currentScreenshot;
+                response = await this.requestNextTurn(followUp.input, newResponseId, context, stepContext);
+                newResponseId = response.id;
+                continue;
+            }
             const followUp = await this.buildActionFollowUp(call_id, pendingSafetyChecks, actionBatchResult.sawExplicitScreenshotAction, addOutput, eventMeta, context, stepContext);
             if (shouldStop()) {
+                summary.stopped = true;
                 return finalize();
             }
+            const followUpPauseResult = await this.waitIfPaused(context, stepContext, "after_action_follow_up", true);
+            if (followUpPauseResult.status === "stopped" || shouldStop()) {
+                summary.stopped = true;
+                return finalize();
+            }
+            if (followUpPauseResult.refreshScreen) {
+                refreshBeforeNextComputerAction = true;
+            }
             currentScreenshot = followUp.currentScreenshot ?? currentScreenshot;
             response = await this.requestNextTurn(followUp.input, newResponseId, context, stepContext);
             newResponseId = response.id;

package/build/src/core/report/run-log-model.js

@@ -32,6 +32,8 @@ export function mediaKindOf(line) {
     return "none";
 }
 export function isHiddenExecutionLogEvent(line) {
+    if (line.eventType === "run_paused")
+        return true;
     return mediaKindOf(line) === "hidden-debug";
 }
 export function isGalleryMediaEvent(line) {

package/build/src/modes/execution-mode.js

@@ -145,6 +145,19 @@ export class ExecutionMode {
                 });
                 return { success: false, error: 'Stopped by user' };
             }
+            if (runContext.waitIfPaused) {
+                const pauseResult = await runContext.waitIfPaused({
+                    boundary: 'before_instruction',
+                    refreshScreenOnResume: false,
+                    stepContext,
+                });
+                if (pauseResult?.status === 'stopped' || this.shouldStop) {
+                    this.emit(addOutput, 'info', 'Test execution stopped by user.', runContext, stepContext, {
+                        eventType: 'system_message'
+                    });
+                    return { success: false, error: 'Stopped by user' };
+                }
+            }
             const instruction = this.instructions[i];
             this.emit(addOutput, 'user', instruction, runContext, stepContext, {
                 eventType: 'instruction_started',
@@ -256,6 +269,9 @@ export class ExecutionMode {
                 return this.shouldStop;
             };
             const { responseId: newResponseId, summary: turnSummary } = await this.engine.runFullTurn(response, trackAction, instructionContext, stepContext, { currentScreenshot });
+            if (turnSummary.stopped || this.shouldStop) {
+                return { success: false, error: "Stopped by user" };
+            }
             if (turnSummary.assertionStatus === "passed") {
                 this.stats.assertionsPassed++;
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.18.0",
+  "version": "2.19.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",