@loadmill/droid-cua 2.4.0 → 2.6.0

package/build/src/core/execution-engine.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import path from "path";
 import { writeFile } from "fs/promises";
 import { getScreenshotAsBase64, getCurrentPlatform } from "../device/connection.js";
@@ -6,31 +7,99 @@ import { sendCUARequest } from "../device/openai.js";
 import { emitDesktopDebug } from "../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../utils/step-delay.js";
 import { printCliOutput } from "../utils/console-output.js";
-function extractComputerCalls(items) {
+import { annotateScreenshotArtifact, createAnnotatedScreenshotFromBase64 } from "../utils/run-screenshot-recorder.js";
+function extractActionableOutputItems(items) {
     const entries = [];
     for (const item of items) {
-        if (item?.type !== "computer_call")
-            continue;
-        const baseCallId = item.call_id || item.id;
-        if (!baseCallId)
+        if (item?.type === "function_call" && item.name === "preview_click") {
+            entries.push({
+                type: "preview_click",
+                call_id: item.call_id || item.id,
+                arguments: item.arguments,
+            });
             continue;
-        const actions = [];
-        if (Array.isArray(item.actions)) {
-            for (const actionEntry of item.actions) {
-                if (!actionEntry)
-                    continue;
-                const action = actionEntry.action || actionEntry;
-                if (action?.type)
-                    actions.push(action);
-            }
         }
-        else if (item.action?.type) {
-            actions.push(item.action);
+        if (item?.type === "computer_call") {
+            const baseCallId = item.call_id || item.id;
+            if (!baseCallId)
+                continue;
+            const actions = [];
+            if (Array.isArray(item.actions)) {
+                for (const actionEntry of item.actions) {
+                    if (!actionEntry)
+                        continue;
+                    const action = actionEntry.action || actionEntry;
+                    if (action?.type)
+                        actions.push(action);
+                }
+            }
+            else if (item.action?.type) {
+                actions.push(item.action);
+            }
+            entries.push({ type: "computer_call", call_id: baseCallId, actions });
         }
-        entries.push({ call_id: baseCallId, actions });
     }
     return entries;
 }
+function buildPreviewClickValidationError(argumentsText) {
+    if (!argumentsText || typeof argumentsText !== "string") {
+        return "Invalid preview_click arguments: expected a JSON object with numeric x and y.";
+    }
+    return `Invalid preview_click arguments: ${argumentsText}`;
+}
+function parsePreviewClickArguments(argumentsText) {
+    let parsedArguments;
+    try {
+        parsedArguments = JSON.parse(argumentsText);
+    }
+    catch {
+        return {
+            ok: false,
+            message: buildPreviewClickValidationError("arguments could not be parsed as JSON; expected numeric x and y."),
+        };
+    }
+    const x = parsedArguments?.x;
+    const y = parsedArguments?.y;
+    if (!Number.isFinite(x) || !Number.isFinite(y)) {
+        return {
+            ok: false,
+            message: buildPreviewClickValidationError("x and y must both be numeric."),
+        };
+    }
+    return {
+        ok: true,
+        x,
+        y,
+    };
+}
+function buildFunctionCallOutputItem(callId, output) {
+    return {
+        type: "function_call_output",
+        call_id: callId,
+        output,
+    };
+}
+function buildPreviewOutputText({ requestedX, requestedY, renderedX, renderedY, wasClamped }) {
+    const baseText = `Marker placed at (${renderedX}, ${renderedY}) on the current screen. The click has not been executed.`;
+    if (!wasClamped) {
+        return baseText;
+    }
+    return `${baseText} Requested point (${requestedX}, ${requestedY}) was off-screen; marker rendered at (${renderedX}, ${renderedY}).`;
+}
+function buildPreviewOutputContent({ text, annotatedScreenshotBase64 = null }) {
+    const content = [{
+            type: "input_text",
+            text,
+        }];
+    if (annotatedScreenshotBase64) {
+        content.push({
+            type: "input_image",
+            image_url: `data:image/png;base64,${annotatedScreenshotBase64}`,
+            detail: "high",
+        });
+    }
+    return content;
+}
 function getScopeAndIds(context = null, stepContext = null) {
     const scope = context?.sessionId ? "design" : "execution";
     const ids = scope === "design"
@@ -80,6 +149,37 @@ function buildStrictModePlan(actions = [], strictMode = false) {
         runtimeNote
     };
 }
+function normalizeArtifactPaths(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        if (Array.isArray(result.annotationSourcePaths)) {
+            return normalizeArtifactPaths(result.annotationSourcePaths);
+        }
+        return normalizeArtifactPaths(result.artifactPaths);
+    }
+    if (Array.isArray(result)) {
+        return result.filter((value) => typeof value === "string" && value.trim());
+    }
+    if (typeof result === "string" && result.trim()) {
+        return [result];
+    }
+    return [];
+}
+function screenshotPathFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotPath === "string" && result.screenshotPath.trim()
+            ? result.screenshotPath
+            : null;
+    }
+    return typeof result === "string" && result.trim() ? result : null;
+}
+function screenshotDataUrlFromRecorderResult(result) {
+    if (result && typeof result === "object" && !Array.isArray(result)) {
+        return typeof result.screenshotDataUrl === "string" && result.screenshotDataUrl.trim()
+            ? result.screenshotDataUrl
+            : null;
+    }
+    return null;
+}
 export class ExecutionEngine {
     constructor(session, options = {}) {
         this.session = session;
@@ -89,26 +189,45 @@ export class ExecutionEngine {
         this.strictMode = options.strictMode === true;
         this.stepDelayMs = getConfiguredStepDelayMs();
         this.reportedScreenshotWriteError = false;
+        this.reportedScreenshotAnnotateError = false;
+        this.lastRecordedScreenshotPaths = [];
         this.getScreenshotAsBase64 = options.getScreenshotAsBase64 || getScreenshotAsBase64;
         this.handleModelAction = options.handleModelAction || handleModelAction;
         this.sendCUARequest = options.sendCUARequest || sendCUARequest;
         this.getCurrentPlatform = options.getCurrentPlatform || getCurrentPlatform;
     }
-    async recordScreenshot(screenshotBase64, metadata = {}) {
+    async saveScreenshotArtifact(screenshotBase64, metadata = {}, options = {}) {
         if (typeof screenshotBase64 !== "string" || !screenshotBase64) {
             return null;
         }
+        const updateLastRecordedScreenshotPaths = options.updateLastRecordedScreenshotPaths !== false;
+        const previousRecordedScreenshotPaths = updateLastRecordedScreenshotPaths
+            ? null
+            : [...this.lastRecordedScreenshotPaths];
         try {
             if (this.screenshotRecorder?.saveScreenshot) {
-                return await this.screenshotRecorder.saveScreenshot(screenshotBase64, metadata);
+                const result = await this.screenshotRecorder.saveScreenshot(screenshotBase64, metadata);
+                if (updateLastRecordedScreenshotPaths) {
+                    this.lastRecordedScreenshotPaths = normalizeArtifactPaths(result);
+                }
+                return result;
             }
             if (this.recordScreenshots && this.screenshotDir) {
                 const framePath = path.join(this.screenshotDir, `frame_${String(Date.now())}.png`);
                 await writeFile(framePath, Buffer.from(screenshotBase64, "base64"));
+                if (updateLastRecordedScreenshotPaths) {
+                    this.lastRecordedScreenshotPaths = [framePath];
+                }
                 return framePath;
             }
         }
         catch (error) {
+            if (updateLastRecordedScreenshotPaths) {
+                this.lastRecordedScreenshotPaths = [];
+            }
+            else {
+                this.lastRecordedScreenshotPaths = previousRecordedScreenshotPaths;
+            }
             if (!this.reportedScreenshotWriteError) {
                 this.reportedScreenshotWriteError = true;
                 const scope = typeof metadata.sessionId === "string" ? "design" : "execution";
@@ -131,6 +250,100 @@ export class ExecutionEngine {
         }
         return null;
     }
+    async recordScreenshot(screenshotBase64, metadata = {}) {
+        return await this.saveScreenshotArtifact(screenshotBase64, metadata);
+    }
+    async recordPreviewScreenshot(screenshotBase64, metadata = {}) {
+        return await this.saveScreenshotArtifact(screenshotBase64, metadata, {
+            updateLastRecordedScreenshotPaths: false,
+        });
+    }
+    async annotateLatestScreenshot(action, metadata = {}) {
+        if (!(action?.type === "click" || action?.type === "double_click") || this.lastRecordedScreenshotPaths.length === 0) {
+            return null;
+        }
+        try {
+            const marker = {
+                x: action.x,
+                y: action.y,
+                style: "click",
+            };
+            const sourcePath = this.lastRecordedScreenshotPaths.find((filePath) => typeof filePath === "string" && filePath.trim());
+            if (sourcePath && this.screenshotRecorder?.saveAnnotatedScreenshot) {
+                return await this.screenshotRecorder.saveAnnotatedScreenshot(sourcePath, marker, metadata);
+            }
+            await Promise.all(this.lastRecordedScreenshotPaths.map((filePath) => annotateScreenshotArtifact(filePath, marker)));
+            return null;
+        }
+        catch (error) {
+            if (!this.reportedScreenshotAnnotateError) {
+                this.reportedScreenshotAnnotateError = true;
+                const scope = typeof metadata.sessionId === "string" ? "design" : "execution";
+                const ids = scope === "design"
+                    ? {
+                        sessionId: metadata.sessionId,
+                        stepId: metadata.stepId,
+                        instructionIndex: metadata.instructionIndex
+                    }
+                    : {
+                        runId: metadata.runId,
+                        stepId: metadata.stepId,
+                        instructionIndex: metadata.instructionIndex
+                    };
+                emitDesktopDebug("run.screenshot_annotate.error", scope, ids, {
+                    actionType: action?.type ?? null,
+                    x: action?.x ?? null,
+                    y: action?.y ?? null,
+                    message: error instanceof Error ? error.message : "Failed to annotate screenshot"
+                });
+            }
+        }
+        return null;
+    }
+    async handlePreviewClick(callId, argumentsText, context = null, stepContext = null) {
+        const parsedArguments = parsePreviewClickArguments(argumentsText);
+        if (!parsedArguments.ok) {
+            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({ text: parsedArguments.message }));
+        }
+        try {
+            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+            const previewResult = await createAnnotatedScreenshotFromBase64(screenshotBase64, {
+                x: parsedArguments.x,
+                y: parsedArguments.y,
+                style: "preview",
+            });
+            if (!previewResult?.screenshotBase64 || !previewResult?.marker) {
+                return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                    text: "preview_click failed: could not annotate the current screenshot.",
+                }));
+            }
+            const previewEventId = randomUUID();
+            await this.recordPreviewScreenshot(previewResult.screenshotBase64, {
+                runId: context?.runId,
+                sessionId: context?.sessionId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                callId,
+                eventId: previewEventId,
+                captureSource: "preview-click",
+            });
+            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                text: buildPreviewOutputText({
+                    requestedX: previewResult.marker.requestedX,
+                    requestedY: previewResult.marker.requestedY,
+                    renderedX: previewResult.marker.x,
+                    renderedY: previewResult.marker.y,
+                    wasClamped: previewResult.marker.wasClamped,
+                }),
+                annotatedScreenshotBase64: previewResult.screenshotBase64,
+            }));
+        }
+        catch (error) {
+            return buildFunctionCallOutputItem(callId, buildPreviewOutputContent({
+                text: `preview_click failed: ${error instanceof Error ? error.message : "unable to capture or annotate the current screen."}`,
+            }));
+        }
+    }
     /**
      * Run a full turn with the CUA model
      * Executes actions until the model stops requesting actions
@@ -156,7 +369,7 @@ export class ExecutionEngine {
                 return newResponseId;
             }
             const items = response.output || [];
-            const computerCalls = extractComputerCalls(items);
+            const actionableItems = extractActionableOutputItems(items);
             // ── Collect pending safety checks ──
             const pendingSafetyChecks = items
                 .filter(item => item.type === "pending_safety_check")
@@ -203,117 +416,18 @@ export class ExecutionEngine {
                     });
                 }
             }
-            if (computerCalls.length === 0) {
+            if (actionableItems.length === 0) {
                 // No actions = turn complete
                 break;
             }
-            // ── Process model actions ──
-            for (const { call_id, actions } of computerCalls) {
-                if (!call_id)
-                    continue;
-                const { actionsToExecute, droppedActions, truncationReason, runtimeNote } = buildStrictModePlan(actions, this.strictMode);
-                let sawExplicitScreenshotAction = false;
-                for (const action of actionsToExecute) {
-                    if (shouldStop()) {
-                        return newResponseId;
-                    }
-                    if (action.type === "screenshot") {
-                        sawExplicitScreenshotAction = true;
-                        addOutput({
-                            type: 'info',
-                            text: 'Capturing screen',
-                            eventType: 'screenshot_captured',
-                            actionType: 'screenshot',
-                            ...eventMeta({
-                                payload: {
-                                    callId: call_id,
-                                    source: 'explicit_action'
-                                }
-                            })
-                        });
-                    }
-                    else {
-                        await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
-                            ...context,
-                            shouldStop,
-                            stepId: stepContext?.stepId,
-                            instructionIndex: stepContext?.instructionIndex
-                        });
-                        // Track action and check for interruption
-                        if (trackAction) {
-                            trackAction(action);
-                        }
-                        if (shouldStop()) {
-                            // User interrupted - stop execution immediately
-                            return newResponseId;
-                        }
-                        // Add delay after UI-changing actions to let the interface update
-                        // before taking the screenshot (except for explicit wait actions which have their own delay)
-                        if (action.type !== "wait") {
-                            await new Promise(resolve => setTimeout(resolve, this.stepDelayMs));
-                        }
-                    }
-                }
-                if (shouldStop()) {
-                    return newResponseId;
-                }
-                const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
-                emitDesktopDebug("device.screenshot", "device", {
-                    runId: context?.runId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex
-                }, {
-                    source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
-                    callId: call_id,
-                    width: this.session.deviceInfo?.scaled_width,
-                    height: this.session.deviceInfo?.scaled_height,
-                    base64Length: screenshotBase64.length
-                });
-                await this.recordScreenshot(screenshotBase64, {
-                    runId: context?.runId,
-                    sessionId: context?.sessionId,
-                    stepId: stepContext?.stepId,
-                    instructionIndex: stepContext?.instructionIndex,
-                    callId: call_id,
-                    captureSource: sawExplicitScreenshotAction ? "call-output-explicit-action" : "call-output-post-action"
-                });
-                if (runtimeNote) {
-                    const executedActionTypes = actionsToExecute
-                        .map((action) => action?.type)
-                        .filter(Boolean);
-                    const droppedActionTypes = droppedActions
-                        .map((action) => action?.type)
-                        .filter(Boolean);
-                    emitDesktopDebug("cua.strict_mode.truncation", scope, ids, {
-                        callId: call_id,
-                        executedCount: actionsToExecute.length,
-                        droppedCount: droppedActions.length,
-                        executedActionTypes,
-                        droppedActionTypes,
-                        reason: truncationReason,
-                        note: runtimeNote
-                    });
-                }
-                // Build next input: screenshot + any carryover reasoning
-                const selectedCuaModel = process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
-                const input = [{
-                        type: "computer_call_output",
-                        call_id,
-                        output: {
-                            type: "computer_screenshot",
-                            image_url: `data:image/png;base64,${screenshotBase64}`,
-                        },
-                        ...(selectedCuaModel === "computer-use-preview"
-                            ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
-                            : {}),
-                        ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
-                    }];
-                if (runtimeNote) {
-                    input.push({
-                        role: "user",
-                        content: runtimeNote
-                    });
-                }
+            const actionableItem = actionableItems[0];
+            if (!actionableItem?.call_id) {
+                break;
+            }
+            if (actionableItem.type === "preview_click") {
+                const input = [
+                    await this.handlePreviewClick(actionableItem.call_id, actionableItem.arguments, context, stepContext),
+                ];
                 if (shouldStop()) {
                     return newResponseId;
                 }
@@ -330,9 +444,145 @@ export class ExecutionEngine {
                     }
                 });
                 newResponseId = response.id;
-                // Each tool output advances the response chain; process newly returned calls next.
-                break;
+                continue;
+            }
+            const { call_id, actions } = actionableItem;
+            const { actionsToExecute, droppedActions, truncationReason, runtimeNote } = buildStrictModePlan(actions, this.strictMode);
+            let sawExplicitScreenshotAction = false;
+            for (const action of actionsToExecute) {
+                const actionEventId = randomUUID();
+                if (shouldStop()) {
+                    return newResponseId;
+                }
+                if (action.type === "screenshot") {
+                    sawExplicitScreenshotAction = true;
+                }
+                else {
+                    const annotatedScreenshotResult = await this.annotateLatestScreenshot(action, {
+                        runId: context?.runId,
+                        sessionId: context?.sessionId,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex,
+                        eventId: actionEventId,
+                        captureSource: "action-click-annotation"
+                    });
+                    const annotatedScreenshotPath = screenshotPathFromRecorderResult(annotatedScreenshotResult);
+                    const annotatedScreenshotDataUrl = screenshotDataUrlFromRecorderResult(annotatedScreenshotResult);
+                    await this.handleModelAction(this.session.deviceId, action, this.session.deviceInfo.scale, {
+                        ...context,
+                        shouldStop,
+                        stepId: stepContext?.stepId,
+                        instructionIndex: stepContext?.instructionIndex,
+                        eventId: actionEventId,
+                        ...(annotatedScreenshotPath ? { screenshotPath: annotatedScreenshotPath } : {}),
+                        ...(context?.includeScreenshotDataUrl && annotatedScreenshotDataUrl ? { screenshotDataUrl: annotatedScreenshotDataUrl } : {})
+                    });
+                    if (trackAction) {
+                        trackAction(action);
+                    }
+                    if (shouldStop()) {
+                        return newResponseId;
+                    }
+                    if (action.type !== "wait") {
+                        await new Promise(resolve => setTimeout(resolve, this.stepDelayMs));
+                    }
+                }
             }
+            if (shouldStop()) {
+                return newResponseId;
+            }
+            const screenshotBase64 = await this.getScreenshotAsBase64(this.session.deviceId, this.session.deviceInfo);
+            emitDesktopDebug("device.screenshot", "device", {
+                runId: context?.runId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex
+            }, {
+                source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+                callId: call_id,
+                width: this.session.deviceInfo?.scaled_width,
+                height: this.session.deviceInfo?.scaled_height,
+                base64Length: screenshotBase64.length
+            });
+            const screenshotEventId = randomUUID();
+            const captureSource = sawExplicitScreenshotAction ? "call-output-explicit-action" : "call-output-post-action";
+            const screenshotResult = await this.recordScreenshot(screenshotBase64, {
+                runId: context?.runId,
+                sessionId: context?.sessionId,
+                stepId: stepContext?.stepId,
+                instructionIndex: stepContext?.instructionIndex,
+                eventId: screenshotEventId,
+                captureSource
+            });
+            const screenshotPath = screenshotPathFromRecorderResult(screenshotResult);
+            addOutput({
+                type: 'info',
+                text: 'Capturing screen',
+                eventType: 'screenshot_captured',
+                actionType: 'screenshot',
+                ...eventMeta({
+                    payload: {
+                        eventId: screenshotEventId,
+                        captureSource,
+                        source: sawExplicitScreenshotAction ? "explicit_action" : "post_action",
+                        ...(typeof screenshotPath === "string" && screenshotPath ? { screenshotPath } : {}),
+                        ...(context?.includeScreenshotDataUrl
+                            ? { screenshotDataUrl: `data:image/png;base64,${screenshotBase64}` }
+                            : {})
+                    }
+                })
+            });
+            if (runtimeNote) {
+                const executedActionTypes = actionsToExecute
+                    .map((action) => action?.type)
+                    .filter(Boolean);
+                const droppedActionTypes = droppedActions
+                    .map((action) => action?.type)
+                    .filter(Boolean);
+                emitDesktopDebug("cua.strict_mode.truncation", scope, ids, {
+                    callId: call_id,
+                    executedCount: actionsToExecute.length,
+                    droppedCount: droppedActions.length,
+                    executedActionTypes,
+                    droppedActionTypes,
+                    reason: truncationReason,
+                    note: runtimeNote
+                });
+            }
+            const selectedCuaModel = process.env.OPENAI_CUA_MODEL === "computer-use-preview" ? "computer-use-preview" : "gpt-5.4";
+            const input = [{
+                    type: "computer_call_output",
+                    call_id,
+                    output: {
+                        type: "computer_screenshot",
+                        image_url: `data:image/png;base64,${screenshotBase64}`,
+                    },
+                    ...(selectedCuaModel === "computer-use-preview"
+                        ? { current_url: this.getCurrentPlatform() === "ios" ? "ios://simulator" : "android://device" }
+                        : {}),
+                    ...(pendingSafetyChecks.length > 0 ? { acknowledged_safety_checks: pendingSafetyChecks } : {})
+                }];
+            if (runtimeNote) {
+                input.push({
+                    role: "user",
+                    content: runtimeNote
+                });
+            }
+            if (shouldStop()) {
+                return newResponseId;
+            }
+            response = await this.sendCUARequest({
+                messages: input,
+                previousResponseId: newResponseId,
+                deviceInfo: this.session.deviceInfo,
+                debugContext: {
+                    scope,
+                    runId: context?.runId,
+                    sessionId: context?.sessionId,
+                    stepId: stepContext?.stepId,
+                    instructionIndex: stepContext?.instructionIndex
+                }
+            });
+            newResponseId = response.id;
         }
         // ── At end, if last output was only reasoning ──
         const finalItems = response.output || [];

package/build/src/device/android/actions.js

@@ -1,6 +1,7 @@
 import { logger } from "../../utils/logger.js";
 import { emitDesktopDebug, truncateForDebug } from "../../utils/desktop-debug.js";
 import { getConfiguredStepDelayMs } from "../../utils/step-delay.js";
+import { resolveScrollGesture } from "../scroll-gesture.js";
 import { execAdb } from "./tools.js";
 function adbShell(deviceId, command) {
     return execAdb(["-s", deviceId, "shell", command]);
@@ -39,6 +40,9 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
         stepId: context?.stepId,
         instructionIndex: context?.instructionIndex,
         payload: {
+            eventId: context?.eventId,
+            screenshotPath: context?.screenshotPath,
+            screenshotDataUrl: context?.screenshotDataUrl,
             platform: 'android',
             ...payload
         }
@@ -70,17 +74,20 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             case "double_click":
                 const realX = Math.round(x / scale);
                 const realY = Math.round(y / scale);
-                addOutput({ type: 'action', text: `Clicking at (${realX}, ${realY})`, ...meta({ x: realX, y: realY, unit: 'px' }) });
+                addOutput({ type: 'action', text: 'Click', ...meta({ x: realX, y: realY, unit: 'px' }) });
                 await adbShell(deviceId, `input tap ${realX} ${realY}`);
                 break;
             case "scroll":
-                const scrollX = Math.round(action.scroll_x / scale);
-                const scrollY = Math.round(action.scroll_y / scale);
-                addOutput({ type: 'action', text: `Scrolling by (${scrollX}, ${scrollY})`, ...meta({ scrollX, scrollY, unit: 'px' }) });
-                const startX = 500;
-                const startY = 500;
-                const endX = startX + scrollX;
-                const endY = startY - scrollY; // <--- INVERT Y
+                const { scrollX, scrollY, startX, startY, endX, endY, hasAnchor } = resolveScrollGesture(action, {
+                    scale,
+                    fallbackStartX: 500,
+                    fallbackStartY: 500
+                });
+                addOutput({
+                    type: 'action',
+                    text: `Scrolling from (${startX}, ${startY}) to (${endX}, ${endY}) by (${scrollX}, ${scrollY})`,
+                    ...meta({ scrollX, scrollY, startX, startY, endX, endY, anchorSource: hasAnchor ? 'action' : 'fallback', unit: 'px' })
+                });
                 await adbShell(deviceId, `input swipe ${startX} ${startY} ${endX} ${endY} 500`);
                 break;
             case "drag":
@@ -150,6 +157,7 @@ export async function handleModelAction(deviceId, action, scale = 1.0, context =
             stepId: context?.stepId,
             instructionIndex: context?.instructionIndex,
             payload: {
+                eventId: context?.eventId,
                 message: error.message,
                 platform: 'android'
             }