mobile-debug-mcp 0.23.0 → 0.24.0

package/dist/interact/index.js

@@ -4,6 +4,7 @@ import { iOSInteract } from './ios.js';
 export { AndroidInteract, iOSInteract };
 import { resolveTargetDevice } from '../utils/resolve-device.js';
 import { ToolsObserve } from '../observe/index.js';
+import { nextActionId } from '../server/common.js';
 export class ToolsInteract {
     static _maxResolvedUiElements = 256;
     static _resolvedUiElements = new Map();
@@ -74,6 +75,39 @@ export class ToolsInteract {
             ToolsInteract._resolvedUiElements.delete(oldestElementId);
         }
     }
+    static async _captureFingerprint(platform, deviceId) {
+        try {
+            const fingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
+            return fingerprint?.fingerprint ?? null;
+        }
+        catch {
+            return null;
+        }
+    }
+    static _resolvedTargetFromElement(elementId, element, index) {
+        return {
+            elementId,
+            text: element.text ?? null,
+            resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
+            accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
+            class: element.type ?? element.class ?? null,
+            bounds: ToolsInteract._normalizeBounds(element.bounds),
+            index
+        };
+    }
+    static _actionFailure(actionId, timestamp, actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter) {
+        return {
+            action_id: actionId,
+            timestamp,
+            action_type: actionType,
+            target: { selector, resolved },
+            success: false,
+            failure_code: failureCode,
+            retryable,
+            ui_fingerprint_before: uiFingerprintBefore,
+            ui_fingerprint_after: uiFingerprintAfter
+        };
+    }
     static _resetResolvedUiElementsForTests() {
         ToolsInteract._resolvedUiElements.clear();
     }
@@ -151,87 +185,53 @@ export class ToolsInteract {
         return await interact.tap(x, y, resolved.id);
     }
     static async tapElementHandler({ elementId }) {
-        const action = 'tap';
+        const timestamp = Date.now();
+        const actionType = 'tap_element';
+        const actionId = nextActionId(actionType, timestamp);
+        const selector = { elementId };
         const resolved = ToolsInteract._resolvedUiElements.get(elementId);
         if (!resolved) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_found',
-                    message: 'Element ID was not found in the current UI context'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, null);
         }
+        const fingerprintBefore = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
         const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
         const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform;
         const treeDeviceId = tree?.device?.id || resolved.deviceId;
         const elements = Array.isArray(tree?.elements) ? tree.elements : [];
         const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
         if (!currentMatch) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_found',
-                    message: 'Element ID is not present in the current UI context'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
         }
+        const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
         if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_visible',
-                    message: 'Element is not visible'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
         }
         if (currentMatch.el.enabled === false) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_enabled',
-                    message: 'Element is not enabled'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
         }
         const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
         if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_visible',
-                    message: 'Element does not have valid visible bounds'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
         }
         const x = Math.floor((bounds[0] + bounds[2]) / 2);
         const y = Math.floor((bounds[1] + bounds[3]) / 2);
         const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId });
         if (!tapResult.success) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'tap_failed',
-                    message: tapResult.error || 'Tap failed'
-                }
-            };
+            const fingerprintAfterFailure = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'UNKNOWN', false, fingerprintBefore, fingerprintAfterFailure);
         }
+        const fingerprintAfter = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
         return {
+            action_id: actionId,
+            timestamp,
+            action_type: actionType,
+            target: {
+                selector,
+                resolved: resolvedTarget
+            },
             success: true,
-            elementId,
-            action
+            ui_fingerprint_before: fingerprintBefore,
+            ui_fingerprint_after: fingerprintAfter
         };
     }
     static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
@@ -670,6 +670,82 @@ export class ToolsInteract {
         }
         return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start };
     }
+    static async expectScreenHandler({ platform, fingerprint, screen, deviceId }) {
+        const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
+        const observedScreen = {
+            fingerprint: observedFingerprint?.fingerprint ?? null,
+            screen: observedFingerprint?.activity ?? null
+        };
+        let observedScreenLabel = observedScreen.screen;
+        if (!fingerprint && screen && platform !== 'ios') {
+            try {
+                const current = await ToolsObserve.getCurrentScreenHandler({ deviceId });
+                observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel;
+            }
+            catch {
+                // Keep fingerprint-derived activity when current-screen lookup is unavailable.
+            }
+        }
+        const expectedScreen = {
+            fingerprint: fingerprint ?? null,
+            screen: screen ?? null
+        };
+        let success = false;
+        if (fingerprint) {
+            success = observedScreen.fingerprint === fingerprint;
+        }
+        else if (screen) {
+            const candidates = new Set();
+            if (observedScreen.screen)
+                candidates.add(observedScreen.screen);
+            if (observedScreenLabel)
+                candidates.add(observedScreenLabel);
+            success = candidates.has(screen);
+        }
+        return {
+            success,
+            observed_screen: {
+                fingerprint: observedScreen.fingerprint,
+                screen: observedScreenLabel
+            },
+            expected_screen: expectedScreen,
+            confidence: success ? 1 : 0
+        };
+    }
+    static async expectElementVisibleHandler({ selector, element_id, timeout_ms = 5000, poll_interval_ms = 300, platform, deviceId }) {
+        const result = await ToolsInteract.waitForUIHandler({
+            selector,
+            condition: 'visible',
+            timeout_ms,
+            poll_interval_ms,
+            platform,
+            deviceId
+        });
+        if (result?.status === 'success' && result?.element) {
+            return {
+                success: true,
+                selector,
+                element_id: result.element.elementId ?? element_id ?? null,
+                element: {
+                    elementId: result.element.elementId ?? null,
+                    text: result.element.text ?? null,
+                    resource_id: result.element.resource_id ?? null,
+                    accessibility_id: result.element.accessibility_id ?? null,
+                    class: result.element.class ?? null,
+                    bounds: result.element.bounds ?? null,
+                    index: typeof result.element.index === 'number' ? result.element.index : null
+                }
+            };
+        }
+        const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT';
+        return {
+            success: false,
+            selector,
+            element_id: element_id ?? null,
+            failure_code: errorCode,
+            retryable: errorCode === 'TIMEOUT'
+        };
+    }
     static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
         const start = Date.now();
         const deadline = start + (timeoutMs || 0);

package/dist/server/common.js

@@ -0,0 +1,66 @@
+import { ToolsObserve } from '../observe/index.js';
+export function wrapResponse(data) {
+    return {
+        content: [{
+                type: 'text',
+                text: JSON.stringify(data, null, 2)
+            }]
+    };
+}
+let actionSequence = 0;
+export function nextActionId(actionType, timestamp) {
+    actionSequence += 1;
+    return `${actionType}_${timestamp}_${actionSequence}`;
+}
+export async function captureActionFingerprint(platform, deviceId) {
+    if (!platform)
+        return null;
+    try {
+        const result = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
+        return result?.fingerprint ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+export function normalizeResolvedTarget(value = null) {
+    if (!value)
+        return null;
+    return {
+        elementId: value.elementId ?? null,
+        text: value.text ?? null,
+        resource_id: value.resource_id ?? null,
+        accessibility_id: value.accessibility_id ?? null,
+        class: value.class ?? null,
+        bounds: value.bounds ?? null,
+        index: value.index ?? null
+    };
+}
+export function inferGenericFailure(message) {
+    if (message && /timeout/i.test(message))
+        return { failureCode: 'TIMEOUT', retryable: true };
+    return { failureCode: 'UNKNOWN', retryable: false };
+}
+export function inferScrollFailure(message) {
+    if (message && /unchanged|no change|end of list/i.test(message))
+        return { failureCode: 'NAVIGATION_NO_CHANGE', retryable: true };
+    if (message && /timeout/i.test(message))
+        return { failureCode: 'TIMEOUT', retryable: true };
+    return { failureCode: 'UNKNOWN', retryable: false };
+}
+export function buildActionExecutionResult({ actionType, selector, resolved, success, uiFingerprintBefore, uiFingerprintAfter, failure }) {
+    const timestamp = Date.now();
+    return {
+        action_id: nextActionId(actionType, timestamp),
+        timestamp,
+        action_type: actionType,
+        target: {
+            selector,
+            resolved: normalizeResolvedTarget(resolved)
+        },
+        success,
+        ...(failure ? { failure_code: failure.failureCode, retryable: failure.retryable } : {}),
+        ui_fingerprint_before: uiFingerprintBefore,
+        ui_fingerprint_after: uiFingerprintAfter
+    };
+}