mobile-debug-mcp 0.22.0 → 0.24.0

package/dist/interact/classify.js

@@ -0,0 +1,35 @@
+/**
+ * Pure deterministic classifier. Applies rules in fixed order.
+ * Same inputs always produce the same output.
+ */
+export function classifyActionOutcome(input) {
+    const { uiChanged, expectedElementVisible, networkRequests, hasLogErrors } = input;
+    // Step 1 — UI signal is positive
+    if (uiChanged || expectedElementVisible === true) {
+        return { outcome: 'success', reasoning: expectedElementVisible === true ? 'expected element is visible' : 'UI changed after action' };
+    }
+    // Step 2 — UI did not change; network signal is required
+    if (networkRequests === null || networkRequests === undefined) {
+        return {
+            outcome: 'unknown',
+            reasoning: 'UI did not change; get_network_activity must be called before classification can proceed',
+            nextAction: 'call_get_network_activity'
+        };
+    }
+    // Step 3 — any network failure
+    const failedRequest = networkRequests.find((r) => r.status === 'failure' || r.status === 'retryable');
+    if (failedRequest) {
+        return { outcome: 'backend_failure', reasoning: `network request ${failedRequest.endpoint} returned ${failedRequest.status}` };
+    }
+    // Step 4 — no network requests at all
+    if (networkRequests.length === 0) {
+        const logNote = hasLogErrors ? ' (log errors present)' : '';
+        return { outcome: 'no_op', reasoning: `no UI change and no network activity${logNote}` };
+    }
+    // Step 5 — network requests exist and all succeeded
+    if (networkRequests.every((r) => r.status === 'success')) {
+        return { outcome: 'ui_failure', reasoning: 'network requests succeeded but UI did not change' };
+    }
+    // Step 6 — fallback
+    return { outcome: 'unknown', reasoning: 'signals are inconclusive' };
+}

package/dist/interact/index.js

@@ -4,6 +4,7 @@ import { iOSInteract } from './ios.js';
 export { AndroidInteract, iOSInteract };
 import { resolveTargetDevice } from '../utils/resolve-device.js';
 import { ToolsObserve } from '../observe/index.js';
+import { nextActionId } from '../server/common.js';
 export class ToolsInteract {
     static _maxResolvedUiElements = 256;
     static _resolvedUiElements = new Map();
@@ -74,6 +75,39 @@ export class ToolsInteract {
             ToolsInteract._resolvedUiElements.delete(oldestElementId);
         }
     }
+    static async _captureFingerprint(platform, deviceId) {
+        try {
+            const fingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
+            return fingerprint?.fingerprint ?? null;
+        }
+        catch {
+            return null;
+        }
+    }
+    static _resolvedTargetFromElement(elementId, element, index) {
+        return {
+            elementId,
+            text: element.text ?? null,
+            resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
+            accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
+            class: element.type ?? element.class ?? null,
+            bounds: ToolsInteract._normalizeBounds(element.bounds),
+            index
+        };
+    }
+    static _actionFailure(actionId, timestamp, actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter) {
+        return {
+            action_id: actionId,
+            timestamp,
+            action_type: actionType,
+            target: { selector, resolved },
+            success: false,
+            failure_code: failureCode,
+            retryable,
+            ui_fingerprint_before: uiFingerprintBefore,
+            ui_fingerprint_after: uiFingerprintAfter
+        };
+    }
     static _resetResolvedUiElementsForTests() {
         ToolsInteract._resolvedUiElements.clear();
     }
@@ -151,87 +185,53 @@ export class ToolsInteract {
         return await interact.tap(x, y, resolved.id);
     }
     static async tapElementHandler({ elementId }) {
-        const action = 'tap';
+        const timestamp = Date.now();
+        const actionType = 'tap_element';
+        const actionId = nextActionId(actionType, timestamp);
+        const selector = { elementId };
         const resolved = ToolsInteract._resolvedUiElements.get(elementId);
         if (!resolved) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_found',
-                    message: 'Element ID was not found in the current UI context'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, null);
         }
+        const fingerprintBefore = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
         const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
         const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform;
         const treeDeviceId = tree?.device?.id || resolved.deviceId;
         const elements = Array.isArray(tree?.elements) ? tree.elements : [];
         const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
         if (!currentMatch) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_found',
-                    message: 'Element ID is not present in the current UI context'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
         }
+        const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
         if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_visible',
-                    message: 'Element is not visible'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
         }
         if (currentMatch.el.enabled === false) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_enabled',
-                    message: 'Element is not enabled'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
         }
         const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
         if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'element_not_visible',
-                    message: 'Element does not have valid visible bounds'
-                }
-            };
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
         }
         const x = Math.floor((bounds[0] + bounds[2]) / 2);
         const y = Math.floor((bounds[1] + bounds[3]) / 2);
         const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId });
         if (!tapResult.success) {
-            return {
-                success: false,
-                elementId,
-                action,
-                error: {
-                    code: 'tap_failed',
-                    message: tapResult.error || 'Tap failed'
-                }
-            };
+            const fingerprintAfterFailure = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
+            return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'UNKNOWN', false, fingerprintBefore, fingerprintAfterFailure);
         }
+        const fingerprintAfter = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
         return {
+            action_id: actionId,
+            timestamp,
+            action_type: actionType,
+            target: {
+                selector,
+                resolved: resolvedTarget
+            },
             success: true,
-            elementId,
-            action
+            ui_fingerprint_before: fingerprintBefore,
+            ui_fingerprint_after: fingerprintAfter
         };
     }
     static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
@@ -670,6 +670,82 @@ export class ToolsInteract {
         }
         return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start };
     }
+    static async expectScreenHandler({ platform, fingerprint, screen, deviceId }) {
+        const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
+        const observedScreen = {
+            fingerprint: observedFingerprint?.fingerprint ?? null,
+            screen: observedFingerprint?.activity ?? null
+        };
+        let observedScreenLabel = observedScreen.screen;
+        if (!fingerprint && screen && platform !== 'ios') {
+            try {
+                const current = await ToolsObserve.getCurrentScreenHandler({ deviceId });
+                observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel;
+            }
+            catch {
+                // Keep fingerprint-derived activity when current-screen lookup is unavailable.
+            }
+        }
+        const expectedScreen = {
+            fingerprint: fingerprint ?? null,
+            screen: screen ?? null
+        };
+        let success = false;
+        if (fingerprint) {
+            success = observedScreen.fingerprint === fingerprint;
+        }
+        else if (screen) {
+            const candidates = new Set();
+            if (observedScreen.screen)
+                candidates.add(observedScreen.screen);
+            if (observedScreenLabel)
+                candidates.add(observedScreenLabel);
+            success = candidates.has(screen);
+        }
+        return {
+            success,
+            observed_screen: {
+                fingerprint: observedScreen.fingerprint,
+                screen: observedScreenLabel
+            },
+            expected_screen: expectedScreen,
+            confidence: success ? 1 : 0
+        };
+    }
+    static async expectElementVisibleHandler({ selector, element_id, timeout_ms = 5000, poll_interval_ms = 300, platform, deviceId }) {
+        const result = await ToolsInteract.waitForUIHandler({
+            selector,
+            condition: 'visible',
+            timeout_ms,
+            poll_interval_ms,
+            platform,
+            deviceId
+        });
+        if (result?.status === 'success' && result?.element) {
+            return {
+                success: true,
+                selector,
+                element_id: result.element.elementId ?? element_id ?? null,
+                element: {
+                    elementId: result.element.elementId ?? null,
+                    text: result.element.text ?? null,
+                    resource_id: result.element.resource_id ?? null,
+                    accessibility_id: result.element.accessibility_id ?? null,
+                    class: result.element.class ?? null,
+                    bounds: result.element.bounds ?? null,
+                    index: typeof result.element.index === 'number' ? result.element.index : null
+                }
+            };
+        }
+        const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT';
+        return {
+            success: false,
+            selector,
+            element_id: element_id ?? null,
+            failure_code: errorCode,
+            retryable: errorCode === 'TIMEOUT'
+        };
+    }
     static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
         const start = Date.now();
         const deadline = start + (timeoutMs || 0);

package/dist/network/index.js

@@ -0,0 +1,232 @@
+import { execAdb, parseLogLine } from '../utils/android/utils.js';
+import { execCommand } from '../utils/ios/utils.js';
+// ─── Module state ─────────────────────────────────────────────────────────────
+// lastActionTimestamp: set when an action tool fires (tap, swipe, etc.)
+// lastConsumedTimestamp: advanced after each get_network_activity call to prevent duplicates
+let lastActionTimestamp = 0;
+let lastConsumedTimestamp = 0;
+export function notifyActionStart() {
+    lastActionTimestamp = Date.now();
+    lastConsumedTimestamp = 0;
+}
+/** Exposed for unit tests only. */
+export function _setTimestampsForTests(actionTs, consumedTs) {
+    lastActionTimestamp = actionTs;
+    lastConsumedTimestamp = consumedTs;
+}
+// ─── Parsing constants ────────────────────────────────────────────────────────
+const URL_RE = /https?:\/\/[^\s"'\]\)><]+/;
+const PATH_RE = /\/[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)+/;
+const METHOD_RE = /\b(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b/;
+const NETWORK_ERROR_PATTERNS = [
+    { re: /timed?\s*out|timeout/i, code: 'timeout' },
+    { re: /dns|name[\s_]resolution|host\s*not\s*found|nodename/i, code: 'dns_error' },
+    { re: /\btls\b|\bssl\b|certificate|handshake/i, code: 'tls_error' },
+    { re: /connection\s*refused/i, code: 'connection_refused' },
+    { re: /connection\s*reset|reset\s*by\s*peer/i, code: 'connection_reset' },
+];
+const BACKGROUND_TOKENS = ['/analytics', '/metrics', '/tracking', '/log', '/events', '/telemetry', '/ping', '/beacon'];
+const BACKGROUND_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.css', '.js', '.svg', '.ico', '.woff', '.ttf'];
+const FILESYSTEM_PREFIXES = ['/data/', '/system/', '/apex/', '/proc/', '/dev/', '/vendor/', '/product/', '/storage/', '/sdcard/', '/mnt/', '/odm/', '/cache/', '/metadata/', '/acct/', '/sys/'];
+const FILESYSTEM_EXTENSIONS = ['.apk', '.apex', '.odex', '.vdex', '.dex', '.so', '.jar', '.bin', '.img', '.db', '.sqlite', '.c', '.cc', '.cpp', '.cxx', '.h', '.hpp', '.m', '.mm', '.kt', '.java', '.swift'];
+// ─── Parsing helpers ─────────────────────────────────────────────────────────
+function extractUrl(text) {
+    const m = text.match(URL_RE);
+    return m ? m[0] : null;
+}
+function isPlausibleEndpointPath(path) {
+    const lower = path.toLowerCase();
+    if (!lower.startsWith('/'))
+        return false;
+    if (FILESYSTEM_PREFIXES.some((prefix) => lower.startsWith(prefix)))
+        return false;
+    if (FILESYSTEM_EXTENSIONS.some((ext) => lower.endsWith(ext)))
+        return false;
+    return true;
+}
+function extractPath(text) {
+    const m = text.match(PATH_RE);
+    if (!m)
+        return null;
+    return isPlausibleEndpointPath(m[0]) ? m[0] : null;
+}
+function toStatusCode(value) {
+    if (!value)
+        return null;
+    const code = Number(value);
+    return code >= 100 && code <= 599 ? code : null;
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function extractStatusCode(text, url, path, method) {
+    const directHttpMatch = text.match(/\bHTTP\/\d(?:\.\d)?\s+([1-5]\d{2})\b/i) || text.match(/\bHTTP\s+([1-5]\d{2})\b/i);
+    if (directHttpMatch)
+        return toStatusCode(directHttpMatch[1]);
+    const endpointToken = url || path;
+    const hasEndpointContext = endpointToken !== null;
+    if (!hasEndpointContext && method === null)
+        return null;
+    const labeledMatch = text.match(/\b(?:status(?:\s*code)?|response(?:\s*code)?)\s*[:=]?\s*([1-5]\d{2})\b/i);
+    if (labeledMatch && hasEndpointContext)
+        return toStatusCode(labeledMatch[1]);
+    if (endpointToken) {
+        const escapedEndpoint = escapeRegExp(endpointToken);
+        const endpointThenCode = new RegExp(`${escapedEndpoint}[^\\n]*?\\b([1-5]\\d{2})\\b`, 'i');
+        const codeThenEndpoint = new RegExp(`\\b([1-5]\\d{2})\\b[^\\n]*?${escapedEndpoint}`, 'i');
+        const contextualMatch = text.match(endpointThenCode) || text.match(codeThenEndpoint);
+        if (contextualMatch)
+            return toStatusCode(contextualMatch[1]);
+    }
+    if (method !== null && path !== null) {
+        const methodPathCodeMatch = text.match(/\b(?:GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b[^\n]*?\b([1-5]\d{2})\b/i);
+        if (methodPathCodeMatch)
+            return toStatusCode(methodPathCodeMatch[1]);
+    }
+    return null;
+}
+function extractMethod(text) {
+    const m = text.match(METHOD_RE);
+    return m ? m[1] : null;
+}
+function detectNetworkError(text) {
+    for (const { re, code } of NETWORK_ERROR_PATTERNS) {
+        if (re.test(text))
+            return code;
+    }
+    return null;
+}
+export function normalizeEndpoint(raw) {
+    try {
+        const u = new URL(raw.startsWith('/') ? `https://x${raw}` : raw);
+        const p = u.pathname.toLowerCase().replace(/\/+$/, '');
+        return p || '/';
+    }
+    catch {
+        return raw.toLowerCase().replace(/\?.*$/, '').replace(/\/+$/, '') || '/';
+    }
+}
+export function classifyStatus(statusCode, networkError) {
+    if (networkError !== null)
+        return 'retryable';
+    if (statusCode === null)
+        return 'success'; // request detected, no failure signal
+    if (statusCode >= 200 && statusCode <= 299)
+        return 'success';
+    if (statusCode >= 400 && statusCode <= 499)
+        return 'failure';
+    return 'retryable'; // 5xx, 1xx, 3xx
+}
+function meetsEmissionCriteria(url, path, statusCode, method) {
+    if (url !== null)
+        return true; // condition 1: full http/https URL
+    if (statusCode !== null)
+        return true; // condition 2: valid HTTP status code
+    if (method !== null && path !== null)
+        return true; // condition 3: method + path
+    return false;
+}
+function classifyEventType(endpoint) {
+    const lower = endpoint.toLowerCase();
+    if (BACKGROUND_TOKENS.some(t => lower.includes(t)))
+        return 'background';
+    if (BACKGROUND_EXTENSIONS.some(e => lower.endsWith(e)))
+        return 'background';
+    return 'primary';
+}
+function filterToSignificantEvents(events) {
+    if (events.length === 0)
+        return events;
+    const hasPrimary = events.some(e => classifyEventType(e.endpoint) === 'primary');
+    return hasPrimary ? events.filter(e => classifyEventType(e.endpoint) === 'primary') : events;
+}
+/** Exported for unit testing. */
+export function parseMessageToEvent(message) {
+    const url = extractUrl(message);
+    const path = url ? null : extractPath(message);
+    const method = extractMethod(message);
+    const statusCode = extractStatusCode(message, url, path, method);
+    const networkError = detectNetworkError(message);
+    if (!meetsEmissionCriteria(url, path, statusCode, method))
+        return null;
+    const rawEndpoint = url || path || 'unknown';
+    return {
+        endpoint: normalizeEndpoint(rawEndpoint),
+        method: method || 'unknown',
+        statusCode,
+        networkError,
+        status: classifyStatus(statusCode, networkError),
+        durationMs: 0
+    };
+}
+// ─── Android ─────────────────────────────────────────────────────────────────
+async function getAndroidEvents(sinceMs, deviceId) {
+    try {
+        const stdout = await execAdb(['logcat', '-d', '-v', 'threadtime', '*:V', '-t', '2000'], deviceId);
+        const lines = stdout ? stdout.split(/\r?\n/).filter(Boolean) : [];
+        const events = [];
+        for (const line of lines) {
+            const parsed = parseLogLine(line);
+            if (parsed._iso) {
+                const ts = new Date(parsed._iso).getTime();
+                if (ts > 0 && ts <= sinceMs)
+                    continue;
+            }
+            const event = parseMessageToEvent(parsed.message || line);
+            if (event)
+                events.push(event);
+        }
+        return events;
+    }
+    catch {
+        return [];
+    }
+}
+// ─── iOS ─────────────────────────────────────────────────────────────────────
+async function getIOSEvents(sinceMs, deviceId = 'booted') {
+    try {
+        const lookbackSeconds = Math.max(15, Math.ceil((Date.now() - sinceMs) / 1000) + 5);
+        const args = [
+            'simctl', 'spawn', deviceId, 'log', 'show',
+            '--last', `${lookbackSeconds}s`,
+            '--style', 'syslog',
+            '--predicate', 'eventMessage contains "http" OR eventMessage contains "URLSession" OR eventMessage contains "Task <" OR eventMessage contains "HTTP/"'
+        ];
+        const result = await execCommand(args, deviceId);
+        const lines = result.output ? result.output.split(/\r?\n/).filter(Boolean) : [];
+        const events = [];
+        for (const line of lines) {
+            const tsMatch = line.match(/^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})/);
+            if (tsMatch) {
+                const ts = new Date(tsMatch[1]).getTime();
+                if (ts > 0 && ts <= sinceMs)
+                    continue;
+            }
+            const event = parseMessageToEvent(line);
+            if (event)
+                events.push(event);
+        }
+        return events;
+    }
+    catch {
+        return [];
+    }
+}
+// ─── Public API ───────────────────────────────────────────────────────────────
+export class ToolsNetwork {
+    static notifyActionStart() {
+        notifyActionStart();
+    }
+    static async getNetworkActivity(params) {
+        const { platform, deviceId } = params;
+        const sinceMs = lastConsumedTimestamp > lastActionTimestamp
+            ? lastConsumedTimestamp
+            : lastActionTimestamp > 0 ? lastActionTimestamp : Date.now() - 30000;
+        const raw = platform === 'android'
+            ? await getAndroidEvents(sinceMs, deviceId)
+            : await getIOSEvents(sinceMs, deviceId);
+        const requests = filterToSignificantEvents(raw);
+        lastConsumedTimestamp = Date.now();
+        return { requests, count: requests.length };
+    }
+}

package/dist/server/common.js

@@ -0,0 +1,66 @@
+import { ToolsObserve } from '../observe/index.js';
+export function wrapResponse(data) {
+    return {
+        content: [{
+                type: 'text',
+                text: JSON.stringify(data, null, 2)
+            }]
+    };
+}
+let actionSequence = 0;
+export function nextActionId(actionType, timestamp) {
+    actionSequence += 1;
+    return `${actionType}_${timestamp}_${actionSequence}`;
+}
+export async function captureActionFingerprint(platform, deviceId) {
+    if (!platform)
+        return null;
+    try {
+        const result = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
+        return result?.fingerprint ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+export function normalizeResolvedTarget(value = null) {
+    if (!value)
+        return null;
+    return {
+        elementId: value.elementId ?? null,
+        text: value.text ?? null,
+        resource_id: value.resource_id ?? null,
+        accessibility_id: value.accessibility_id ?? null,
+        class: value.class ?? null,
+        bounds: value.bounds ?? null,
+        index: value.index ?? null
+    };
+}
+export function inferGenericFailure(message) {
+    if (message && /timeout/i.test(message))
+        return { failureCode: 'TIMEOUT', retryable: true };
+    return { failureCode: 'UNKNOWN', retryable: false };
+}
+export function inferScrollFailure(message) {
+    if (message && /unchanged|no change|end of list/i.test(message))
+        return { failureCode: 'NAVIGATION_NO_CHANGE', retryable: true };
+    if (message && /timeout/i.test(message))
+        return { failureCode: 'TIMEOUT', retryable: true };
+    return { failureCode: 'UNKNOWN', retryable: false };
+}
+export function buildActionExecutionResult({ actionType, selector, resolved, success, uiFingerprintBefore, uiFingerprintAfter, failure }) {
+    const timestamp = Date.now();
+    return {
+        action_id: nextActionId(actionType, timestamp),
+        timestamp,
+        action_type: actionType,
+        target: {
+            selector,
+            resolved: normalizeResolvedTarget(resolved)
+        },
+        success,
+        ...(failure ? { failure_code: failure.failureCode, retryable: failure.retryable } : {}),
+        ui_fingerprint_before: uiFingerprintBefore,
+        ui_fingerprint_after: uiFingerprintAfter
+    };
+}