mobile-debug-mcp 0.14.0 → 0.15.0

package/dist/android/interact.js

@@ -1,6 +1,6 @@
 import { execAdb, getAndroidDeviceMetadata, getDeviceInfo } from "./utils.js";
-import { AndroidObserve } from "./observe.js";
-import { scrollToElementShared } from "../tools/scroll_to_element.js";
+import { AndroidObserve } from "../observe/index.js";
+import { scrollToElementShared } from "../intera../interact/shared/scroll_to_element.js";
 export class AndroidInteract {
     observe = new AndroidObserve();
     async waitForElement(text, timeout, deviceId) {

package/dist/android/observe.js

@@ -4,6 +4,7 @@ import { getAdbCmd, execAdb, getAndroidDeviceMetadata, getDeviceInfo, delay, get
 import { createWriteStream } from "fs";
 import { promises as fsPromises } from "fs";
 import path from "path";
+import { computeScreenFingerprint } from "../intera../interact/shared/fingerprint.js";
 const activeLogStreams = new Map();
 export class AndroidObserve {
     async getDeviceMetadata(appId, deviceId) {
@@ -229,6 +230,18 @@ export class AndroidObserve {
             };
         }
     }
+    async getScreenFingerprint(deviceId) {
+        try {
+            const tree = await this.getUITree(deviceId);
+            if (!tree || tree.error)
+                return { fingerprint: null, error: tree.error };
+            const current = await this.getCurrentScreen(deviceId).catch(() => null);
+            return computeScreenFingerprint(tree, current, 'android', 50);
+        }
+        catch (e) {
+            return { fingerprint: null, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
     async startLogStream(packageName, level = 'error', deviceId, sessionId = 'default') {
         try {
             const pidOutput = await execAdb(['shell', 'pidof', packageName], deviceId).catch(() => '');

package/dist/cli/ios/run-ios-smoke.js

@@ -1,5 +1,5 @@
-import { iOSObserve } from '../../ios/observe.js';
-import { iOSManage } from '../../ios/manage.js';
+import { iOSObserve } from '../../observe/index.js';
+import { iOSManage } from '../../manage/index.js';
 async function main() {
     const appId = process.argv[2] || 'com.apple.springboard';
     const deviceId = 'booted';

package/dist/cli/ios/run-ios-ui-tree-tap.js

@@ -1,5 +1,5 @@
-import { iOSObserve } from '../../ios/observe.js';
-import { iOSInteract } from '../../ios/interact.js';
+import { iOSObserve } from '../../observe/index.js';
+import { iOSInteract } from '../../interact/index.js';
 async function main() {
     const deviceId = 'booted';
     const obs = new iOSObserve();

package/dist/interact/android.js

@@ -0,0 +1,91 @@
+import { execAdb, getAndroidDeviceMetadata, getDeviceInfo } from "../utils/android/utils.js";
+import { AndroidObserve } from "../observe/index.js";
+import { scrollToElementShared } from "../interact/shared/scroll_to_element.js";
+export class AndroidInteract {
+    observe = new AndroidObserve();
+    async waitForElement(text, timeout, deviceId) {
+        const metadata = await getAndroidDeviceMetadata("", deviceId);
+        const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
+        const startTime = Date.now();
+        while (Date.now() - startTime < timeout) {
+            try {
+                const tree = await this.observe.getUITree(deviceId);
+                if (tree.error) {
+                    return { device: deviceInfo, found: false, error: tree.error };
+                }
+                const element = tree.elements.find(e => e.text === text);
+                if (element) {
+                    return { device: deviceInfo, found: true, element };
+                }
+            }
+            catch (e) {
+                // Ignore errors during polling and retry
+                console.error("Error polling UI tree:", e);
+            }
+            const elapsed = Date.now() - startTime;
+            const remaining = timeout - elapsed;
+            if (remaining <= 0)
+                break;
+            await new Promise(resolve => setTimeout(resolve, Math.min(500, remaining)));
+        }
+        return { device: deviceInfo, found: false };
+    }
+    async tap(x, y, deviceId) {
+        const metadata = await getAndroidDeviceMetadata("", deviceId);
+        const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
+        try {
+            await execAdb(['shell', 'input', 'tap', x.toString(), y.toString()], deviceId);
+            return { device: deviceInfo, success: true, x, y };
+        }
+        catch (e) {
+            return { device: deviceInfo, success: false, x, y, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    async swipe(x1, y1, x2, y2, duration, deviceId) {
+        const metadata = await getAndroidDeviceMetadata("", deviceId);
+        const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
+        try {
+            await execAdb(['shell', 'input', 'swipe', x1.toString(), y1.toString(), x2.toString(), y2.toString(), duration.toString()], deviceId);
+            return { device: deviceInfo, success: true, start: [x1, y1], end: [x2, y2], duration };
+        }
+        catch (e) {
+            return { device: deviceInfo, success: false, start: [x1, y1], end: [x2, y2], duration, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    async typeText(text, deviceId) {
+        const metadata = await getAndroidDeviceMetadata("", deviceId);
+        const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
+        try {
+            // Encode spaces as %s to ensure proper input handling by adb shell input text
+            const encodedText = text.replace(/\s/g, '%s');
+            // Note: 'input text' might fail with some characters or if keyboard isn't ready, but it's the standard ADB way.
+            await execAdb(['shell', 'input', 'text', encodedText], deviceId);
+            return { device: deviceInfo, success: true, text };
+        }
+        catch (e) {
+            return { device: deviceInfo, success: false, text, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    async pressBack(deviceId) {
+        const metadata = await getAndroidDeviceMetadata("", deviceId);
+        const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
+        try {
+            await execAdb(['shell', 'input', 'keyevent', '4'], deviceId);
+            return { device: deviceInfo, success: true };
+        }
+        catch (e) {
+            return { device: deviceInfo, success: false, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    async scrollToElement(selector, direction = 'down', maxScrolls = 10, scrollAmount = 0.7, deviceId) {
+        return await scrollToElementShared({
+            selector,
+            direction,
+            maxScrolls,
+            scrollAmount,
+            deviceId,
+            fetchTree: async () => await this.observe.getUITree(deviceId),
+            swipe: async (x1, y1, x2, y2, duration, devId) => await this.swipe(x1, y1, x2, y2, duration, devId)
+        });
+    }
+}

package/dist/interact/index.js

@@ -0,0 +1,37 @@
+import { AndroidInteract } from './android.js';
+import { iOSInteract } from './ios.js';
+export { AndroidInteract, iOSInteract };
+import { resolveTargetDevice } from '../utils/resolve-device.js';
+export class ToolsInteract {
+    static async getInteractionService(platform, deviceId) {
+        const effectivePlatform = platform || 'android';
+        const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
+        const interact = effectivePlatform === 'android' ? new AndroidInteract() : new iOSInteract();
+        return { interact: interact, resolved, platform: effectivePlatform };
+    }
+    static async waitForElementHandler({ platform, text, timeout, deviceId }) {
+        const effectiveTimeout = timeout ?? 10000;
+        const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
+        return await interact.waitForElement(text, effectiveTimeout, resolved.id);
+    }
+    static async tapHandler({ platform, x, y, deviceId }) {
+        const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
+        return await interact.tap(x, y, resolved.id);
+    }
+    static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
+        const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
+        return await interact.swipe(x1, y1, x2, y2, duration, resolved.id);
+    }
+    static async typeTextHandler({ text, deviceId }) {
+        const resolved = await resolveTargetDevice({ platform: 'android', deviceId });
+        return await new AndroidInteract().typeText(text, resolved.id);
+    }
+    static async pressBackHandler({ deviceId }) {
+        const resolved = await resolveTargetDevice({ platform: 'android', deviceId });
+        return await new AndroidInteract().pressBack(resolved.id);
+    }
+    static async scrollToElementHandler({ platform, selector, direction = 'down', maxScrolls = 10, scrollAmount = 0.7, deviceId }) {
+        const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
+        return await interact.scrollToElement(selector, direction, maxScrolls, scrollAmount, resolved.id);
+    }
+}

package/dist/interact/ios.js

@@ -0,0 +1,120 @@
+import { spawn } from "child_process";
+import { getIOSDeviceMetadata, getIdbCmd, isIDBInstalled } from "../utils/ios/utils.js";
+import { iOSObserve } from "../observe/index.js";
+import { scrollToElementShared } from "../interact/shared/scroll_to_element.js";
+export class iOSInteract {
+    observe = new iOSObserve();
+    async waitForElement(text, timeout, deviceId = "booted") {
+        const device = await getIOSDeviceMetadata(deviceId);
+        const startTime = Date.now();
+        while (Date.now() - startTime < timeout) {
+            try {
+                const tree = await this.observe.getUITree(deviceId);
+                if (tree.error) {
+                    return { device, found: false, error: tree.error };
+                }
+                const element = tree.elements.find(e => e.text === text);
+                if (element) {
+                    return { device, found: true, element };
+                }
+            }
+            catch (e) {
+                // Ignore errors during polling and retry
+                console.error("Error polling UI tree:", e);
+            }
+            const elapsed = Date.now() - startTime;
+            const remaining = timeout - elapsed;
+            if (remaining <= 0)
+                break;
+            await new Promise(resolve => setTimeout(resolve, Math.min(500, remaining)));
+        }
+        return { device, found: false };
+    }
+    async tap(x, y, deviceId = "booted") {
+        const device = await getIOSDeviceMetadata(deviceId);
+        // Use shared helper to detect idb
+        const idbExists = await isIDBInstalled();
+        if (!idbExists) {
+            return {
+                device,
+                success: false,
+                x,
+                y,
+                error: "iOS tap requires 'idb' (iOS Device Bridge)."
+            };
+        }
+        try {
+            const targetUdid = (device.id && device.id !== 'booted') ? device.id : undefined;
+            const args = ['ui', 'tap', x.toString(), y.toString()];
+            if (targetUdid) {
+                args.push('--udid', targetUdid);
+            }
+            await new Promise((resolve, reject) => {
+                const proc = spawn(getIdbCmd(), args);
+                let stderr = '';
+                proc.stderr.on('data', d => stderr += d.toString());
+                proc.on('close', code => {
+                    if (code === 0)
+                        resolve();
+                    else
+                        reject(new Error(`idb ui tap failed: ${stderr}`));
+                });
+                proc.on('error', err => reject(err));
+            });
+            return { device, success: true, x, y };
+        }
+        catch (e) {
+            return { device, success: false, x, y, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    async swipe(x1, y1, x2, y2, duration, deviceId = "booted") {
+        const device = await getIOSDeviceMetadata(deviceId);
+        // Use shared helper to detect idb
+        const idbExists = await isIDBInstalled();
+        if (!idbExists) {
+            return {
+                device,
+                success: false,
+                start: [x1, y1],
+                end: [x2, y2],
+                duration,
+                error: "iOS swipe requires 'idb' (iOS Device Bridge)."
+            };
+        }
+        try {
+            const targetUdid = (device.id && device.id !== 'booted') ? device.id : undefined;
+            // idb 'ui swipe' does not accept a duration parameter; use coordinates only
+            const args = ['ui', 'swipe', x1.toString(), y1.toString(), x2.toString(), y2.toString()];
+            if (targetUdid) {
+                args.push('--udid', targetUdid);
+            }
+            await new Promise((resolve, reject) => {
+                const proc = spawn(getIdbCmd(), args);
+                let stderr = '';
+                proc.stderr.on('data', d => stderr += d.toString());
+                proc.on('close', code => {
+                    if (code === 0)
+                        resolve();
+                    else
+                        reject(new Error(`idb ui swipe failed: ${stderr}`));
+                });
+                proc.on('error', err => reject(err));
+            });
+            return { device, success: true, start: [x1, y1], end: [x2, y2], duration };
+        }
+        catch (e) {
+            return { device, success: false, start: [x1, y1], end: [x2, y2], duration, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
+    async scrollToElement(selector, direction = 'down', maxScrolls = 10, scrollAmount = 0.7, deviceId = 'booted') {
+        return await scrollToElementShared({
+            selector,
+            direction,
+            maxScrolls,
+            scrollAmount,
+            deviceId,
+            fetchTree: async () => await this.observe.getUITree(deviceId),
+            swipe: async (x1, y1, x2, y2, duration, devId) => await this.swipe(x1, y1, x2, y2, duration, devId)
+        });
+    }
+}

package/dist/interact/shared/fingerprint.js

@@ -0,0 +1,72 @@
+import crypto from 'crypto';
+const ANDROID_STRUCTURAL_TYPES = ['Window', 'Application', 'View', 'ViewGroup', 'LinearLayout', 'FrameLayout', 'RelativeLayout', 'ScrollView', 'RecyclerView', 'TextView', 'ImageView'];
+const IOS_STRUCTURAL_TYPES = ['Window', 'Application', 'View', 'ViewController', 'UITableView', 'UICollectionView', 'UILabel', 'UIImageView', 'UIView', 'UIWindow', 'UIStackView', 'UITextView', 'UITableViewCell'];
+function isDynamicText(t) {
+    if (!t)
+        return false;
+    const txt = t.trim();
+    if (!txt)
+        return false;
+    if (/\b\d{1,2}:\d{2}\b/.test(txt))
+        return true;
+    if (/\b\d{4}-\d{2}-\d{2}\b/.test(txt))
+        return true;
+    if (/^\d+(?:\.\d+)?%$/.test(txt))
+        return true;
+    if (/^\d+$/.test(txt))
+        return true;
+    if (/^[\d,]{1,10}$/.test(txt))
+        return true;
+    return false;
+}
+function normalizeElement(e) {
+    return {
+        type: (e.type || '').toString(),
+        resourceId: (e.resourceId || '').toString(),
+        text: typeof e.text === 'string' ? (isDynamicText(e.text) ? '' : e.text.trim().toLowerCase()) : '',
+        contentDesc: (e.contentDescription || '').toString(),
+        bounds: Array.isArray(e.bounds) ? e.bounds.slice(0, 4).map((n) => Number(n) || 0) : [0, 0, 0, 0]
+    };
+}
+export function computeScreenFingerprint(tree, current, platform, limit = 50) {
+    try {
+        if (!tree || tree.error)
+            return { fingerprint: null, error: tree.error };
+        const activity = current && (current.activity || current.shortActivity) ? (current.activity || current.shortActivity) : '';
+        const candidates = (tree.elements || []).filter(e => {
+            if (!e)
+                return false;
+            if (!e.visible)
+                return false;
+            const hasStableText = typeof e.text === 'string' && e.text.trim().length > 0;
+            const hasResource = !!e.resourceId;
+            const interactable = !!e.clickable || !!e.enabled;
+            const structuralList = platform === 'android' ? ANDROID_STRUCTURAL_TYPES : IOS_STRUCTURAL_TYPES;
+            const structurallySignificant = hasStableText || hasResource || structuralList.includes(e.type || '');
+            return interactable || structurallySignificant;
+        });
+        const normalized = candidates.map(normalizeElement);
+        const filteredNormalized = normalized.filter(e => (e.text && e.text.length > 0) || (e.resourceId && e.resourceId.length > 0) || (e.contentDesc && e.contentDesc.length > 0));
+        filteredNormalized.sort((a, b) => {
+            const ay = (a.bounds && a.bounds[1]) || 0;
+            const by = (b.bounds && b.bounds[1]) || 0;
+            if (ay !== by)
+                return ay - by;
+            const ax = (a.bounds && a.bounds[0]) || 0;
+            const bx = (b.bounds && b.bounds[0]) || 0;
+            return ax - bx;
+        });
+        const limited = filteredNormalized.slice(0, Math.max(0, limit));
+        const payload = {
+            activity: platform === 'android' ? (activity || '') : '',
+            resolution: tree.resolution || { width: 0, height: 0 },
+            elements: limited.map(e => ({ type: e.type, resourceId: e.resourceId, text: e.text, contentDesc: e.contentDesc }))
+        };
+        const combined = JSON.stringify(payload);
+        const hash = crypto.createHash('sha256').update(combined).digest('hex');
+        return { fingerprint: hash, activity: activity };
+    }
+    catch (e) {
+        return { fingerprint: null, error: e instanceof Error ? e.message : String(e) };
+    }
+}

package/dist/interact/shared/scroll_to_element.js

@@ -0,0 +1,98 @@
+export async function scrollToElementShared(opts) {
+    const { selector, direction = 'down', maxScrolls = 10, scrollAmount = 0.7, deviceId, fetchTree, swipe, stabilizationDelayMs = 350 } = opts;
+    const matchElement = (el) => {
+        if (!el)
+            return false;
+        if (selector.text !== undefined && selector.text !== el.text)
+            return false;
+        if (selector.resourceId !== undefined && selector.resourceId !== el.resourceId)
+            return false;
+        if (selector.contentDesc !== undefined && selector.contentDesc !== el.contentDescription)
+            return false;
+        if (selector.className !== undefined && selector.className !== el.type)
+            return false;
+        return true;
+    };
+    const isVisible = (el, resolution) => {
+        if (!el)
+            return false;
+        if (el.visible === false)
+            return false;
+        if (!el.bounds || !resolution || !resolution.width || !resolution.height)
+            return (el.visible === undefined ? true : !!el.visible);
+        const [left, top, right, bottom] = el.bounds;
+        const withinY = bottom > 0 && top < resolution.height;
+        const withinX = right > 0 && left < resolution.width;
+        return withinX && withinY;
+    };
+    const findVisibleMatch = (elements, resolution) => {
+        if (!Array.isArray(elements))
+            return null;
+        for (const e of elements) {
+            if (matchElement(e) && isVisible(e, resolution))
+                return e;
+        }
+        return null;
+    };
+    // Initial check
+    let tree = await fetchTree();
+    if (tree.error)
+        return { success: false, reason: tree.error, scrollsPerformed: 0 };
+    let found = findVisibleMatch(tree.elements, tree.resolution);
+    if (found) {
+        return { success: true, element: { text: found.text, resourceId: found.resourceId, bounds: found.bounds }, scrollsPerformed: 0 };
+    }
+    const fingerprintOf = (t) => {
+        try {
+            return JSON.stringify((t.elements || []).map((e) => ({ text: e.text, resourceId: e.resourceId, bounds: e.bounds })));
+        }
+        catch {
+            return '';
+        }
+    };
+    let prevFingerprint = fingerprintOf(tree);
+    const width = (tree.resolution && tree.resolution.width) ? tree.resolution.width : 0;
+    const height = (tree.resolution && tree.resolution.height) ? tree.resolution.height : 0;
+    const centerX = Math.round(width / 2) || 50;
+    const clampPct = (v) => Math.max(0.05, Math.min(0.95, v));
+    const computeCoords = () => {
+        const defaultStart = direction === 'down' ? 0.8 : 0.2;
+        const startPct = clampPct(defaultStart);
+        const endPct = clampPct(defaultStart + (direction === 'down' ? -scrollAmount : scrollAmount));
+        const x1 = centerX;
+        const x2 = centerX;
+        const y1 = Math.round((height || 100) * startPct);
+        const y2 = Math.round((height || 100) * endPct);
+        return { x1, y1, x2, y2 };
+    };
+    const duration = 300;
+    let scrollsPerformed = 0;
+    for (let i = 0; i < maxScrolls; i++) {
+        const { x1, y1, x2, y2 } = computeCoords();
+        try {
+            await swipe(x1, y1, x2, y2, duration, deviceId);
+        }
+        catch (e) {
+            // Log swipe failures to aid debugging but don't fail the overall flow
+            try {
+                console.warn(`scrollToElement swipe failed: ${e instanceof Error ? e.message : String(e)}`);
+            }
+            catch { }
+        }
+        scrollsPerformed++;
+        await new Promise(resolve => setTimeout(resolve, stabilizationDelayMs));
+        tree = await fetchTree();
+        if (tree.error)
+            return { success: false, reason: tree.error, scrollsPerformed: scrollsPerformed };
+        found = findVisibleMatch(tree.elements, tree.resolution);
+        if (found) {
+            return { success: true, element: { text: found.text, resourceId: found.resourceId, bounds: found.bounds }, scrollsPerformed };
+        }
+        const fp = fingerprintOf(tree);
+        if (fp === prevFingerprint) {
+            return { success: false, reason: 'UI unchanged after scroll; likely end of list', scrollsPerformed: scrollsPerformed };
+        }
+        prevFingerprint = fp;
+    }
+    return { success: false, reason: 'Element not found after scrolling', scrollsPerformed: scrollsPerformed };
+}

package/dist/ios/interact.js

@@ -1,7 +1,7 @@
 import { spawn } from "child_process";
 import { getIOSDeviceMetadata, getIdbCmd, isIDBInstalled } from "./utils.js";
-import { iOSObserve } from "./observe.js";
-import { scrollToElementShared } from "../tools/scroll_to_element.js";
+import { iOSObserve } from "../observe/index.js";
+import { scrollToElementShared } from "../intera../interact/shared/scroll_to_element.js";
 export class iOSInteract {
     observe = new iOSObserve();
     async waitForElement(text, timeout, deviceId = "booted") {

package/dist/ios/observe.js

@@ -4,6 +4,7 @@ import { execCommand, getIOSDeviceMetadata, validateBundleId, getIdbCmd, getXcru
 import { createWriteStream, promises as fsPromises } from 'fs';
 import path from 'path';
 import { parseLogLine } from '../android/utils.js';
+import { computeScreenFingerprint } from '../intera../interact/shared/fingerprint.js';
 // --- Helper Functions Specific to Observe ---
 const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
 function parseIDBFrame(frame) {
@@ -237,6 +238,17 @@ export class iOSObserve {
             };
         }
     }
+    async getScreenFingerprint(deviceId = 'booted') {
+        try {
+            const tree = await this.getUITree(deviceId);
+            if (!tree || tree.error)
+                return { fingerprint: null, error: tree && tree.error };
+            return computeScreenFingerprint(tree, null, 'ios', 50);
+        }
+        catch (e) {
+            return { fingerprint: null, error: e instanceof Error ? e.message : String(e) };
+        }
+    }
     // --- Log stream methods ---
     async startLogStream(bundleId, deviceId = 'booted', sessionId = 'default') {
         try {