mobile-debug-mcp 0.21.4 → 0.22.0

package/src/interact/index.ts

@@ -1,9 +1,11 @@
+import { createHash } from 'crypto'
 import { AndroidInteract } from './android.js';
 import { iOSInteract } from './ios.js';
 export { AndroidInteract, iOSInteract };
 
 import { resolveTargetDevice } from '../utils/resolve-device.js'
 import { ToolsObserve } from '../observe/index.js'
+import type { TapElementResponse } from '../types.js'
 
 interface ScreenFingerprintResponse { fingerprint: string | null }
 
@@ -29,8 +31,159 @@ interface UiElement {
   _interactable?: boolean
 }
 
+interface ResolvedUiElementContext {
+  elementId: string
+  platform: 'android' | 'ios'
+  deviceId?: string
+  bounds: [number, number, number, number] | null
+  index: number
+}
+
 
 export class ToolsInteract {
+  private static readonly _maxResolvedUiElements = 256
+  private static _resolvedUiElements = new Map<string, ResolvedUiElementContext>()
+
+  private static _normalize(s: any): string {
+    if (s === null || s === undefined) return ''
+    try { return String(s).toLowerCase().trim() } catch { return '' }
+  }
+
+  private static _normalizeBounds(bounds: any): [number, number, number, number] | null {
+    if (!Array.isArray(bounds) || bounds.length < 4) return null
+    const normalized = bounds.slice(0, 4).map((value: any) => Number(value))
+    if (normalized.some((value: number) => Number.isNaN(value))) return null
+    return normalized as [number, number, number, number]
+  }
+
+  private static _isVisibleElement(el: UiElement): boolean {
+    const bounds = ToolsInteract._normalizeBounds(el.bounds)
+    return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
+  }
+
+  private static _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
+    const identity = {
+      platform,
+      deviceId: deviceId || '',
+      text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
+      resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
+      accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
+      class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
+      bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
+      index
+    }
+    return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`
+  }
+
+  private static _buildResolvedElement(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number) {
+    const bounds = ToolsInteract._normalizeBounds(el.bounds)
+    const elementId = ToolsInteract._computeElementId(platform, deviceId, el, index)
+
+    ToolsInteract._rememberResolvedElement(elementId, {
+      elementId,
+      platform,
+      deviceId,
+      bounds,
+      index
+    })
+
+    return {
+      text: el.text ?? null,
+      resource_id: el.resourceId ?? el.resourceID ?? el.id ?? null,
+      accessibility_id: el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? null,
+      class: el.type ?? el.class ?? null,
+      bounds,
+      index,
+      elementId
+    }
+  }
+
+  private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
+    if (ToolsInteract._resolvedUiElements.has(elementId)) {
+      ToolsInteract._resolvedUiElements.delete(elementId)
+    }
+
+    ToolsInteract._resolvedUiElements.set(elementId, context)
+
+    while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
+      const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value
+      if (!oldestElementId) break
+      ToolsInteract._resolvedUiElements.delete(oldestElementId)
+    }
+  }
+
+  static _resetResolvedUiElementsForTests() {
+    ToolsInteract._resolvedUiElements.clear()
+  }
+
+  private static _findCurrentResolvedElement(
+    elements: UiElement[],
+    platform: 'android' | 'ios',
+    deviceId: string | undefined,
+    resolved: ResolvedUiElementContext
+  ): { el: UiElement, index: number } | null {
+    const indexedCandidate = elements[resolved.index]
+    if (indexedCandidate && ToolsInteract._computeElementId(platform, deviceId, indexedCandidate, resolved.index) === resolved.elementId) {
+      return { el: indexedCandidate, index: resolved.index }
+    }
+
+    return null
+  }
+
+  private static _resolveActionableAncestor(elements: UiElement[], chosen: { el: UiElement, idx: number } | null): { el: UiElement, idx: number } | null {
+    if (!chosen) return null
+    if (chosen.el.clickable || chosen.el.focusable) return chosen
+
+    let current = chosen
+    let safety = 0
+
+    while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable) && current.el.parentId !== undefined && current.el.parentId !== null) {
+      const parentId = current.el.parentId
+      let parentIndex: number | null = null
+
+      if (typeof parentId === 'number') parentIndex = parentId
+      else if (typeof parentId === 'string' && /^\d+$/.test(parentId)) parentIndex = Number(parentId)
+
+      if (parentIndex !== null && elements[parentIndex]) {
+        current = { el: elements[parentIndex], idx: parentIndex }
+        if (current.el.clickable || current.el.focusable) return current
+      } else if (typeof parentId === 'string') {
+        const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId)
+        if (foundIndex === -1) break
+        current = { el: elements[foundIndex], idx: foundIndex }
+        if (current.el.clickable || current.el.focusable) return current
+      } else {
+        break
+      }
+
+      safety++
+    }
+
+    const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
+    if (!childBounds) return null
+    const [cl, ct, cr, cb] = childBounds
+
+    let best: { el: UiElement, idx: number } | null = null
+    let bestArea = Infinity
+
+    for (let i = 0; i < elements.length; i++) {
+      const el = elements[i]
+      if (!el || !(el.clickable || el.focusable)) continue
+      const bounds = ToolsInteract._normalizeBounds(el.bounds)
+      if (!bounds) continue
+      const [pl, pt, pr, pb] = bounds
+      if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
+        const area = (pr - pl) * (pb - pt)
+        if (area < bestArea) {
+          bestArea = area
+          best = { el, idx: i }
+        }
+      }
+    }
+
+    return best
+  }
+
 
   private static async getInteractionService(platform?: 'android' | 'ios', deviceId?: string) {
     const effectivePlatform = platform || 'android'
@@ -44,6 +197,99 @@ export class ToolsInteract {
     return await interact.tap(x, y, resolved.id)
   }
 
+  static async tapElementHandler({ elementId }: { elementId: string }): Promise<TapElementResponse> {
+    const action = 'tap' as const
+    const resolved = ToolsInteract._resolvedUiElements.get(elementId)
+    if (!resolved) {
+      return {
+        success: false,
+        elementId,
+        action,
+        error: {
+          code: 'element_not_found',
+          message: 'Element ID was not found in the current UI context'
+        }
+      }
+    }
+
+    const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId }) as any
+    const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform
+    const treeDeviceId = tree?.device?.id || resolved.deviceId
+    const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
+    const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
+
+    if (!currentMatch) {
+      return {
+        success: false,
+        elementId,
+        action,
+        error: {
+          code: 'element_not_found',
+          message: 'Element ID is not present in the current UI context'
+        }
+      }
+    }
+
+    if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
+      return {
+        success: false,
+        elementId,
+        action,
+        error: {
+          code: 'element_not_visible',
+          message: 'Element is not visible'
+        }
+      }
+    }
+
+    if (currentMatch.el.enabled === false) {
+      return {
+        success: false,
+        elementId,
+        action,
+        error: {
+          code: 'element_not_enabled',
+          message: 'Element is not enabled'
+        }
+      }
+    }
+
+    const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
+    if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
+      return {
+        success: false,
+        elementId,
+        action,
+        error: {
+          code: 'element_not_visible',
+          message: 'Element does not have valid visible bounds'
+        }
+      }
+    }
+
+    const x = Math.floor((bounds[0] + bounds[2]) / 2)
+    const y = Math.floor((bounds[1] + bounds[3]) / 2)
+    const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId })
+
+    if (!tapResult.success) {
+      return {
+        success: false,
+        elementId,
+        action,
+        error: {
+          code: 'tap_failed',
+          message: tapResult.error || 'Tap failed'
+        }
+      }
+    }
+
+    return {
+      success: true,
+      elementId,
+      action
+    }
+  }
+
   static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }: { platform?: 'android' | 'ios', x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string }) {
     const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId)
     return await interact.swipe(x1, y1, x2, y2, duration, resolved.id)
@@ -68,7 +314,7 @@ export class ToolsInteract {
     // Try to use observe layer to fetch the current UI tree and perform a fast semantic search
     const start = Date.now()
     const deadline = start + timeoutMs
-    const normalize = (s: any) => (s === null || s === undefined) ? '' : String(s).toLowerCase().trim()
+    const normalize = ToolsInteract._normalize
 
     const q = normalize(query)
     if (!q) return { found: false, error: 'Empty query' }
@@ -221,16 +467,20 @@ export class ToolsInteract {
   static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string }) {
     const overallStart = Date.now()
 
-    // Validate selector: require at least one of text, resource_id, or accessibility_id
-    if (!selector || (typeof selector === 'object' && Object.keys(selector).length === 0)) {
-      return { status: 'timeout', error: { code: 'INVALID_SELECTOR', message: 'At least one selector field must be provided (text, resource_id, or accessibility_id)' }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 } }
-    }
+    // Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
+    const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
+    const hasResId = typeof selector?.resource_id === 'string' && selector.resource_id.trim().length > 0;
+    const hasAccId = typeof selector?.accessibility_id === 'string' && selector.accessibility_id.trim().length > 0;
 
-    const hasText = selector && typeof (selector as any).text === 'string' && (selector as any).text.trim().length > 0
-    const hasResId = selector && typeof (selector as any).resource_id === 'string' && (selector as any).resource_id.trim().length > 0
-    const hasAccId = selector && typeof (selector as any).accessibility_id === 'string' && (selector as any).accessibility_id.trim().length > 0
     if (!hasText && !hasResId && !hasAccId) {
-      return { status: 'timeout', error: { code: 'INVALID_SELECTOR', message: 'Selector must include at least one of: text, resource_id, accessibility_id' }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 } }
+      return {
+        status: 'timeout',
+        error: {
+          code: 'INVALID_SELECTOR',
+          message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
+        },
+        metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }
+      };
     }
 
     // Validate condition
@@ -251,11 +501,11 @@ export class ToolsInteract {
     let totalPollCount = 0
 
     // Precompute normalized selector values and helpers (constant across polls)
-    const normalize = (s: any) => (s === null || s === undefined) ? '' : String(s).toLowerCase().trim()
-    const containsFlag = !!selector.contains
-    const selText = normalize((selector as any).text)
-    const selRid = normalize((selector as any).resource_id)
-    const selAid = normalize((selector as any).accessibility_id)
+    const normalize = ToolsInteract._normalize
+    const containsFlag = !!selector?.contains
+    const selText = normalize(selector?.text)
+    const selRid = normalize(selector?.resource_id)
+    const selAid = normalize(selector?.accessibility_id)
 
     try {
       while (attempts < maxAttempts) {
@@ -310,26 +560,28 @@ export class ToolsInteract {
 
             // Evaluate condition
             const matchedCount = matches.length
-            const pickIndexProvided = (match && typeof (match as any).index === 'number')
-            const pickIndex: number = pickIndexProvided ? Number((match as any).index) : 0
+            const pickIndex = (typeof match?.index === 'number') ? match!.index as number : undefined
             let chosen: { el: any, idx: number } | null = null
-            if (matches.length === 0) {
-              chosen = null
-            } else if (pickIndexProvided) {
-              // If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
-              if (pickIndex >= 0 && pickIndex < matches.length) chosen = matches[pickIndex]
-              else chosen = null
+            if (matches.length > 0) {
+              if (pickIndex !== undefined) {
+                // If a specific index is requested but out of bounds, treat as not matched for this poll (deterministic)
+                if (pickIndex >= 0 && pickIndex < matches.length) chosen = matches[pickIndex]
+                else chosen = null
+              } else {
+                chosen = matches[0]
+              }
             } else {
-              chosen = matches[0]
+              chosen = null
             }
 
             let conditionMet = false
+            let matchedElement = chosen
             if (condition === 'exists') {
               // when an index is specified, existence requires that specific index be present
-              conditionMet = pickIndexProvided ? (chosen !== null) : (matchedCount >= 1)
+              conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1)
             } else if (condition === 'not_exists') {
               // when an index is specified, not_exists is true if that index is absent
-              conditionMet = pickIndexProvided ? (chosen === null) : (matchedCount === 0)
+              conditionMet = (pickIndex !== undefined) ? (chosen === null) : (matchedCount === 0)
             } else if (condition === 'visible') {
               if (chosen) {
                 const b = chosen.el.bounds
@@ -337,11 +589,12 @@ export class ToolsInteract {
                 conditionMet = visibleFlag
               } else conditionMet = false
             } else if (condition === 'clickable') {
-              if (chosen) {
-                const b = chosen.el.bounds
-                const visibleFlag = !!chosen.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
-                const enabled = !!chosen.el.enabled
-                const clickable = !!chosen.el.clickable || !!chosen.el._interactable
+              matchedElement = chosen ? (ToolsInteract._resolveActionableAncestor(elements, chosen as { el: UiElement, idx: number }) || chosen) : null
+              if (matchedElement) {
+                const b = matchedElement.el.bounds
+                const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
+                const enabled = !!matchedElement.el.enabled
+                const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable
                 conditionMet = visibleFlag && enabled && clickable
               } else conditionMet = false
             }
@@ -350,14 +603,9 @@ export class ToolsInteract {
               const now = Date.now()
               const latency_ms = now - overallStart
               // Build element output per spec
-              const outEl = chosen ? {
-                text: chosen.el.text ?? null,
-                resource_id: chosen.el.resourceId ?? chosen.el.resourceID ?? chosen.el.id ?? null,
-                accessibility_id: chosen.el.contentDescription ?? chosen.el.contentDesc ?? chosen.el.accessibilityLabel ?? chosen.el.label ?? null,
-                class: chosen.el.type ?? chosen.el.class ?? null,
-                bounds: Array.isArray(chosen.el.bounds) && chosen.el.bounds.length >= 4 ? chosen.el.bounds : null,
-                index: chosen.idx
-              } : null
+              const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
+              const resolvedDeviceId = tree?.device?.id || deviceId
+              const outEl = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
 
               return {
                 status: 'success',

package/src/observe/ios.ts

@@ -9,6 +9,15 @@ import { computeScreenFingerprint } from '../utils/ui/index.js'
 import { parsePngSize } from '../utils/image.js'
 
 const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+let iosExecCommand = execCommand
+
+export function _setIOSExecCommandForTests(fn: typeof execCommand) {
+  iosExecCommand = fn
+}
+
+export function _resetIOSExecCommandForTests() {
+  iosExecCommand = execCommand
+}
 
 interface IDBElement {
   AXFrame?: { x: number | string, y: number | string, width: number | string, height: number | string, w?: number | string, h?: number | string };
@@ -157,7 +166,7 @@ export class iOSObserve {
         const parts = appId.split('.')
         const simpleName = parts[parts.length - 1]
         try {
-          const pgrepRes = await execCommand(['simctl','spawn', deviceId, 'pgrep', '-f', simpleName], deviceId)
+          const pgrepRes = await iosExecCommand(['simctl','spawn', deviceId, 'pgrep', '-f', simpleName], deviceId)
           const out = pgrepRes && pgrepRes.output ? pgrepRes.output.trim() : ''
           const firstLine = out.split(/\r?\n/).find(Boolean)
           if (firstLine) {
@@ -169,7 +178,7 @@ export class iOSObserve {
         }
       }
       const effectivePid = pid || detectedPid || null
-      const result = await execCommand(args, deviceId)
+      const result = await iosExecCommand(args, deviceId)
       const device = await getIOSDeviceMetadata(deviceId)
       const rawLines = result.output ? result.output.split(/\r?\n/).filter(Boolean) : []
 
@@ -281,7 +290,7 @@ export class iOSObserve {
     const tmpFile = `/tmp/mcp-ios-screenshot-${Date.now()}.png`
 
     try {
-      await execCommand(['simctl', 'io', deviceId, 'screenshot', tmpFile], deviceId)
+      await iosExecCommand(['simctl', 'io', deviceId, 'screenshot', tmpFile], deviceId)
       
       const buffer = await fs.readFile(tmpFile)
       const base64 = buffer.toString('base64')