mobile-debug-mcp 0.25.0 → 0.26.0

package/src/server/tool-definitions.ts

@@ -240,7 +240,7 @@ Failure Handling:
   },
   {
     name: 'capture_debug_snapshot',
-    description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON.',
+    description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON with snapshot_revision, captured_at_ms, and loading_state when detectable.',
     inputSchema: {
       type: 'object',
       properties: {
@@ -291,7 +291,7 @@ Failure Handling:
   },
   {
     name: 'get_ui_tree',
-    description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content.',
+    description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content with snapshot metadata when available.',
     inputSchema: {
       type: 'object',
       properties: {
@@ -363,6 +363,34 @@ Recommended Usage:
       required: ['previousFingerprint']
     }
   },
+  {
+    name: 'wait_for_ui_change',
+    description: `Purpose:
+Wait for a non-navigation UI mutation or in-place update to become stable.
+
+Inputs:
+- expected_change (optional): hierarchy_diff, text_change, or state_change
+- timeout_ms (optional)
+- stability_window_ms (optional)
+
+Guidance:
+- Prefer wait_for_screen_change for navigation transitions.
+- Prefer wait_for_ui_change for in-place mutations and non-navigation updates.
+- Use the returned snapshot_revision as the observed synchronization point when available.
+
+Failure Handling:
+- TIMEOUT means the UI did not change in a stable way within the allotted time.`,
+    inputSchema: {
+      type: 'object',
+      properties: {
+        platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override (android|ios)' },
+        deviceId: { type: 'string', description: 'Optional device id/udid to target' },
+        expected_change: { type: 'string', enum: ['hierarchy_diff', 'text_change', 'state_change'], description: 'Optional type of UI change to wait for' },
+        timeout_ms: { type: 'number', description: 'Timeout in ms to wait for change (default 60000)', default: 60000 },
+        stability_window_ms: { type: 'number', description: 'How long the change must remain stable before success (default 250)', default: 250 }
+      }
+    }
+  },
   {
     name: 'expect_screen',
     description: `Purpose:

package/src/server/tool-handlers.ts

@@ -288,6 +288,16 @@ async function handleWaitForUI(args: ToolCallArgs) {
   return wrapResponse(res)
 }
 
+async function handleWaitForUIChange(args: ToolCallArgs) {
+  const platform = getStringArg(args, 'platform') as PlatformArg | undefined
+  const deviceId = getStringArg(args, 'deviceId')
+  const timeout_ms = getNumberArg(args, 'timeout_ms') ?? 60000
+  const stability_window_ms = getNumberArg(args, 'stability_window_ms') ?? 250
+  const expected_change = getStringArg(args, 'expected_change') as 'hierarchy_diff' | 'text_change' | 'state_change' | undefined
+  const res = await ToolsInteract.waitForUIChangeHandler({ platform, deviceId, timeout_ms, stability_window_ms, expected_change })
+  return wrapResponse(res)
+}
+
 async function handleFindElement(args: ToolCallArgs) {
   const query = requireStringArg(args, 'query')
   const exact = getBooleanArg(args, 'exact') ?? false
@@ -473,6 +483,7 @@ export const toolHandlers: Record<string, ToolHandler> = {
   get_current_screen: handleGetCurrentScreen,
   get_screen_fingerprint: handleGetScreenFingerprint,
   wait_for_screen_change: handleWaitForScreenChange,
+  wait_for_ui_change: handleWaitForUIChange,
   expect_screen: handleExpectScreen,
   expect_element_visible: handleExpectElementVisible,
   expect_state: handleExpectState,

package/src/server-core.ts

@@ -13,7 +13,7 @@ export { wrapResponse, toolDefinitions, handleToolCall }
 
 export const serverInfo = {
   name: 'mobile-debug-mcp',
-  version: '0.25.0'
+  version: '0.26.0'
 }
 
 export function createServer() {

package/src/types.ts

@@ -94,6 +94,27 @@ export interface UIElementState {
   } | null;
 }
 
+export interface SelectorConfidence {
+  score: number;
+  reason: string;
+}
+
+export interface UIResolutionSelector {
+  value: string | null;
+  confidence: SelectorConfidence | null;
+}
+
+export interface UIElementSemanticMetadata {
+  is_clickable: boolean;
+  is_container: boolean;
+}
+
+export interface LoadingState {
+  active: boolean;
+  signal: string;
+  source: string;
+}
+
 export interface CaptureAndroidScreenResponse {
   device: DeviceInfo;
   screenshot: string; // base64 encoded string
@@ -132,6 +153,11 @@ export interface UIElement {
   center?: [number, number];
   depth?: number;
   state?: UIElementState | null;
+  stable_id?: string | null;
+  role?: string | null;
+  test_tag?: string | null;
+  selector?: UIResolutionSelector | null;
+  semantic?: UIElementSemanticMetadata | null;
 }
 
 export interface GetUITreeResponse {
@@ -142,6 +168,9 @@ export interface GetUITreeResponse {
     height: number;
   };
   elements: UIElement[];
+  snapshot_revision: number;
+  captured_at_ms: number;
+  loading_state?: LoadingState | null;
   error?: string;
 }
 
@@ -163,12 +192,15 @@ export interface SnapshotSemanticResponse {
 
 export interface CaptureDebugSnapshotRawResponse {
   timestamp: number;
+  snapshot_revision: number;
+  captured_at_ms: number;
   reason: string;
   activity: string | null;
   fingerprint: string | null;
   screenshot: string | null;
-  ui_tree: unknown | null;
+  ui_tree: GetUITreeResponse | null;
   logs: StructuredLogEntry[];
+  loading_state?: LoadingState | null;
   device?: DeviceInfo;
   screenshot_error?: string;
   activity_error?: string;
@@ -215,6 +247,11 @@ export interface ActionTargetResolved {
   bounds: [number, number, number, number] | null;
   index: number | null;
   state?: UIElementState | null;
+  stable_id?: string | null;
+  role?: string | null;
+  test_tag?: string | null;
+  selector?: UIResolutionSelector | null;
+  semantic?: UIElementSemanticMetadata | null;
 }
 
 export interface ActionExecutionResult {
@@ -301,6 +338,17 @@ export interface ExpectStateResponse {
   retryable?: boolean;
 }
 
+export interface WaitForUIChangeResponse {
+  success: boolean;
+  observed_change: 'hierarchy_diff' | 'text_change' | 'state_change' | null;
+  snapshot_revision?: number;
+  timeout: boolean;
+  elapsed_ms: number;
+  expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change';
+  reason?: string;
+  loading_state?: LoadingState | null;
+}
+
 export interface SwipeResponse {
   device: DeviceInfo;
   success: boolean;

package/src/utils/android/utils.ts

@@ -1,4 +1,4 @@
-import { DeviceInfo, UIElement, UIElementState } from "../../types.js"
+import { DeviceInfo, UIElement, UIElementSemanticMetadata, UIElementState, UIResolutionSelector, SelectorConfidence } from "../../types.js"
 import { promises as fsPromises, existsSync } from 'fs'
 import path from 'path'
 import { detectJavaHome } from '../java.js'
@@ -336,6 +336,52 @@ function parseNumberAttr(value: unknown): number | null {
   return Number.isFinite(parsed) ? parsed : null
 }
 
+function normalizeClassName(value: unknown): string {
+  return typeof value === 'string' ? value.trim().toLowerCase() : ''
+}
+
+function inferAndroidRole(className: string): string | null {
+  if (/seekbar|slider|progress/.test(className)) return 'slider'
+  if (/switch|toggle/.test(className)) return 'switch'
+  if (/checkbox/.test(className)) return 'checkbox'
+  if (/radiobutton|radio/.test(className)) return 'radio'
+  if (/edittext|textfield|search/.test(className)) return 'text_field'
+  if (/button|fab/.test(className)) return 'button'
+  if (/imageview|icon/.test(className)) return 'image'
+  if (/recyclerview|scroll|layout|viewgroup|frame/.test(className)) return 'container'
+  return null
+}
+
+function buildAndroidSelectorConfidence(source: 'resource_id' | 'content_desc' | 'text' | 'class' | 'none'): SelectorConfidence | null {
+  switch (source) {
+    case 'resource_id':
+      return { score: 1, reason: 'resource_id' }
+    case 'content_desc':
+      return { score: 0.9, reason: 'content_description' }
+    case 'text':
+      return { score: 0.6, reason: 'text_match' }
+    case 'class':
+      return { score: 0.35, reason: 'class_match' }
+    default:
+      return null
+  }
+}
+
+function buildAndroidSelector(text: string | null, contentDescription: string | null, resourceId: string | null, className: string): UIResolutionSelector | null {
+  if (resourceId) return { value: resourceId, confidence: buildAndroidSelectorConfidence('resource_id') }
+  if (contentDescription) return { value: contentDescription, confidence: buildAndroidSelectorConfidence('content_desc') }
+  if (text) return { value: text, confidence: buildAndroidSelectorConfidence('text') }
+  if (className) return { value: className, confidence: buildAndroidSelectorConfidence('class') }
+  return null
+}
+
+function buildAndroidSemantic(clickable: boolean, className: string): UIElementSemanticMetadata {
+  return {
+    is_clickable: clickable,
+    is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
+  }
+}
+
 function isSliderLikeAndroid(node: any): boolean {
   const className = String(node['@_class'] || '').toLowerCase()
   return /seekbar|slider|range|progress/i.test(className)
@@ -401,29 +447,41 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
 
   let currentIndex = -1;
 
-    if (node['@_class']) {
-      const text = node['@_text'] || null;
-      const contentDescription = node['@_content-desc'] || null;
-      const clickable = node['@_clickable'] === 'true';
-      const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
-      const state = extractAndroidState(node);
-
-      const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
-
-      if (isUseful) {
-        const element: UIElement = {
+  if (node['@_class']) {
+    const text = node['@_text'] || null;
+    const contentDescription = node['@_content-desc'] || null;
+    const clickable = node['@_clickable'] === 'true';
+    const className = String(node['@_class'] || 'unknown');
+    const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
+    const state = extractAndroidState(node);
+    const role = inferAndroidRole(normalizeClassName(className));
+    const resourceId = typeof node['@_resource-id'] === 'string' && node['@_resource-id'].trim().length > 0 ? node['@_resource-id'] : null
+    const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null)
+    const testTag = stableId
+    const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className))
+    const semantic = buildAndroidSemantic(clickable, normalizeClassName(className))
+
+    const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
+
+    if (isUseful) {
+      const element: UIElement = {
         text,
         contentDescription,
-        type: node['@_class'] || 'unknown',
-        resourceId: node['@_resource-id'] || null,
+        type: className,
+        resourceId,
         clickable,
         enabled: node['@_enabled'] === 'true',
-          visible: true,
-          bounds,
-          center: getCenter(bounds),
-          depth,
-          state
-        };
+        visible: true,
+        bounds,
+        center: getCenter(bounds),
+        depth,
+        state,
+        stable_id: stableId,
+        role,
+        test_tag: testTag,
+        selector,
+        semantic
+      };
 
       if (parentIndex !== -1) {
         element.parentId = parentIndex;

package/test/unit/interact/wait_for_ui_change.test.ts

@@ -0,0 +1,76 @@
+import assert from 'assert'
+import { ToolsInteract } from '../../../src/interact/index.js'
+import { ToolsObserve } from '../../../src/observe/index.js'
+
+async function run() {
+  const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
+
+  try {
+    let calls = 0
+    ;(ToolsObserve as any).getUITreeHandler = async () => {
+      calls++
+      if (calls === 1) {
+        return {
+          device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
+          screen: 'Loading',
+          resolution: { width: 1080, height: 2400 },
+          elements: [{ text: 'Loading', type: 'TextView', bounds: [0, 0, 100, 40], visible: true }],
+          snapshot_revision: 1,
+          captured_at_ms: 1000
+        }
+      }
+
+      return {
+        device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
+        screen: 'Loaded',
+        resolution: { width: 1080, height: 2400 },
+        elements: [{ text: 'Loaded', type: 'TextView', bounds: [0, 0, 100, 40], visible: true }],
+        snapshot_revision: 2,
+        captured_at_ms: 2000
+      }
+    }
+
+    const success = await ToolsInteract.waitForUIChangeHandler({
+      platform: 'android',
+      deviceId: 'mock',
+      expected_change: 'text_change',
+      timeout_ms: 1500,
+      stability_window_ms: 1
+    })
+
+    assert.strictEqual(success.success, true)
+    assert.strictEqual(success.observed_change, 'text_change')
+    assert.strictEqual(success.snapshot_revision, 2)
+    assert.strictEqual(success.timeout, false)
+
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
+      screen: 'Static',
+      resolution: { width: 1080, height: 2400 },
+      elements: [{ text: 'Static', type: 'TextView', bounds: [0, 0, 100, 40], visible: true }],
+      snapshot_revision: 9,
+      captured_at_ms: 3000
+    })
+
+    const timeout = await ToolsInteract.waitForUIChangeHandler({
+      platform: 'android',
+      deviceId: 'mock',
+      expected_change: 'state_change',
+      timeout_ms: 700,
+      stability_window_ms: 1
+    })
+
+    assert.strictEqual(timeout.success, false)
+    assert.strictEqual(timeout.observed_change, null)
+    assert.strictEqual(timeout.timeout, true)
+
+    console.log('wait_for_ui_change tests passed')
+  } finally {
+    ;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler
+  }
+}
+
+run().catch((error) => {
+  console.error(error)
+  process.exit(1)
+})

package/test/unit/observe/state_extraction.test.ts

@@ -8,6 +8,7 @@ async function run() {
     '@_class': 'android.widget.SeekBar',
     '@_text': '',
     '@_content-desc': 'Duration',
+    '@_resource-id': 'com.example:id/duration',
     '@_clickable': 'true',
     '@_enabled': 'true',
     '@_selected': 'true',
@@ -21,18 +22,64 @@ async function run() {
   assert.strictEqual(androidElements[0].state?.raw_value, 7)
   assert.strictEqual(androidElements[0].state?.value, 50)
   assert.deepStrictEqual(androidElements[0].state?.value_range, { min: 0, max: 14 })
+  assert.strictEqual(androidElements[0].stable_id, 'com.example:id/duration')
+  assert.strictEqual(androidElements[0].role, 'slider')
+  assert.strictEqual(androidElements[0].test_tag, 'com.example:id/duration')
+  assert.deepStrictEqual(androidElements[0].selector, {
+    value: 'com.example:id/duration',
+    confidence: { score: 1, reason: 'resource_id' }
+  })
+  assert.deepStrictEqual(androidElements[0].semantic, { is_clickable: true, is_container: false })
+
+  const androidFallbackElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.Button',
+    '@_text': '',
+    '@_content-desc': 'Save',
+    '@_clickable': 'true',
+    '@_enabled': 'true',
+    '@_bounds': '[0,0][100,50]'
+  }, androidFallbackElements)
+
+  assert.strictEqual(androidFallbackElements.length, 1)
+  assert.strictEqual(androidFallbackElements[0].resourceId, null)
+  assert.strictEqual(androidFallbackElements[0].stable_id, 'Save')
+  assert.deepStrictEqual(androidFallbackElements[0].selector, {
+    value: 'Save',
+    confidence: { score: 0.9, reason: 'content_description' }
+  })
 
   const iosElements: any[] = []
   traverseIDBNode({
     AXElementType: 'Slider',
     AXLabel: 'Playback speed',
     AXValue: '0.75',
+    AXUniqueId: 'playback_speed_slider',
     AXTraits: ['UIAccessibilityTraitAdjustable']
   }, iosElements)
 
   assert.strictEqual(iosElements.length, 1)
   assert.strictEqual(iosElements[0].state?.value, 75)
   assert.strictEqual(iosElements[0].state?.raw_value, 0.75)
+  assert.strictEqual(iosElements[0].stable_id, 'playback_speed_slider')
+  assert.strictEqual(iosElements[0].role, 'slider')
+  assert.strictEqual(iosElements[0].test_tag, 'playback_speed_slider')
+  assert.deepStrictEqual(iosElements[0].selector, {
+    value: 'playback_speed_slider',
+    confidence: { score: 1, reason: 'accessibility_identifier' }
+  })
+  assert.deepStrictEqual(iosElements[0].semantic, { is_clickable: true, is_container: false })
+
+  const iosFallbackElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Button',
+    AXLabel: 'Save',
+    AXTraits: ['UIAccessibilityTraitButton'],
+    AXUniqueId: 'fallback_unique_id'
+  }, iosFallbackElements)
+
+  assert.strictEqual(iosFallbackElements.length, 1)
+  assert.strictEqual(iosFallbackElements[0].stable_id, 'fallback_unique_id')
 
   console.log('state extraction tests passed')
 }

package/test/unit/server/response_shapes.test.ts

@@ -8,6 +8,7 @@ import { ToolsObserve } from '../../../src/observe/index.js'
 async function run() {
   const originalInstallAppHandler = (ToolsManage as any).installAppHandler
   const originalWaitForUIHandler = (ToolsInteract as any).waitForUIHandler
+  const originalWaitForUIChangeHandler = (ToolsInteract as any).waitForUIChangeHandler
   const originalTapElementHandler = (ToolsInteract as any).tapElementHandler
   const originalTapHandler = (ToolsInteract as any).tapHandler
   const originalExpectScreenHandler = (ToolsInteract as any).expectScreenHandler
@@ -145,12 +146,16 @@ async function run() {
     ;(ToolsObserve as any).getUITreeHandler = async () => ({
       device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
       resolution: { width: 1080, height: 2400 },
+      screen: 'Notifications',
       elements: [{
         text: 'Notifications',
         depth: 0,
         center: { x: 50, y: 20 },
         state: { checked: true, selected: 'Notifications' }
-      }]
+      }],
+      snapshot_revision: 12,
+      captured_at_ms: 1710000000123,
+      loading_state: { active: true, signal: 'progress_indicator', source: 'ui_tree' }
     })
 
     ;(ToolsInteract as any).expectStateHandler = async () => ({
@@ -227,8 +232,12 @@ async function run() {
 
     ;(ToolsObserve as any).getUITreeHandler = async () => ({
       device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
+      screen: 'Login',
       resolution: { width: 1080, height: 2400 },
-      elements: [{ text: 'Login', depth: 0, center: { x: 50, y: 20 } }]
+      elements: [{ text: 'Login', depth: 0, center: { x: 50, y: 20 } }],
+      snapshot_revision: 12,
+      captured_at_ms: 1710000000123,
+      loading_state: { active: true, signal: 'progress_indicator', source: 'ui_tree' }
     })
 
     const uiTreeResponse = await handleToolCall('get_ui_tree', { platform: 'android' })
@@ -236,16 +245,21 @@ async function run() {
     assert.strictEqual(uiTreePayload.elements.length, 1)
     assert.strictEqual(uiTreePayload.resolution.height, 2400)
     assert.strictEqual(uiTreePayload.elements[0].text, 'Login')
+    assert.strictEqual(uiTreePayload.snapshot_revision, 12)
+    assert.strictEqual(uiTreePayload.loading_state.signal, 'progress_indicator')
 
     ;(ToolsObserve as any).captureDebugSnapshotHandler = async () => ({
       raw: {
         timestamp: 1710000000000,
+        snapshot_revision: 12,
+        captured_at_ms: 1710000000123,
         reason: 'manual',
         activity: 'com.example.MainActivity',
         fingerprint: 'fp_raw',
         screenshot: 'base64',
-        ui_tree: { screen: 'Home', elements: [] },
+        ui_tree: { screen: 'Home', elements: [], snapshot_revision: 12, captured_at_ms: 1710000000123, loading_state: { active: true, signal: 'spinner', source: 'snapshot' } },
         logs: [],
+        loading_state: { active: true, signal: 'spinner', source: 'snapshot' },
         device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true }
       },
       semantic: {
@@ -260,13 +274,33 @@ async function run() {
     const snapshotResponse = await handleToolCall('capture_debug_snapshot', { platform: 'android' })
     const snapshotPayload = JSON.parse((snapshotResponse as any).content[0].text)
     assert.strictEqual(snapshotPayload.raw.fingerprint, 'fp_raw')
+    assert.strictEqual(snapshotPayload.raw.snapshot_revision, 12)
+    assert.strictEqual(snapshotPayload.raw.loading_state.signal, 'spinner')
     assert.strictEqual(snapshotPayload.semantic.screen, 'Home')
     assert.strictEqual(snapshotPayload.semantic.confidence, 0.8)
 
+    ;(ToolsInteract as any).waitForUIChangeHandler = async () => ({
+      success: true,
+      observed_change: 'text_change',
+      snapshot_revision: 13,
+      timeout: false,
+      elapsed_ms: 1550,
+      expected_change: 'text_change',
+      loading_state: { active: false, signal: 'spinner', source: 'ui_tree' },
+      reason: 'UI change observed'
+    })
+
+    const waitForUIChangeResponse = await handleToolCall('wait_for_ui_change', { expected_change: 'text_change' })
+    const waitForUIChangePayload = JSON.parse((waitForUIChangeResponse as any).content[0].text)
+    assert.strictEqual(waitForUIChangePayload.success, true)
+    assert.strictEqual(waitForUIChangePayload.observed_change, 'text_change')
+    assert.strictEqual(waitForUIChangePayload.snapshot_revision, 13)
+
     console.log('server response-shape tests passed')
   } finally {
     ;(ToolsManage as any).installAppHandler = originalInstallAppHandler
     ;(ToolsInteract as any).waitForUIHandler = originalWaitForUIHandler
+    ;(ToolsInteract as any).waitForUIChangeHandler = originalWaitForUIChangeHandler
     ;(ToolsInteract as any).tapElementHandler = originalTapElementHandler
     ;(ToolsInteract as any).tapHandler = originalTapHandler
     ;(ToolsInteract as any).expectScreenHandler = originalExpectScreenHandler