mobile-debug-mcp 0.24.8 → 0.25.1

package/src/types.ts

@@ -79,6 +79,36 @@ export interface GetCrashResponse {
   crashes: string[];
 }
 
+export interface UIElementState {
+  checked?: boolean | null;
+  selected?: boolean | string | { id: string; label?: string } | null;
+  focused?: boolean | null;
+  expanded?: boolean | null;
+  enabled?: boolean | null;
+  text_value?: string | null;
+  value?: number | string | null;
+  raw_value?: number | string | null;
+  value_range?: {
+    min: number;
+    max: number;
+  } | null;
+}
+
+export interface SelectorConfidence {
+  score: number;
+  reason: string;
+}
+
+export interface UIResolutionSelector {
+  value: string | null;
+  confidence: SelectorConfidence | null;
+}
+
+export interface UIElementSemanticMetadata {
+  is_clickable: boolean;
+  is_container: boolean;
+}
+
 export interface CaptureAndroidScreenResponse {
   device: DeviceInfo;
   screenshot: string; // base64 encoded string
@@ -116,6 +146,12 @@ export interface UIElement {
   children?: number[];
   center?: [number, number];
   depth?: number;
+  state?: UIElementState | null;
+  stable_id?: string | null;
+  role?: string | null;
+  test_tag?: string | null;
+  selector?: UIResolutionSelector | null;
+  semantic?: UIElementSemanticMetadata | null;
 }
 
 export interface GetUITreeResponse {
@@ -151,7 +187,7 @@ export interface CaptureDebugSnapshotRawResponse {
   activity: string | null;
   fingerprint: string | null;
   screenshot: string | null;
-  ui_tree: unknown | null;
+  ui_tree: GetUITreeResponse | null;
   logs: StructuredLogEntry[];
   device?: DeviceInfo;
   screenshot_error?: string;
@@ -198,6 +234,12 @@ export interface ActionTargetResolved {
   class: string | null;
   bounds: [number, number, number, number] | null;
   index: number | null;
+  state?: UIElementState | null;
+  stable_id?: string | null;
+  role?: string | null;
+  test_tag?: string | null;
+  selector?: UIResolutionSelector | null;
+  semantic?: UIElementSemanticMetadata | null;
 }
 
 export interface ActionExecutionResult {
@@ -260,6 +302,30 @@ export interface ExpectElementVisibleResponse {
   retryable?: boolean;
 }
 
+export interface ExpectStateResponse {
+  success: boolean;
+  selector?: {
+    text?: string;
+    resource_id?: string;
+    accessibility_id?: string;
+    contains?: boolean;
+  };
+  element_id: string | null;
+  expected_state: {
+    property: string;
+    expected: boolean | number | string | Record<string, unknown>;
+  };
+  element?: (ActionTargetResolved & { state?: UIElementState | null }) | null;
+  observed_state?: {
+    property: string;
+    value: boolean | number | string | Record<string, unknown> | null;
+    raw_value?: boolean | number | string | null;
+  };
+  reason?: string;
+  failure_code?: 'ELEMENT_NOT_FOUND' | 'UNKNOWN';
+  retryable?: boolean;
+}
+
 export interface SwipeResponse {
   device: DeviceInfo;
   success: boolean;

package/src/utils/android/utils.ts

@@ -1,4 +1,4 @@
-import { DeviceInfo, UIElement } from "../../types.js"
+import { DeviceInfo, UIElement, UIElementSemanticMetadata, UIElementState, UIResolutionSelector, SelectorConfidence } from "../../types.js"
 import { promises as fsPromises, existsSync } from 'fs'
 import path from 'path'
 import { detectJavaHome } from '../java.js'
@@ -323,6 +323,114 @@ export function getCenter(bounds: [number, number, number, number]): [number, nu
   return [Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)];
 }
 
+function parseBooleanAttr(value: unknown): boolean | null {
+  if (value === true || value === 'true') return true
+  if (value === false || value === 'false') return false
+  return null
+}
+
+function parseNumberAttr(value: unknown): number | null {
+  if (typeof value === 'number' && Number.isFinite(value)) return value
+  if (typeof value !== 'string') return null
+  const parsed = Number(value)
+  return Number.isFinite(parsed) ? parsed : null
+}
+
+function normalizeClassName(value: unknown): string {
+  return typeof value === 'string' ? value.trim().toLowerCase() : ''
+}
+
+function inferAndroidRole(className: string): string | null {
+  if (/seekbar|slider|progress/.test(className)) return 'slider'
+  if (/switch|toggle/.test(className)) return 'switch'
+  if (/checkbox/.test(className)) return 'checkbox'
+  if (/radiobutton|radio/.test(className)) return 'radio'
+  if (/edittext|textfield|search/.test(className)) return 'text_field'
+  if (/button|fab/.test(className)) return 'button'
+  if (/imageview|icon/.test(className)) return 'image'
+  if (/recyclerview|scroll|layout|viewgroup|frame/.test(className)) return 'container'
+  return null
+}
+
+function buildAndroidSelectorConfidence(source: 'resource_id' | 'content_desc' | 'text' | 'class' | 'none'): SelectorConfidence | null {
+  switch (source) {
+    case 'resource_id':
+      return { score: 1, reason: 'resource_id' }
+    case 'content_desc':
+      return { score: 0.9, reason: 'content_description' }
+    case 'text':
+      return { score: 0.6, reason: 'text_match' }
+    case 'class':
+      return { score: 0.35, reason: 'class_match' }
+    default:
+      return null
+  }
+}
+
+function buildAndroidSelector(text: string | null, contentDescription: string | null, resourceId: string | null, className: string): UIResolutionSelector | null {
+  if (resourceId) return { value: resourceId, confidence: buildAndroidSelectorConfidence('resource_id') }
+  if (contentDescription) return { value: contentDescription, confidence: buildAndroidSelectorConfidence('content_desc') }
+  if (text) return { value: text, confidence: buildAndroidSelectorConfidence('text') }
+  if (className) return { value: className, confidence: buildAndroidSelectorConfidence('class') }
+  return null
+}
+
+function buildAndroidSemantic(clickable: boolean, className: string): UIElementSemanticMetadata {
+  return {
+    is_clickable: clickable,
+    is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
+  }
+}
+
+function isSliderLikeAndroid(node: any): boolean {
+  const className = String(node['@_class'] || '').toLowerCase()
+  return /seekbar|slider|range|progress/i.test(className)
+}
+
+function extractAndroidState(node: any): UIElementState | null {
+  const checked = parseBooleanAttr(node['@_checked'])
+  const selectedFlag = parseBooleanAttr(node['@_selected'])
+  const focused = parseBooleanAttr(node['@_focused'])
+  const expanded = parseBooleanAttr(node['@_expanded'])
+  const enabled = parseBooleanAttr(node['@_enabled'])
+  const textValue = typeof node['@_text'] === 'string' && node['@_text'].trim().length > 0 ? node['@_text'] : null
+  const state: UIElementState = {}
+
+  if (checked !== null) state.checked = checked
+  if (selectedFlag !== null) {
+    state.selected = textValue || node['@_content-desc'] || true
+  }
+  if (focused !== null) state.focused = focused
+  if (expanded !== null) state.expanded = expanded
+  if (enabled !== null) state.enabled = enabled
+
+  if (textValue && /edittext|textfield|search/i.test(String(node['@_class'] || ''))) {
+    state.text_value = textValue
+  }
+
+  if (isSliderLikeAndroid(node)) {
+    const rawProgress = parseNumberAttr(node['@_progress'])
+    const max = parseNumberAttr(node['@_max'])
+    const fallbackValue = rawProgress ?? parseNumberAttr(node['@_value']) ?? parseNumberAttr(node['@_content-desc'])
+    const numericValue = rawProgress ?? fallbackValue
+    if (numericValue !== null) {
+      state.raw_value = numericValue
+      state.value_range = max !== null && max > 0 ? { min: 0, max } : null
+      state.value = max !== null && max > 0 ? Math.round((numericValue / max) * 100) : numericValue
+    }
+  } else {
+    const numericValue = parseNumberAttr(node['@_value'])
+    if (numericValue !== null) {
+      state.value = numericValue
+      state.raw_value = numericValue
+    } else if (textValue) {
+      state.value = textValue
+    }
+  }
+
+  return Object.keys(state).length > 0 ? state : null
+}
+
 export async function getScreenResolution(deviceId?: string): Promise<{ width: number; height: number }> {
   try {
     const output = await execAdb(['shell', 'wm', 'size'], deviceId);
@@ -343,7 +451,15 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
     const text = node['@_text'] || null;
     const contentDescription = node['@_content-desc'] || null;
     const clickable = node['@_clickable'] === 'true';
+    const className = String(node['@_class'] || 'unknown');
     const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
+    const state = extractAndroidState(node);
+    const role = inferAndroidRole(normalizeClassName(className));
+    const resourceId = typeof node['@_resource-id'] === 'string' && node['@_resource-id'].trim().length > 0 ? node['@_resource-id'] : null
+    const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null)
+    const testTag = stableId
+    const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className))
+    const semantic = buildAndroidSemantic(clickable, normalizeClassName(className))
 
     const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
 
@@ -351,14 +467,20 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
       const element: UIElement = {
         text,
         contentDescription,
-        type: node['@_class'] || 'unknown',
-        resourceId: node['@_resource-id'] || null,
+        type: className,
+        resourceId,
         clickable,
         enabled: node['@_enabled'] === 'true',
         visible: true,
         bounds,
         center: getCenter(bounds),
-        depth
+        depth,
+        state,
+        stable_id: stableId,
+        role,
+        test_tag: testTag,
+        selector,
+        semantic
       };
 
       if (parentIndex !== -1) {

package/test/unit/observe/state_extraction.test.ts

@@ -0,0 +1,90 @@
+import assert from 'assert'
+import { traverseNode } from '../../../src/utils/android/utils.js'
+import { traverseIDBNode } from '../../../src/observe/ios.js'
+
+async function run() {
+  const androidElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.SeekBar',
+    '@_text': '',
+    '@_content-desc': 'Duration',
+    '@_resource-id': 'com.example:id/duration',
+    '@_clickable': 'true',
+    '@_enabled': 'true',
+    '@_selected': 'true',
+    '@_progress': '7',
+    '@_max': '14',
+    '@_bounds': '[0,0][200,40]'
+  }, androidElements)
+
+  assert.strictEqual(androidElements.length, 1)
+  assert.deepStrictEqual(androidElements[0].state?.selected, 'Duration')
+  assert.strictEqual(androidElements[0].state?.raw_value, 7)
+  assert.strictEqual(androidElements[0].state?.value, 50)
+  assert.deepStrictEqual(androidElements[0].state?.value_range, { min: 0, max: 14 })
+  assert.strictEqual(androidElements[0].stable_id, 'com.example:id/duration')
+  assert.strictEqual(androidElements[0].role, 'slider')
+  assert.strictEqual(androidElements[0].test_tag, 'com.example:id/duration')
+  assert.deepStrictEqual(androidElements[0].selector, {
+    value: 'com.example:id/duration',
+    confidence: { score: 1, reason: 'resource_id' }
+  })
+  assert.deepStrictEqual(androidElements[0].semantic, { is_clickable: true, is_container: false })
+
+  const androidFallbackElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.Button',
+    '@_text': '',
+    '@_content-desc': 'Save',
+    '@_clickable': 'true',
+    '@_enabled': 'true',
+    '@_bounds': '[0,0][100,50]'
+  }, androidFallbackElements)
+
+  assert.strictEqual(androidFallbackElements.length, 1)
+  assert.strictEqual(androidFallbackElements[0].resourceId, null)
+  assert.strictEqual(androidFallbackElements[0].stable_id, 'Save')
+  assert.deepStrictEqual(androidFallbackElements[0].selector, {
+    value: 'Save',
+    confidence: { score: 0.9, reason: 'content_description' }
+  })
+
+  const iosElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Slider',
+    AXLabel: 'Playback speed',
+    AXValue: '0.75',
+    AXUniqueId: 'playback_speed_slider',
+    AXTraits: ['UIAccessibilityTraitAdjustable']
+  }, iosElements)
+
+  assert.strictEqual(iosElements.length, 1)
+  assert.strictEqual(iosElements[0].state?.value, 75)
+  assert.strictEqual(iosElements[0].state?.raw_value, 0.75)
+  assert.strictEqual(iosElements[0].stable_id, 'playback_speed_slider')
+  assert.strictEqual(iosElements[0].role, 'slider')
+  assert.strictEqual(iosElements[0].test_tag, 'playback_speed_slider')
+  assert.deepStrictEqual(iosElements[0].selector, {
+    value: 'playback_speed_slider',
+    confidence: { score: 1, reason: 'accessibility_identifier' }
+  })
+  assert.deepStrictEqual(iosElements[0].semantic, { is_clickable: true, is_container: false })
+
+  const iosFallbackElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Button',
+    AXLabel: 'Save',
+    AXTraits: ['UIAccessibilityTraitButton'],
+    AXUniqueId: 'fallback_unique_id'
+  }, iosFallbackElements)
+
+  assert.strictEqual(iosFallbackElements.length, 1)
+  assert.strictEqual(iosFallbackElements[0].stable_id, 'fallback_unique_id')
+
+  console.log('state extraction tests passed')
+}
+
+run().catch((error) => {
+  console.error(error)
+  process.exit(1)
+})

package/test/unit/server/response_shapes.test.ts

@@ -12,6 +12,7 @@ async function run() {
   const originalTapHandler = (ToolsInteract as any).tapHandler
   const originalExpectScreenHandler = (ToolsInteract as any).expectScreenHandler
   const originalExpectElementVisibleHandler = (ToolsInteract as any).expectElementVisibleHandler
+  const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
   const originalStartApp = AndroidManage.prototype.startApp
   const originalCaptureScreenshotHandler = (ToolsObserve as any).captureScreenshotHandler
   const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
@@ -130,8 +131,8 @@ async function run() {
       selector: { text: 'Ready' },
       element_id: 'el_ready',
       expected_condition: 'visible',
-      element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0 },
-      observed: { status: 'success', matched_count: 1, condition_satisfied: true, selected_index: 0, last_matched_element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0 } },
+      element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0, state: { enabled: true } },
+      observed: { status: 'success', matched_count: 1, condition_satisfied: true, selected_index: 0, last_matched_element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0, state: { enabled: true } } },
       reason: 'selector is visible'
     })
 
@@ -141,6 +142,42 @@ async function run() {
     assert.strictEqual(expectElementPayload.element_id, 'el_ready')
     assert.strictEqual(expectElementPayload.expected_condition, 'visible')
 
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
+      resolution: { width: 1080, height: 2400 },
+      elements: [{
+        text: 'Notifications',
+        depth: 0,
+        center: { x: 50, y: 20 },
+        state: { checked: true, selected: 'Notifications' }
+      }]
+    })
+
+    ;(ToolsInteract as any).expectStateHandler = async () => ({
+      success: true,
+      selector: { text: 'Notifications' },
+      element_id: 'el_notifications',
+      expected_state: { property: 'checked', expected: true },
+      element: {
+        elementId: 'el_notifications',
+        text: 'Notifications',
+        resource_id: null,
+        accessibility_id: null,
+        class: 'Switch',
+        bounds: [0, 0, 10, 10],
+        index: 0,
+        state: { checked: true, selected: 'Notifications' }
+      },
+      observed_state: { property: 'checked', value: true, raw_value: true },
+      reason: 'checked matches expected value'
+    })
+
+    const expectStateResponse = await handleToolCall('expect_state', { selector: { text: 'Notifications' }, property: 'checked', expected: true })
+    const expectStatePayload = JSON.parse((expectStateResponse as any).content[0].text)
+    assert.strictEqual(expectStatePayload.success, true)
+    assert.strictEqual(expectStatePayload.expected_state.property, 'checked')
+    assert.strictEqual(expectStatePayload.observed_state.value, true)
+
     ;(ToolsInteract as any).tapHandler = async () => {
       throw new Error('boom')
     }
@@ -234,6 +271,7 @@ async function run() {
     ;(ToolsInteract as any).tapHandler = originalTapHandler
     ;(ToolsInteract as any).expectScreenHandler = originalExpectScreenHandler
     ;(ToolsInteract as any).expectElementVisibleHandler = originalExpectElementVisibleHandler
+    ;(ToolsInteract as any).expectStateHandler = originalExpectStateHandler
     AndroidManage.prototype.startApp = originalStartApp
     ;(ToolsObserve as any).captureScreenshotHandler = originalCaptureScreenshotHandler
     ;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler