mobile-debug-mcp 0.24.7 → 0.25.0

package/src/types.ts

@@ -79,6 +79,21 @@ export interface GetCrashResponse {
   crashes: string[];
 }
 
+export interface UIElementState {
+  checked?: boolean | null;
+  selected?: boolean | string | { id: string; label?: string } | null;
+  focused?: boolean | null;
+  expanded?: boolean | null;
+  enabled?: boolean | null;
+  text_value?: string | null;
+  value?: number | string | null;
+  raw_value?: number | string | null;
+  value_range?: {
+    min: number;
+    max: number;
+  } | null;
+}
+
 export interface CaptureAndroidScreenResponse {
   device: DeviceInfo;
   screenshot: string; // base64 encoded string
@@ -116,6 +131,7 @@ export interface UIElement {
   children?: number[];
   center?: [number, number];
   depth?: number;
+  state?: UIElementState | null;
 }
 
 export interface GetUITreeResponse {
@@ -198,6 +214,7 @@ export interface ActionTargetResolved {
   class: string | null;
   bounds: [number, number, number, number] | null;
   index: number | null;
+  state?: UIElementState | null;
 }
 
 export interface ActionExecutionResult {
@@ -260,6 +277,30 @@ export interface ExpectElementVisibleResponse {
   retryable?: boolean;
 }
 
+export interface ExpectStateResponse {
+  success: boolean;
+  selector?: {
+    text?: string;
+    resource_id?: string;
+    accessibility_id?: string;
+    contains?: boolean;
+  };
+  element_id: string | null;
+  expected_state: {
+    property: string;
+    expected: boolean | number | string | Record<string, unknown>;
+  };
+  element?: (ActionTargetResolved & { state?: UIElementState | null }) | null;
+  observed_state?: {
+    property: string;
+    value: boolean | number | string | Record<string, unknown> | null;
+    raw_value?: boolean | number | string | null;
+  };
+  reason?: string;
+  failure_code?: 'ELEMENT_NOT_FOUND' | 'UNKNOWN';
+  retryable?: boolean;
+}
+
 export interface SwipeResponse {
   device: DeviceInfo;
   success: boolean;

package/src/utils/android/utils.ts

@@ -1,4 +1,4 @@
-import { DeviceInfo, UIElement } from "../../types.js"
+import { DeviceInfo, UIElement, UIElementState } from "../../types.js"
 import { promises as fsPromises, existsSync } from 'fs'
 import path from 'path'
 import { detectJavaHome } from '../java.js'
@@ -323,6 +323,68 @@ export function getCenter(bounds: [number, number, number, number]): [number, nu
   return [Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)];
 }
 
+function parseBooleanAttr(value: unknown): boolean | null {
+  if (value === true || value === 'true') return true
+  if (value === false || value === 'false') return false
+  return null
+}
+
+function parseNumberAttr(value: unknown): number | null {
+  if (typeof value === 'number' && Number.isFinite(value)) return value
+  if (typeof value !== 'string') return null
+  const parsed = Number(value)
+  return Number.isFinite(parsed) ? parsed : null
+}
+
+function isSliderLikeAndroid(node: any): boolean {
+  const className = String(node['@_class'] || '').toLowerCase()
+  return /seekbar|slider|range|progress/i.test(className)
+}
+
+function extractAndroidState(node: any): UIElementState | null {
+  const checked = parseBooleanAttr(node['@_checked'])
+  const selectedFlag = parseBooleanAttr(node['@_selected'])
+  const focused = parseBooleanAttr(node['@_focused'])
+  const expanded = parseBooleanAttr(node['@_expanded'])
+  const enabled = parseBooleanAttr(node['@_enabled'])
+  const textValue = typeof node['@_text'] === 'string' && node['@_text'].trim().length > 0 ? node['@_text'] : null
+  const state: UIElementState = {}
+
+  if (checked !== null) state.checked = checked
+  if (selectedFlag !== null) {
+    state.selected = textValue || node['@_content-desc'] || true
+  }
+  if (focused !== null) state.focused = focused
+  if (expanded !== null) state.expanded = expanded
+  if (enabled !== null) state.enabled = enabled
+
+  if (textValue && /edittext|textfield|search/i.test(String(node['@_class'] || ''))) {
+    state.text_value = textValue
+  }
+
+  if (isSliderLikeAndroid(node)) {
+    const rawProgress = parseNumberAttr(node['@_progress'])
+    const max = parseNumberAttr(node['@_max'])
+    const fallbackValue = rawProgress ?? parseNumberAttr(node['@_value']) ?? parseNumberAttr(node['@_content-desc'])
+    const numericValue = rawProgress ?? fallbackValue
+    if (numericValue !== null) {
+      state.raw_value = numericValue
+      state.value_range = max !== null && max > 0 ? { min: 0, max } : null
+      state.value = max !== null && max > 0 ? Math.round((numericValue / max) * 100) : numericValue
+    }
+  } else {
+    const numericValue = parseNumberAttr(node['@_value'])
+    if (numericValue !== null) {
+      state.value = numericValue
+      state.raw_value = numericValue
+    } else if (textValue) {
+      state.value = textValue
+    }
+  }
+
+  return Object.keys(state).length > 0 ? state : null
+}
+
 export async function getScreenResolution(deviceId?: string): Promise<{ width: number; height: number }> {
   try {
     const output = await execAdb(['shell', 'wm', 'size'], deviceId);
@@ -339,27 +401,29 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
 
   let currentIndex = -1;
 
-  if (node['@_class']) {
-    const text = node['@_text'] || null;
-    const contentDescription = node['@_content-desc'] || null;
-    const clickable = node['@_clickable'] === 'true';
-    const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
+    if (node['@_class']) {
+      const text = node['@_text'] || null;
+      const contentDescription = node['@_content-desc'] || null;
+      const clickable = node['@_clickable'] === 'true';
+      const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
+      const state = extractAndroidState(node);
 
-    const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
+      const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
 
-    if (isUseful) {
-      const element: UIElement = {
+      if (isUseful) {
+        const element: UIElement = {
         text,
         contentDescription,
         type: node['@_class'] || 'unknown',
         resourceId: node['@_resource-id'] || null,
         clickable,
         enabled: node['@_enabled'] === 'true',
-        visible: true,
-        bounds,
-        center: getCenter(bounds),
-        depth
-      };
+          visible: true,
+          bounds,
+          center: getCenter(bounds),
+          depth,
+          state
+        };
 
       if (parentIndex !== -1) {
         element.parentId = parentIndex;

package/test/unit/observe/find_element.test.ts

@@ -75,14 +75,73 @@ async function run() {
     assert.ok(pass4, 'Child text should resolve to a clickable parent ancestor')
     process.stdout.write('Test 4: ' + (pass4 ? 'PASS' : 'FAIL') + '\n');
 
-    // Test 5: not found
-    ;(ToolsObserve as any).getUITreeHandler = async () => ({ device: { platform: 'android', id: 'mock' }, screen: '', resolution: { width: 1080, height: 1920 }, elements: [] })
-    const res5: any = await ToolsInteract.findElementHandler({ query: 'nope', exact: false, platform: 'android', timeoutMs: 300 })
+    // Test 5: duration label should resolve to the nearby slider control
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 1920 },
+      elements: [
+        { text: 'Duration: 5 min', type: 'android.widget.TextView', contentDescription: null, clickable: false, enabled: true, visible: true, bounds: [10,10,260,50], resourceId: null },
+        { text: null, type: 'android.view.View', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,60,1040,140], resourceId: null }
+      ]
+    })
+
+    const res5: any = await ToolsInteract.findElementHandler({ query: 'duration', exact: false, platform: 'android', timeoutMs: 300 })
     process.stdout.write('res5 ' + JSON.stringify(res5, null, 2) + '\n');
-    const pass5 = res5.found === false
-    assert.ok(pass5, 'Missing elements should return found=false')
+    const pass5 = res5.found === true && res5.element && res5.element.clickable === true && res5.element.bounds && res5.element.bounds.top === 60 && res5.element.bounds.bottom === 140
+    assert.ok(pass5, 'Duration label should resolve to the slider control below it')
     process.stdout.write('Test 5: ' + (pass5 ? 'PASS' : 'FAIL') + '\n');
 
+    // Test 6: prefer track-like control over a closer texty sibling
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 1920 },
+      elements: [
+        { text: 'Duration: 5 min', type: 'android.widget.TextView', contentDescription: null, clickable: false, enabled: true, visible: true, bounds: [10,10,260,50], resourceId: null },
+        { text: 'Reset', type: 'android.widget.Button', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,60,150,120], resourceId: 'btn_reset' },
+        { text: null, type: 'android.view.View', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,130,1040,210], resourceId: null }
+      ]
+    })
+
+    const res6: any = await ToolsInteract.findElementHandler({ query: 'duration', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res6 ' + JSON.stringify(res6, null, 2) + '\n');
+    const pass6 = res6.found === true && res6.element && res6.element.clickable === true && res6.element.bounds && res6.element.bounds.top === 130 && res6.element.bounds.bottom === 210
+    assert.ok(pass6, 'Duration lookup should prefer the track-like control over a closer text button')
+    process.stdout.write('Test 6: ' + (pass6 ? 'PASS' : 'FAIL') + '\n');
+    const pass6b = res6.element && res6.element.telemetry && res6.element.telemetry.sliderLike === true && res6.element.interactionHint && res6.element.interactionHint.kind === 'slider'
+    assert.ok(pass6b, 'Duration lookup should include slider-specific telemetry')
+    process.stdout.write('Test 6b: ' + (pass6b ? 'PASS' : 'FAIL') + '\n');
+
+    // Test 7: prefer vertical track-like control over a closer text button
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 2400 },
+      elements: [
+        { text: 'Duration: 5 min', type: 'android.widget.TextView', contentDescription: null, clickable: false, enabled: true, visible: true, bounds: [10,10,260,50], resourceId: null },
+        { text: 'Reset', type: 'android.widget.Button', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,60,150,120], resourceId: 'btn_reset' },
+        { text: null, type: 'android.view.View', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [270,20,350,1040], resourceId: null }
+      ]
+    })
+
+    const res7: any = await ToolsInteract.findElementHandler({ query: 'duration', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res7 ' + JSON.stringify(res7, null, 2) + '\n');
+    const pass7 = res7.found === true && res7.element && res7.element.clickable === true && res7.element.bounds && res7.element.bounds.left === 270 && res7.element.bounds.right === 350
+    assert.ok(pass7, 'Duration lookup should prefer a vertical track-like control')
+    process.stdout.write('Test 7: ' + (pass7 ? 'PASS' : 'FAIL') + '\n');
+    const pass7b = res7.element && res7.element.interactionHint && res7.element.interactionHint.axis === 'vertical'
+    assert.ok(pass7b, 'Vertical sliders should report a vertical interaction axis')
+    process.stdout.write('Test 7b: ' + (pass7b ? 'PASS' : 'FAIL') + '\n');
+
+    // Test 8: not found
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({ device: { platform: 'android', id: 'mock' }, screen: '', resolution: { width: 1080, height: 1920 }, elements: [] })
+    const res8: any = await ToolsInteract.findElementHandler({ query: 'nope', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res8 ' + JSON.stringify(res8, null, 2) + '\n');
+    const pass8 = res8.found === false
+    assert.ok(pass8, 'Missing elements should return found=false')
+    process.stdout.write('Test 8: ' + (pass8 ? 'PASS' : 'FAIL') + '\n');
+
   } finally {
     ;(ToolsObserve as any).getUITreeHandler = origGetTree
   }

package/test/unit/observe/state_extraction.test.ts

@@ -0,0 +1,43 @@
+import assert from 'assert'
+import { traverseNode } from '../../../src/utils/android/utils.js'
+import { traverseIDBNode } from '../../../src/observe/ios.js'
+
+async function run() {
+  const androidElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.SeekBar',
+    '@_text': '',
+    '@_content-desc': 'Duration',
+    '@_clickable': 'true',
+    '@_enabled': 'true',
+    '@_selected': 'true',
+    '@_progress': '7',
+    '@_max': '14',
+    '@_bounds': '[0,0][200,40]'
+  }, androidElements)
+
+  assert.strictEqual(androidElements.length, 1)
+  assert.deepStrictEqual(androidElements[0].state?.selected, 'Duration')
+  assert.strictEqual(androidElements[0].state?.raw_value, 7)
+  assert.strictEqual(androidElements[0].state?.value, 50)
+  assert.deepStrictEqual(androidElements[0].state?.value_range, { min: 0, max: 14 })
+
+  const iosElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Slider',
+    AXLabel: 'Playback speed',
+    AXValue: '0.75',
+    AXTraits: ['UIAccessibilityTraitAdjustable']
+  }, iosElements)
+
+  assert.strictEqual(iosElements.length, 1)
+  assert.strictEqual(iosElements[0].state?.value, 75)
+  assert.strictEqual(iosElements[0].state?.raw_value, 0.75)
+
+  console.log('state extraction tests passed')
+}
+
+run().catch((error) => {
+  console.error(error)
+  process.exit(1)
+})

package/test/unit/server/response_shapes.test.ts

@@ -12,6 +12,7 @@ async function run() {
   const originalTapHandler = (ToolsInteract as any).tapHandler
   const originalExpectScreenHandler = (ToolsInteract as any).expectScreenHandler
   const originalExpectElementVisibleHandler = (ToolsInteract as any).expectElementVisibleHandler
+  const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
   const originalStartApp = AndroidManage.prototype.startApp
   const originalCaptureScreenshotHandler = (ToolsObserve as any).captureScreenshotHandler
   const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
@@ -130,8 +131,8 @@ async function run() {
       selector: { text: 'Ready' },
       element_id: 'el_ready',
       expected_condition: 'visible',
-      element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0 },
-      observed: { status: 'success', matched_count: 1, condition_satisfied: true, selected_index: 0, last_matched_element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0 } },
+      element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0, state: { enabled: true } },
+      observed: { status: 'success', matched_count: 1, condition_satisfied: true, selected_index: 0, last_matched_element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0, state: { enabled: true } } },
       reason: 'selector is visible'
     })
 
@@ -141,6 +142,42 @@ async function run() {
     assert.strictEqual(expectElementPayload.element_id, 'el_ready')
     assert.strictEqual(expectElementPayload.expected_condition, 'visible')
 
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
+      resolution: { width: 1080, height: 2400 },
+      elements: [{
+        text: 'Notifications',
+        depth: 0,
+        center: { x: 50, y: 20 },
+        state: { checked: true, selected: 'Notifications' }
+      }]
+    })
+
+    ;(ToolsInteract as any).expectStateHandler = async () => ({
+      success: true,
+      selector: { text: 'Notifications' },
+      element_id: 'el_notifications',
+      expected_state: { property: 'checked', expected: true },
+      element: {
+        elementId: 'el_notifications',
+        text: 'Notifications',
+        resource_id: null,
+        accessibility_id: null,
+        class: 'Switch',
+        bounds: [0, 0, 10, 10],
+        index: 0,
+        state: { checked: true, selected: 'Notifications' }
+      },
+      observed_state: { property: 'checked', value: true, raw_value: true },
+      reason: 'checked matches expected value'
+    })
+
+    const expectStateResponse = await handleToolCall('expect_state', { selector: { text: 'Notifications' }, property: 'checked', expected: true })
+    const expectStatePayload = JSON.parse((expectStateResponse as any).content[0].text)
+    assert.strictEqual(expectStatePayload.success, true)
+    assert.strictEqual(expectStatePayload.expected_state.property, 'checked')
+    assert.strictEqual(expectStatePayload.observed_state.value, true)
+
     ;(ToolsInteract as any).tapHandler = async () => {
       throw new Error('boom')
     }
@@ -234,6 +271,7 @@ async function run() {
     ;(ToolsInteract as any).tapHandler = originalTapHandler
     ;(ToolsInteract as any).expectScreenHandler = originalExpectScreenHandler
     ;(ToolsInteract as any).expectElementVisibleHandler = originalExpectElementVisibleHandler
+    ;(ToolsInteract as any).expectStateHandler = originalExpectStateHandler
     AndroidManage.prototype.startApp = originalStartApp
     ;(ToolsObserve as any).captureScreenshotHandler = originalCaptureScreenshotHandler
     ;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler