mobile-debug-mcp 0.25.0 → 0.25.1

package/src/interact/index.ts

@@ -40,6 +40,11 @@ interface UiElement {
   _interactable?: boolean
   _sliderLike?: boolean
   state?: UIElementState | null
+  stable_id?: string | null
+  role?: string | null
+  test_tag?: string | null
+  selector?: { value: string | null, confidence: { score: number, reason: string } | null } | null
+  semantic?: { is_clickable: boolean, is_container: boolean } | null
 }
 
 interface ResolvedUiElementContext {
@@ -157,7 +162,13 @@ export class ToolsInteract {
       class: el.type ?? el.class ?? null,
       bounds,
       index,
-      elementId
+      elementId,
+      state: el.state ?? null,
+      stable_id: el.stable_id ?? null,
+      role: el.role ?? null,
+      test_tag: el.test_tag ?? null,
+      selector: el.selector ?? null,
+      semantic: el.semantic ?? null
     }
   }
 
@@ -197,7 +208,12 @@ export class ToolsInteract {
       class: element.type ?? element.class ?? null,
       bounds: ToolsInteract._normalizeBounds(element.bounds),
       index,
-      state: element.state ?? null
+      state: element.state ?? null,
+      stable_id: element.stable_id ?? null,
+      role: element.role ?? null,
+      test_tag: element.test_tag ?? null,
+      selector: element.selector ?? null,
+      semantic: element.semantic ?? null
     }
   }
 
@@ -621,6 +637,11 @@ export class ToolsInteract {
       bounds: boundsObj,
       clickable: !!best.clickable,
       enabled: !!best.enabled,
+      stable_id: best.stable_id ?? null,
+      role: best.role ?? null,
+      test_tag: best.test_tag ?? null,
+      selector: best.selector ?? null,
+      semantic: best.semantic ?? null,
       tapCoordinates,
       telemetry: {
         matchedIndex: best?._index ?? null,
@@ -1040,7 +1061,12 @@ export class ToolsInteract {
           class: result.element.class ?? null,
           bounds: result.element.bounds ?? null,
           index: typeof result.element.index === 'number' ? result.element.index : null,
-          state: (result.element as any).state ?? null
+          state: (result.element as any).state ?? null,
+          stable_id: (result.element as any).stable_id ?? null,
+          role: (result.element as any).role ?? null,
+          test_tag: (result.element as any).test_tag ?? null,
+          selector: (result.element as any).selector ?? null,
+          semantic: (result.element as any).semantic ?? null
         },
         observed: {
           status: result.status,
@@ -1055,7 +1081,12 @@ export class ToolsInteract {
             class: result.element.class ?? null,
             bounds: result.element.bounds ?? null,
             index: typeof result.element.index === 'number' ? result.element.index : null,
-            state: (result.element as any).state ?? null
+            state: (result.element as any).state ?? null,
+            stable_id: (result.element as any).stable_id ?? null,
+            role: (result.element as any).role ?? null,
+            test_tag: (result.element as any).test_tag ?? null,
+            selector: (result.element as any).selector ?? null,
+            semantic: (result.element as any).semantic ?? null
           }
         },
         reason: 'selector is visible'

package/src/observe/index.ts

@@ -21,6 +21,12 @@ interface SnapshotTreeElementLike {
   clickable?: boolean
   enabled?: boolean
   visible?: boolean
+  state?: unknown
+  stable_id?: string | null
+  role?: string | null
+  test_tag?: string | null
+  selector?: unknown
+  semantic?: unknown
 }
 
 interface SnapshotTreeLike {

package/src/observe/ios.ts

@@ -1,6 +1,6 @@
 import { spawn } from "child_process"
 import { promises as fs } from "fs"
-import { GetLogsResponse, CaptureIOSScreenshotResponse, GetUITreeResponse, UIElement, DeviceInfo, UIElementState } from "../types.js"
+import { GetLogsResponse, CaptureIOSScreenshotResponse, GetUITreeResponse, UIElement, DeviceInfo, UIElementSemanticMetadata, UIElementState, UIResolutionSelector, SelectorConfidence } from "../types.js"
 import { execCommand, getIOSDeviceMetadata, validateBundleId, getIdbCmd, getXcrunCmd, isIDBInstalled } from "../utils/ios/utils.js"
 import { createWriteStream, promises as fsPromises } from 'fs'
 import path from 'path'
@@ -22,6 +22,9 @@ export function _resetIOSExecCommandForTests() {
 interface IDBElement {
   AXFrame?: { x: number | string, y: number | string, width: number | string, height: number | string, w?: number | string, h?: number | string };
   frame?: { x: number | string, y: number | string, width: number | string, height: number | string, w?: number | string, h?: number | string };
+  AXIdentifier?: string;
+  accessibilityIdentifier?: string;
+  identifier?: string;
   AXUniqueId?: string;
   AXLabel?: string;
   AXValue?: string;
@@ -63,6 +66,59 @@ function parseIOSNumber(value: unknown): number | null {
   return Number.isFinite(parsed) ? parsed : null
 }
 
+function normalizeIOSType(value: unknown): string {
+  return typeof value === 'string' ? value.trim().toLowerCase() : ''
+}
+
+function inferIOSRole(type: string, traits: string[]): string | null {
+  if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait))) return 'slider'
+  if (/button/.test(type) || traits.some((trait) => /button/.test(trait))) return 'button'
+  if (/cell/.test(type)) return 'cell'
+  if (/switch/.test(type)) return 'switch'
+  if (/text field|textfield|search field/.test(type)) return 'text_field'
+  if (/image/.test(type)) return 'image'
+  if (/window|application|group|scroll view|collection view/.test(type)) return 'container'
+  return null
+}
+
+function getIOSStableId(node: IDBElement): string | null {
+  const candidates = [node.AXIdentifier, node.accessibilityIdentifier, node.identifier, node.AXUniqueId]
+  for (const candidate of candidates) {
+    if (typeof candidate === 'string' && candidate.trim().length > 0) return candidate
+  }
+  return null
+}
+
+function buildIOSSelectorConfidence(source: 'identifier' | 'label' | 'value' | 'type' | 'none'): SelectorConfidence | null {
+  switch (source) {
+    case 'identifier':
+      return { score: 1, reason: 'accessibility_identifier' }
+    case 'label':
+      return { score: 0.9, reason: 'label_match' }
+    case 'value':
+      return { score: 0.75, reason: 'value_match' }
+    case 'type':
+      return { score: 0.35, reason: 'type_match' }
+    default:
+      return null
+  }
+}
+
+function buildIOSSelector(type: string, label: string | null, value: string | null, stableId: string | null): UIResolutionSelector | null {
+  if (stableId) return { value: stableId, confidence: buildIOSSelectorConfidence('identifier') }
+  if (label) return { value: label, confidence: buildIOSSelectorConfidence('label') }
+  if (value) return { value: value, confidence: buildIOSSelectorConfidence('value') }
+  if (type) return { value: type, confidence: buildIOSSelectorConfidence('type') }
+  return null
+}
+
+function buildIOSSemantic(type: string, traits: string[]): UIElementSemanticMetadata {
+  return {
+    is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
+    is_container: /window|application|group|scroll view|collection view/.test(type)
+  }
+}
+
 function isIOSAdjustable(node: IDBElement, type: string, traits: string[]): boolean {
   return /slider|adjustable|stepper|progress/i.test(type) || traits.some((trait) => /adjustable|slider|progress/i.test(trait))
 }
@@ -124,6 +180,11 @@ export function traverseIDBNode(node: IDBElement, elements: UIElement[], parentI
   const frame = node.AXFrame || node.frame;
   const traits = node.AXTraits || [];
   const state = extractIOSState(node, type, label, value, traits);
+  const normalizedType = normalizeIOSType(type)
+  const stableId = getIOSStableId(node)
+  const selector = buildIOSSelector(type, label, value, stableId)
+  const semantic = buildIOSSemantic(normalizedType, traits)
+  const role = inferIOSRole(normalizedType, traits)
   
   const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
   
@@ -135,14 +196,19 @@ export function traverseIDBNode(node: IDBElement, elements: UIElement[], parentI
       text: label,
       contentDescription: value,
       type: type,
-      resourceId: node.AXUniqueId || null,
+      resourceId: stableId,
       clickable: clickable,
       enabled: true,
       visible: true,
       bounds: bounds,
       center: getCenter(bounds),
       depth: depth,
-      state
+      state,
+      stable_id: stableId,
+      role,
+      test_tag: stableId,
+      selector,
+      semantic
     };
 
     if (parentIndex !== -1) {

package/src/server-core.ts

@@ -13,7 +13,7 @@ export { wrapResponse, toolDefinitions, handleToolCall }
 
 export const serverInfo = {
   name: 'mobile-debug-mcp',
-  version: '0.25.0'
+  version: '0.25.1'
 }
 
 export function createServer() {

package/src/types.ts

@@ -94,6 +94,21 @@ export interface UIElementState {
   } | null;
 }
 
+export interface SelectorConfidence {
+  score: number;
+  reason: string;
+}
+
+export interface UIResolutionSelector {
+  value: string | null;
+  confidence: SelectorConfidence | null;
+}
+
+export interface UIElementSemanticMetadata {
+  is_clickable: boolean;
+  is_container: boolean;
+}
+
 export interface CaptureAndroidScreenResponse {
   device: DeviceInfo;
   screenshot: string; // base64 encoded string
@@ -132,6 +147,11 @@ export interface UIElement {
   center?: [number, number];
   depth?: number;
   state?: UIElementState | null;
+  stable_id?: string | null;
+  role?: string | null;
+  test_tag?: string | null;
+  selector?: UIResolutionSelector | null;
+  semantic?: UIElementSemanticMetadata | null;
 }
 
 export interface GetUITreeResponse {
@@ -167,7 +187,7 @@ export interface CaptureDebugSnapshotRawResponse {
   activity: string | null;
   fingerprint: string | null;
   screenshot: string | null;
-  ui_tree: unknown | null;
+  ui_tree: GetUITreeResponse | null;
   logs: StructuredLogEntry[];
   device?: DeviceInfo;
   screenshot_error?: string;
@@ -215,6 +235,11 @@ export interface ActionTargetResolved {
   bounds: [number, number, number, number] | null;
   index: number | null;
   state?: UIElementState | null;
+  stable_id?: string | null;
+  role?: string | null;
+  test_tag?: string | null;
+  selector?: UIResolutionSelector | null;
+  semantic?: UIElementSemanticMetadata | null;
 }
 
 export interface ActionExecutionResult {

package/src/utils/android/utils.ts

@@ -1,4 +1,4 @@
-import { DeviceInfo, UIElement, UIElementState } from "../../types.js"
+import { DeviceInfo, UIElement, UIElementSemanticMetadata, UIElementState, UIResolutionSelector, SelectorConfidence } from "../../types.js"
 import { promises as fsPromises, existsSync } from 'fs'
 import path from 'path'
 import { detectJavaHome } from '../java.js'
@@ -336,6 +336,52 @@ function parseNumberAttr(value: unknown): number | null {
   return Number.isFinite(parsed) ? parsed : null
 }
 
+function normalizeClassName(value: unknown): string {
+  return typeof value === 'string' ? value.trim().toLowerCase() : ''
+}
+
+function inferAndroidRole(className: string): string | null {
+  if (/seekbar|slider|progress/.test(className)) return 'slider'
+  if (/switch|toggle/.test(className)) return 'switch'
+  if (/checkbox/.test(className)) return 'checkbox'
+  if (/radiobutton|radio/.test(className)) return 'radio'
+  if (/edittext|textfield|search/.test(className)) return 'text_field'
+  if (/button|fab/.test(className)) return 'button'
+  if (/imageview|icon/.test(className)) return 'image'
+  if (/recyclerview|scroll|layout|viewgroup|frame/.test(className)) return 'container'
+  return null
+}
+
+function buildAndroidSelectorConfidence(source: 'resource_id' | 'content_desc' | 'text' | 'class' | 'none'): SelectorConfidence | null {
+  switch (source) {
+    case 'resource_id':
+      return { score: 1, reason: 'resource_id' }
+    case 'content_desc':
+      return { score: 0.9, reason: 'content_description' }
+    case 'text':
+      return { score: 0.6, reason: 'text_match' }
+    case 'class':
+      return { score: 0.35, reason: 'class_match' }
+    default:
+      return null
+  }
+}
+
+function buildAndroidSelector(text: string | null, contentDescription: string | null, resourceId: string | null, className: string): UIResolutionSelector | null {
+  if (resourceId) return { value: resourceId, confidence: buildAndroidSelectorConfidence('resource_id') }
+  if (contentDescription) return { value: contentDescription, confidence: buildAndroidSelectorConfidence('content_desc') }
+  if (text) return { value: text, confidence: buildAndroidSelectorConfidence('text') }
+  if (className) return { value: className, confidence: buildAndroidSelectorConfidence('class') }
+  return null
+}
+
+function buildAndroidSemantic(clickable: boolean, className: string): UIElementSemanticMetadata {
+  return {
+    is_clickable: clickable,
+    is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
+  }
+}
+
 function isSliderLikeAndroid(node: any): boolean {
   const className = String(node['@_class'] || '').toLowerCase()
   return /seekbar|slider|range|progress/i.test(className)
@@ -401,29 +447,41 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
 
   let currentIndex = -1;
 
-    if (node['@_class']) {
-      const text = node['@_text'] || null;
-      const contentDescription = node['@_content-desc'] || null;
-      const clickable = node['@_clickable'] === 'true';
-      const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
-      const state = extractAndroidState(node);
-
-      const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
-
-      if (isUseful) {
-        const element: UIElement = {
+  if (node['@_class']) {
+    const text = node['@_text'] || null;
+    const contentDescription = node['@_content-desc'] || null;
+    const clickable = node['@_clickable'] === 'true';
+    const className = String(node['@_class'] || 'unknown');
+    const bounds = parseBounds(node['@_bounds'] || '[0,0][0,0]');
+    const state = extractAndroidState(node);
+    const role = inferAndroidRole(normalizeClassName(className));
+    const resourceId = typeof node['@_resource-id'] === 'string' && node['@_resource-id'].trim().length > 0 ? node['@_resource-id'] : null
+    const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null)
+    const testTag = stableId
+    const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className))
+    const semantic = buildAndroidSemantic(clickable, normalizeClassName(className))
+
+    const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
+
+    if (isUseful) {
+      const element: UIElement = {
         text,
         contentDescription,
-        type: node['@_class'] || 'unknown',
-        resourceId: node['@_resource-id'] || null,
+        type: className,
+        resourceId,
         clickable,
         enabled: node['@_enabled'] === 'true',
-          visible: true,
-          bounds,
-          center: getCenter(bounds),
-          depth,
-          state
-        };
+        visible: true,
+        bounds,
+        center: getCenter(bounds),
+        depth,
+        state,
+        stable_id: stableId,
+        role,
+        test_tag: testTag,
+        selector,
+        semantic
+      };
 
       if (parentIndex !== -1) {
         element.parentId = parentIndex;

package/test/unit/observe/state_extraction.test.ts

@@ -8,6 +8,7 @@ async function run() {
     '@_class': 'android.widget.SeekBar',
     '@_text': '',
     '@_content-desc': 'Duration',
+    '@_resource-id': 'com.example:id/duration',
     '@_clickable': 'true',
     '@_enabled': 'true',
     '@_selected': 'true',
@@ -21,18 +22,64 @@ async function run() {
   assert.strictEqual(androidElements[0].state?.raw_value, 7)
   assert.strictEqual(androidElements[0].state?.value, 50)
   assert.deepStrictEqual(androidElements[0].state?.value_range, { min: 0, max: 14 })
+  assert.strictEqual(androidElements[0].stable_id, 'com.example:id/duration')
+  assert.strictEqual(androidElements[0].role, 'slider')
+  assert.strictEqual(androidElements[0].test_tag, 'com.example:id/duration')
+  assert.deepStrictEqual(androidElements[0].selector, {
+    value: 'com.example:id/duration',
+    confidence: { score: 1, reason: 'resource_id' }
+  })
+  assert.deepStrictEqual(androidElements[0].semantic, { is_clickable: true, is_container: false })
+
+  const androidFallbackElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.Button',
+    '@_text': '',
+    '@_content-desc': 'Save',
+    '@_clickable': 'true',
+    '@_enabled': 'true',
+    '@_bounds': '[0,0][100,50]'
+  }, androidFallbackElements)
+
+  assert.strictEqual(androidFallbackElements.length, 1)
+  assert.strictEqual(androidFallbackElements[0].resourceId, null)
+  assert.strictEqual(androidFallbackElements[0].stable_id, 'Save')
+  assert.deepStrictEqual(androidFallbackElements[0].selector, {
+    value: 'Save',
+    confidence: { score: 0.9, reason: 'content_description' }
+  })
 
   const iosElements: any[] = []
   traverseIDBNode({
     AXElementType: 'Slider',
     AXLabel: 'Playback speed',
     AXValue: '0.75',
+    AXUniqueId: 'playback_speed_slider',
     AXTraits: ['UIAccessibilityTraitAdjustable']
   }, iosElements)
 
   assert.strictEqual(iosElements.length, 1)
   assert.strictEqual(iosElements[0].state?.value, 75)
   assert.strictEqual(iosElements[0].state?.raw_value, 0.75)
+  assert.strictEqual(iosElements[0].stable_id, 'playback_speed_slider')
+  assert.strictEqual(iosElements[0].role, 'slider')
+  assert.strictEqual(iosElements[0].test_tag, 'playback_speed_slider')
+  assert.deepStrictEqual(iosElements[0].selector, {
+    value: 'playback_speed_slider',
+    confidence: { score: 1, reason: 'accessibility_identifier' }
+  })
+  assert.deepStrictEqual(iosElements[0].semantic, { is_clickable: true, is_container: false })
+
+  const iosFallbackElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Button',
+    AXLabel: 'Save',
+    AXTraits: ['UIAccessibilityTraitButton'],
+    AXUniqueId: 'fallback_unique_id'
+  }, iosFallbackElements)
+
+  assert.strictEqual(iosFallbackElements.length, 1)
+  assert.strictEqual(iosFallbackElements[0].stable_id, 'fallback_unique_id')
 
   console.log('state extraction tests passed')
 }