mobile-debug-mcp 0.25.1 → 0.26.1

package/src/interact/classify.ts

@@ -1,5 +1,6 @@
 export type ActionOutcome = 'success' | 'no_op' | 'backend_failure' | 'ui_failure' | 'unknown'
 export type NetworkRequestStatus = 'success' | 'failure' | 'retryable'
+export type ActionCategory = 'local_state' | 'side_effect'
 
 export interface NetworkRequest {
   endpoint: string
@@ -9,6 +10,8 @@ export interface NetworkRequest {
 export interface ClassifyActionOutcomeInput {
   uiChanged: boolean
   expectedElementVisible?: boolean | null
+  /** Concrete action_type from the runtime action result (for example: tap, type_text, start_app). */
+  actionType?: string | null
   /** null = get_network_activity has not been called yet */
   networkRequests?: NetworkRequest[] | null
   hasLogErrors?: boolean | null
@@ -17,8 +20,29 @@ export interface ClassifyActionOutcomeInput {
 export interface ClassifyActionOutcomeResult {
   outcome: ActionOutcome
   reasoning: string
-  /** Present when the caller must call get_network_activity before a final classification is possible */
-  nextAction?: 'call_get_network_activity'
+}
+
+const ACTION_CATEGORY_BY_TYPE: Record<string, ActionCategory> = {
+  tap: 'local_state',
+  tap_element: 'local_state',
+  swipe: 'local_state',
+  scroll_to_element: 'local_state',
+  type_text: 'local_state',
+  press_back: 'local_state',
+  start_app: 'side_effect',
+  restart_app: 'side_effect',
+  terminate_app: 'side_effect',
+  reset_app_data: 'side_effect',
+  install_app: 'side_effect',
+  build_app: 'side_effect',
+  build_and_install: 'side_effect'
+}
+
+function inferActionCategory(actionType?: string | null): ActionCategory | null {
+  if (typeof actionType !== 'string') return null
+  const normalized = actionType.trim().toLowerCase()
+  if (!normalized) return null
+  return ACTION_CATEGORY_BY_TYPE[normalized] ?? 'side_effect'
 }
 
 /**
@@ -26,39 +50,55 @@ export interface ClassifyActionOutcomeResult {
  * Same inputs always produce the same output.
  */
 export function classifyActionOutcome(input: ClassifyActionOutcomeInput): ClassifyActionOutcomeResult {
-  const { uiChanged, expectedElementVisible, networkRequests, hasLogErrors } = input
+  const { uiChanged, expectedElementVisible, actionType, networkRequests, hasLogErrors } = input
+  const actionCategory = inferActionCategory(actionType)
 
   // Step 1 — UI signal is positive
   if (uiChanged || expectedElementVisible === true) {
     return { outcome: 'success', reasoning: expectedElementVisible === true ? 'expected element is visible' : 'UI changed after action' }
   }
 
-  // Step 2 — UI did not change; network signal is required
-  if (networkRequests === null || networkRequests === undefined) {
+  // Step 2 — no action type means we cannot choose a safe routing path
+  if (actionCategory === null) {
     return {
       outcome: 'unknown',
-      reasoning: 'UI did not change; get_network_activity must be called before classification can proceed',
-      nextAction: 'call_get_network_activity'
+      reasoning: 'actionType was not supplied; pass the runtime action_type so the classifier can distinguish local-state and side-effect routing'
     }
   }
 
-  // Step 3 — any network failure
-  const failedRequest = networkRequests.find((r) => r.status === 'failure' || r.status === 'retryable')
+  const failedRequest = networkRequests?.find((r) => r.status === 'failure' || r.status === 'retryable')
   if (failedRequest) {
     return { outcome: 'backend_failure', reasoning: `network request ${failedRequest.endpoint} returned ${failedRequest.status}` }
   }
 
-  // Step 4 — no network requests at all
+  // Step 3 — local-state actions should be verified with state-specific signals first
+  if (actionCategory === 'local_state') {
+    const logNote = hasLogErrors ? ' (log errors present)' : ''
+    return {
+      outcome: 'no_op',
+      reasoning: `local-state action${logNote}; use expect_state, refreshed snapshot comparison, or expect_element_visible instead of defaulting to network inspection`
+    }
+  }
+
+  // Step 4 — side-effect actions may legitimately need network or log inspection
+  if (networkRequests === null || networkRequests === undefined) {
+    return {
+      outcome: 'unknown',
+      reasoning: 'side-effect action without network data; inspect network or log signals only if the outcome is still ambiguous'
+    }
+  }
+
+  // Step 5 — no network requests at all
   if (networkRequests.length === 0) {
     const logNote = hasLogErrors ? ' (log errors present)' : ''
-    return { outcome: 'no_op', reasoning: `no UI change and no network activity${logNote}` }
+    return { outcome: 'no_op', reasoning: `side-effect action and no network activity${logNote}` }
   }
 
-  // Step 5 — network requests exist and all succeeded
+  // Step 6 — network requests exist and all succeeded
   if (networkRequests.every((r) => r.status === 'success')) {
     return { outcome: 'ui_failure', reasoning: 'network requests succeeded but UI did not change' }
   }
 
-  // Step 6 — fallback
+  // Step 7 — fallback
   return { outcome: 'unknown', reasoning: 'signals are inconclusive' }
 }

package/src/interact/index.ts

@@ -5,6 +5,7 @@ export { AndroidInteract, iOSInteract };
 
 import { resolveTargetDevice } from '../utils/resolve-device.js'
 import { ToolsObserve } from '../observe/index.js'
+import { computeSnapshotSignature } from '../observe/snapshot-metadata.js'
 import { nextActionId } from '../server/common.js'
 import type {
   ActionFailureCode,
@@ -12,6 +13,7 @@ import type {
   ExpectElementVisibleResponse,
   ExpectStateResponse,
   ExpectScreenResponse,
+  WaitForUIChangeResponse,
   UIElementState,
   TapElementResponse
 } from '../types.js'
@@ -60,9 +62,16 @@ interface UiResolution {
   height?: number
 }
 
+interface UiChangeSignatureSet {
+  hierarchy: string | null
+  text: string | null
+  state: string | null
+}
+
 
 export class ToolsInteract {
   private static readonly _maxResolvedUiElements = 256
+  private static readonly _uiChangeKinds: Array<'hierarchy_diff' | 'text_change' | 'state_change'> = ['hierarchy_diff', 'text_change', 'state_change']
   private static readonly _sliderSearchLookahead = 8
   private static readonly _sliderNegativeGapTolerancePx = 32
   private static readonly _sliderPositiveGapLimitPx = 640
@@ -85,6 +94,10 @@ export class ToolsInteract {
     return normalized as [number, number, number, number]
   }
 
+  private static _hash(value: unknown): string {
+    return createHash('sha256').update(JSON.stringify(value)).digest('hex')
+  }
+
   private static _matchesSelector(el: UiElement, selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }): boolean {
     if (!selector) return false
     const normalize = ToolsInteract._normalize
@@ -195,6 +208,66 @@ export class ToolsInteract {
     }
   }
 
+  private static _buildUiChangeSignatures(tree: any): UiChangeSignatureSet {
+    const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
+    const textPayload: Array<{ text: string, contentDescription: string, resourceId: string }> = []
+    const statePayload: Array<{
+      checked: boolean | null
+      selected: boolean | string | { id: string; label?: string } | null
+      focused: boolean | null
+      expanded: boolean | null
+      enabled: boolean | null
+      text_value: string | null
+      value: number | string | null
+      raw_value: number | string | null
+      value_range: UIElementState['value_range']
+    }> = []
+
+    for (const el of elements) {
+      textPayload.push({
+        text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
+        contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
+        resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
+      })
+
+      statePayload.push({
+        checked: el?.state?.checked ?? null,
+        selected: el?.state?.selected ?? null,
+        focused: el?.state?.focused ?? null,
+        expanded: el?.state?.expanded ?? null,
+        enabled: el?.state?.enabled ?? null,
+        text_value: el?.state?.text_value ?? null,
+        value: el?.state?.value ?? null,
+        raw_value: el?.state?.raw_value ?? null,
+        value_range: el?.state?.value_range ?? null
+      })
+    }
+
+    return {
+      hierarchy: computeSnapshotSignature(tree),
+      text: ToolsInteract._hash({
+        screen: ToolsInteract._normalize(tree?.screen),
+        elements: textPayload
+      }),
+      state: ToolsInteract._hash({
+        screen: ToolsInteract._normalize(tree?.screen),
+        elements: statePayload
+      })
+    }
+  }
+
+  private static _matchesUiChange(expected: 'hierarchy_diff' | 'text_change' | 'state_change' | undefined, initial: UiChangeSignatureSet, current: UiChangeSignatureSet): 'hierarchy_diff' | 'text_change' | 'state_change' | null {
+    const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds
+
+    for (const changeKind of candidates) {
+      if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy) return changeKind
+      if (changeKind === 'text_change' && initial.text !== current.text) return changeKind
+      if (changeKind === 'state_change' && initial.state !== current.state) return changeKind
+    }
+
+    return null
+  }
+
   private static _resolvedTargetFromElement(
     elementId: string,
     element: UiElement,
@@ -955,6 +1028,84 @@ export class ToolsInteract {
     }
   }
 
+  static async waitForUIChangeHandler({
+    platform,
+    deviceId,
+    timeout_ms = 60000,
+    stability_window_ms = 250,
+    expected_change
+  }: {
+    platform?: 'android' | 'ios',
+    deviceId?: string,
+    timeout_ms?: number,
+    stability_window_ms?: number,
+    expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change'
+  }): Promise<WaitForUIChangeResponse> {
+    const start = Date.now()
+    const pollIntervalMs = 300
+    const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 250)
+    let baseline: UiChangeSignatureSet | null = null
+    let lastObservedRevision: number | null = null
+    let lastLoadingState: any = null
+
+    while (Date.now() - start < timeout_ms) {
+      try {
+        const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
+        const signatures = ToolsInteract._buildUiChangeSignatures(tree)
+        lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
+        lastLoadingState = tree?.loading_state ?? lastLoadingState
+
+        if (!baseline) {
+          baseline = signatures
+        } else {
+          const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
+          if (observedChange) {
+            if (stabilityWindow > 0) {
+              await new Promise(resolve => setTimeout(resolve, stabilityWindow))
+              const confirmTree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
+              const confirmSignatures = ToolsInteract._buildUiChangeSignatures(confirmTree)
+              const confirmChange = ToolsInteract._matchesUiChange(expected_change, baseline, confirmSignatures)
+              if (!confirmChange || confirmSignatures.hierarchy !== signatures.hierarchy || confirmSignatures.text !== signatures.text || confirmSignatures.state !== signatures.state) {
+                lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision
+                lastLoadingState = confirmTree?.loading_state ?? lastLoadingState
+                await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
+                continue
+              }
+              lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision
+              lastLoadingState = confirmTree?.loading_state ?? lastLoadingState
+            }
+
+            return {
+              success: true,
+              observed_change: observedChange,
+              snapshot_revision: lastObservedRevision ?? undefined,
+              timeout: false,
+              elapsed_ms: Date.now() - start,
+              expected_change,
+              loading_state: lastLoadingState ?? null,
+              reason: 'UI change observed'
+            }
+          }
+        }
+      } catch {
+        // Keep polling until timeout; the observable surface should be best-effort.
+      }
+
+      await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
+    }
+
+    return {
+      success: false,
+      observed_change: null,
+      snapshot_revision: lastObservedRevision ?? undefined,
+      timeout: true,
+      elapsed_ms: Date.now() - start,
+      expected_change,
+      loading_state: lastLoadingState ?? null,
+      reason: 'timeout'
+    }
+  }
+
   static async expectScreenHandler({
     platform,
     fingerprint,

package/src/observe/android.ts

@@ -7,6 +7,7 @@ import { promises as fsPromises } from "fs"
 import path from "path"
 import { computeScreenFingerprint } from "../utils/ui/index.js"
 import { parsePngSize } from "../utils/image.js"
+import { deriveSnapshotMetadata } from "./snapshot-metadata.js"
 
 const activeLogStreams: Map<string, { proc: any, file: string }> = new Map()
 
@@ -74,20 +75,29 @@ export class AndroidObserve {
           }
       }
 
+      const snapshotMetadata = deriveSnapshotMetadata(`android:${deviceInfo.id}`, {
+        screen: "",
+        resolution,
+        elements
+      }, 'ui_tree')
+
       return {
         device: deviceInfo,
         screen: "",
         resolution,
-        elements
+        elements,
+        ...snapshotMetadata
       };
     } catch (e) {
       const errorMessage = `Failed to get UI tree. ADB Path: '${getAdbCmd()}'. Error: ${e instanceof Error ? e.message : String(e)}`;
       console.error(errorMessage);
+      const snapshotMetadata = deriveSnapshotMetadata(`android:${deviceInfo.id}`, null, 'ui_tree')
       return {
           device: deviceInfo,
           screen: "",
           resolution: { width: 0, height: 0 },
           elements: [],
+          ...snapshotMetadata,
           error: errorMessage
       };
     }

package/src/observe/index.ts

@@ -5,6 +5,7 @@ import type {
   CaptureDebugSnapshotRawResponse,
   SnapshotSemanticResponse
 } from '../types.js'
+import { deriveSnapshotMetadata } from './snapshot-metadata.js'
 
 export { AndroidObserve } from './android.js'
 export { iOSObserve } from './ios.js'
@@ -245,7 +246,17 @@ export class ToolsObserve {
 
   static async captureDebugSnapshotHandler({ reason, includeLogs = true, logLines = 200, platform, appId, deviceId, sessionId }: { reason?: string; includeLogs?: boolean; logLines?: number; platform?: 'android' | 'ios'; appId?: string; deviceId?: string; sessionId?: string } = {}) {
     const timestamp = Date.now()
-    const raw: CaptureDebugSnapshotRawResponse = { timestamp, reason: reason || '', activity: null, fingerprint: null, screenshot: null, ui_tree: null, logs: [] }
+    const raw: CaptureDebugSnapshotRawResponse = {
+      timestamp,
+      snapshot_revision: 0,
+      captured_at_ms: timestamp,
+      reason: reason || '',
+      activity: null,
+      fingerprint: null,
+      screenshot: null,
+      ui_tree: null,
+      logs: []
+    }
 
     // Parallel fetches for performance: screenshot, current screen, fingerprint, ui tree, and log stream/get logs
     const sid = sessionId || 'default'
@@ -335,6 +346,20 @@ export class ToolsObserve {
       }
     }
 
+    const snapshotDeviceKey = raw.ui_tree?.device
+      ? `${raw.ui_tree.device.platform}:${raw.ui_tree.device.id}`
+      : `${platform || 'unknown'}:${deviceId || 'default'}`
+    const snapshotMetadata = deriveSnapshotMetadata(
+      snapshotDeviceKey,
+      raw.ui_tree,
+      'snapshot',
+      raw.ui_tree?.snapshot_revision ? null : (raw.fingerprint || raw.activity || null)
+    )
+
+    raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision
+    raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms
+    raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state
+
     const semantic = deriveSnapshotSemantic(raw)
     return semantic ? { raw, semantic } : { raw }
   }

package/src/observe/ios.ts

@@ -7,6 +7,7 @@ import path from 'path'
 import { parseLogLine } from '../utils/android/utils.js'
 import { computeScreenFingerprint } from '../utils/ui/index.js'
 import { parsePngSize } from '../utils/image.js'
+import { deriveSnapshotMetadata } from './snapshot-metadata.js'
 
 const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
 let iosExecCommand = execCommand
@@ -485,16 +486,19 @@ export class iOSObserve {
 
   async getUITree(deviceId: string = "booted"): Promise<GetUITreeResponse> {
     const device = await getIOSDeviceMetadata(deviceId);
+    const deviceKey = `ios:${device.id}`
     
     const idbExists = await isIDBInstalled();
     if (!idbExists) {
+       const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree')
        return {
           device,
           screen: "",
           resolution: { width: 0, height: 0 },
           elements: [],
+          ...snapshotMetadata,
           error: "iOS UI tree retrieval requires 'idb' (iOS Device Bridge). Please install it via Homebrew: `brew tap facebook/fb && brew install idb-companion` and `pip3 install fb-idb`."
-       };
+        };
     }
 
     const targetUdid = (device.id && device.id !== 'booted') ? device.id : undefined;
@@ -540,15 +544,17 @@ export class iOSObserve {
          console.error(`Attempt ${attempts} failed: ${e}`);
       }
       
-      if (attempts === maxAttempts) {
-          return {
-              device,
-              screen: "",
-              resolution: { width: 0, height: 0 },
-              elements: [],
-              error: `Failed to retrieve valid UI dump after ${maxAttempts} attempts.`
-          };
-      }
+       if (attempts === maxAttempts) {
+           const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree')
+           return {
+               device,
+               screen: "",
+               resolution: { width: 0, height: 0 },
+               elements: [],
+               ...snapshotMetadata,
+               error: `Failed to retrieve valid UI dump after ${maxAttempts} attempts.`
+           };
+       }
     }
 
     try {
@@ -569,20 +575,29 @@ export class iOSObserve {
             height = rootBounds[3] - rootBounds[1];
         }
 
+        const snapshotMetadata = deriveSnapshotMetadata(deviceKey, {
+          screen: "",
+          resolution: { width, height },
+          elements
+        }, 'ui_tree')
+
         return {
             device,
             screen: "",
             resolution: { width, height },
-            elements
+            elements,
+            ...snapshotMetadata
         };
     } catch (e) {
-         return {
+         const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree')
+          return {
             device,
             screen: "",
             resolution: { width: 0, height: 0 },
             elements: [],
+            ...snapshotMetadata,
             error: `Failed to parse idb output: ${e instanceof Error ? e.message : String(e)}`
-         };
+          };
     }
   }
 

package/src/observe/snapshot-metadata.ts

@@ -0,0 +1,107 @@
+import crypto from 'crypto'
+import type { GetUITreeResponse, LoadingState, UIElement } from '../types.js'
+
+interface SnapshotState {
+  revision: number
+  signature: string | null
+}
+
+const snapshotStateByDevice = new Map<string, SnapshotState>()
+
+function normalize(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  return String(value).trim().toLowerCase()
+}
+
+function normalizeBounds(bounds: unknown): [number, number, number, number] | null {
+  if (!Array.isArray(bounds) || bounds.length < 4) return null
+  const normalized = bounds.slice(0, 4).map((value) => Number(value))
+  if (normalized.some((value) => Number.isNaN(value))) return null
+  return normalized as [number, number, number, number]
+}
+
+function stableElementSignature(element: UIElement) {
+  return {
+    text: normalize(element.text),
+    contentDescription: normalize(element.contentDescription),
+    resourceId: normalize(element.resourceId),
+    type: normalize(element.type),
+    stable_id: normalize(element.stable_id),
+    role: normalize(element.role),
+    test_tag: normalize(element.test_tag),
+    selector: normalize(element.selector?.value),
+    clickable: !!element.clickable,
+    enabled: !!element.enabled,
+    visible: !!element.visible,
+    state: element.state ?? null,
+    bounds: normalizeBounds(element.bounds)
+  }
+}
+
+export function computeSnapshotSignature(tree: Pick<GetUITreeResponse, 'elements' | 'screen' | 'resolution' | 'error'> | null | undefined): string | null {
+  if (!tree || tree.error) return null
+
+  const payload = {
+    screen: normalize(tree.screen),
+    resolution: tree.resolution || { width: 0, height: 0 },
+    elements: Array.isArray(tree.elements) ? tree.elements.map((element) => stableElementSignature(element)) : []
+  }
+
+  return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex')
+}
+
+export function detectLoadingState(tree: Pick<GetUITreeResponse, 'elements' | 'error'> | null | undefined, source: string): LoadingState | null {
+  if (!tree || tree.error || !Array.isArray(tree.elements)) return null
+
+  for (const element of tree.elements) {
+    if (!element?.visible) continue
+    const text = normalize(element?.text ?? element?.contentDescription ?? '')
+    const type = normalize(element?.type ?? '')
+    const combined = `${type} ${text}`
+    if (/progress|spinner|loading|please wait|busy|loading indicator|skeleton|pending/.test(combined)) {
+      const signal = /progress/.test(combined)
+        ? 'progress_indicator'
+        : /spinner/.test(combined)
+          ? 'spinner'
+          : /busy/.test(combined)
+            ? 'busy_indicator'
+            : /skeleton/.test(combined)
+              ? 'skeleton'
+              : 'loading_indicator'
+      return { active: true, signal, source }
+    }
+  }
+
+  return null
+}
+
+export function deriveSnapshotMetadata(
+  deviceKey: string,
+  tree: Pick<GetUITreeResponse, 'elements' | 'screen' | 'resolution' | 'error'> | null | undefined,
+  source: string,
+  signatureOverride?: string | null
+) {
+  const signature = signatureOverride ?? computeSnapshotSignature(tree)
+  const previous = snapshotStateByDevice.get(deviceKey)
+
+  let revision = 1
+  if (previous) {
+    if (signature === null) {
+      revision = previous.revision
+    } else {
+      revision = previous.signature === signature ? previous.revision : previous.revision + 1
+    }
+  }
+
+  snapshotStateByDevice.set(deviceKey, { revision, signature })
+
+  return {
+    snapshot_revision: revision,
+    captured_at_ms: Date.now(),
+    loading_state: detectLoadingState(tree, source)
+  }
+}
+
+export function resetSnapshotMetadataForTests() {
+  snapshotStateByDevice.clear()
+}