mobile-debug-mcp 0.22.0 → 0.24.0

package/docs/tools/observe.md

@@ -1,10 +1,20 @@
 # Observe (logs, screenshots, UI trees)
 
-Tools that retrieve device state, logs, screenshots and UI hierarchies.
+Tools that retrieve device state, logs, screenshots, fingerprints, and UI hierarchies.
+
+These tools are primarily for:
+
+- building context before an action
+- supporting synchronization
+- diagnostics when verification fails
+
+They are **not** the primary success signal when an applicable `expect_*` tool exists.
 
 ## get_logs
 
-Fetch recent logs as structured entries optimized for AI agents. Use logs as a debugging aid only — prefer UI validation (wait_for_ui) first.
+Fetch recent logs as structured entries optimized for AI agents.
+
+Use logs as a debugging aid only. Prefer `expect_*` for verification, and use logs after verification fails or when an error is suspected.
 
 Input (example):
 
@@ -18,7 +28,7 @@ Defaults:
 
 When to use get_logs:
 
-- After a UI validation (wait_for_ui) fails to confirm the expected outcome.
+- After deterministic verification fails.
 - When you suspect a crash, error, or silent failure that the UI doesn't expose.
 - To provide additional debugging context correlated with an action.
 
@@ -41,7 +51,7 @@ Notes:
 - Errors are returned as structured objects with `error.code` and `error.message`. Possible codes: LOGS_UNAVAILABLE, INVALID_FILTER, PLATFORM_NOT_SUPPORTED, INTERNAL_ERROR.
 
 ## capture_screenshot
-Capture screen. Returns JSON metadata then an image/png block with base64 PNG data.
+Capture the current screen. Returns JSON metadata followed by one or more image blocks.
 
 Input:
 
@@ -52,13 +62,17 @@ Input:
 Response (metadata):
 
 ```json
-{ "device": { "platform": "android", "id": "emulator-5554" }, "width": 1080, "height": 2400 }
+{ "device": { "platform": "android", "id": "emulator-5554" }, "result": { "resolution": { "width": 1080, "height": 2400 }, "mimeType": "image/webp" } }
 ```
 
+Notes:
+- The image block may use WebP, PNG, or a compatibility fallback such as JPEG.
+- Best used for inspection and debugging, not as a primary verification mechanism.
+
 ---
 
 ## get_ui_tree
-Returns parsed UI hierarchy.
+Return the parsed UI hierarchy for the current screen.
 
 Input:
 
@@ -69,9 +83,13 @@ Input:
 Response (example):
 
 ```json
-{ "device": { "platform": "android", "id": "emulator-5554" }, "elements": [ { "text": "Sign in", "type": "android.widget.Button", "resourceId": "com.example:id/signin", "clickable": true, "bounds": [0,0,100,50] } ] }
+{ "device": { "platform": "android", "id": "emulator-5554" }, "screen": "", "resolution": { "width": 1080, "height": 2400 }, "elements": [ { "text": "Sign in", "type": "android.widget.Button", "resourceId": "com.example:id/signin", "clickable": true, "bounds": [0,0,100,50] } ] }
 ```
 
+Notes:
+- Useful for inspection, selector development, and fallback debugging.
+- Prefer `wait_for_ui` for deterministic element resolution in interactive flows.
+
 ---
 
 ## get_current_screen
@@ -136,7 +154,7 @@ Notes:
 ---
 
 ## get_screen_fingerprint
-Generate a stable fingerprint representing the visible screen. Useful for detecting navigation changes, preventing loops, and synchronisation.
+Generate a stable fingerprint representing the visible screen. Useful for detecting navigation changes, preventing loops, and synchronization.
 
 Input (optional):
 
@@ -157,6 +175,10 @@ Notes:
 - Sorts deterministically (top-to-bottom, left-to-right) and limits elements to 50.
 - Returns fingerprint: null and an error message if the UI tree or activity cannot be retrieved.
 
+Guidance:
+- Use as a baseline for `wait_for_screen_change`.
+- Use fingerprints to define expected screens for `expect_screen`.
+
 ---
 
 ## start_log_stream / read_log_stream / stop_log_stream

package/docs/tools/system.md

@@ -44,5 +44,5 @@ Behavior notes:
 
 Usage guidance:
 - Call before build/install flows to avoid wasted build attempts on misconfigured systems.
+- Call early in a session when device or toolchain availability is uncertain.
 - If `success: false`, attempt recovery steps or report issues to the user.
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobile-debug-mcp",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "MCP server for mobile app debugging (Android + iOS), with focus on security and reliability",
   "type": "module",
   "bin": {

package/src/interact/classify.ts

@@ -0,0 +1,64 @@
+export type ActionOutcome = 'success' | 'no_op' | 'backend_failure' | 'ui_failure' | 'unknown'
+export type NetworkRequestStatus = 'success' | 'failure' | 'retryable'
+
+export interface NetworkRequest {
+  endpoint: string
+  status: NetworkRequestStatus
+}
+
+export interface ClassifyActionOutcomeInput {
+  uiChanged: boolean
+  expectedElementVisible?: boolean | null
+  /** null = get_network_activity has not been called yet */
+  networkRequests?: NetworkRequest[] | null
+  hasLogErrors?: boolean | null
+}
+
+export interface ClassifyActionOutcomeResult {
+  outcome: ActionOutcome
+  reasoning: string
+  /** Present when the caller must call get_network_activity before a final classification is possible */
+  nextAction?: 'call_get_network_activity'
+}
+
+/**
+ * Pure deterministic classifier. Applies rules in fixed order.
+ * Same inputs always produce the same output.
+ */
+export function classifyActionOutcome(input: ClassifyActionOutcomeInput): ClassifyActionOutcomeResult {
+  const { uiChanged, expectedElementVisible, networkRequests, hasLogErrors } = input
+
+  // Step 1 — UI signal is positive
+  if (uiChanged || expectedElementVisible === true) {
+    return { outcome: 'success', reasoning: expectedElementVisible === true ? 'expected element is visible' : 'UI changed after action' }
+  }
+
+  // Step 2 — UI did not change; network signal is required
+  if (networkRequests === null || networkRequests === undefined) {
+    return {
+      outcome: 'unknown',
+      reasoning: 'UI did not change; get_network_activity must be called before classification can proceed',
+      nextAction: 'call_get_network_activity'
+    }
+  }
+
+  // Step 3 — any network failure
+  const failedRequest = networkRequests.find((r) => r.status === 'failure' || r.status === 'retryable')
+  if (failedRequest) {
+    return { outcome: 'backend_failure', reasoning: `network request ${failedRequest.endpoint} returned ${failedRequest.status}` }
+  }
+
+  // Step 4 — no network requests at all
+  if (networkRequests.length === 0) {
+    const logNote = hasLogErrors ? ' (log errors present)' : ''
+    return { outcome: 'no_op', reasoning: `no UI change and no network activity${logNote}` }
+  }
+
+  // Step 5 — network requests exist and all succeeded
+  if (networkRequests.every((r) => r.status === 'success')) {
+    return { outcome: 'ui_failure', reasoning: 'network requests succeeded but UI did not change' }
+  }
+
+  // Step 6 — fallback
+  return { outcome: 'unknown', reasoning: 'signals are inconclusive' }
+}

package/src/interact/index.ts

@@ -5,7 +5,14 @@ export { AndroidInteract, iOSInteract };
 
 import { resolveTargetDevice } from '../utils/resolve-device.js'
 import { ToolsObserve } from '../observe/index.js'
-import type { TapElementResponse } from '../types.js'
+import { nextActionId } from '../server/common.js'
+import type {
+  ActionFailureCode,
+  ActionTargetResolved,
+  ExpectElementVisibleResponse,
+  ExpectScreenResponse,
+  TapElementResponse
+} from '../types.js'
 
 interface ScreenFingerprintResponse { fingerprint: string | null }
 
@@ -112,6 +119,55 @@ export class ToolsInteract {
     }
   }
 
+  private static async _captureFingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null> {
+    try {
+      const fingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
+      return fingerprint?.fingerprint ?? null
+    } catch {
+      return null
+    }
+  }
+
+  private static _resolvedTargetFromElement(
+    elementId: string,
+    element: UiElement,
+    index: number
+  ): ActionTargetResolved {
+    return {
+      elementId,
+      text: element.text ?? null,
+      resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
+      accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
+      class: element.type ?? element.class ?? null,
+      bounds: ToolsInteract._normalizeBounds(element.bounds),
+      index
+    }
+  }
+
+  private static _actionFailure(
+    actionId: string,
+    timestamp: number,
+    actionType: string,
+    selector: Record<string, unknown> | null,
+    resolved: ActionTargetResolved | null,
+    failureCode: ActionFailureCode,
+    retryable: boolean,
+    uiFingerprintBefore: string | null,
+    uiFingerprintAfter?: string | null
+  ): TapElementResponse {
+    return {
+      action_id: actionId,
+      timestamp,
+      action_type: actionType,
+      target: { selector, resolved },
+      success: false,
+      failure_code: failureCode,
+      retryable,
+      ui_fingerprint_before: uiFingerprintBefore,
+      ui_fingerprint_after: uiFingerprintAfter
+    }
+  }
+
   static _resetResolvedUiElementsForTests() {
     ToolsInteract._resolvedUiElements.clear()
   }
@@ -198,20 +254,17 @@ export class ToolsInteract {
   }
 
   static async tapElementHandler({ elementId }: { elementId: string }): Promise<TapElementResponse> {
-    const action = 'tap' as const
+    const timestamp = Date.now()
+    const actionType = 'tap_element'
+    const actionId = nextActionId(actionType, timestamp)
+    const selector = { elementId }
     const resolved = ToolsInteract._resolvedUiElements.get(elementId)
     if (!resolved) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_found',
-          message: 'Element ID was not found in the current UI context'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, null)
     }
 
+    const fingerprintBefore = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId)
+
     const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId }) as any
     const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform
     const treeDeviceId = tree?.device?.id || resolved.deviceId
@@ -219,52 +272,22 @@ export class ToolsInteract {
     const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
 
     if (!currentMatch) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_found',
-          message: 'Element ID is not present in the current UI context'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore)
     }
 
+    const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
+
     if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_visible',
-          message: 'Element is not visible'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
     }
 
     if (currentMatch.el.enabled === false) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_enabled',
-          message: 'Element is not enabled'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
     }
 
     const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
     if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_visible',
-          message: 'Element does not have valid visible bounds'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
     }
 
     const x = Math.floor((bounds[0] + bounds[2]) / 2)
@@ -272,21 +295,22 @@ export class ToolsInteract {
     const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId })
 
     if (!tapResult.success) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'tap_failed',
-          message: tapResult.error || 'Tap failed'
-        }
-      }
+      const fingerprintAfterFailure = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId)
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'UNKNOWN', false, fingerprintBefore, fingerprintAfterFailure)
     }
 
+    const fingerprintAfter = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId)
     return {
+      action_id: actionId,
+      timestamp,
+      action_type: actionType,
+      target: {
+        selector,
+        resolved: resolvedTarget
+      },
       success: true,
-      elementId,
-      action
+      ui_fingerprint_before: fingerprintBefore,
+      ui_fingerprint_after: fingerprintAfter
     }
   }
 
@@ -692,6 +716,110 @@ export class ToolsInteract {
     return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start }
   }
 
+  static async expectScreenHandler({
+    platform,
+    fingerprint,
+    screen,
+    deviceId
+  }: {
+    platform?: 'android' | 'ios',
+    fingerprint?: string,
+    screen?: string,
+    deviceId?: string
+  }): Promise<ExpectScreenResponse> {
+    const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
+    const observedScreen = {
+      fingerprint: observedFingerprint?.fingerprint ?? null,
+      screen: observedFingerprint?.activity ?? null
+    }
+
+    let observedScreenLabel = observedScreen.screen
+    if (!fingerprint && screen && platform !== 'ios') {
+      try {
+        const current = await ToolsObserve.getCurrentScreenHandler({ deviceId }) as any
+        observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel
+      } catch {
+        // Keep fingerprint-derived activity when current-screen lookup is unavailable.
+      }
+    }
+
+    const expectedScreen = {
+      fingerprint: fingerprint ?? null,
+      screen: screen ?? null
+    }
+
+    let success = false
+    if (fingerprint) {
+      success = observedScreen.fingerprint === fingerprint
+    } else if (screen) {
+      const candidates = new Set<string>()
+      if (observedScreen.screen) candidates.add(observedScreen.screen)
+      if (observedScreenLabel) candidates.add(observedScreenLabel)
+      success = candidates.has(screen)
+    }
+
+    return {
+      success,
+      observed_screen: {
+        fingerprint: observedScreen.fingerprint,
+        screen: observedScreenLabel
+      },
+      expected_screen: expectedScreen,
+      confidence: success ? 1 : 0
+    }
+  }
+
+  static async expectElementVisibleHandler({
+    selector,
+    element_id,
+    timeout_ms = 5000,
+    poll_interval_ms = 300,
+    platform,
+    deviceId
+  }: {
+    selector: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
+    element_id?: string,
+    timeout_ms?: number,
+    poll_interval_ms?: number,
+    platform?: 'android' | 'ios',
+    deviceId?: string
+  }): Promise<ExpectElementVisibleResponse> {
+    const result = await ToolsInteract.waitForUIHandler({
+      selector,
+      condition: 'visible',
+      timeout_ms,
+      poll_interval_ms,
+      platform,
+      deviceId
+    }) as any
+
+    if (result?.status === 'success' && result?.element) {
+      return {
+        success: true,
+        selector,
+        element_id: result.element.elementId ?? element_id ?? null,
+        element: {
+          elementId: result.element.elementId ?? null,
+          text: result.element.text ?? null,
+          resource_id: result.element.resource_id ?? null,
+          accessibility_id: result.element.accessibility_id ?? null,
+          class: result.element.class ?? null,
+          bounds: result.element.bounds ?? null,
+          index: typeof result.element.index === 'number' ? result.element.index : null
+        }
+      }
+    }
+
+    const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT'
+    return {
+      success: false,
+      selector,
+      element_id: element_id ?? null,
+      failure_code: errorCode,
+      retryable: errorCode === 'TIMEOUT'
+    }
+  }
+
   static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }: { type?: 'ui' | 'log' | 'screen' | 'idle', query?: string, timeoutMs?: number, pollIntervalMs?: number, includeSnapshotOnFailure?: boolean, match?: 'present'|'absent', stability_ms?: number, observationDelayMs?: number, platform?: 'android' | 'ios', deviceId?: string }) {
     const start = Date.now()
     const deadline = start + (timeoutMs || 0)