mobile-debug-mcp 0.23.0 → 0.24.1

package/docs/tools/observe.md

@@ -1,10 +1,20 @@
 # Observe (logs, screenshots, UI trees)
 
-Tools that retrieve device state, logs, screenshots and UI hierarchies.
+Tools that retrieve device state, logs, screenshots, fingerprints, and UI hierarchies.
+
+These tools are primarily for:
+
+- building context before an action
+- supporting synchronization
+- diagnostics when verification fails
+
+They are **not** the primary success signal when an applicable `expect_*` tool exists.
 
 ## get_logs
 
-Fetch recent logs as structured entries optimized for AI agents. Use logs as a debugging aid only — prefer UI validation (wait_for_ui) first.
+Fetch recent logs as structured entries optimized for AI agents.
+
+Use logs as a debugging aid only. Prefer `expect_*` for verification, and use logs after verification fails or when an error is suspected.
 
 Input (example):
 
@@ -18,7 +28,7 @@ Defaults:
 
 When to use get_logs:
 
-- After a UI validation (wait_for_ui) fails to confirm the expected outcome.
+- After deterministic verification fails.
 - When you suspect a crash, error, or silent failure that the UI doesn't expose.
 - To provide additional debugging context correlated with an action.
 
@@ -41,7 +51,7 @@ Notes:
 - Errors are returned as structured objects with `error.code` and `error.message`. Possible codes: LOGS_UNAVAILABLE, INVALID_FILTER, PLATFORM_NOT_SUPPORTED, INTERNAL_ERROR.
 
 ## capture_screenshot
-Capture screen. Returns JSON metadata then an image/png block with base64 PNG data.
+Capture the current screen. Returns JSON metadata followed by one or more image blocks.
 
 Input:
 
@@ -52,13 +62,17 @@ Input:
 Response (metadata):
 
 ```json
-{ "device": { "platform": "android", "id": "emulator-5554" }, "width": 1080, "height": 2400 }
+{ "device": { "platform": "android", "id": "emulator-5554" }, "result": { "resolution": { "width": 1080, "height": 2400 }, "mimeType": "image/webp" } }
 ```
 
+Notes:
+- The image block may use WebP, PNG, or a compatibility fallback such as JPEG.
+- Best used for inspection and debugging, not as a primary verification mechanism.
+
 ---
 
 ## get_ui_tree
-Returns parsed UI hierarchy.
+Return the parsed UI hierarchy for the current screen.
 
 Input:
 
@@ -69,9 +83,13 @@ Input:
 Response (example):
 
 ```json
-{ "device": { "platform": "android", "id": "emulator-5554" }, "elements": [ { "text": "Sign in", "type": "android.widget.Button", "resourceId": "com.example:id/signin", "clickable": true, "bounds": [0,0,100,50] } ] }
+{ "device": { "platform": "android", "id": "emulator-5554" }, "screen": "", "resolution": { "width": 1080, "height": 2400 }, "elements": [ { "text": "Sign in", "type": "android.widget.Button", "resourceId": "com.example:id/signin", "clickable": true, "bounds": [0,0,100,50] } ] }
 ```
 
+Notes:
+- Useful for inspection, selector development, and fallback debugging.
+- Prefer `wait_for_ui` for deterministic element resolution in interactive flows.
+
 ---
 
 ## get_current_screen
@@ -136,7 +154,7 @@ Notes:
 ---
 
 ## get_screen_fingerprint
-Generate a stable fingerprint representing the visible screen. Useful for detecting navigation changes, preventing loops, and synchronisation.
+Generate a stable fingerprint representing the visible screen. Useful for detecting navigation changes, preventing loops, and synchronization.
 
 Input (optional):
 
@@ -157,6 +175,10 @@ Notes:
 - Sorts deterministically (top-to-bottom, left-to-right) and limits elements to 50.
 - Returns fingerprint: null and an error message if the UI tree or activity cannot be retrieved.
 
+Guidance:
+- Use as a baseline for `wait_for_screen_change`.
+- Use fingerprints to define expected screens for `expect_screen`.
+
 ---
 
 ## start_log_stream / read_log_stream / stop_log_stream

package/docs/tools/system.md

@@ -44,5 +44,5 @@ Behavior notes:
 
 Usage guidance:
 - Call before build/install flows to avoid wasted build attempts on misconfigured systems.
+- Call early in a session when device or toolchain availability is uncertain.
 - If `success: false`, attempt recovery steps or report issues to the user.
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobile-debug-mcp",
-  "version": "0.23.0",
+  "version": "0.24.1",
   "description": "MCP server for mobile app debugging (Android + iOS), with focus on security and reliability",
   "type": "module",
   "bin": {

package/src/interact/index.ts

@@ -5,7 +5,14 @@ export { AndroidInteract, iOSInteract };
 
 import { resolveTargetDevice } from '../utils/resolve-device.js'
 import { ToolsObserve } from '../observe/index.js'
-import type { TapElementResponse } from '../types.js'
+import { nextActionId } from '../server/common.js'
+import type {
+  ActionFailureCode,
+  ActionTargetResolved,
+  ExpectElementVisibleResponse,
+  ExpectScreenResponse,
+  TapElementResponse
+} from '../types.js'
 
 interface ScreenFingerprintResponse { fingerprint: string | null }
 
@@ -112,6 +119,55 @@ export class ToolsInteract {
     }
   }
 
+  private static async _captureFingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null> {
+    try {
+      const fingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
+      return fingerprint?.fingerprint ?? null
+    } catch {
+      return null
+    }
+  }
+
+  private static _resolvedTargetFromElement(
+    elementId: string,
+    element: UiElement,
+    index: number
+  ): ActionTargetResolved {
+    return {
+      elementId,
+      text: element.text ?? null,
+      resource_id: element.resourceId ?? element.resourceID ?? element.id ?? null,
+      accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
+      class: element.type ?? element.class ?? null,
+      bounds: ToolsInteract._normalizeBounds(element.bounds),
+      index
+    }
+  }
+
+  private static _actionFailure(
+    actionId: string,
+    timestamp: number,
+    actionType: string,
+    selector: Record<string, unknown> | null,
+    resolved: ActionTargetResolved | null,
+    failureCode: ActionFailureCode,
+    retryable: boolean,
+    uiFingerprintBefore: string | null,
+    uiFingerprintAfter?: string | null
+  ): TapElementResponse {
+    return {
+      action_id: actionId,
+      timestamp,
+      action_type: actionType,
+      target: { selector, resolved },
+      success: false,
+      failure_code: failureCode,
+      retryable,
+      ui_fingerprint_before: uiFingerprintBefore,
+      ui_fingerprint_after: uiFingerprintAfter
+    }
+  }
+
   static _resetResolvedUiElementsForTests() {
     ToolsInteract._resolvedUiElements.clear()
   }
@@ -198,20 +254,17 @@ export class ToolsInteract {
   }
 
   static async tapElementHandler({ elementId }: { elementId: string }): Promise<TapElementResponse> {
-    const action = 'tap' as const
+    const timestamp = Date.now()
+    const actionType = 'tap_element'
+    const actionId = nextActionId(actionType, timestamp)
+    const selector = { elementId }
     const resolved = ToolsInteract._resolvedUiElements.get(elementId)
     if (!resolved) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_found',
-          message: 'Element ID was not found in the current UI context'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, null)
     }
 
+    const fingerprintBefore = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId)
+
     const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId }) as any
     const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform
     const treeDeviceId = tree?.device?.id || resolved.deviceId
@@ -219,52 +272,22 @@ export class ToolsInteract {
     const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
 
     if (!currentMatch) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_found',
-          message: 'Element ID is not present in the current UI context'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore)
     }
 
+    const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
+
     if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_visible',
-          message: 'Element is not visible'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
     }
 
     if (currentMatch.el.enabled === false) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_enabled',
-          message: 'Element is not enabled'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
     }
 
     const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
     if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'element_not_visible',
-          message: 'Element does not have valid visible bounds'
-        }
-      }
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
     }
 
     const x = Math.floor((bounds[0] + bounds[2]) / 2)
@@ -272,21 +295,22 @@ export class ToolsInteract {
     const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId })
 
     if (!tapResult.success) {
-      return {
-        success: false,
-        elementId,
-        action,
-        error: {
-          code: 'tap_failed',
-          message: tapResult.error || 'Tap failed'
-        }
-      }
+      const fingerprintAfterFailure = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId)
+      return ToolsInteract._actionFailure(actionId, timestamp, actionType, selector, resolvedTarget, 'UNKNOWN', false, fingerprintBefore, fingerprintAfterFailure)
     }
 
+    const fingerprintAfter = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId)
     return {
+      action_id: actionId,
+      timestamp,
+      action_type: actionType,
+      target: {
+        selector,
+        resolved: resolvedTarget
+      },
       success: true,
-      elementId,
-      action
+      ui_fingerprint_before: fingerprintBefore,
+      ui_fingerprint_after: fingerprintAfter
     }
   }
 
@@ -692,6 +716,110 @@ export class ToolsInteract {
     return { success: false, reason: 'timeout', lastFingerprint, elapsedMs: Date.now() - start }
   }
 
+  static async expectScreenHandler({
+    platform,
+    fingerprint,
+    screen,
+    deviceId
+  }: {
+    platform?: 'android' | 'ios',
+    fingerprint?: string,
+    screen?: string,
+    deviceId?: string
+  }): Promise<ExpectScreenResponse> {
+    const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
+    const observedScreen = {
+      fingerprint: observedFingerprint?.fingerprint ?? null,
+      screen: observedFingerprint?.activity ?? null
+    }
+
+    let observedScreenLabel = observedScreen.screen
+    if (!fingerprint && screen && platform !== 'ios') {
+      try {
+        const current = await ToolsObserve.getCurrentScreenHandler({ deviceId }) as any
+        observedScreenLabel = current?.shortActivity || current?.activity || observedScreenLabel
+      } catch {
+        // Keep fingerprint-derived activity when current-screen lookup is unavailable.
+      }
+    }
+
+    const expectedScreen = {
+      fingerprint: fingerprint ?? null,
+      screen: screen ?? null
+    }
+
+    let success = false
+    if (fingerprint) {
+      success = observedScreen.fingerprint === fingerprint
+    } else if (screen) {
+      const candidates = new Set<string>()
+      if (observedScreen.screen) candidates.add(observedScreen.screen)
+      if (observedScreenLabel) candidates.add(observedScreenLabel)
+      success = candidates.has(screen)
+    }
+
+    return {
+      success,
+      observed_screen: {
+        fingerprint: observedScreen.fingerprint,
+        screen: observedScreenLabel
+      },
+      expected_screen: expectedScreen,
+      confidence: success ? 1 : 0
+    }
+  }
+
+  static async expectElementVisibleHandler({
+    selector,
+    element_id,
+    timeout_ms = 5000,
+    poll_interval_ms = 300,
+    platform,
+    deviceId
+  }: {
+    selector: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
+    element_id?: string,
+    timeout_ms?: number,
+    poll_interval_ms?: number,
+    platform?: 'android' | 'ios',
+    deviceId?: string
+  }): Promise<ExpectElementVisibleResponse> {
+    const result = await ToolsInteract.waitForUIHandler({
+      selector,
+      condition: 'visible',
+      timeout_ms,
+      poll_interval_ms,
+      platform,
+      deviceId
+    }) as any
+
+    if (result?.status === 'success' && result?.element) {
+      return {
+        success: true,
+        selector,
+        element_id: result.element.elementId ?? element_id ?? null,
+        element: {
+          elementId: result.element.elementId ?? null,
+          text: result.element.text ?? null,
+          resource_id: result.element.resource_id ?? null,
+          accessibility_id: result.element.accessibility_id ?? null,
+          class: result.element.class ?? null,
+          bounds: result.element.bounds ?? null,
+          index: typeof result.element.index === 'number' ? result.element.index : null
+        }
+      }
+    }
+
+    const errorCode = result?.error?.code === 'INTERNAL_ERROR' ? 'UNKNOWN' : 'TIMEOUT'
+    return {
+      success: false,
+      selector,
+      element_id: element_id ?? null,
+      failure_code: errorCode,
+      retryable: errorCode === 'TIMEOUT'
+    }
+  }
+
   static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }: { type?: 'ui' | 'log' | 'screen' | 'idle', query?: string, timeoutMs?: number, pollIntervalMs?: number, includeSnapshotOnFailure?: boolean, match?: 'present'|'absent', stability_ms?: number, observationDelayMs?: number, platform?: 'android' | 'ios', deviceId?: string }) {
     const start = Date.now()
     const deadline = start + (timeoutMs || 0)

package/src/manage/android.ts

@@ -8,6 +8,10 @@ import { detectJavaHome } from '../utils/java.js'
 import { InstallAppResponse, StartAppResponse, TerminateAppResponse, RestartAppResponse, ResetAppDataResponse } from '../types.js'
 
 export class AndroidManage {
+  private isTestOnlyInstallFailure(output: string | undefined): boolean {
+    return typeof output === 'string' && output.includes('INSTALL_FAILED_TEST_ONLY')
+  }
+
   async build(projectPath: string, _variant?: string): Promise<{ artifactPath: string, output?: string } | { error: string }> {
     void _variant
     try {
@@ -92,6 +96,14 @@ export class AndroidManage {
         if (res.code === 0) {
           return { device: deviceInfo, installed: true, output: res.stdout }
         }
+
+        const installOutput = `${res.stdout}\n${res.stderr}`.trim()
+        if (this.isTestOnlyInstallFailure(installOutput)) {
+          const retryRes = await spawnAdb(['install', '-r', '-t', apkToInstall], deviceId)
+          if (retryRes.code === 0) {
+            return { device: deviceInfo, installed: true, output: retryRes.stdout }
+          }
+        }
       } catch (e) {
         console.debug('[android-run] adb install failed, attempting push+pm fallback:', e instanceof Error ? e.message : String(e))
       }
@@ -99,9 +111,21 @@ export class AndroidManage {
       const basename = path.basename(apkToInstall)
       const remotePath = `/data/local/tmp/${basename}`
       await execAdb(['push', apkToInstall, remotePath], deviceId)
-      const pmOut = await execAdb(['shell', 'pm', 'install', '-r', remotePath], deviceId)
-      try { await execAdb(['shell', 'rm', remotePath], deviceId) } catch {}
-      return { device: deviceInfo, installed: true, output: pmOut }
+      let finalPmRes = await spawnAdb(['shell', 'pm', 'install', '-r', remotePath], deviceId)
+      try {
+        if (finalPmRes.code === 0) {
+          return { device: deviceInfo, installed: true, output: finalPmRes.stdout }
+        }
+        if (this.isTestOnlyInstallFailure(`${finalPmRes.stdout}\n${finalPmRes.stderr}`)) {
+          finalPmRes = await spawnAdb(['shell', 'pm', 'install', '-r', '-t', remotePath], deviceId)
+          if (finalPmRes.code === 0) {
+            return { device: deviceInfo, installed: true, output: finalPmRes.stdout }
+          }
+        }
+        throw new Error(finalPmRes.stderr || finalPmRes.stdout || 'pm install failed')
+      } finally {
+        try { await execAdb(['shell', 'rm', remotePath], deviceId) } catch {}
+      }
     } catch (e) {
       // gather diagnostics for attempted adb operations
       const basename = path.basename(apkToInstall)

package/src/server/common.ts

@@ -0,0 +1,95 @@
+import type {
+  ActionExecutionResult,
+  ActionFailureCode,
+  ActionTargetResolved
+} from '../types.js'
+import { ToolsObserve } from '../observe/index.js'
+
+export function wrapResponse<T>(data: T) {
+  return {
+    content: [{
+      type: 'text' as const,
+      text: JSON.stringify(data, null, 2)
+    }]
+  }
+}
+
+export type ToolCallArgs = Record<string, unknown>
+export type ToolCallResult = Awaited<ReturnType<typeof wrapResponse>> | {
+  content: Array<{ type: 'text' | 'image'; text?: string; data?: string; mimeType?: string }>
+}
+export type ToolHandler = (args: ToolCallArgs) => Promise<ToolCallResult>
+
+let actionSequence = 0
+
+export function nextActionId(actionType: string, timestamp: number) {
+  actionSequence += 1
+  return `${actionType}_${timestamp}_${actionSequence}`
+}
+
+export async function captureActionFingerprint(platform?: 'android' | 'ios', deviceId?: string): Promise<string | null> {
+  if (!platform) return null
+  try {
+    const result = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
+    return result?.fingerprint ?? null
+  } catch {
+    return null
+  }
+}
+
+export function normalizeResolvedTarget(value: Partial<ActionTargetResolved> | null = null): ActionTargetResolved | null {
+  if (!value) return null
+  return {
+    elementId: value.elementId ?? null,
+    text: value.text ?? null,
+    resource_id: value.resource_id ?? null,
+    accessibility_id: value.accessibility_id ?? null,
+    class: value.class ?? null,
+    bounds: value.bounds ?? null,
+    index: value.index ?? null
+  }
+}
+
+export function inferGenericFailure(message: string | undefined): { failureCode: ActionFailureCode; retryable: boolean } {
+  if (message && /timeout/i.test(message)) return { failureCode: 'TIMEOUT', retryable: true }
+  return { failureCode: 'UNKNOWN', retryable: false }
+}
+
+export function inferScrollFailure(message: string | undefined): { failureCode: ActionFailureCode; retryable: boolean } {
+  if (message && /unchanged|no change|end of list/i.test(message)) return { failureCode: 'NAVIGATION_NO_CHANGE', retryable: true }
+  if (message && /timeout/i.test(message)) return { failureCode: 'TIMEOUT', retryable: true }
+  return { failureCode: 'UNKNOWN', retryable: false }
+}
+
+export function buildActionExecutionResult({
+  actionType,
+  selector,
+  resolved,
+  success,
+  uiFingerprintBefore,
+  uiFingerprintAfter,
+  failure
+}: {
+  actionType: string
+  selector: Record<string, unknown> | null
+  resolved?: Partial<ActionTargetResolved> | null
+  success: boolean
+  uiFingerprintBefore: string | null
+  uiFingerprintAfter: string | null
+  failure?: { failureCode: ActionFailureCode; retryable: boolean }
+}): ActionExecutionResult {
+  const timestamp = Date.now()
+  return {
+    action_id: nextActionId(actionType, timestamp),
+    timestamp,
+    action_type: actionType,
+    target: {
+      selector,
+      resolved: normalizeResolvedTarget(resolved)
+    },
+    success,
+    ...(failure ? { failure_code: failure.failureCode, retryable: failure.retryable } : {}),
+    ui_fingerprint_before: uiFingerprintBefore,
+    ui_fingerprint_after: uiFingerprintAfter
+  }
+}