mobile-debug-mcp 0.23.0 → 0.24.0

package/src/types.ts

@@ -132,14 +132,67 @@ export interface TapResponse {
   error?: string;
 }
 
-export interface TapElementResponse {
+export type ActionFailureCode =
+  | 'ELEMENT_NOT_FOUND'
+  | 'ELEMENT_NOT_INTERACTABLE'
+  | 'TIMEOUT'
+  | 'NAVIGATION_NO_CHANGE'
+  | 'AMBIGUOUS_TARGET'
+  | 'STALE_REFERENCE'
+  | 'UNKNOWN'
+
+export interface ActionTargetResolved {
+  elementId: string | null;
+  text: string | null;
+  resource_id: string | null;
+  accessibility_id: string | null;
+  class: string | null;
+  bounds: [number, number, number, number] | null;
+  index: number | null;
+}
+
+export interface ActionExecutionResult {
+  action_id: string;
+  timestamp: number;
+  action_type: string;
+  target: {
+    selector: Record<string, unknown> | null;
+    resolved: ActionTargetResolved | null;
+  };
+  success: boolean;
+  failure_code?: ActionFailureCode;
+  retryable?: boolean;
+  ui_fingerprint_before?: string | null;
+  ui_fingerprint_after?: string | null;
+}
+
+export interface TapElementResponse extends ActionExecutionResult {}
+
+export interface ExpectScreenResponse {
+  success: boolean;
+  observed_screen: {
+    fingerprint: string | null;
+    screen: string | null;
+  };
+  expected_screen: {
+    fingerprint: string | null;
+    screen: string | null;
+  };
+  confidence: number;
+}
+
+export interface ExpectElementVisibleResponse {
   success: boolean;
-  elementId: string;
-  action: 'tap';
-  error?: {
-    code: 'element_not_found' | 'element_not_visible' | 'element_not_enabled' | 'tap_failed';
-    message: string;
+  selector: {
+    text?: string;
+    resource_id?: string;
+    accessibility_id?: string;
+    contains?: boolean;
   };
+  element_id: string | null;
+  element?: ActionTargetResolved | null;
+  failure_code?: 'TIMEOUT' | 'ELEMENT_NOT_FOUND' | 'UNKNOWN';
+  retryable?: boolean;
 }
 
 export interface SwipeResponse {

package/test/unit/interact/expect_tools.test.ts

@@ -0,0 +1,77 @@
+import assert from 'assert'
+import { ToolsInteract } from '../../../src/interact/index.js'
+import * as Observe from '../../../src/observe/index.js'
+
+async function run() {
+  console.log('Starting expect_* unit tests...')
+  const originalGetScreenFingerprintHandler = (Observe as any).ToolsObserve.getScreenFingerprintHandler
+  const originalGetCurrentScreenHandler = (Observe as any).ToolsObserve.getCurrentScreenHandler
+  const originalWaitForUIHandler = (ToolsInteract as any).waitForUIHandler
+
+  try {
+    ;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => ({ fingerprint: 'fp_home', activity: 'com.example.HomeActivity' })
+    let expectScreen = await ToolsInteract.expectScreenHandler({ platform: 'android', fingerprint: 'fp_home' })
+    assert.deepStrictEqual(expectScreen, {
+      success: true,
+      observed_screen: { fingerprint: 'fp_home', screen: 'com.example.HomeActivity' },
+      expected_screen: { fingerprint: 'fp_home', screen: null },
+      confidence: 1
+    })
+
+    ;(Observe as any).ToolsObserve.getCurrentScreenHandler = async () => ({
+      activity: 'com.example.HomeActivity',
+      shortActivity: 'HomeActivity'
+    })
+    expectScreen = await ToolsInteract.expectScreenHandler({ platform: 'android', screen: 'HomeActivity' })
+    assert.strictEqual(expectScreen.success, true)
+    assert.strictEqual(expectScreen.observed_screen.screen, 'HomeActivity')
+    assert.strictEqual(expectScreen.confidence, 1)
+
+    ;(ToolsInteract as any).waitForUIHandler = async () => ({
+      status: 'success',
+      element: {
+        text: 'Ready',
+        resource_id: 'rid_ready',
+        accessibility_id: null,
+        class: 'TextView',
+        bounds: [0, 0, 10, 10],
+        index: 0,
+        elementId: 'el_ready'
+      }
+    })
+    const expectElementVisible = await ToolsInteract.expectElementVisibleHandler({
+      selector: { text: 'Ready' },
+      platform: 'android'
+    })
+    assert.strictEqual(expectElementVisible.success, true)
+    assert.strictEqual(expectElementVisible.element_id, 'el_ready')
+    assert.strictEqual(expectElementVisible.element?.resource_id, 'rid_ready')
+
+    ;(ToolsInteract as any).waitForUIHandler = async () => ({
+      status: 'timeout',
+      error: { code: 'ELEMENT_NOT_FOUND', message: 'Condition visible not satisfied within timeout' }
+    })
+    const timeoutResult = await ToolsInteract.expectElementVisibleHandler({
+      selector: { text: 'Missing' },
+      platform: 'android'
+    })
+    assert.deepStrictEqual(timeoutResult, {
+      success: false,
+      selector: { text: 'Missing' },
+      element_id: null,
+      failure_code: 'TIMEOUT',
+      retryable: true
+    })
+
+    console.log('expect_* unit tests passed')
+  } finally {
+    ;(Observe as any).ToolsObserve.getScreenFingerprintHandler = originalGetScreenFingerprintHandler
+    ;(Observe as any).ToolsObserve.getCurrentScreenHandler = originalGetCurrentScreenHandler
+    ;(ToolsInteract as any).waitForUIHandler = originalWaitForUIHandler
+  }
+}
+
+run().catch((error) => {
+  console.error(error)
+  process.exit(1)
+})

package/test/unit/interact/tap_element.test.ts

@@ -5,11 +5,18 @@ import * as Observe from '../../../src/observe/index.js'
 async function run() {
   console.log('Starting tap_element unit tests...')
   const originalGetUITreeHandler = (Observe as any).ToolsObserve.getUITreeHandler
+  const originalGetScreenFingerprintHandler = (Observe as any).ToolsObserve.getScreenFingerprintHandler
   const originalTapHandler = (ToolsInteract as any).tapHandler
   const originalComputeElementId = (ToolsInteract as any)._computeElementId
   ;(ToolsInteract as any)._resetResolvedUiElementsForTests()
 
   try {
+    let fingerprintCalls = 0
+    ;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => {
+      fingerprintCalls++
+      return { fingerprint: 'fp_mock' }
+    }
+
     ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
       device: { platform: 'android', id: 'mock-device' },
       elements: [
@@ -34,7 +41,12 @@ async function run() {
     }
 
     const tapSuccess = await ToolsInteract.tapElementHandler({ elementId: successElementId })
-    assert.deepStrictEqual(tapSuccess, { success: true, elementId: successElementId, action: 'tap' })
+    assert.strictEqual(tapSuccess.success, true)
+    assert.strictEqual(tapSuccess.action_type, 'tap_element')
+    assert.strictEqual(tapSuccess.target.selector?.elementId, successElementId)
+    assert.strictEqual(tapSuccess.target.resolved?.elementId, successElementId)
+    assert.strictEqual(tapSuccess.ui_fingerprint_before, 'fp_mock')
+    assert.strictEqual(tapSuccess.ui_fingerprint_after, 'fp_mock')
     assert.deepStrictEqual(tapped, { platform: 'android', x: 10, y: 10, deviceId: 'mock-device' })
 
     ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
@@ -52,7 +64,8 @@ async function run() {
     })
     const hiddenResult = await ToolsInteract.tapElementHandler({ elementId: waitHidden.element.elementId })
     assert.strictEqual(hiddenResult.success, false)
-    assert.strictEqual(hiddenResult.error?.code, 'element_not_visible')
+    assert.strictEqual(hiddenResult.failure_code, 'ELEMENT_NOT_INTERACTABLE')
+    assert.strictEqual(hiddenResult.retryable, true)
 
     ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
       device: { platform: 'android', id: 'mock-device' },
@@ -69,7 +82,7 @@ async function run() {
     })
     const disabledResult = await ToolsInteract.tapElementHandler({ elementId: waitDisabled.element.elementId })
     assert.strictEqual(disabledResult.success, false)
-    assert.strictEqual(disabledResult.error?.code, 'element_not_enabled')
+    assert.strictEqual(disabledResult.failure_code, 'ELEMENT_NOT_INTERACTABLE')
 
     ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
       device: { platform: 'android', id: 'mock-device' },
@@ -77,7 +90,7 @@ async function run() {
     })
     const notFoundResult = await ToolsInteract.tapElementHandler({ elementId: successElementId })
     assert.strictEqual(notFoundResult.success, false)
-    assert.strictEqual(notFoundResult.error?.code, 'element_not_found')
+    assert.strictEqual(notFoundResult.failure_code, 'STALE_REFERENCE')
 
     ;(ToolsInteract as any)._resetResolvedUiElementsForTests()
     const targetIndex = 25
@@ -124,7 +137,7 @@ async function run() {
     })
     const shiftedIndexResult = await ToolsInteract.tapElementHandler({ elementId: indexedWait.element.elementId })
     assert.strictEqual(shiftedIndexResult.success, false)
-    assert.strictEqual(shiftedIndexResult.error?.code, 'element_not_found')
+    assert.strictEqual(shiftedIndexResult.failure_code, 'STALE_REFERENCE')
 
     ;(ToolsInteract as any)._resetResolvedUiElementsForTests()
     const cacheLimit = (ToolsInteract as any)._maxResolvedUiElements as number
@@ -151,15 +164,19 @@ async function run() {
     }
 
     assert.ok(oldestElementId, 'Oldest element ID should be captured')
+    const fingerprintCallsBeforeEvictedTap = fingerprintCalls
     const evictedResult = await ToolsInteract.tapElementHandler({ elementId: oldestElementId as string })
     assert.strictEqual(evictedResult.success, false)
-    assert.strictEqual(evictedResult.error?.code, 'element_not_found')
+    assert.strictEqual(evictedResult.failure_code, 'STALE_REFERENCE')
+    assert.strictEqual(evictedResult.ui_fingerprint_before, null)
+    assert.strictEqual(fingerprintCalls, fingerprintCallsBeforeEvictedTap)
 
     console.log('tap_element unit tests passed')
   } finally {
     ;(ToolsInteract as any)._resetResolvedUiElementsForTests()
     ;(ToolsInteract as any)._computeElementId = originalComputeElementId
     ;(Observe as any).ToolsObserve.getUITreeHandler = originalGetUITreeHandler
+    ;(Observe as any).ToolsObserve.getScreenFingerprintHandler = originalGetScreenFingerprintHandler
     ;(ToolsInteract as any).tapHandler = originalTapHandler
   }
 }

package/test/unit/server/contract.test.ts

@@ -8,6 +8,8 @@ async function run() {
   assert.strictEqual(serverInfo.name, 'mobile-debug-mcp')
   assert.strictEqual(names.length, uniqueNames.size, 'tool names should be unique')
   assert(names.includes('wait_for_ui'))
+  assert(names.includes('expect_screen'))
+  assert(names.includes('expect_element_visible'))
   assert(names.includes('capture_screenshot'))
   assert(names.includes('get_ui_tree'))
   assert(names.includes('tap_element'))
@@ -16,6 +18,14 @@ async function run() {
   assert(waitForUI, 'wait_for_ui should be registered')
   assert.strictEqual((waitForUI as any).inputSchema.properties.timeout_ms.default, 60000)
   assert.strictEqual((waitForUI as any).inputSchema.properties.condition.default, 'exists')
+  assert.match((waitForUI as any).description, /resolve elements/i)
+  assert.match((waitForUI as any).description, /must not be used alone to confirm action success/i)
+  assert.match((waitForUI as any).description, /follow with expect_\*/i)
+
+  const waitForScreenChange = toolDefinitions.find((tool) => tool.name === 'wait_for_screen_change')
+  assert(waitForScreenChange, 'wait_for_screen_change should be registered')
+  assert.match((waitForScreenChange as any).description, /does not verify correctness of the resulting state/i)
+  assert.match((waitForScreenChange as any).description, /follow with expect_screen/i)
 
   const captureDebugSnapshot = toolDefinitions.find((tool) => tool.name === 'capture_debug_snapshot')
   assert(captureDebugSnapshot, 'capture_debug_snapshot should be registered')
@@ -33,6 +43,22 @@ async function run() {
   const tapElement = toolDefinitions.find((tool) => tool.name === 'tap_element')
   assert(tapElement, 'tap_element should be registered')
   assert.deepStrictEqual((tapElement as any).inputSchema.required, ['elementId'])
+  assert.match((tapElement as any).description, /RESOLVE → ACT → WAIT \(if needed\) → EXPECT/)
+  assert.match((tapElement as any).description, /If needed, wait for transition using wait_for_\*/)
+  assert.match((tapElement as any).description, /Verify outcome using expect_\*/)
+
+  const expectScreen = toolDefinitions.find((tool) => tool.name === 'expect_screen')
+  assert(expectScreen, 'expect_screen should be registered')
+  assert.match((expectScreen as any).description, /Primary and authoritative verification tool/i)
+  assert.match((expectScreen as any).description, /final verification step/i)
+  assert.match((expectScreen as any).description, /Returns structured binary success\/failure only/i)
+
+  const expectElementVisible = toolDefinitions.find((tool) => tool.name === 'expect_element_visible')
+  assert(expectElementVisible, 'expect_element_visible should be registered')
+  assert.deepStrictEqual((expectElementVisible as any).inputSchema.required, ['selector'])
+  assert.match((expectElementVisible as any).description, /Primary and authoritative verification tool/i)
+  assert.match((expectElementVisible as any).description, /selector is the primary input/i)
+  assert.match((expectElementVisible as any).description, /Returns structured binary success\/failure only/i)
 
   await assert.rejects(() => handleToolCall('unknown_tool'), /Unknown tool: unknown_tool/)
 

package/test/unit/server/response_shapes.test.ts

@@ -1,6 +1,7 @@
 import assert from 'assert'
 import { handleToolCall } from '../../../src/server-core.js'
 import { ToolsManage } from '../../../src/manage/index.js'
+import { AndroidManage } from '../../../src/manage/index.js'
 import { ToolsInteract } from '../../../src/interact/index.js'
 import { ToolsObserve } from '../../../src/observe/index.js'
 
@@ -8,8 +9,13 @@ async function run() {
   const originalInstallAppHandler = (ToolsManage as any).installAppHandler
   const originalWaitForUIHandler = (ToolsInteract as any).waitForUIHandler
   const originalTapElementHandler = (ToolsInteract as any).tapElementHandler
+  const originalTapHandler = (ToolsInteract as any).tapHandler
+  const originalExpectScreenHandler = (ToolsInteract as any).expectScreenHandler
+  const originalExpectElementVisibleHandler = (ToolsInteract as any).expectElementVisibleHandler
+  const originalStartApp = AndroidManage.prototype.startApp
   const originalCaptureScreenshotHandler = (ToolsObserve as any).captureScreenshotHandler
   const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
+  const originalGetScreenFingerprintHandler = (ToolsObserve as any).getScreenFingerprintHandler
 
   try {
     ;(ToolsManage as any).installAppHandler = async () => ({
@@ -40,16 +46,70 @@ async function run() {
     assert.strictEqual(waitForUIPayload.element.elementId, 'el_ready')
 
     ;(ToolsInteract as any).tapElementHandler = async () => ({
+      action_id: 'tap_element_1',
+      timestamp: 1234567890,
+      action_type: 'tap_element',
+      target: {
+        selector: { elementId: 'el_ready' },
+        resolved: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'Button', bounds: [0, 0, 10, 10], index: 0 }
+      },
       success: true,
-      elementId: 'el_ready',
-      action: 'tap'
+      ui_fingerprint_before: 'fp_before',
+      ui_fingerprint_after: 'fp_after'
     })
 
     const tapElementResponse = await handleToolCall('tap_element', { elementId: 'el_ready' })
     const tapElementPayload = JSON.parse((tapElementResponse as any).content[0].text)
     assert.strictEqual(tapElementPayload.success, true)
-    assert.strictEqual(tapElementPayload.elementId, 'el_ready')
-    assert.strictEqual(tapElementPayload.action, 'tap')
+    assert.strictEqual(tapElementPayload.action_type, 'tap_element')
+    assert.strictEqual(tapElementPayload.target.resolved.elementId, 'el_ready')
+    assert.strictEqual(tapElementPayload.ui_fingerprint_before, 'fp_before')
+
+    ;(ToolsObserve as any).getScreenFingerprintHandler = async () => ({ fingerprint: 'fp_mock', activity: 'MainActivity' })
+    ;(ToolsInteract as any).tapHandler = async () => ({ success: true, x: 1, y: 2 })
+    const tapResponse = await handleToolCall('tap', { platform: 'android', x: 1, y: 2 })
+    const tapPayload = JSON.parse((tapResponse as any).content[0].text)
+    assert.strictEqual(tapPayload.success, true)
+    assert.strictEqual(tapPayload.action_type, 'tap')
+    assert.deepStrictEqual(tapPayload.target.selector, { x: 1, y: 2 })
+    assert.strictEqual(tapPayload.ui_fingerprint_before, 'fp_mock')
+
+    AndroidManage.prototype.startApp = async function () {
+      return {
+        device: { platform: 'android', id: 'emulator-5554', osVersion: '14', model: 'Pixel', simulator: true },
+        appStarted: true,
+        launchTimeMs: 123
+      } as any
+    }
+    const startAppResponse = await handleToolCall('start_app', { platform: 'android', appId: 'com.example.app' })
+    const startAppPayload = JSON.parse((startAppResponse as any).content[0].text)
+    assert.strictEqual(startAppPayload.success, true)
+    assert.strictEqual(startAppPayload.action_type, 'start_app')
+    assert.deepStrictEqual(startAppPayload.target.selector, { appId: 'com.example.app' })
+
+    ;(ToolsInteract as any).expectScreenHandler = async () => ({
+      success: true,
+      observed_screen: { fingerprint: 'fp_after', screen: 'MainActivity' },
+      expected_screen: { fingerprint: 'fp_after', screen: null },
+      confidence: 1
+    })
+
+    const expectScreenResponse = await handleToolCall('expect_screen', { fingerprint: 'fp_after' })
+    const expectScreenPayload = JSON.parse((expectScreenResponse as any).content[0].text)
+    assert.strictEqual(expectScreenPayload.success, true)
+    assert.strictEqual(expectScreenPayload.confidence, 1)
+
+    ;(ToolsInteract as any).expectElementVisibleHandler = async () => ({
+      success: true,
+      selector: { text: 'Ready' },
+      element_id: 'el_ready',
+      element: { elementId: 'el_ready', text: 'Ready', resource_id: null, accessibility_id: null, class: 'TextView', bounds: [0, 0, 10, 10], index: 0 }
+    })
+
+    const expectElementResponse = await handleToolCall('expect_element_visible', { selector: { text: 'Ready' } })
+    const expectElementPayload = JSON.parse((expectElementResponse as any).content[0].text)
+    assert.strictEqual(expectElementPayload.success, true)
+    assert.strictEqual(expectElementPayload.element_id, 'el_ready')
 
     ;(ToolsObserve as any).captureScreenshotHandler = async () => ({
       device: { platform: 'ios', id: 'booted', osVersion: '18.0', model: 'Simulator', simulator: true },
@@ -82,8 +142,13 @@ async function run() {
     ;(ToolsManage as any).installAppHandler = originalInstallAppHandler
     ;(ToolsInteract as any).waitForUIHandler = originalWaitForUIHandler
     ;(ToolsInteract as any).tapElementHandler = originalTapElementHandler
+    ;(ToolsInteract as any).tapHandler = originalTapHandler
+    ;(ToolsInteract as any).expectScreenHandler = originalExpectScreenHandler
+    ;(ToolsInteract as any).expectElementVisibleHandler = originalExpectElementVisibleHandler
+    AndroidManage.prototype.startApp = originalStartApp
     ;(ToolsObserve as any).captureScreenshotHandler = originalCaptureScreenshotHandler
     ;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler
+    ;(ToolsObserve as any).getScreenFingerprintHandler = originalGetScreenFingerprintHandler
   }
 }