mobile-debug-mcp 0.26.3 → 0.26.4

package/src/server/tool-handlers.ts

@@ -277,6 +277,31 @@ async function handleExpectState(args: ToolCallArgs) {
   return wrapResponse(res)
 }
 
+async function handleAdjustControl(args: ToolCallArgs) {
+  const selector = getObjectArg<ExpectElementSelectorArg>(args, 'selector')
+  const element_id = getStringArg(args, 'element_id')
+  const property = getStringArg(args, 'property') ?? 'value'
+  const targetValue = requireNumberArg(args, 'targetValue')
+  const tolerance = getNumberArg(args, 'tolerance') ?? 0
+  const maxAttempts = getNumberArg(args, 'maxAttempts') ?? 3
+  const platform = getStringArg(args, 'platform') as PlatformArg | undefined
+  const deviceId = getStringArg(args, 'deviceId')
+  if (!selector && !element_id) {
+    throw new Error('Missing selector or element_id argument')
+  }
+  const res = await ToolsInteract.adjustControlHandler({
+    selector: selector ?? undefined,
+    element_id: element_id ?? undefined,
+    property,
+    targetValue,
+    tolerance,
+    maxAttempts,
+    platform,
+    deviceId
+  })
+  return wrapResponse(res)
+}
+
 async function handleWaitForUI(args: ToolCallArgs) {
   const selector = getObjectArg<ExpectElementSelectorArg>(args, 'selector')
   const condition = (getStringArg(args, 'condition') as 'exists' | 'not_exists' | 'visible' | 'clickable' | undefined) ?? 'exists'
@@ -496,6 +521,7 @@ export const toolHandlers: Record<string, ToolHandler> = {
   expect_screen: handleExpectScreen,
   expect_element_visible: handleExpectElementVisible,
   expect_state: handleExpectState,
+  adjust_control: handleAdjustControl,
   wait_for_ui: handleWaitForUI,
   find_element: handleFindElement,
   tap: handleTap,

package/src/server-core.ts

@@ -13,7 +13,7 @@ export { wrapResponse, toolDefinitions, handleToolCall }
 
 export const serverInfo = {
   name: 'mobile-debug-mcp',
-  version: '0.26.3'
+  version: '0.26.4'
 }
 
 export function createServer() {

package/src/types.ts

@@ -413,6 +413,23 @@ export interface ExpectStateResponse {
   retryable?: boolean;
 }
 
+export interface AdjustControlResponse extends ActionExecutionResult {
+  target_state: {
+    property: string;
+    target_value: number;
+    tolerance: number;
+  };
+  actual_state: {
+    property: string;
+    value: number | null;
+    raw_value?: number | null;
+  } | null;
+  within_tolerance: boolean;
+  converged: boolean;
+  attempts: number;
+  adjustment_mode: 'semantic' | 'gesture' | 'coordinate';
+}
+
 export interface WaitForUIChangeResponse {
   success: boolean;
   observed_change: 'hierarchy_diff' | 'text_change' | 'state_change' | null;

package/src/utils/android/utils.ts

@@ -341,7 +341,7 @@ function normalizeClassName(value: unknown): string {
 }
 
 function inferAndroidRole(className: string): string | null {
-  if (/seekbar|slider|progress/.test(className)) return 'slider'
+  if (/seekbar|slider/.test(className)) return 'slider'
   if (/switch|toggle/.test(className)) return 'switch'
   if (/checkbox/.test(className)) return 'checkbox'
   if (/radiobutton|radio/.test(className)) return 'radio'
@@ -384,7 +384,7 @@ function buildAndroidSemantic(clickable: boolean, className: string): UIElementS
 
 function isSliderLikeAndroid(node: any): boolean {
   const className = String(node['@_class'] || '').toLowerCase()
-  return /seekbar|slider|range|progress/i.test(className)
+  return /seekbar|slider|range/i.test(className)
 }
 
 function extractAndroidState(node: any): UIElementState | null {

package/test/unit/interact/adjust_control.test.ts

@@ -0,0 +1,365 @@
+import assert from 'assert'
+import { ToolsInteract } from '../../../src/interact/index.js'
+import * as Observe from '../../../src/observe/index.js'
+
+async function run() {
+  console.log('Starting adjust_control unit tests...')
+
+    const originalGetUITreeHandler = (Observe as any).ToolsObserve.getUITreeHandler
+    const originalGetScreenFingerprintHandler = (Observe as any).ToolsObserve.getScreenFingerprintHandler
+    const originalTapHandler = (ToolsInteract as any).tapHandler
+    const originalSwipeHandler = (ToolsInteract as any).swipeHandler
+    const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
+
+  try {
+    ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+      screen: '',
+      resolution: { width: 1080, height: 2400 },
+      elements: [
+        {
+          text: 'Duration',
+          type: 'android.widget.SeekBar',
+          contentDescription: null,
+          clickable: true,
+          enabled: true,
+          visible: true,
+          bounds: [0, 0, 200, 40],
+          resourceId: 'seek_duration',
+          state: {
+            value: 10,
+            raw_value: 10,
+            value_range: { min: 0, max: 100 }
+          }
+        }
+      ]
+    })
+
+    ;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => ({ fingerprint: 'fp_slider', activity: 'MainActivity' })
+
+    const wait = await ToolsInteract.waitForUIHandler({
+      selector: { text: 'Duration' },
+      condition: 'clickable',
+      timeout_ms: 200,
+      poll_interval_ms: 50,
+      platform: 'android'
+    })
+    assert.strictEqual(wait.status, 'success')
+    assert.ok(wait.element?.elementId)
+
+    const tapCalls: Array<{ platform?: string, x: number, y: number, deviceId?: string }> = []
+    const swipeCalls: Array<{ platform?: string, x1: number, y1: number, x2: number, y2: number, duration: number, deviceId?: string }> = []
+    ;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
+      tapCalls.push({ platform, x, y, deviceId })
+      return {
+        device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+        success: true,
+        x,
+        y
+      }
+    }
+    ;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
+      swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
+      return {
+        device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+        success: true,
+        start: [x1, y1],
+        end: [x2, y2],
+        duration
+      }
+    }
+
+    ;(ToolsInteract as any).expectStateHandler = async () => ({
+      success: true,
+      selector: { text: 'Duration' },
+      element_id: wait.element.elementId,
+      expected_state: { property: 'value', expected: 30 },
+      element: {
+        elementId: wait.element.elementId,
+        text: 'Duration',
+        resource_id: 'seek_duration',
+        accessibility_id: null,
+        class: 'android.widget.SeekBar',
+        bounds: [0, 0, 200, 40],
+        index: 0,
+        state: { value: 30, raw_value: 30, value_range: { min: 0, max: 100 } }
+      },
+      observed_state: { property: 'value', value: 30, raw_value: 30 },
+      reason: 'value matches expected value'
+    })
+
+    const adjust = await ToolsInteract.adjustControlHandler({
+      element_id: wait.element.elementId,
+      property: 'value',
+      targetValue: 30,
+      tolerance: 0.5,
+      maxAttempts: 2,
+      platform: 'android'
+    })
+
+    assert.strictEqual(adjust.success, true)
+    assert.strictEqual(adjust.converged, true)
+    assert.strictEqual(adjust.within_tolerance, true)
+    assert.strictEqual(adjust.adjustment_mode, 'coordinate')
+    assert.strictEqual(adjust.target_state.target_value, 30)
+    assert.strictEqual(adjust.attempts, 1)
+    assert.strictEqual(tapCalls.length, 1)
+    assert.strictEqual(swipeCalls.length, 0)
+    assert.ok(tapCalls[0].x <= 66, 'tap should bias inward from the exact target point')
+    assert.strictEqual(adjust.action_type, 'adjust_control')
+    assert.strictEqual(adjust.target.selector.elementId, wait.element.elementId)
+
+    ;(ToolsInteract as any).expectStateHandler = async () => ({
+      success: true,
+      selector: { text: 'Duration' },
+      element_id: wait.element.elementId,
+      expected_state: { property: 'value', expected: 2 },
+      element: {
+        elementId: wait.element.elementId,
+        text: 'Duration',
+        resource_id: 'seek_duration',
+        accessibility_id: null,
+        class: 'android.widget.SeekBar',
+        bounds: [0, 0, 200, 40],
+        index: 0,
+        state: { value: 2, raw_value: 2, value_range: { min: 0, max: 100 } }
+      },
+      observed_state: { property: 'value', value: 2, raw_value: 2 },
+      reason: 'value matches expected value'
+    })
+
+    const lowEndAdjust = await ToolsInteract.adjustControlHandler({
+      element_id: wait.element.elementId,
+      property: 'value',
+      targetValue: 2,
+      tolerance: 0.5,
+      maxAttempts: 2,
+      platform: 'android'
+    })
+
+    assert.strictEqual(lowEndAdjust.success, true)
+    assert.strictEqual(lowEndAdjust.converged, true)
+    assert.strictEqual(lowEndAdjust.within_tolerance, true)
+    assert.strictEqual(lowEndAdjust.attempts, 1)
+    assert.strictEqual(tapCalls.length, 2)
+    assert.strictEqual(swipeCalls.length, 0)
+    assert.ok(tapCalls[1].x >= 22, 'low-end tap should stay inside the first step instead of hugging the edge')
+
+    ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+      screen: '',
+      resolution: { width: 1080, height: 2400 },
+      elements: [
+        {
+          text: 'Duration',
+          type: 'android.widget.SeekBar',
+          contentDescription: null,
+          clickable: true,
+          enabled: true,
+          visible: true,
+          bounds: [0, 0, 200, 40],
+          resourceId: 'seek_duration',
+          state: {
+            value: 18,
+            raw_value: 18,
+            value_range: { min: 0, max: 20 }
+          }
+        }
+      ]
+    })
+
+    ;(ToolsInteract as any).expectStateHandler = async () => ({
+      success: true,
+      selector: { text: 'Duration' },
+      element_id: wait.element.elementId,
+      expected_state: { property: 'value', expected: 20 },
+      element: {
+        elementId: wait.element.elementId,
+        text: 'Duration',
+        resource_id: 'seek_duration',
+        accessibility_id: null,
+        class: 'android.widget.SeekBar',
+        bounds: [0, 0, 200, 40],
+        index: 0,
+        state: { value: 20, raw_value: 20, value_range: { min: 0, max: 20 } }
+      },
+      observed_state: { property: 'value', value: 20, raw_value: 20 },
+      reason: 'value matches expected value'
+    })
+
+    const highEndAdjust = await ToolsInteract.adjustControlHandler({
+      element_id: wait.element.elementId,
+      property: 'value',
+      targetValue: 20,
+      tolerance: 0.5,
+      maxAttempts: 2,
+      platform: 'android'
+    })
+
+    assert.strictEqual(highEndAdjust.success, true)
+    assert.strictEqual(highEndAdjust.converged, true)
+    assert.strictEqual(highEndAdjust.within_tolerance, true)
+    assert.strictEqual(highEndAdjust.attempts, 1)
+    assert.strictEqual(tapCalls.length, 3)
+    assert.strictEqual(swipeCalls.length, 0)
+    assert.ok(tapCalls[2].x >= 180, 'high-end tap should bias into the last step without hitting the edge')
+
+    ;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+      screen: '',
+      resolution: { width: 1440, height: 3200 },
+      elements: [
+        {
+          text: 'Precision',
+          type: 'android.widget.SeekBar',
+          contentDescription: null,
+          clickable: true,
+          enabled: true,
+          visible: true,
+          bounds: [0, 0, 3000, 40],
+          resourceId: 'seek_precision',
+          state: {
+            value: 9000,
+            raw_value: 9000,
+            value_range: { min: 0, max: 10000 }
+          }
+        }
+      ]
+    })
+
+    ;(ToolsInteract as any).expectStateHandler = async () => ({
+      success: true,
+      selector: { text: 'Precision' },
+      element_id: wait.element.elementId,
+      expected_state: { property: 'value', expected: 9999 },
+      element: {
+        elementId: wait.element.elementId,
+        text: 'Precision',
+        resource_id: 'seek_precision',
+        accessibility_id: null,
+        class: 'android.widget.SeekBar',
+        bounds: [0, 0, 3000, 40],
+        index: 0,
+        state: { value: 9999, raw_value: 9999, value_range: { min: 0, max: 10000 } }
+      },
+      observed_state: { property: 'value', value: 9999, raw_value: 9999 },
+      reason: 'value matches expected value'
+    })
+
+    const precisionAdjust = await ToolsInteract.adjustControlHandler({
+      selector: { text: 'Precision' },
+      property: 'value',
+      targetValue: 9999,
+      tolerance: 0.5,
+      maxAttempts: 2,
+      platform: 'android'
+    })
+
+    assert.strictEqual(precisionAdjust.success, true)
+    assert.strictEqual(precisionAdjust.converged, true)
+    assert.strictEqual(precisionAdjust.within_tolerance, true)
+    assert.strictEqual(precisionAdjust.attempts, 1)
+    assert.strictEqual(tapCalls.length, 4)
+    assert.strictEqual(swipeCalls.length, 0)
+    assert.ok(tapCalls[3].x > 2750, 'wide, high-range control should not be clamped to a 3% endpoint margin')
+
+    let treeFetches = 0
+    let retryVerificationCount = 0
+    ;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
+      treeFetches++
+      return {
+        device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+        screen: '',
+        resolution: { width: 1080, height: 2400 },
+        elements: [
+          {
+            text: 'Duration',
+            type: 'android.widget.SeekBar',
+            contentDescription: null,
+            clickable: true,
+            enabled: true,
+            visible: true,
+            bounds: [0, 0, 200, 40],
+            resourceId: 'seek_duration',
+            state: {
+              value: 10,
+              raw_value: 10,
+              value_range: { min: 0, max: 20 }
+            }
+          }
+        ]
+      }
+    }
+
+    ;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
+      tapCalls.push({ platform, x, y, deviceId })
+      return {
+        device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+        success: false,
+        error: 'tap failed'
+      }
+    }
+
+    ;(ToolsInteract as any).swipeHandler = async ({ platform, x1, y1, x2, y2, duration, deviceId }: any) => {
+      swipeCalls.push({ platform, x1, y1, x2, y2, duration, deviceId })
+      return {
+        device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
+        success: true,
+        start: [x1, y1],
+        end: [x2, y2],
+        duration
+      }
+    }
+
+    ;(ToolsInteract as any).expectStateHandler = async () => {
+      retryVerificationCount++
+      const value = retryVerificationCount === 1 ? 11 : 12
+      return {
+        success: true,
+        selector: { text: 'Duration' },
+        element_id: wait.element.elementId,
+        expected_state: { property: 'value', expected: 12 },
+        element: {
+          elementId: wait.element.elementId,
+          text: 'Duration',
+          resource_id: 'seek_duration',
+          accessibility_id: null,
+          class: 'android.widget.SeekBar',
+          bounds: [0, 0, 200, 40],
+          index: 0,
+          state: { value, raw_value: value, value_range: { min: 0, max: 20 } }
+        },
+        observed_state: { property: 'value', value, raw_value: value },
+        reason: value === 12 ? 'value matches expected value' : 'value still below target'
+      }
+    }
+
+    const cachedResolveAdjust = await ToolsInteract.adjustControlHandler({
+      element_id: wait.element.elementId,
+      property: 'value',
+      targetValue: 12,
+      tolerance: 0.5,
+      maxAttempts: 2,
+      platform: 'android'
+    })
+
+    assert.strictEqual(cachedResolveAdjust.success, true)
+    assert.strictEqual(cachedResolveAdjust.converged, true)
+    assert.strictEqual(cachedResolveAdjust.within_tolerance, true)
+    assert.strictEqual(cachedResolveAdjust.attempts, 3)
+    assert.strictEqual(treeFetches, 1, 'second attempt should reuse the resolved element instead of refetching the UI tree')
+
+    console.log('adjust_control unit tests passed')
+  } finally {
+    ;(Observe as any).ToolsObserve.getUITreeHandler = originalGetUITreeHandler
+    ;(Observe as any).ToolsObserve.getScreenFingerprintHandler = originalGetScreenFingerprintHandler
+    ;(ToolsInteract as any).tapHandler = originalTapHandler
+    ;(ToolsInteract as any).swipeHandler = originalSwipeHandler
+    ;(ToolsInteract as any).expectStateHandler = originalExpectStateHandler
+  }
+}
+
+run().catch((error) => {
+  console.error(error)
+  process.exit(1)
+})

package/test/unit/observe/state_extraction.test.ts

@@ -31,6 +31,20 @@ async function run() {
   })
   assert.deepStrictEqual(androidElements[0].semantic, { is_clickable: true, is_container: false })
 
+  const androidProgressElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.ProgressBar',
+    '@_text': 'Loading progress',
+    '@_content-desc': 'Loading progress',
+    '@_enabled': 'true',
+    '@_progress': '40',
+    '@_max': '100',
+    '@_bounds': '[0,0][200,40]'
+  }, androidProgressElements)
+
+  assert.notStrictEqual(androidProgressElements[0]?.role, 'slider')
+  assert.notStrictEqual(androidProgressElements[0]?.state?.value, 40)
+
   const androidFallbackElements: any[] = []
   traverseNode({
     '@_class': 'android.widget.Button',
@@ -70,6 +84,16 @@ async function run() {
   })
   assert.deepStrictEqual(iosElements[0].semantic, { is_clickable: true, is_container: false })
 
+  const iosProgressElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'ProgressIndicator',
+    AXLabel: 'Loading progress',
+    AXValue: '0.4',
+    AXTraits: ['UIAccessibilityTraitUpdatesFrequently']
+  }, iosProgressElements)
+
+  assert.notStrictEqual(iosProgressElements[0]?.role, 'slider')
+
   const iosFallbackElements: any[] = []
   traverseIDBNode({
     AXElementType: 'Button',

package/test/unit/server/contract.test.ts

@@ -16,6 +16,7 @@ async function run() {
   assert(names.includes('capture_screenshot'))
   assert(names.includes('get_ui_tree'))
   assert(names.includes('tap_element'))
+  assert(names.includes('adjust_control'))
 
   const waitForUI = toolDefinitions.find((tool) => tool.name === 'wait_for_ui')
   assert(waitForUI, 'wait_for_ui should be registered')
@@ -66,6 +67,13 @@ async function run() {
   assert.match((expectElementVisible as any).description, /selector is the primary input/i)
   assert.match((expectElementVisible as any).description, /Returns structured binary success\/failure only/i)
 
+  const adjustControl = toolDefinitions.find((tool) => tool.name === 'adjust_control')
+  assert(adjustControl, 'adjust_control should be registered')
+  assert.deepStrictEqual((adjustControl as any).inputSchema.required, ['targetValue'])
+  assert.strictEqual((adjustControl as any).inputSchema.properties.targetValue.type, 'number')
+  assert.match((adjustControl as any).description, /numeric control value/i)
+  assert.match((adjustControl as any).description, /expect_state/i)
+
   const classifyActionOutcome = toolDefinitions.find((tool) => tool.name === 'classify_action_outcome')
   assert(classifyActionOutcome, 'classify_action_outcome should be registered')
   assert.match((classifyActionOutcome as any).description, /action_type/i)

package/test/unit/server/response_shapes.test.ts

@@ -14,6 +14,7 @@ async function run() {
   const originalExpectScreenHandler = (ToolsInteract as any).expectScreenHandler
   const originalExpectElementVisibleHandler = (ToolsInteract as any).expectElementVisibleHandler
   const originalExpectStateHandler = (ToolsInteract as any).expectStateHandler
+  const originalAdjustControlHandler = (ToolsInteract as any).adjustControlHandler
   const originalStartApp = AndroidManage.prototype.startApp
   const originalCaptureScreenshotHandler = (ToolsObserve as any).captureScreenshotHandler
   const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
@@ -191,6 +192,43 @@ async function run() {
     assert.strictEqual(expectStatePayload.expected_state.property, 'checked')
     assert.strictEqual(expectStatePayload.observed_state.value, true)
 
+    ;(ToolsInteract as any).adjustControlHandler = async () => ({
+      action_id: 'adjust_control_1',
+      timestamp: '2026-04-29T08:00:00.000Z',
+      action_type: 'adjust_control',
+      lifecycle_state: 'pending_verification',
+      source_module: 'interact',
+      target: {
+        selector: { elementId: 'el_duration' },
+        resolved: {
+          elementId: 'el_duration',
+          text: 'Duration',
+          resource_id: null,
+          accessibility_id: null,
+          class: 'android.view.View',
+          bounds: [0, 0, 100, 20],
+          index: 0
+        }
+      },
+      success: true,
+      ui_fingerprint_before: 'fp_before',
+      ui_fingerprint_after: 'fp_after',
+      target_state: { property: 'value', target_value: 30, tolerance: 0.5 },
+      actual_state: { property: 'value', value: 30, raw_value: 30 },
+      within_tolerance: true,
+      converged: true,
+      attempts: 1,
+      adjustment_mode: 'semantic'
+    })
+
+    const adjustControlResponse = await handleToolCall('adjust_control', { element_id: 'el_duration', targetValue: 30, tolerance: 0.5, property: 'value' })
+    const adjustControlPayload = JSON.parse((adjustControlResponse as any).content[0].text)
+    assert.strictEqual(adjustControlPayload.success, true)
+    assert.strictEqual(adjustControlPayload.action_type, 'adjust_control')
+    assert.strictEqual(adjustControlPayload.target_state.target_value, 30)
+    assert.strictEqual(adjustControlPayload.within_tolerance, true)
+    assert.strictEqual(adjustControlPayload.converged, true)
+
     ;(ToolsInteract as any).tapHandler = async () => {
       throw new Error('boom')
     }
@@ -319,6 +357,7 @@ async function run() {
     ;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler
     ;(ToolsObserve as any).getScreenFingerprintHandler = originalGetScreenFingerprintHandler
     ;(ToolsObserve as any).captureDebugSnapshotHandler = originalCaptureDebugSnapshotHandler
+    ;(ToolsInteract as any).adjustControlHandler = originalAdjustControlHandler
   }
 }