mobile-debug-mcp 0.24.7 → 0.24.8

package/README.md

@@ -3,11 +3,11 @@
 A minimal, secure MCP server for AI-assisted mobile development. Build, install, and inspect Android/iOS apps from an MCP-compatible client.
 
 > **Support:**
-> * Android support
-> * iOS only tested on simulator
-> * KMP support
-> * Flutter iOS projects not fetching logs
-> * React native not tested
+> * KMP
+> * Android
+> * iOS
+> * Flutter - not tested
+> * React native - not tested
 
 ## Requirements
 

package/dist/interact/index.js

@@ -7,6 +7,14 @@ import { ToolsObserve } from '../observe/index.js';
 import { nextActionId } from '../server/common.js';
 export class ToolsInteract {
     static _maxResolvedUiElements = 256;
+    static _sliderSearchLookahead = 8;
+    static _sliderNegativeGapTolerancePx = 32;
+    static _sliderPositiveGapLimitPx = 640;
+    static _sliderTrackMinLengthPx = 220;
+    static _sliderTrackMaxThicknessPx = 180;
+    static _sliderTrackLengthRatio = 0.18;
+    static _sliderTrackThicknessRatio = 0.08;
+    static _sliderLabelWidthRatio = 1.5;
     static _resolvedUiElements = new Map();
     static _normalize(s) {
         if (s === null || s === undefined)
@@ -174,6 +182,63 @@ export class ToolsInteract {
         }
         return best;
     }
+    static _resolveNearbyActionableControl(elements, chosen, screen) {
+        if (!chosen)
+            return null;
+        const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
+        if (!labelBounds)
+            return null;
+        const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds;
+        const labelWidth = labelRight - labelLeft;
+        const labelHeight = labelBottom - labelTop;
+        const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0;
+        const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0;
+        const minTrackLengthPx = Math.max(ToolsInteract._sliderTrackMinLengthPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0);
+        const maxTrackThicknessPx = Math.max(ToolsInteract._sliderTrackMaxThicknessPx, screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0, screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0);
+        let best = null;
+        let bestScore = Infinity;
+        for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
+            const candidate = elements[i];
+            if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false)
+                continue;
+            const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds);
+            if (!candidateBounds)
+                continue;
+            const [left, top, right] = candidateBounds;
+            const width = right - left;
+            const height = candidateBounds[3] - top;
+            const verticalGap = top - labelBottom;
+            if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx)
+                continue;
+            const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left);
+            if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx)
+                continue;
+            const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '');
+            const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '');
+            const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '');
+            let score = verticalGap;
+            const horizontalTrackLike = width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
+                height <= maxTrackThicknessPx;
+            const verticalTrackLike = height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
+                width <= maxTrackThicknessPx;
+            const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike;
+            if (!candidateText && !candidateContent)
+                score -= 18;
+            if (trackLike)
+                score -= 30;
+            if (/view|layout|group|frame/i.test(candidateClass))
+                score -= 10;
+            if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio)
+                score -= 8;
+            if (candidateText || candidateContent)
+                score += 20;
+            if (score < bestScore) {
+                bestScore = score;
+                best = { el: candidate, idx: i, sliderLike: trackLike };
+            }
+        }
+        return best;
+    }
     static async getInteractionService(platform, deviceId) {
         const effectivePlatform = platform || 'android';
         const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
@@ -262,6 +327,7 @@ export class ToolsInteract {
             return { found: false, error: 'Empty query' };
         let best = null;
         let bestScore = 0;
+        let lastTree = null;
         const scoreElement = (el) => {
             if (!el || !el.visible)
                 return 0;
@@ -305,6 +371,7 @@ export class ToolsInteract {
         while (Date.now() <= deadline) {
             try {
                 const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
+                lastTree = tree;
                 if (tree && Array.isArray(tree.elements)) {
                     const elements = tree.elements;
                     for (let i = 0; i < elements.length; i++) {
@@ -342,8 +409,8 @@ export class ToolsInteract {
             return { found: false, error: 'Element not found' };
         // If the best match is not interactable, try to resolve an actionable ancestor.
         try {
-            const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
-            const elements = (tree && Array.isArray(tree.elements)) ? tree.elements : [];
+            const elements = (lastTree && Array.isArray(lastTree.elements)) ? lastTree.elements : [];
+            const screen = lastTree?.resolution && typeof lastTree.resolution === 'object' ? lastTree.resolution : null;
             let chosen = best;
             const childBounds = Array.isArray(chosen?.bounds) ? chosen.bounds : null;
             // Strategy 1: if parentId references an index, climb that chain
@@ -412,6 +479,15 @@ export class ToolsInteract {
                 // small score bump to reflect actionability
                 bestScore = Math.min(1, bestScore + 0.02);
             }
+            if (best && !(best.clickable || best.focusable)) {
+                const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best, idx: best._index ?? elements.indexOf(best) }, screen);
+                if (nearbyActionable) {
+                    best = nearbyActionable.el;
+                    best._index = nearbyActionable.idx;
+                    best._interactable = true;
+                    best._sliderLike = nearbyActionable.sliderLike;
+                }
+            }
         }
         catch (e) {
             console.error('Error resolving ancestor:', e);
@@ -431,9 +507,19 @@ export class ToolsInteract {
             tapCoordinates,
             telemetry: {
                 matchedIndex: best?._index ?? null,
-                matchedInteractable: !!best?._interactable
+                matchedInteractable: !!best?._interactable,
+                sliderLike: !!best?._sliderLike
             }
         };
+        if (best?._sliderLike) {
+            const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left);
+            const interactionHint = {
+                kind: 'slider',
+                axis: isVertical ? 'vertical' : 'horizontal',
+                trackBounds: boundsObj
+            };
+            outEl.interactionHint = interactionHint;
+        }
         const scoreVal = Math.min(1, Number(bestScore.toFixed(3)));
         return { found: true, element: outEl, score: scoreVal, confidence: scoreVal };
     }

package/dist/server-core.js

@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
 export { wrapResponse, toolDefinitions, handleToolCall };
 export const serverInfo = {
     name: 'mobile-debug-mcp',
-    version: '0.24.7'
+    version: '0.24.8'
 };
 export function createServer() {
     const server = new Server(serverInfo, {

package/docs/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 All notable changes to the **Mobile Debug MCP** project will be documented in this file.
 
+## [0.24.8]
+- Improved slider interaction
+
 ## [0.24.7]
 - Aligned runtime metadata with the published package version.
 - Fixed stale CLI helper paths in npm scripts and the `idb` healthcheck helper.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobile-debug-mcp",
-  "version": "0.24.7",
+  "version": "0.24.8",
   "description": "MCP server for mobile app debugging (Android + iOS), with focus on security and reliability",
   "type": "module",
   "bin": {

package/src/interact/index.ts

@@ -36,6 +36,7 @@ interface UiElement {
   parentId?: number | string | null
   _index?: number
   _interactable?: boolean
+  _sliderLike?: boolean
 }
 
 interface ResolvedUiElementContext {
@@ -46,9 +47,22 @@ interface ResolvedUiElementContext {
   index: number
 }
 
+interface UiResolution {
+  width?: number
+  height?: number
+}
+
 
 export class ToolsInteract {
   private static readonly _maxResolvedUiElements = 256
+  private static readonly _sliderSearchLookahead = 8
+  private static readonly _sliderNegativeGapTolerancePx = 32
+  private static readonly _sliderPositiveGapLimitPx = 640
+  private static readonly _sliderTrackMinLengthPx = 220
+  private static readonly _sliderTrackMaxThicknessPx = 180
+  private static readonly _sliderTrackLengthRatio = 0.18
+  private static readonly _sliderTrackThicknessRatio = 0.08
+  private static readonly _sliderLabelWidthRatio = 1.5
   private static _resolvedUiElements = new Map<string, ResolvedUiElementContext>()
 
   private static _normalize(s: any): string {
@@ -240,6 +254,78 @@ export class ToolsInteract {
     return best
   }
 
+  private static _resolveNearbyActionableControl(
+    elements: UiElement[],
+    chosen: { el: UiElement, idx: number } | null,
+    screen?: UiResolution | null
+  ): { el: UiElement, idx: number, sliderLike?: boolean } | null {
+    if (!chosen) return null
+
+    const labelBounds = ToolsInteract._normalizeBounds(chosen.el.bounds)
+    if (!labelBounds) return null
+
+    const [labelLeft, labelTop, labelRight, labelBottom] = labelBounds
+    const labelWidth = labelRight - labelLeft
+    const labelHeight = labelBottom - labelTop
+    const screenWidth = Number(screen?.width) > 0 ? Number(screen?.width) : 0
+    const screenHeight = Number(screen?.height) > 0 ? Number(screen?.height) : 0
+    const minTrackLengthPx = Math.max(
+      ToolsInteract._sliderTrackMinLengthPx,
+      screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackLengthRatio) : 0,
+      screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackLengthRatio) : 0
+    )
+    const maxTrackThicknessPx = Math.max(
+      ToolsInteract._sliderTrackMaxThicknessPx,
+      screenWidth > 0 ? Math.floor(screenWidth * ToolsInteract._sliderTrackThicknessRatio) : 0,
+      screenHeight > 0 ? Math.floor(screenHeight * ToolsInteract._sliderTrackThicknessRatio) : 0
+    )
+
+    let best: { el: UiElement, idx: number, sliderLike?: boolean } | null = null
+    let bestScore = Infinity
+
+    for (let i = chosen.idx + 1; i < Math.min(elements.length, chosen.idx + ToolsInteract._sliderSearchLookahead); i++) {
+      const candidate = elements[i]
+      if (!candidate || !(candidate.clickable || candidate.focusable) || candidate.visible === false) continue
+
+      const candidateBounds = ToolsInteract._normalizeBounds(candidate.bounds)
+      if (!candidateBounds) continue
+
+      const [left, top, right] = candidateBounds
+      const width = right - left
+      const height = candidateBounds[3] - top
+      const verticalGap = top - labelBottom
+      if (verticalGap < -ToolsInteract._sliderNegativeGapTolerancePx || verticalGap > ToolsInteract._sliderPositiveGapLimitPx) continue
+
+      const horizontalOverlap = Math.min(labelRight, right) - Math.max(labelLeft, left)
+      if (horizontalOverlap < -ToolsInteract._sliderNegativeGapTolerancePx) continue
+
+      const candidateText = ToolsInteract._normalize(candidate.text ?? candidate.label ?? candidate.value ?? '')
+      const candidateContent = ToolsInteract._normalize(candidate.contentDescription ?? candidate.contentDesc ?? candidate.accessibilityLabel ?? '')
+      const candidateClass = ToolsInteract._normalize(candidate.type ?? candidate.class ?? '')
+
+      let score = verticalGap
+      const horizontalTrackLike =
+        width >= Math.max(minTrackLengthPx, Math.floor(labelWidth * ToolsInteract._sliderLabelWidthRatio)) &&
+        height <= maxTrackThicknessPx
+      const verticalTrackLike =
+        height >= Math.max(minTrackLengthPx, Math.floor(labelHeight * ToolsInteract._sliderLabelWidthRatio)) &&
+        width <= maxTrackThicknessPx
+      const trackLike = /slider|seek|range/i.test(candidateClass) || horizontalTrackLike || verticalTrackLike
+      if (!candidateText && !candidateContent) score -= 18
+      if (trackLike) score -= 30
+      if (/view|layout|group|frame/i.test(candidateClass)) score -= 10
+      if (width > labelWidth * ToolsInteract._sliderLabelWidthRatio) score -= 8
+      if (candidateText || candidateContent) score += 20
+
+      if (score < bestScore) {
+        bestScore = score
+        best = { el: candidate, idx: i, sliderLike: trackLike }
+      }
+    }
+
+    return best
+  }
+
 
   private static async getInteractionService(platform?: 'android' | 'ios', deviceId?: string) {
     const effectivePlatform = platform || 'android'
@@ -347,6 +433,7 @@ export class ToolsInteract {
 
     let best: UiElement | null = null
     let bestScore = 0
+    let lastTree: any = null
 
     const scoreElement = (el: UiElement | null) => {
       if (!el || !el.visible) return 0
@@ -380,7 +467,8 @@ export class ToolsInteract {
 
     while (Date.now() <= deadline) {
       try {
-        const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
+      const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId })
+      lastTree = tree
         if (tree && Array.isArray((tree as any).elements)) {
           const elements = ((tree as any).elements as UiElement[])
           for (let i = 0; i < elements.length; i++) {
@@ -407,8 +495,8 @@ export class ToolsInteract {
 
     // If the best match is not interactable, try to resolve an actionable ancestor.
     try {
-      const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
-      const elements = (tree && Array.isArray(tree.elements)) ? (tree.elements as UiElement[]) : []
+      const elements = (lastTree && Array.isArray(lastTree.elements)) ? (lastTree.elements as UiElement[]) : []
+      const screen = lastTree?.resolution && typeof lastTree.resolution === 'object' ? lastTree.resolution as UiResolution : null
       let chosen = best as any
       const childBounds = Array.isArray(chosen?.bounds) ? chosen.bounds : null
 
@@ -465,6 +553,16 @@ export class ToolsInteract {
         // small score bump to reflect actionability
         bestScore = Math.min(1, bestScore + 0.02)
       }
+
+      if (best && !(best.clickable || best.focusable)) {
+        const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best, idx: best._index ?? elements.indexOf(best) }, screen)
+        if (nearbyActionable) {
+          best = nearbyActionable.el
+          best._index = nearbyActionable.idx
+          best._interactable = true
+          best._sliderLike = nearbyActionable.sliderLike
+        }
+      }
     } catch (e) { console.error('Error resolving ancestor:', e) }
 
     if (!best) return { found: false, error: 'Element not found' }
@@ -483,8 +581,18 @@ export class ToolsInteract {
       tapCoordinates,
       telemetry: {
         matchedIndex: best?._index ?? null,
-        matchedInteractable: !!best?._interactable
+        matchedInteractable: !!best?._interactable,
+        sliderLike: !!best?._sliderLike
+      }
+    }
+    if (best?._sliderLike) {
+      const isVertical = !!boundsObj && (boundsObj.bottom - boundsObj.top) > (boundsObj.right - boundsObj.left)
+      const interactionHint = {
+        kind: 'slider',
+        axis: isVertical ? 'vertical' : 'horizontal',
+        trackBounds: boundsObj
       }
+      ;(outEl as any).interactionHint = interactionHint
     }
     const scoreVal = Math.min(1, Number(bestScore.toFixed(3)))
     return { found: true, element: outEl, score: scoreVal, confidence: scoreVal }

package/src/server-core.ts

@@ -13,7 +13,7 @@ export { wrapResponse, toolDefinitions, handleToolCall }
 
 export const serverInfo = {
   name: 'mobile-debug-mcp',
-  version: '0.24.7'
+  version: '0.24.8'
 }
 
 export function createServer() {

package/test/unit/observe/find_element.test.ts

@@ -75,14 +75,73 @@ async function run() {
     assert.ok(pass4, 'Child text should resolve to a clickable parent ancestor')
     process.stdout.write('Test 4: ' + (pass4 ? 'PASS' : 'FAIL') + '\n');
 
-    // Test 5: not found
-    ;(ToolsObserve as any).getUITreeHandler = async () => ({ device: { platform: 'android', id: 'mock' }, screen: '', resolution: { width: 1080, height: 1920 }, elements: [] })
-    const res5: any = await ToolsInteract.findElementHandler({ query: 'nope', exact: false, platform: 'android', timeoutMs: 300 })
+    // Test 5: duration label should resolve to the nearby slider control
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 1920 },
+      elements: [
+        { text: 'Duration: 5 min', type: 'android.widget.TextView', contentDescription: null, clickable: false, enabled: true, visible: true, bounds: [10,10,260,50], resourceId: null },
+        { text: null, type: 'android.view.View', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,60,1040,140], resourceId: null }
+      ]
+    })
+
+    const res5: any = await ToolsInteract.findElementHandler({ query: 'duration', exact: false, platform: 'android', timeoutMs: 300 })
     process.stdout.write('res5 ' + JSON.stringify(res5, null, 2) + '\n');
-    const pass5 = res5.found === false
-    assert.ok(pass5, 'Missing elements should return found=false')
+    const pass5 = res5.found === true && res5.element && res5.element.clickable === true && res5.element.bounds && res5.element.bounds.top === 60 && res5.element.bounds.bottom === 140
+    assert.ok(pass5, 'Duration label should resolve to the slider control below it')
     process.stdout.write('Test 5: ' + (pass5 ? 'PASS' : 'FAIL') + '\n');
 
+    // Test 6: prefer track-like control over a closer texty sibling
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 1920 },
+      elements: [
+        { text: 'Duration: 5 min', type: 'android.widget.TextView', contentDescription: null, clickable: false, enabled: true, visible: true, bounds: [10,10,260,50], resourceId: null },
+        { text: 'Reset', type: 'android.widget.Button', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,60,150,120], resourceId: 'btn_reset' },
+        { text: null, type: 'android.view.View', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,130,1040,210], resourceId: null }
+      ]
+    })
+
+    const res6: any = await ToolsInteract.findElementHandler({ query: 'duration', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res6 ' + JSON.stringify(res6, null, 2) + '\n');
+    const pass6 = res6.found === true && res6.element && res6.element.clickable === true && res6.element.bounds && res6.element.bounds.top === 130 && res6.element.bounds.bottom === 210
+    assert.ok(pass6, 'Duration lookup should prefer the track-like control over a closer text button')
+    process.stdout.write('Test 6: ' + (pass6 ? 'PASS' : 'FAIL') + '\n');
+    const pass6b = res6.element && res6.element.telemetry && res6.element.telemetry.sliderLike === true && res6.element.interactionHint && res6.element.interactionHint.kind === 'slider'
+    assert.ok(pass6b, 'Duration lookup should include slider-specific telemetry')
+    process.stdout.write('Test 6b: ' + (pass6b ? 'PASS' : 'FAIL') + '\n');
+
+    // Test 7: prefer vertical track-like control over a closer text button
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 2400 },
+      elements: [
+        { text: 'Duration: 5 min', type: 'android.widget.TextView', contentDescription: null, clickable: false, enabled: true, visible: true, bounds: [10,10,260,50], resourceId: null },
+        { text: 'Reset', type: 'android.widget.Button', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [10,60,150,120], resourceId: 'btn_reset' },
+        { text: null, type: 'android.view.View', contentDescription: null, clickable: true, enabled: true, visible: true, bounds: [270,20,350,1040], resourceId: null }
+      ]
+    })
+
+    const res7: any = await ToolsInteract.findElementHandler({ query: 'duration', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res7 ' + JSON.stringify(res7, null, 2) + '\n');
+    const pass7 = res7.found === true && res7.element && res7.element.clickable === true && res7.element.bounds && res7.element.bounds.left === 270 && res7.element.bounds.right === 350
+    assert.ok(pass7, 'Duration lookup should prefer a vertical track-like control')
+    process.stdout.write('Test 7: ' + (pass7 ? 'PASS' : 'FAIL') + '\n');
+    const pass7b = res7.element && res7.element.interactionHint && res7.element.interactionHint.axis === 'vertical'
+    assert.ok(pass7b, 'Vertical sliders should report a vertical interaction axis')
+    process.stdout.write('Test 7b: ' + (pass7b ? 'PASS' : 'FAIL') + '\n');
+
+    // Test 8: not found
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({ device: { platform: 'android', id: 'mock' }, screen: '', resolution: { width: 1080, height: 1920 }, elements: [] })
+    const res8: any = await ToolsInteract.findElementHandler({ query: 'nope', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res8 ' + JSON.stringify(res8, null, 2) + '\n');
+    const pass8 = res8.found === false
+    assert.ok(pass8, 'Missing elements should return found=false')
+    process.stdout.write('Test 8: ' + (pass8 ? 'PASS' : 'FAIL') + '\n');
+
   } finally {
     ;(ToolsObserve as any).getUITreeHandler = origGetTree
   }