mobile-debug-mcp 0.24.8 → 0.25.1

package/README.md

@@ -1,6 +1,6 @@
 # Mobile Debug Tools
 
-A minimal, secure MCP server for AI-assisted mobile development. Build, install, and inspect Android/iOS apps from an MCP-compatible client.
+A minimal, secure MCP server for AI-assisted mobile development. Build, install, interact and inspect Android/iOS apps from an MCP-compatible client.
 
 > **Support:**
 > * KMP

package/dist/interact/index.js

@@ -34,6 +34,43 @@ export class ToolsInteract {
             return null;
         return normalized;
     }
+    static _matchesSelector(el, selector) {
+        if (!selector)
+            return false;
+        const normalize = ToolsInteract._normalize;
+        const containsFlag = !!selector.contains;
+        const text = normalize(el.text ?? el.label ?? el.value ?? '');
+        const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
+        const accessibilityId = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? '');
+        if (selector.text !== undefined && selector.text !== null) {
+            const q = normalize(selector.text);
+            if (containsFlag ? !text.includes(q) : text !== q)
+                return false;
+        }
+        if (selector.resource_id !== undefined && selector.resource_id !== null) {
+            const q = normalize(selector.resource_id);
+            if (containsFlag ? !resourceId.includes(q) : resourceId !== q)
+                return false;
+        }
+        if (selector.accessibility_id !== undefined && selector.accessibility_id !== null) {
+            const q = normalize(selector.accessibility_id);
+            if (containsFlag ? !accessibilityId.includes(q) : accessibilityId !== q)
+                return false;
+        }
+        return true;
+    }
+    static _findFirstMatchingElement(elements, selector) {
+        if (!selector)
+            return null;
+        for (let i = 0; i < elements.length; i++) {
+            const el = elements[i];
+            if (!el)
+                continue;
+            if (ToolsInteract._matchesSelector(el, selector))
+                return { el, idx: i };
+        }
+        return null;
+    }
     static _isVisibleElement(el) {
         const bounds = ToolsInteract._normalizeBounds(el.bounds);
         return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
@@ -68,7 +105,13 @@ export class ToolsInteract {
             class: el.type ?? el.class ?? null,
             bounds,
             index,
-            elementId
+            elementId,
+            state: el.state ?? null,
+            stable_id: el.stable_id ?? null,
+            role: el.role ?? null,
+            test_tag: el.test_tag ?? null,
+            selector: el.selector ?? null,
+            semantic: el.semantic ?? null
         };
     }
     static _rememberResolvedElement(elementId, context) {
@@ -100,7 +143,13 @@ export class ToolsInteract {
             accessibility_id: element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? element.label ?? null,
             class: element.type ?? element.class ?? null,
             bounds: ToolsInteract._normalizeBounds(element.bounds),
-            index
+            index,
+            state: element.state ?? null,
+            stable_id: element.stable_id ?? null,
+            role: element.role ?? null,
+            test_tag: element.test_tag ?? null,
+            selector: element.selector ?? null,
+            semantic: element.semantic ?? null
         };
     }
     static _actionFailure(actionId, timestamp, actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter) {
@@ -504,6 +553,11 @@ export class ToolsInteract {
             bounds: boundsObj,
             clickable: !!best.clickable,
             enabled: !!best.enabled,
+            stable_id: best.stable_id ?? null,
+            role: best.role ?? null,
+            test_tag: best.test_tag ?? null,
+            selector: best.selector ?? null,
+            semantic: best.semantic ?? null,
             tapCoordinates,
             telemetry: {
                 matchedIndex: best?._index ?? null,
@@ -901,7 +955,13 @@ export class ToolsInteract {
                     accessibility_id: result.element.accessibility_id ?? null,
                     class: result.element.class ?? null,
                     bounds: result.element.bounds ?? null,
-                    index: typeof result.element.index === 'number' ? result.element.index : null
+                    index: typeof result.element.index === 'number' ? result.element.index : null,
+                    state: result.element.state ?? null,
+                    stable_id: result.element.stable_id ?? null,
+                    role: result.element.role ?? null,
+                    test_tag: result.element.test_tag ?? null,
+                    selector: result.element.selector ?? null,
+                    semantic: result.element.semantic ?? null
                 },
                 observed: {
                     status: result.status,
@@ -915,7 +975,13 @@ export class ToolsInteract {
                         accessibility_id: result.element.accessibility_id ?? null,
                         class: result.element.class ?? null,
                         bounds: result.element.bounds ?? null,
-                        index: typeof result.element.index === 'number' ? result.element.index : null
+                        index: typeof result.element.index === 'number' ? result.element.index : null,
+                        state: result.element.state ?? null,
+                        stable_id: result.element.stable_id ?? null,
+                        role: result.element.role ?? null,
+                        test_tag: result.element.test_tag ?? null,
+                        selector: result.element.selector ?? null,
+                        semantic: result.element.semantic ?? null
                     }
                 },
                 reason: 'selector is visible'
@@ -939,6 +1005,176 @@ export class ToolsInteract {
             retryable: errorCode === 'TIMEOUT'
         };
     }
+    static async expectStateHandler({ selector, element_id, property, expected, platform, deviceId }) {
+        const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
+        const elements = Array.isArray(tree?.elements) ? tree.elements : [];
+        const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
+        const treeDeviceId = tree?.device?.id || deviceId;
+        let matched = null;
+        if (element_id) {
+            const resolved = ToolsInteract._resolvedUiElements.get(element_id);
+            if (resolved) {
+                const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
+                if (current)
+                    matched = { el: current.el, idx: current.index };
+            }
+        }
+        if (!matched && selector) {
+            matched = ToolsInteract._findFirstMatchingElement(elements, selector);
+        }
+        if (!matched) {
+            return {
+                success: false,
+                selector,
+                element_id: element_id ?? null,
+                expected_state: { property, expected },
+                reason: 'element not found',
+                failure_code: 'ELEMENT_NOT_FOUND',
+                retryable: true
+            };
+        }
+        const resolvedElement = ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx), matched.el, matched.idx);
+        const observedState = matched.el.state ?? null;
+        const actual = observedState?.[property] ?? null;
+        const compareBoolean = (value) => typeof value === 'boolean' ? value : null;
+        const compareString = (value) => typeof value === 'string' ? value : null;
+        const compareNumber = (value) => typeof value === 'number' && Number.isFinite(value) ? value : null;
+        let success = false;
+        let reason = '';
+        let rawValue = null;
+        let observedValue = actual;
+        switch (property) {
+            case 'checked':
+            case 'focused':
+            case 'expanded':
+            case 'enabled': {
+                const expectedBool = compareBoolean(expected);
+                const actualBool = compareBoolean(actual);
+                if (expectedBool === null) {
+                    reason = `expected ${property} must be boolean`;
+                }
+                else if (actualBool === null) {
+                    reason = `${property} state unavailable`;
+                }
+                else {
+                    rawValue = actualBool;
+                    success = actualBool === expectedBool;
+                    reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`;
+                }
+                observedValue = actualBool;
+                break;
+            }
+            case 'value':
+            case 'raw_value': {
+                const expectedNumber = compareNumber(expected);
+                const actualNumber = compareNumber(actual);
+                if (expectedNumber !== null && actualNumber !== null) {
+                    success = actualNumber === expectedNumber;
+                    rawValue = actualNumber;
+                    observedValue = actualNumber;
+                    reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`;
+                    break;
+                }
+                const expectedString = typeof expected === 'string' ? expected : null;
+                const actualString = compareString(actual);
+                if (expectedString !== null && actualString !== null) {
+                    success = actualString === expectedString;
+                    rawValue = actualString;
+                    observedValue = actualString;
+                    reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`;
+                }
+                else {
+                    reason = 'value state unavailable';
+                }
+                break;
+            }
+            case 'selected': {
+                const expectedBool = typeof expected === 'boolean' ? expected : null;
+                const expectedString = typeof expected === 'string'
+                    ? expected
+                    : expected && typeof expected === 'object'
+                        ? String(expected.id ?? expected.label ?? '')
+                        : null;
+                if (!observedState || observedState.selected === undefined || observedState.selected === null) {
+                    reason = 'selected state unavailable';
+                    break;
+                }
+                if (expectedBool !== null) {
+                    const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null;
+                    if (actualBool === null) {
+                        reason = 'selected state is not boolean';
+                        break;
+                    }
+                    rawValue = actualBool;
+                    observedValue = actualBool;
+                    success = actualBool === expectedBool;
+                    reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`;
+                    break;
+                }
+                const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
+                    ? String(observedState.selected.id ?? observedState.selected.label ?? '')
+                    : String(observedState.selected);
+                const actualString = actualSelected.trim();
+                if (!expectedString) {
+                    reason = 'expected selected must be boolean, string, or object with id/label';
+                    break;
+                }
+                rawValue = actualString;
+                observedValue = actualString;
+                success = actualString === expectedString;
+                reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`;
+                break;
+            }
+            case 'text_value': {
+                const expectedString = typeof expected === 'string' ? expected : null;
+                const actualString = compareString(actual);
+                if (!expectedString) {
+                    reason = 'expected text_value must be string';
+                }
+                else if (!actualString) {
+                    reason = 'text_value state unavailable';
+                }
+                else {
+                    success = actualString === expectedString;
+                    rawValue = actualString;
+                    observedValue = actualString;
+                    reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`;
+                }
+                break;
+            }
+            default: {
+                if (actual !== null && actual !== undefined) {
+                    success = actual === expected;
+                    observedValue = actual;
+                    rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null;
+                    reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`;
+                }
+                else {
+                    reason = `unsupported or unavailable state property: ${property}`;
+                }
+            }
+        }
+        if (!success && !reason) {
+            reason = `${property} did not match expected value`;
+        }
+        return {
+            success,
+            selector,
+            element_id: element_id ?? resolvedElement.elementId,
+            expected_state: { property, expected },
+            element: {
+                ...resolvedElement,
+                state: observedState
+            },
+            observed_state: {
+                property,
+                value: observedValue,
+                ...(rawValue !== null ? { raw_value: rawValue } : {})
+            },
+            reason,
+            ...(success ? {} : { failure_code: 'UNKNOWN', retryable: false })
+        };
+    }
     static async waitForUICore({ type = 'ui', query, timeoutMs = 30000, pollIntervalMs = 300, includeSnapshotOnFailure = true, match = 'present', stability_ms = 700, observationDelayMs = 0, platform, deviceId }) {
         const start = Date.now();
         const deadline = start + (timeoutMs || 0);

package/dist/observe/ios.js

@@ -37,7 +37,118 @@ function getCenter(bounds) {
     const [x1, y1, x2, y2] = bounds;
     return [Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)];
 }
-function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
+function parseIOSNumber(value) {
+    if (typeof value === 'number' && Number.isFinite(value))
+        return value;
+    if (typeof value !== 'string')
+        return null;
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+function normalizeIOSType(value) {
+    return typeof value === 'string' ? value.trim().toLowerCase() : '';
+}
+function inferIOSRole(type, traits) {
+    if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait)))
+        return 'slider';
+    if (/button/.test(type) || traits.some((trait) => /button/.test(trait)))
+        return 'button';
+    if (/cell/.test(type))
+        return 'cell';
+    if (/switch/.test(type))
+        return 'switch';
+    if (/text field|textfield|search field/.test(type))
+        return 'text_field';
+    if (/image/.test(type))
+        return 'image';
+    if (/window|application|group|scroll view|collection view/.test(type))
+        return 'container';
+    return null;
+}
+function getIOSStableId(node) {
+    const candidates = [node.AXIdentifier, node.accessibilityIdentifier, node.identifier, node.AXUniqueId];
+    for (const candidate of candidates) {
+        if (typeof candidate === 'string' && candidate.trim().length > 0)
+            return candidate;
+    }
+    return null;
+}
+function buildIOSSelectorConfidence(source) {
+    switch (source) {
+        case 'identifier':
+            return { score: 1, reason: 'accessibility_identifier' };
+        case 'label':
+            return { score: 0.9, reason: 'label_match' };
+        case 'value':
+            return { score: 0.75, reason: 'value_match' };
+        case 'type':
+            return { score: 0.35, reason: 'type_match' };
+        default:
+            return null;
+    }
+}
+function buildIOSSelector(type, label, value, stableId) {
+    if (stableId)
+        return { value: stableId, confidence: buildIOSSelectorConfidence('identifier') };
+    if (label)
+        return { value: label, confidence: buildIOSSelectorConfidence('label') };
+    if (value)
+        return { value: value, confidence: buildIOSSelectorConfidence('value') };
+    if (type)
+        return { value: type, confidence: buildIOSSelectorConfidence('type') };
+    return null;
+}
+function buildIOSSemantic(type, traits) {
+    return {
+        is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
+        is_container: /window|application|group|scroll view|collection view/.test(type)
+    };
+}
+function isIOSAdjustable(node, type, traits) {
+    return /slider|adjustable|stepper|progress/i.test(type) || traits.some((trait) => /adjustable|slider|progress/i.test(trait));
+}
+function extractIOSState(node, type, label, value, traits) {
+    const state = {};
+    const normalizedTraits = traits.map((trait) => String(trait).toLowerCase());
+    if (normalizedTraits.some((trait) => /selected/.test(trait))) {
+        state.selected = label || value || true;
+    }
+    if (normalizedTraits.some((trait) => /focused/.test(trait))) {
+        state.focused = true;
+    }
+    if (normalizedTraits.some((trait) => /enabled/.test(trait))) {
+        state.enabled = true;
+    }
+    if (normalizedTraits.some((trait) => /disabled/.test(trait))) {
+        state.enabled = false;
+    }
+    if (value && /textfield|search|text/i.test(type)) {
+        state.text_value = value;
+    }
+    if (isIOSAdjustable(node, type, traits)) {
+        const rawValue = parseIOSNumber(value);
+        if (rawValue !== null) {
+            state.raw_value = rawValue;
+            state.value = rawValue >= 0 && rawValue <= 1 ? Math.round(rawValue * 100) : rawValue;
+        }
+        else if (value) {
+            state.raw_value = value;
+            state.value = value;
+        }
+    }
+    else if (value) {
+        const numericValue = parseIOSNumber(value);
+        if (numericValue !== null) {
+            state.value = numericValue;
+            state.raw_value = numericValue;
+        }
+        else {
+            state.value = value;
+        }
+    }
+    return Object.keys(state).length > 0 ? state : null;
+}
+export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
     if (!node)
         return -1;
     let currentIndex = -1;
@@ -46,6 +157,12 @@ function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
     const value = node.AXValue || null;
     const frame = node.AXFrame || node.frame;
     const traits = node.AXTraits || [];
+    const state = extractIOSState(node, type, label, value, traits);
+    const normalizedType = normalizeIOSType(type);
+    const stableId = getIOSStableId(node);
+    const selector = buildIOSSelector(type, label, value, stableId);
+    const semantic = buildIOSSemantic(normalizedType, traits);
+    const role = inferIOSRole(normalizedType, traits);
     const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
     const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
     if (isUseful) {
@@ -54,13 +171,19 @@ function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
             text: label,
             contentDescription: value,
             type: type,
-            resourceId: node.AXUniqueId || null,
+            resourceId: stableId,
             clickable: clickable,
             enabled: true,
             visible: true,
             bounds: bounds,
             center: getCenter(bounds),
-            depth: depth
+            depth: depth,
+            state,
+            stable_id: stableId,
+            role,
+            test_tag: stableId,
+            selector,
+            semantic
         };
         if (parentIndex !== -1) {
             element.parentId = parentIndex;

package/dist/server/common.js

@@ -79,7 +79,8 @@ export function normalizeResolvedTarget(value = null) {
         accessibility_id: value.accessibility_id ?? null,
         class: value.class ?? null,
         bounds: value.bounds ?? null,
-        index: value.index ?? null
+        index: value.index ?? null,
+        state: value.state ?? null
     };
 }
 export function inferGenericFailure(message) {

package/dist/server/tool-definitions.js

@@ -468,6 +468,61 @@ Failure Handling:
             required: ['selector']
         }
     },
+    {
+        name: 'expect_state',
+        description: `Purpose:
+Verify a readable UI state property on the currently visible element.
+
+Inputs:
+- selector or element_id
+- property
+- expected
+- platform/deviceId (optional)
+
+Supported properties:
+- checked, selected, focused, expanded, enabled, text_value, value, raw_value
+
+Verification Guidance:
+- Use this when the UI element is visible but its state must also be confirmed
+- Prefer the canonical property names above
+- The tool compares the normalized readable state and returns the observed value when available
+
+Constraints:
+- Returns structured success/failure only
+- Does not infer a state when the property is unavailable
+
+Failure Handling:
+- ELEMENT_NOT_FOUND → re-resolve the element or wait for UI stabilization
+- UNKNOWN → capture a snapshot and stop`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                selector: {
+                    type: 'object',
+                    properties: {
+                        text: { type: 'string' },
+                        resource_id: { type: 'string' },
+                        accessibility_id: { type: 'string' },
+                        contains: { type: 'boolean', default: false }
+                    }
+                },
+                element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
+                property: { type: 'string', description: 'Readable state property to verify.' },
+                expected: {
+                    description: 'Expected normalized state value.',
+                    oneOf: [
+                        { type: 'boolean' },
+                        { type: 'number' },
+                        { type: 'string' },
+                        { type: 'object' }
+                    ]
+                },
+                platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
+                deviceId: { type: 'string', description: 'Optional device serial/udid' }
+            },
+            required: ['property', 'expected']
+        }
+    },
     {
         name: 'wait_for_ui',
         description: `Purpose:

package/dist/server/tool-handlers.js

@@ -208,6 +208,22 @@ async function handleExpectElementVisible(args) {
     const res = await ToolsInteract.expectElementVisibleHandler({ selector, element_id, timeout_ms, poll_interval_ms, platform, deviceId });
     return wrapResponse(res);
 }
+async function handleExpectState(args) {
+    const selector = getObjectArg(args, 'selector');
+    const element_id = getStringArg(args, 'element_id');
+    const property = requireStringArg(args, 'property');
+    const platform = getStringArg(args, 'platform');
+    const deviceId = getStringArg(args, 'deviceId');
+    if (!selector && !element_id) {
+        throw new Error('Missing selector or element_id argument');
+    }
+    if (!Object.prototype.hasOwnProperty.call(args, 'expected')) {
+        throw new Error('Missing expected argument');
+    }
+    const expected = args.expected;
+    const res = await ToolsInteract.expectStateHandler({ selector: selector ?? undefined, element_id: element_id ?? undefined, property, expected, platform, deviceId });
+    return wrapResponse(res);
+}
 async function handleWaitForUI(args) {
     const selector = getObjectArg(args, 'selector');
     const condition = getStringArg(args, 'condition') ?? 'exists';
@@ -395,6 +411,7 @@ export const toolHandlers = {
     wait_for_screen_change: handleWaitForScreenChange,
     expect_screen: handleExpectScreen,
     expect_element_visible: handleExpectElementVisible,
+    expect_state: handleExpectState,
     wait_for_ui: handleWaitForUI,
     find_element: handleFindElement,
     tap: handleTap,

package/dist/server-core.js

@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
 export { wrapResponse, toolDefinitions, handleToolCall };
 export const serverInfo = {
     name: 'mobile-debug-mcp',
-    version: '0.24.8'
+    version: '0.25.1'
 };
 export function createServer() {
     const server = new Server(serverInfo, {