mobile-debug-mcp 0.26.3 → 0.26.5

package/dist/interact/index.js

@@ -219,6 +219,87 @@ export class ToolsInteract {
             reason: candidate.reason
         };
     }
+    static _isAdjustableControl(el) {
+        if (!el)
+            return false;
+        const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
+        const role = ToolsInteract._normalize(el.role ?? '');
+        return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
+    }
+    static _isSemanticActionable(el) {
+        if (!el?.semantic)
+            return false;
+        if (el.semantic.adjustable)
+            return true;
+        return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0;
+    }
+    static _readNumericControlValue(el, property) {
+        if (!el?.state)
+            return null;
+        const stateValue = el.state[property];
+        if (typeof stateValue === 'number' && Number.isFinite(stateValue))
+            return stateValue;
+        if (property === 'value' || property === 'raw_value') {
+            const fallback = el.state.raw_value ?? el.state.value;
+            if (typeof fallback === 'number' && Number.isFinite(fallback))
+                return fallback;
+        }
+        return null;
+    }
+    static _buildControlPoint(bounds, ratio, axis) {
+        const clampedRatio = Math.max(0, Math.min(1, ratio));
+        const [left, top, right, bottom] = bounds;
+        const width = Math.max(1, right - left);
+        const height = Math.max(1, bottom - top);
+        const insetX = Math.max(8, Math.floor(width * 0.08));
+        const insetY = Math.max(8, Math.floor(height * 0.08));
+        if (axis === 'vertical') {
+            const usableHeight = Math.max(1, height - (insetY * 2));
+            return {
+                x: Math.floor((left + right) / 2),
+                y: Math.floor(bottom - insetY - (usableHeight * clampedRatio))
+            };
+        }
+        const usableWidth = Math.max(1, width - (insetX * 2));
+        return {
+            x: Math.floor(left + insetX + (usableWidth * clampedRatio)),
+            y: Math.floor((top + bottom) / 2)
+        };
+    }
+    static _buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis) {
+        const range = Math.max(1, max - min);
+        const targetRatio = (targetValue - min) / range;
+        const stepRatio = 1 / range;
+        const centerBias = stepRatio / 2;
+        const direction = currentValue === null ? 0 : Math.sign(targetValue - currentValue);
+        const controlLengthPx = axis === 'vertical' ? Math.max(1, bounds[3] - bounds[1]) : Math.max(1, bounds[2] - bounds[0]);
+        const edgeWindow = Math.max(3, Math.floor(range * 0.1));
+        const isNearLowEdge = targetValue - min <= edgeWindow;
+        const isNearHighEdge = max - targetValue <= edgeWindow;
+        const directionBias = direction > 0
+            ? -stepRatio * 0.15
+            : direction < 0
+                ? stepRatio * 0.65
+                : 0;
+        const pixelBasedMargin = Math.min(0.03, Math.max(0.005, 2 / controlLengthPx));
+        const endpointMargin = Math.max(stepRatio * 0.5, pixelBasedMargin);
+        const edgeBias = isNearLowEdge
+            ? endpointMargin
+            : isNearHighEdge
+                ? Math.max(stepRatio * 0.4, endpointMargin * 0.75)
+                : 0;
+        const safeRatio = Math.min(1 - (endpointMargin * 0.25), Math.max(endpointMargin, targetRatio + centerBias + directionBias + edgeBias));
+        return ToolsInteract._buildControlPoint(bounds, safeRatio, axis);
+    }
+    static _controlAxis(el, bounds) {
+        const type = ToolsInteract._normalize(el.type ?? el.class ?? '');
+        const role = ToolsInteract._normalize(el.role ?? '');
+        if (/vertical/.test(type) || /vertical/.test(role))
+            return 'vertical';
+        if (/horizontal/.test(type) || /horizontal/.test(role))
+            return 'horizontal';
+        return (bounds[3] - bounds[1]) > (bounds[2] - bounds[0]) ? 'vertical' : 'horizontal';
+    }
     static _actionFailure(actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter, sourceModule = 'interact') {
         return buildActionExecutionResult({
             actionType,
@@ -244,11 +325,11 @@ export class ToolsInteract {
     static _resolveActionableAncestor(elements, chosen) {
         if (!chosen)
             return null;
-        if (chosen.el.clickable || chosen.el.focusable)
+        if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el))
             return chosen;
         let current = chosen;
         let safety = 0;
-        while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable) && current.el.parentId !== undefined && current.el.parentId !== null) {
+        while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
             const parentId = current.el.parentId;
             let parentIndex = null;
             if (typeof parentId === 'number')
@@ -257,7 +338,7 @@ export class ToolsInteract {
                 parentIndex = Number(parentId);
             if (parentIndex !== null && elements[parentIndex]) {
                 current = { el: elements[parentIndex], idx: parentIndex };
-                if (current.el.clickable || current.el.focusable)
+                if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
                     return current;
             }
             else if (typeof parentId === 'string') {
@@ -265,7 +346,7 @@ export class ToolsInteract {
                 if (foundIndex === -1)
                     break;
                 current = { el: elements[foundIndex], idx: foundIndex };
-                if (current.el.clickable || current.el.focusable)
+                if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
                     return current;
             }
             else {
@@ -281,7 +362,7 @@ export class ToolsInteract {
         let bestArea = Infinity;
         for (let i = 0; i < elements.length; i++) {
             const el = elements[i];
-            if (!el || !(el.clickable || el.focusable))
+            if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el)))
                 continue;
             const bounds = ToolsInteract._normalizeBounds(el.bounds);
             if (!bounds)
@@ -410,6 +491,388 @@ export class ToolsInteract {
             sourceModule: 'interact'
         });
     }
+    static async adjustControlHandler({ selector, element_id, property = 'value', targetValue, tolerance = 0, maxAttempts = 3, platform, deviceId }) {
+        const actionType = 'adjust_control';
+        const targetSelector = selector ?? (element_id ? { elementId: element_id } : null);
+        const normalizedTolerance = Number.isFinite(tolerance) ? Math.max(0, tolerance) : 0;
+        const attemptsLimit = Math.max(1, Math.floor(Number(maxAttempts) || 1));
+        const sourcePlatform = platform || 'android';
+        let resolvedPlatform = sourcePlatform;
+        let resolvedDeviceId = deviceId;
+        const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
+        let semanticFallbackElement = null;
+        const buildFailure = (failureCode, reason, resolved, device, actualState, attempts, adjustmentMode = 'gesture', retryable = false, uiFingerprintAfter = null) => {
+            const base = buildActionExecutionResult({
+                actionType,
+                sourceModule: 'interact',
+                device,
+                selector: targetSelector,
+                resolved,
+                success: false,
+                uiFingerprintBefore: fingerprintBefore,
+                uiFingerprintAfter,
+                failure: { failureCode, retryable },
+                details: {
+                    target_value: targetValue,
+                    tolerance: normalizedTolerance,
+                    property,
+                    attempts,
+                    adjustment_mode: adjustmentMode,
+                    actual_state: actualState,
+                    converged: false,
+                    within_tolerance: false,
+                    reason
+                }
+            });
+            return {
+                ...base,
+                target_state: {
+                    property,
+                    target_value: targetValue,
+                    tolerance: normalizedTolerance
+                },
+                actual_state: actualState,
+                within_tolerance: false,
+                converged: false,
+                attempts,
+                adjustment_mode: adjustmentMode
+            };
+        };
+        const resolveCurrentMatch = async () => {
+            const tree = await ToolsObserve.getUITreeHandler({ platform: resolvedPlatform, deviceId: resolvedDeviceId });
+            resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : resolvedPlatform;
+            resolvedDeviceId = tree?.device?.id || resolvedDeviceId;
+            const elements = Array.isArray(tree?.elements) ? tree.elements : [];
+            if (element_id) {
+                const stored = ToolsInteract._resolvedUiElements.get(element_id);
+                if (!stored) {
+                    return null;
+                }
+                const current = ToolsInteract._findCurrentResolvedElement(elements, resolvedPlatform, resolvedDeviceId, stored);
+                if (!current) {
+                    return null;
+                }
+                return {
+                    tree,
+                    device: tree?.device,
+                    match: { el: current.el, idx: current.index },
+                    resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, current.el, current.index), current.el, current.index)
+                };
+            }
+            if (semanticFallbackElement) {
+                const fallbackBounds = ToolsInteract._normalizeBounds(Array.isArray(semanticFallbackElement.bounds)
+                    ? semanticFallbackElement.bounds
+                    : semanticFallbackElement.bounds && typeof semanticFallbackElement.bounds === 'object'
+                        ? [
+                            Number(semanticFallbackElement.bounds.left),
+                            Number(semanticFallbackElement.bounds.top),
+                            Number(semanticFallbackElement.bounds.right),
+                            Number(semanticFallbackElement.bounds.bottom)
+                        ]
+                        : null);
+                let matchedIndex = -1;
+                if (fallbackBounds) {
+                    matchedIndex = elements.findIndex((el) => {
+                        const bounds = ToolsInteract._normalizeBounds(el.bounds);
+                        return !!bounds && bounds[0] === fallbackBounds[0] && bounds[1] === fallbackBounds[1] && bounds[2] === fallbackBounds[2] && bounds[3] === fallbackBounds[3];
+                    });
+                }
+                if (matchedIndex === -1 && fallbackBounds) {
+                    const fallbackCenterX = Math.floor((fallbackBounds[0] + fallbackBounds[2]) / 2);
+                    const fallbackCenterY = Math.floor((fallbackBounds[1] + fallbackBounds[3]) / 2);
+                    let bestDistance = Infinity;
+                    for (let i = 0; i < elements.length; i++) {
+                        const el = elements[i];
+                        if (!ToolsInteract._isAdjustableControl(el))
+                            continue;
+                        const bounds = ToolsInteract._normalizeBounds(el.bounds);
+                        if (!bounds)
+                            continue;
+                        const centerX = Math.floor((bounds[0] + bounds[2]) / 2);
+                        const centerY = Math.floor((bounds[1] + bounds[3]) / 2);
+                        const distance = Math.abs(centerX - fallbackCenterX) + Math.abs(centerY - fallbackCenterY);
+                        if (distance < bestDistance) {
+                            bestDistance = distance;
+                            matchedIndex = i;
+                        }
+                    }
+                }
+                if (matchedIndex >= 0 && elements[matchedIndex]) {
+                    const matched = { el: elements[matchedIndex], idx: matchedIndex };
+                    return {
+                        tree,
+                        device: tree?.device,
+                        match: matched,
+                        resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
+                    };
+                }
+            }
+            if (selector) {
+                const matched = ToolsInteract._findFirstMatchingElement(elements, selector);
+                if (!matched) {
+                    return null;
+                }
+                return {
+                    tree,
+                    device: tree?.device,
+                    match: matched,
+                    resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, matched.el, matched.idx), matched.el, matched.idx)
+                };
+            }
+            return null;
+        };
+        if (!selector && !element_id) {
+            return buildFailure('ELEMENT_NOT_FOUND', 'selector or element_id is required', null, undefined, null, 0, 'gesture', false);
+        }
+        if (selector && !element_id) {
+            const waitResult = await ToolsInteract.waitForUIHandler({
+                selector,
+                condition: 'clickable',
+                timeout_ms: 5000,
+                poll_interval_ms: 300,
+                platform: resolvedPlatform,
+                deviceId: resolvedDeviceId
+            });
+            if (waitResult?.status !== 'success' || !waitResult?.element?.elementId) {
+                const semanticQuery = selector.text ?? selector.resource_id ?? selector.accessibility_id ?? '';
+                if (!semanticQuery) {
+                    return buildFailure(waitResult?.error?.code === 'ELEMENT_NOT_FOUND' ? 'ELEMENT_NOT_FOUND' : 'TIMEOUT', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', waitResult?.error?.code === 'ELEMENT_NOT_FOUND');
+                }
+                const fallback = await ToolsInteract.findElementHandler({
+                    query: semanticQuery,
+                    exact: false,
+                    timeoutMs: 3000,
+                    platform: resolvedPlatform,
+                    deviceId: resolvedDeviceId
+                });
+                if (!fallback.found || !fallback.element) {
+                    return buildFailure('ELEMENT_NOT_FOUND', waitResult?.error?.message ?? 'adjustable control not found', null, waitResult?.device, null, 0, 'gesture', true);
+                }
+                semanticFallbackElement = fallback.element;
+            }
+            else {
+                element_id = waitResult.element.elementId;
+                semanticFallbackElement = null;
+            }
+        }
+        let lastObservedState = null;
+        let lastAdjustmentMode = 'gesture';
+        let resolvedTarget = null;
+        let currentDevice = undefined;
+        let attemptCount = 0;
+        let cachedResolvedMatch = null;
+        for (let attempt = 0; attempt < attemptsLimit; attempt++) {
+            const resolved = cachedResolvedMatch
+                ? {
+                    tree: null,
+                    device: currentDevice,
+                    match: cachedResolvedMatch,
+                    resolvedTarget: ToolsInteract._resolvedTargetFromElement(ToolsInteract._computeElementId(resolvedPlatform, resolvedDeviceId, cachedResolvedMatch.el, cachedResolvedMatch.idx), cachedResolvedMatch.el, cachedResolvedMatch.idx)
+                }
+                : await resolveCurrentMatch();
+            if (!resolved || !resolved.match || !resolved.resolvedTarget) {
+                return buildFailure('STALE_REFERENCE', 'adjustable control could not be resolved', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
+            }
+            currentDevice = resolved.device;
+            resolvedTarget = resolved.resolvedTarget;
+            const currentEl = resolved.match.el;
+            cachedResolvedMatch = resolved.match;
+            const bounds = ToolsInteract._normalizeBounds(currentEl.bounds);
+            const valueRange = currentEl.state?.value_range ?? null;
+            const currentValue = ToolsInteract._readNumericControlValue(currentEl, property);
+            const actualState = currentValue !== null
+                ? { property, value: currentValue, raw_value: typeof currentEl.state?.raw_value === 'number' ? currentEl.state.raw_value : undefined }
+                : null;
+            lastObservedState = actualState;
+            if (property !== 'value' && property !== 'raw_value') {
+                return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
+            }
+            if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
+                const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
+                const base = buildActionExecutionResult({
+                    actionType,
+                    sourceModule: 'interact',
+                    device: currentDevice,
+                    selector: targetSelector,
+                    resolved: resolvedTarget,
+                    success: true,
+                    uiFingerprintBefore: fingerprintBefore,
+                    uiFingerprintAfter,
+                    details: {
+                        target_value: targetValue,
+                        tolerance: normalizedTolerance,
+                        property,
+                        attempts: attemptCount,
+                        adjustment_mode: 'semantic',
+                        actual_state: actualState,
+                        converged: true,
+                        within_tolerance: true,
+                        reason: 'control already within tolerance'
+                    }
+                });
+                return {
+                    ...base,
+                    target_state: {
+                        property,
+                        target_value: targetValue,
+                        tolerance: normalizedTolerance
+                    },
+                    actual_state: actualState,
+                    within_tolerance: true,
+                    converged: true,
+                    attempts: attemptCount,
+                    adjustment_mode: 'semantic'
+                };
+            }
+            if (!ToolsInteract._isAdjustableControl(currentEl)) {
+                return buildFailure('ELEMENT_NOT_INTERACTABLE', 'target is not an adjustable control', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
+            }
+            if (!bounds) {
+                return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjustable control has no bounds', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
+            }
+            const min = typeof valueRange?.min === 'number' ? valueRange.min : null;
+            const max = typeof valueRange?.max === 'number' ? valueRange.max : null;
+            if (min === null || max === null || !Number.isFinite(min) || !Number.isFinite(max) || max <= min) {
+                return buildFailure('ELEMENT_NOT_INTERACTABLE', 'value_range unavailable', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
+            }
+            if (targetValue < min || targetValue > max) {
+                return buildFailure('UNKNOWN', `targetValue ${targetValue} is outside the control range ${min}..${max}`, resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false);
+            }
+            const axis = ToolsInteract._controlAxis(currentEl, bounds);
+            const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis);
+            const currentPoint = currentValue !== null
+                ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
+                : ToolsInteract._buildControlPoint(bounds, 0.5, axis);
+            const runVerification = async () => {
+                const verification = await ToolsInteract.expectStateHandler({
+                    element_id: resolvedTarget?.elementId ?? element_id,
+                    selector: selector ?? undefined,
+                    property,
+                    expected: targetValue,
+                    platform: resolvedPlatform,
+                    deviceId: resolvedDeviceId
+                });
+                const observedValue = typeof verification?.observed_state?.value === 'number'
+                    ? verification.observed_state.value
+                    : typeof verification?.observed_state?.raw_value === 'number'
+                        ? verification.observed_state.raw_value
+                        : null;
+                const observedState = observedValue !== null
+                    ? {
+                        property,
+                        value: observedValue,
+                        raw_value: typeof verification?.observed_state?.raw_value === 'number' ? verification.observed_state.raw_value : undefined
+                    }
+                    : actualState;
+                return {
+                    verification,
+                    observedState,
+                    withinTolerance: observedValue !== null && Math.abs(observedValue - targetValue) <= normalizedTolerance
+                };
+            };
+            lastAdjustmentMode = 'coordinate';
+            const primaryActionResult = await ToolsInteract.tapHandler({
+                platform: resolvedPlatform,
+                x: targetPoint.x,
+                y: targetPoint.y,
+                deviceId: resolvedDeviceId
+            });
+            let actionDevice = primaryActionResult.device ?? currentDevice;
+            attemptCount++;
+            if (!primaryActionResult.success) {
+                lastAdjustmentMode = 'gesture';
+                const fallbackActionResult = await ToolsInteract.swipeHandler({
+                    platform: resolvedPlatform,
+                    x1: currentPoint.x,
+                    y1: currentPoint.y,
+                    x2: targetPoint.x,
+                    y2: targetPoint.y,
+                    duration: 220,
+                    deviceId: resolvedDeviceId
+                });
+                attemptCount++;
+                if (!fallbackActionResult.success) {
+                    return buildFailure('UNKNOWN', fallbackActionResult.error ?? primaryActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? primaryActionResult.device, actualState, attemptCount, lastAdjustmentMode, false);
+                }
+                actionDevice = fallbackActionResult.device ?? actionDevice;
+            }
+            let verificationResult = await runVerification();
+            let observedState = verificationResult.observedState;
+            lastObservedState = observedState;
+            if (!verificationResult.withinTolerance && currentValue !== null) {
+                lastAdjustmentMode = 'gesture';
+                const fallbackActionResult = await ToolsInteract.swipeHandler({
+                    platform: resolvedPlatform,
+                    x1: currentPoint.x,
+                    y1: currentPoint.y,
+                    x2: targetPoint.x,
+                    y2: targetPoint.y,
+                    duration: 220,
+                    deviceId: resolvedDeviceId
+                });
+                attemptCount++;
+                if (!fallbackActionResult.success) {
+                    return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device, observedState ?? actualState, attemptCount, lastAdjustmentMode, false);
+                }
+                verificationResult = await runVerification();
+                observedState = verificationResult.observedState;
+            }
+            const verification = verificationResult.verification;
+            lastObservedState = observedState;
+            if (verificationResult.withinTolerance) {
+                const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
+                const base = buildActionExecutionResult({
+                    actionType,
+                    sourceModule: 'interact',
+                    device: actionDevice ?? currentDevice,
+                    selector: targetSelector,
+                    resolved: resolvedTarget,
+                    success: true,
+                    uiFingerprintBefore: fingerprintBefore,
+                    uiFingerprintAfter,
+                    details: {
+                        target_value: targetValue,
+                        tolerance: normalizedTolerance,
+                        property,
+                        attempts: attemptCount,
+                        adjustment_mode: lastAdjustmentMode,
+                        actual_state: observedState,
+                        converged: true,
+                        within_tolerance: true,
+                        reason: verification?.reason ?? 'control converged to target value'
+                    }
+                });
+                return {
+                    ...base,
+                    target_state: {
+                        property,
+                        target_value: targetValue,
+                        tolerance: normalizedTolerance
+                    },
+                    actual_state: observedState,
+                    within_tolerance: true,
+                    converged: true,
+                    attempts: attemptCount,
+                    adjustment_mode: lastAdjustmentMode
+                };
+            }
+            cachedResolvedMatch = {
+                el: {
+                    ...currentEl,
+                    state: {
+                        ...(currentEl.state ?? null),
+                        ...(observedState ? {
+                            [observedState.property]: observedState.value,
+                            raw_value: observedState.raw_value ?? observedState.value
+                        } : {})
+                    }
+                },
+                idx: resolved.match.idx
+            };
+        }
+        const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
+        return buildFailure('TIMEOUT', 'control did not converge within the allotted attempts', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true, uiFingerprintAfter);
+    }
     static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
         const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
         return await interact.swipe(x1, y1, x2, y2, duration, resolved.id);
@@ -448,11 +911,13 @@ export class ToolsInteract {
             if (r <= l || b <= t)
                 return null;
             // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
-            const interactable = !!(el.clickable || el.enabled || el.focusable);
+            const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el));
             const text = normalize(el.text ?? el.label ?? el.value ?? '');
             const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
             const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
             const className = normalize(el.type ?? el.class ?? '');
+            const semanticRole = normalize(el.semantic?.semantic_role ?? '');
+            const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : [];
             let score = 0;
             let reason = 'best_scoring_candidate';
             if (exact) {
@@ -503,6 +968,30 @@ export class ToolsInteract {
                     reason = 'partial_class_match';
                 }
             }
+            if (!exact) {
+                if (!score && semanticRole && semanticRole.includes(q)) {
+                    score = 0.5;
+                    reason = 'semantic_role_match';
+                }
+                if (semanticActions.some((action) => action.includes(q))) {
+                    score = Math.max(score, score > 0 ? 0.65 : 0.6);
+                    reason = 'semantic_action_match';
+                }
+                if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
+                    score = 0.45;
+                    reason = 'semantic_control_match';
+                }
+            }
+            else {
+                if (!score && semanticRole && semanticRole === q) {
+                    score = 0.5;
+                    reason = 'semantic_role_match';
+                }
+                if (semanticActions.some((action) => action === q)) {
+                    score = Math.max(score, score > 0 ? 0.65 : 0.6);
+                    reason = 'semantic_action_match';
+                }
+            }
             if (score > 0 && interactable)
                 score += 0.05;
             if (score <= 0)
@@ -631,7 +1120,7 @@ export class ToolsInteract {
                     interactable: true
                 };
             }
-            if (best && !(best.el.clickable || best.el.focusable)) {
+            if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
                 const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen);
                 if (nearbyActionable) {
                     best = {

package/dist/observe/ios.js

@@ -52,6 +52,12 @@ function normalizeIOSType(value) {
 function inferIOSRole(type, traits) {
     if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait)))
         return 'slider';
+    if (/stepper/.test(type))
+        return 'stepper';
+    if (/picker|pop up button|dropdown/.test(type))
+        return 'dropdown';
+    if (/segmented control/.test(type))
+        return 'segmented_control';
     if (/button/.test(type) || traits.some((trait) => /button/.test(trait)))
         return 'button';
     if (/cell/.test(type))
@@ -99,14 +105,52 @@ function buildIOSSelector(type, label, value, stableId) {
         return { value: type, confidence: buildIOSSelectorConfidence('type') };
     return null;
 }
-function buildIOSSemantic(type, traits) {
-    return {
+function buildIOSSemantic(type, traits, role, value) {
+    const semantic = {
         is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
         is_container: /window|application|group|scroll view|collection view/.test(type)
     };
+    if (role === 'slider') {
+        semantic.semantic_role = 'slider';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['adjust'];
+        semantic.state_shape = 'continuous';
+    }
+    else if (role === 'stepper') {
+        semantic.semantic_role = 'stepper';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['increment', 'decrement'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (role === 'dropdown') {
+        semantic.semantic_role = 'dropdown';
+        semantic.supported_actions = ['tap', 'expand'];
+        semantic.state_shape = 'semantic';
+    }
+    else if (role === 'segmented_control') {
+        semantic.semantic_role = 'segmented_control';
+        semantic.supported_actions = ['tap'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (traits.some((trait) => /adjustable|slider/i.test(trait)) || /adjustable|slider/.test(type)) {
+        semantic.semantic_role = 'custom_adjustable';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['adjust'];
+        semantic.state_shape = 'continuous';
+    }
+    else if (semantic.is_clickable) {
+        semantic.supported_actions = ['tap'];
+    }
+    if (semantic.state_shape === undefined && semantic.adjustable && value !== null) {
+        const numericValue = parseIOSNumber(value);
+        if (numericValue !== null && numericValue >= 0 && numericValue <= 1) {
+            semantic.state_shape = 'continuous';
+        }
+    }
+    return semantic;
 }
 function isIOSAdjustable(node, type, traits) {
-    return /slider|adjustable|stepper|progress/i.test(type) || traits.some((trait) => /adjustable|slider|progress/i.test(trait));
+    return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait));
 }
 function extractIOSState(node, type, label, value, traits) {
     const state = {};
@@ -162,8 +206,8 @@ export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
     const normalizedType = normalizeIOSType(type);
     const stableId = getIOSStableId(node);
     const selector = buildIOSSelector(type, label, value, stableId);
-    const semantic = buildIOSSemantic(normalizedType, traits);
     const role = inferIOSRole(normalizedType, traits);
+    const semantic = buildIOSSemantic(normalizedType, traits, role, value);
     const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
     const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
     if (isUseful) {