mobile-debug-mcp 0.26.4 → 0.26.5

package/dist/interact/index.js

@@ -226,6 +226,13 @@ export class ToolsInteract {
         const role = ToolsInteract._normalize(el.role ?? '');
         return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role);
     }
+    static _isSemanticActionable(el) {
+        if (!el?.semantic)
+            return false;
+        if (el.semantic.adjustable)
+            return true;
+        return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0;
+    }
     static _readNumericControlValue(el, property) {
         if (!el?.state)
             return null;
@@ -318,11 +325,11 @@ export class ToolsInteract {
     static _resolveActionableAncestor(elements, chosen) {
         if (!chosen)
             return null;
-        if (chosen.el.clickable || chosen.el.focusable)
+        if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el))
             return chosen;
         let current = chosen;
         let safety = 0;
-        while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable) && current.el.parentId !== undefined && current.el.parentId !== null) {
+        while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
             const parentId = current.el.parentId;
             let parentIndex = null;
             if (typeof parentId === 'number')
@@ -331,7 +338,7 @@ export class ToolsInteract {
                 parentIndex = Number(parentId);
             if (parentIndex !== null && elements[parentIndex]) {
                 current = { el: elements[parentIndex], idx: parentIndex };
-                if (current.el.clickable || current.el.focusable)
+                if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
                     return current;
             }
             else if (typeof parentId === 'string') {
@@ -339,7 +346,7 @@ export class ToolsInteract {
                 if (foundIndex === -1)
                     break;
                 current = { el: elements[foundIndex], idx: foundIndex };
-                if (current.el.clickable || current.el.focusable)
+                if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
                     return current;
             }
             else {
@@ -355,7 +362,7 @@ export class ToolsInteract {
         let bestArea = Infinity;
         for (let i = 0; i < elements.length; i++) {
             const el = elements[i];
-            if (!el || !(el.clickable || el.focusable))
+            if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el)))
                 continue;
             const bounds = ToolsInteract._normalizeBounds(el.bounds);
             if (!bounds)
@@ -904,11 +911,13 @@ export class ToolsInteract {
             if (r <= l || b <= t)
                 return null;
             // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
-            const interactable = !!(el.clickable || el.enabled || el.focusable);
+            const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el));
             const text = normalize(el.text ?? el.label ?? el.value ?? '');
             const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '');
             const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '');
             const className = normalize(el.type ?? el.class ?? '');
+            const semanticRole = normalize(el.semantic?.semantic_role ?? '');
+            const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : [];
             let score = 0;
             let reason = 'best_scoring_candidate';
             if (exact) {
@@ -959,6 +968,30 @@ export class ToolsInteract {
                     reason = 'partial_class_match';
                 }
             }
+            if (!exact) {
+                if (!score && semanticRole && semanticRole.includes(q)) {
+                    score = 0.5;
+                    reason = 'semantic_role_match';
+                }
+                if (semanticActions.some((action) => action.includes(q))) {
+                    score = Math.max(score, score > 0 ? 0.65 : 0.6);
+                    reason = 'semantic_action_match';
+                }
+                if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
+                    score = 0.45;
+                    reason = 'semantic_control_match';
+                }
+            }
+            else {
+                if (!score && semanticRole && semanticRole === q) {
+                    score = 0.5;
+                    reason = 'semantic_role_match';
+                }
+                if (semanticActions.some((action) => action === q)) {
+                    score = Math.max(score, score > 0 ? 0.65 : 0.6);
+                    reason = 'semantic_action_match';
+                }
+            }
             if (score > 0 && interactable)
                 score += 0.05;
             if (score <= 0)
@@ -1087,7 +1120,7 @@ export class ToolsInteract {
                     interactable: true
                 };
             }
-            if (best && !(best.el.clickable || best.el.focusable)) {
+            if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
                 const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen);
                 if (nearbyActionable) {
                     best = {

package/dist/observe/ios.js

@@ -52,6 +52,12 @@ function normalizeIOSType(value) {
 function inferIOSRole(type, traits) {
     if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait)))
         return 'slider';
+    if (/stepper/.test(type))
+        return 'stepper';
+    if (/picker|pop up button|dropdown/.test(type))
+        return 'dropdown';
+    if (/segmented control/.test(type))
+        return 'segmented_control';
     if (/button/.test(type) || traits.some((trait) => /button/.test(trait)))
         return 'button';
     if (/cell/.test(type))
@@ -99,11 +105,49 @@ function buildIOSSelector(type, label, value, stableId) {
         return { value: type, confidence: buildIOSSelectorConfidence('type') };
     return null;
 }
-function buildIOSSemantic(type, traits) {
-    return {
+function buildIOSSemantic(type, traits, role, value) {
+    const semantic = {
         is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
         is_container: /window|application|group|scroll view|collection view/.test(type)
     };
+    if (role === 'slider') {
+        semantic.semantic_role = 'slider';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['adjust'];
+        semantic.state_shape = 'continuous';
+    }
+    else if (role === 'stepper') {
+        semantic.semantic_role = 'stepper';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['increment', 'decrement'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (role === 'dropdown') {
+        semantic.semantic_role = 'dropdown';
+        semantic.supported_actions = ['tap', 'expand'];
+        semantic.state_shape = 'semantic';
+    }
+    else if (role === 'segmented_control') {
+        semantic.semantic_role = 'segmented_control';
+        semantic.supported_actions = ['tap'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (traits.some((trait) => /adjustable|slider/i.test(trait)) || /adjustable|slider/.test(type)) {
+        semantic.semantic_role = 'custom_adjustable';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['adjust'];
+        semantic.state_shape = 'continuous';
+    }
+    else if (semantic.is_clickable) {
+        semantic.supported_actions = ['tap'];
+    }
+    if (semantic.state_shape === undefined && semantic.adjustable && value !== null) {
+        const numericValue = parseIOSNumber(value);
+        if (numericValue !== null && numericValue >= 0 && numericValue <= 1) {
+            semantic.state_shape = 'continuous';
+        }
+    }
+    return semantic;
 }
 function isIOSAdjustable(node, type, traits) {
     return /slider|adjustable|stepper/i.test(type) || traits.some((trait) => /adjustable|slider/i.test(trait));
@@ -162,8 +206,8 @@ export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
     const normalizedType = normalizeIOSType(type);
     const stableId = getIOSStableId(node);
     const selector = buildIOSSelector(type, label, value, stableId);
-    const semantic = buildIOSSemantic(normalizedType, traits);
     const role = inferIOSRole(normalizedType, traits);
+    const semantic = buildIOSSemantic(normalizedType, traits, role, value);
     const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
     const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
     if (isUseful) {

package/dist/server-core.js

@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
 export { wrapResponse, toolDefinitions, handleToolCall };
 export const serverInfo = {
     name: 'mobile-debug-mcp',
-    version: '0.26.4'
+    version: '0.26.5'
 };
 export function createServer() {
     const server = new Server(serverInfo, {

package/dist/utils/android/utils.js

@@ -362,6 +362,12 @@ function normalizeClassName(value) {
 function inferAndroidRole(className) {
     if (/seekbar|slider/.test(className))
         return 'slider';
+    if (/stepper|numberpicker/.test(className))
+        return 'stepper';
+    if (/spinner|dropdown/.test(className))
+        return 'dropdown';
+    if (/segment|tablayout/.test(className))
+        return 'segmented_control';
     if (/switch|toggle/.test(className))
         return 'switch';
     if (/checkbox/.test(className))
@@ -403,11 +409,37 @@ function buildAndroidSelector(text, contentDescription, resourceId, className) {
         return { value: className, confidence: buildAndroidSelectorConfidence('class') };
     return null;
 }
-function buildAndroidSemantic(clickable, className) {
-    return {
+function buildAndroidSemantic(clickable, className, role) {
+    const semantic = {
         is_clickable: clickable,
         is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
     };
+    if (role === 'slider') {
+        semantic.semantic_role = 'slider';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['adjust'];
+        semantic.state_shape = 'continuous';
+    }
+    else if (role === 'stepper') {
+        semantic.semantic_role = 'stepper';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['increment', 'decrement'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (role === 'dropdown') {
+        semantic.semantic_role = 'dropdown';
+        semantic.supported_actions = ['tap', 'expand'];
+        semantic.state_shape = 'semantic';
+    }
+    else if (role === 'segmented_control') {
+        semantic.semantic_role = 'segmented_control';
+        semantic.supported_actions = ['tap'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (clickable) {
+        semantic.supported_actions = ['tap'];
+    }
+    return semantic;
 }
 function isSliderLikeAndroid(node) {
     const className = String(node['@_class'] || '').toLowerCase();
@@ -487,7 +519,7 @@ export function traverseNode(node, elements, parentIndex = -1, depth = 0) {
         const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null);
         const testTag = stableId;
         const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className));
-        const semantic = buildAndroidSemantic(clickable, normalizeClassName(className));
+        const semantic = buildAndroidSemantic(clickable, normalizeClassName(className), role);
         const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
         if (isUseful) {
             const element = {

package/docs/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 All notable changes to the **Mobile Debug MCP** project will be documented in this file.
 
+## [0.26.5]
+- Introduces a semantic control model to improve the identification and interaction with custom and composite UI controls. 
+
 ## [0.26.4]
 - Improved slider accuracy
 

package/docs/ROADMAP.md

@@ -44,6 +44,7 @@ Higher task success with fewer retries.
 
 - Stronger State Verification — Complete (Foundational verification layer shipped)
 - Richer Element Identity — Complete (Identity and selector confidence foundations shipped)
+- Better Compose / Custom Control Semantics — Complete (Semantic role enrichment and custom-adjustable inference shipped)
 
 ## Current Focus
 
@@ -54,7 +55,6 @@ Higher task success with fewer retries.
 
 - Environment Auto-Configuration and Toolchain Discovery
 - Adjustable Control Support
-- Better Compose / Custom Control Semantics
 - Signal-Oriented Diagnostic Filtering
 - Long Press Gesture
 # Stronger State Verification
@@ -435,7 +435,7 @@ Strengthens:
 ## Rationale
 Higher priority after agent feedback exposed custom control semantics as a core reliability gap, not a later optimization.
 
-**Status:** Spec Ready
+**Status:** Completed
 
 Semantics become more useful once:
 - identity is stronger
@@ -446,8 +446,8 @@ Semantics become more useful once:
 
 ## Scope
 - Composite control traits
-- Control role enrichment (adjustable, expandable, selectable_group)
-- Interaction contracts metadata
+- Control role enrichment (`slider`, `stepper`, `dropdown`, `segmented_control`, `custom_adjustable`)
+- Interaction contract metadata (`supported_actions`, `adjustable`, `state_shape`)
 - Custom widget gesture affordance hints
 - Semantic confidence annotations
 - Compose-aware selectors for waits (merged semantics and element relationships)
@@ -457,7 +457,7 @@ High.
 
 ## Exit Criteria
 - Semantic traits implemented for major custom control classes
-- Interaction contracts surfaced in snapshot model
+- Interaction contracts surfaced in observation and resolution paths
 - Confidence model defined for derived semantics
 - Custom control manipulation success validated in benchmark flows
 

package/docs/rfcs/009-semantic-control-modeling-for-custom-and-composite-controls.md

@@ -0,0 +1,238 @@
+# RFC 009 — Semantic Control Modeling for Custom and Composite Controls
+
+## 1. Summary
+
+This RFC defines a semantic control model for identifying, exposing, and interacting with custom and composite controls that are poorly represented through raw accessibility or platform UI trees.
+
+It introduces semantic enrichment for controls such as:
+
+- sliders
+- steppers
+- segmented controls
+- dropdowns
+- Compose/SwiftUI custom widgets
+- composite gesture-driven controls
+
+The goal is to improve target resolution, control interaction, and verification reliability for controls whose actionable semantics are not fully captured by raw snapshots.
+
+---
+
+## 2. Problem Statement
+
+Current interaction logic works well when platform semantics are explicit.
+
+It is weaker when controls appear as:
+
+- generic container views
+- unlabeled clickable wrappers
+- nested composite controls
+- custom Compose/SwiftUI components with weak accessibility exposure
+
+Observed problems include:
+
+- controls resolving as parent containers rather than actionable targets
+- missing slider-like controls in snapshots
+- weak distinction between discrete vs continuous controls
+- inability to infer supported interactions from control structure
+- unreliable verification of control state
+
+This causes brittle automation and coordinate fallback behavior.
+
+---
+
+## 3. Goals
+
+This RFC introduces a semantic layer that MUST:
+
+- infer higher-level control semantics from raw UI structures
+- enrich snapshots with semantic control metadata
+- improve actionable target selection (RFC 007)
+- improve adjustable control handling (RFC 008)
+- improve verification for semantic control state
+- reduce coordinate fallback usage
+
+---
+
+## 4. Non-Goals
+
+This RFC does NOT define:
+
+- replacement of raw accessibility trees
+- ML-based semantic inference
+- probabilistic control classification
+- new gesture primitives
+- autonomous planning behavior
+
+Semantic modeling is deterministic enrichment layered over raw signals.
+
+---
+
+## 5. Runtime Surfaces
+
+This RFC applies to existing runtime surfaces:
+
+- findElementHandler
+- _resolveActionableAncestor
+- _buildResolvedElement
+- tapElementHandler
+- scrollToElementHandler
+
+Semantic modeling augments these surfaces; it does not replace them.
+
+---
+
+## 6. Semantic Control Model
+
+Controls MAY progressively expose semantic metadata such as:
+
+```ts
+interface SemanticControl {
+  semantic_role:
+    | "slider"
+    | "stepper"
+    | "dropdown"
+    | "segmented_control"
+    | "custom_adjustable"
+    | "composite_control";
+
+  supported_actions: string[];
+
+  adjustable: boolean;
+
+  state_shape:
+    | "continuous"
+    | "discrete"
+    | "semantic";
+}
+```
+
+The control roles above represent an expected semantic model, not a claim that all such control classes are equally surfaced in the current runtime.
+
+Current runtime support may initially expose simpler semantic signals such as:
+- role hints
+- semantic labels
+- value_range metadata
+- selector confidence or related resolution signals
+
+Richer control roles are progressive extensions over time.
+
+---
+
+## 7. Semantic Inference Rules
+
+Inference MAY use signals such as:
+
+- accessibility role hints
+- value_range metadata
+- child composition patterns
+- repeated selectable child structures
+- platform traits (adjustable, selected, expanded)
+- known control heuristics
+
+Inference MUST be deterministic and explainable.
+
+Raw signals always win on conflict.
+
+Semantic inference confidence, where present, is advisory only and MUST NOT be treated as executable truth.
+
+---
+
+## 8. Resolution Integration (RFC 007)
+
+Semantic metadata SHOULD improve target resolution by:
+
+- preferring actionable child controls over generic containers
+- promoting semantically actionable descendants
+- disambiguating among multiple candidate matches
+
+Semantic signals are advisory enrichment, not executable truth.
+
+---
+
+## 9. Adjustable Control Integration (RFC 008)
+
+Where adjustable=true:
+
+Semantic metadata MAY expose:
+
+- supported adjustment mode
+- discrete vs continuous state model
+- expected verification strategy
+
+This improves convergence for value-setting workflows.
+
+---
+
+## 10. Verification Integration
+
+Verification MAY use semantic control metadata to improve:
+
+- value-state verification
+- discrete selection verification
+- semantic-state checks
+
+Formal verification still remains governed by RFC 005.
+
+---
+
+## 11. Output Contract (Progressive Extension)
+
+Current runtime may expose partial semantic outputs.
+
+Expected progressive shape (future extension model):
+
+```ts
+interface SemanticResolutionMetadata {
+  semantic_role?: string;
+  supported_actions?: string[];
+  adjustable?: boolean;
+  state_shape?: string;
+  confidence?: "low" | "medium" | "high";
+}
+```
+
+These fields are progressive enrichment and MUST NOT be assumed universally present.
+
+Implementations MAY expose only a subset of this model initially. Presence of a richer semantic role does not imply universal runtime support for all control classes.
+
+---
+
+## 12. Failure Modes
+
+Semantic modeling MAY fail due to:
+
+- insufficient raw signals
+- ambiguous composite structures
+- conflicting heuristics
+
+When semantic inference confidence is insufficient:
+
+- raw resolution flow MUST continue
+- semantic fields MAY be omitted
+- no semantic guessing should be forced
+
+---
+
+## 13. Success Metrics
+
+- fewer coordinate fallbacks
+- improved control discovery
+- improved actionable-target precision
+- improved slider/custom-control automation success
+- reduced semantic mismatch failures (RFC 010)
+
+---
+
+## 14. Relationship to Other RFCs
+
+RFC 005 — verification correctness model
+RFC 006 — runtime action execution
+RFC 007 — target resolution
+RFC 008 — adjustable control support
+RFC 010 — recovery uses semantic mismatch failures defined here
+
+---
+
+## 15. Summary
+
+This RFC adds deterministic semantic control enrichment for custom and composite controls, improving resolution, interaction reliability, and verification while remaining layered over existing runtime signals.

package/docs/specs/mcp-tooling-spec-v1.md

@@ -244,6 +244,7 @@ Raw layer contents include:
 - UI hierarchy or accessibility tree
 - normalized readable element state where exposed by the platform
 - platform-native identity hints such as stable identifiers, roles, and test tags
+- semantic control metadata when derivable from the raw tree, including `semantic_role`, `supported_actions`, `adjustable`, and `state_shape`
 - snapshot metadata such as `snapshot_revision` and `captured_at_ms`
 - `loading_state` when a reliable loading signal is detectable
 - screenshot when available
@@ -292,6 +293,27 @@ Semantic output MUST NOT replace classification or verification.
 
 Classification remains a supplementary, post-action interpretation mechanism.
 
+### 9.4 Semantic Control Metadata
+
+When present, semantic control metadata MAY include:
+
+```ts
+{
+  semantic_role?: 'slider' | 'stepper' | 'dropdown' | 'segmented_control' | 'custom_adjustable' | 'composite_control' | null,
+  supported_actions?: string[] | null,
+  adjustable?: boolean | null,
+  state_shape?: 'continuous' | 'discrete' | 'semantic' | null
+}
+```
+
+Rules:
+
+- semantic control metadata is derived and best-effort
+- raw platform roles and state remain authoritative on conflict
+- `adjustable` MAY be inferred from platform traits when no known role matches
+- `state_shape` MUST respect known control roles before value-based heuristics
+- `supported_actions` are hints only and MUST NOT be treated as guaranteed executable actions
+
 ## 10. Classification
 
 Tool: `classify_action_outcome`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobile-debug-mcp",
-  "version": "0.26.4",
+  "version": "0.26.5",
   "description": "MCP server for mobile app debugging (Android + iOS), with focus on security and reliability",
   "type": "module",
   "bin": {

package/src/interact/index.ts

@@ -16,6 +16,7 @@ import type {
   ExpectStateResponse,
   ExpectScreenResponse,
   WaitForUIChangeResponse,
+  UIElementSemanticMetadata,
   UIElementState,
   TapElementResponse
 } from '../types.js'
@@ -48,7 +49,7 @@ interface UiElement {
   role?: string | null
   test_tag?: string | null
   selector?: { value: string | null, confidence: { score: number, reason: string } | null } | null
-  semantic?: { is_clickable: boolean, is_container: boolean } | null
+  semantic?: UIElementSemanticMetadata | null
 }
 
 interface ResolvedUiElementContext {
@@ -342,6 +343,12 @@ export class ToolsInteract {
     return !!el.state?.value_range || /slider|seekbar|stepper|adjustable|range/.test(type) || /slider|seekbar|stepper|adjustable|range/.test(role)
   }
 
+  private static _isSemanticActionable(el: UiElement | null): boolean {
+    if (!el?.semantic) return false
+    if (el.semantic.adjustable) return true
+    return Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.length > 0
+  }
+
   private static _readNumericControlValue(el: UiElement | null, property: string): number | null {
     if (!el?.state) return null
     const stateValue = el.state[property as keyof UIElementState]
@@ -460,12 +467,12 @@ export class ToolsInteract {
 
   private static _resolveActionableAncestor(elements: UiElement[], chosen: { el: UiElement, idx: number } | null): { el: UiElement, idx: number } | null {
     if (!chosen) return null
-    if (chosen.el.clickable || chosen.el.focusable) return chosen
+    if (chosen.el.clickable || chosen.el.focusable || ToolsInteract._isSemanticActionable(chosen.el)) return chosen
 
     let current = chosen
     let safety = 0
 
-    while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable) && current.el.parentId !== undefined && current.el.parentId !== null) {
+    while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
       const parentId = current.el.parentId
       let parentIndex: number | null = null
 
@@ -474,12 +481,12 @@ export class ToolsInteract {
 
       if (parentIndex !== null && elements[parentIndex]) {
         current = { el: elements[parentIndex], idx: parentIndex }
-        if (current.el.clickable || current.el.focusable) return current
+        if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
       } else if (typeof parentId === 'string') {
         const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId)
         if (foundIndex === -1) break
         current = { el: elements[foundIndex], idx: foundIndex }
-        if (current.el.clickable || current.el.focusable) return current
+        if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
       } else {
         break
       }
@@ -496,7 +503,7 @@ export class ToolsInteract {
 
     for (let i = 0; i < elements.length; i++) {
       const el = elements[i]
-      if (!el || !(el.clickable || el.focusable)) continue
+      if (!el || !(el.clickable || el.focusable || ToolsInteract._isSemanticActionable(el))) continue
       const bounds = ToolsInteract._normalizeBounds(el.bounds)
       if (!bounds) continue
       const [pl, pt, pr, pb] = bounds
@@ -1196,12 +1203,14 @@ export class ToolsInteract {
       const [l,t,r,b] = bounds
       if (r <= l || b <= t) return null
       // Do not early-return on non-interactable elements — score them so we can locate their clickable ancestor later
-      const interactable = !!(el.clickable || el.enabled || el.focusable)
+      const interactable = !!(el.clickable || el.enabled || el.focusable || ToolsInteract._isSemanticActionable(el))
 
       const text = normalize(el.text ?? el.label ?? el.value ?? '')
       const content = normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? '')
       const resourceId = normalize(el.resourceId ?? el.resourceID ?? el.id ?? '')
       const className = normalize(el.type ?? el.class ?? '')
+      const semanticRole = normalize(el.semantic?.semantic_role ?? '')
+      const semanticActions = Array.isArray(el.semantic?.supported_actions) ? el.semantic.supported_actions.map((action) => normalize(action)).filter(Boolean) : []
 
       let score = 0
       let reason = 'best_scoring_candidate'
@@ -1243,6 +1252,29 @@ export class ToolsInteract {
           reason = 'partial_class_match'
         }
       }
+      if (!exact) {
+        if (!score && semanticRole && semanticRole.includes(q)) {
+          score = 0.5
+          reason = 'semantic_role_match'
+        }
+        if (semanticActions.some((action) => action.includes(q))) {
+          score = Math.max(score, score > 0 ? 0.65 : 0.6)
+          reason = 'semantic_action_match'
+        }
+        if (score === 0 && el.semantic?.adjustable && /slider|stepper|dropdown|segment|control|adjust/.test(q)) {
+          score = 0.45
+          reason = 'semantic_control_match'
+        }
+      } else {
+        if (!score && semanticRole && semanticRole === q) {
+          score = 0.5
+          reason = 'semantic_role_match'
+        }
+        if (semanticActions.some((action) => action === q)) {
+          score = Math.max(score, score > 0 ? 0.65 : 0.6)
+          reason = 'semantic_action_match'
+        }
+      }
       if (score > 0 && interactable) score += 0.05
       if (score <= 0) return null
       return { el, idx, score, reason, interactable }
@@ -1352,7 +1384,7 @@ export class ToolsInteract {
         }
       }
 
-      if (best && !(best.el.clickable || best.el.focusable)) {
+      if (best && !(best.el.clickable || best.el.focusable || ToolsInteract._isSemanticActionable(best.el))) {
         const nearbyActionable = ToolsInteract._resolveNearbyActionableControl(elements, { el: best.el, idx: best.idx }, screen)
         if (nearbyActionable) {
           best = {

package/src/observe/ios.ts

@@ -73,6 +73,9 @@ function normalizeIOSType(value: unknown): string {
 
 function inferIOSRole(type: string, traits: string[]): string | null {
   if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait))) return 'slider'
+  if (/stepper/.test(type)) return 'stepper'
+  if (/picker|pop up button|dropdown/.test(type)) return 'dropdown'
+  if (/segmented control/.test(type)) return 'segmented_control'
   if (/button/.test(type) || traits.some((trait) => /button/.test(trait))) return 'button'
   if (/cell/.test(type)) return 'cell'
   if (/switch/.test(type)) return 'switch'
@@ -113,11 +116,47 @@ function buildIOSSelector(type: string, label: string | null, value: string | nu
   return null
 }
 
-function buildIOSSemantic(type: string, traits: string[]): UIElementSemanticMetadata {
-  return {
+function buildIOSSemantic(type: string, traits: string[], role: string | null, value: string | null): UIElementSemanticMetadata {
+  const semantic: UIElementSemanticMetadata = {
     is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
     is_container: /window|application|group|scroll view|collection view/.test(type)
   }
+
+  if (role === 'slider') {
+    semantic.semantic_role = 'slider'
+    semantic.adjustable = true
+    semantic.supported_actions = ['adjust']
+    semantic.state_shape = 'continuous'
+  } else if (role === 'stepper') {
+    semantic.semantic_role = 'stepper'
+    semantic.adjustable = true
+    semantic.supported_actions = ['increment', 'decrement']
+    semantic.state_shape = 'discrete'
+  } else if (role === 'dropdown') {
+    semantic.semantic_role = 'dropdown'
+    semantic.supported_actions = ['tap', 'expand']
+    semantic.state_shape = 'semantic'
+  } else if (role === 'segmented_control') {
+    semantic.semantic_role = 'segmented_control'
+    semantic.supported_actions = ['tap']
+    semantic.state_shape = 'discrete'
+  } else if (traits.some((trait) => /adjustable|slider/i.test(trait)) || /adjustable|slider/.test(type)) {
+    semantic.semantic_role = 'custom_adjustable'
+    semantic.adjustable = true
+    semantic.supported_actions = ['adjust']
+    semantic.state_shape = 'continuous'
+  } else if (semantic.is_clickable) {
+    semantic.supported_actions = ['tap']
+  }
+
+  if (semantic.state_shape === undefined && semantic.adjustable && value !== null) {
+    const numericValue = parseIOSNumber(value)
+    if (numericValue !== null && numericValue >= 0 && numericValue <= 1) {
+      semantic.state_shape = 'continuous'
+    }
+  }
+
+  return semantic
 }
 
 function isIOSAdjustable(node: IDBElement, type: string, traits: string[]): boolean {
@@ -184,8 +223,8 @@ export function traverseIDBNode(node: IDBElement, elements: UIElement[], parentI
   const normalizedType = normalizeIOSType(type)
   const stableId = getIOSStableId(node)
   const selector = buildIOSSelector(type, label, value, stableId)
-  const semantic = buildIOSSemantic(normalizedType, traits)
   const role = inferIOSRole(normalizedType, traits)
+  const semantic = buildIOSSemantic(normalizedType, traits, role, value)
   
   const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
   

package/src/server-core.ts

@@ -13,7 +13,7 @@ export { wrapResponse, toolDefinitions, handleToolCall }
 
 export const serverInfo = {
   name: 'mobile-debug-mcp',
-  version: '0.26.4'
+  version: '0.26.5'
 }
 
 export function createServer() {

package/src/types.ts

@@ -107,6 +107,10 @@ export interface UIResolutionSelector {
 export interface UIElementSemanticMetadata {
   is_clickable: boolean;
   is_container: boolean;
+  semantic_role?: 'slider' | 'stepper' | 'dropdown' | 'segmented_control' | 'custom_adjustable' | 'composite_control' | null;
+  supported_actions?: string[] | null;
+  adjustable?: boolean | null;
+  state_shape?: 'continuous' | 'discrete' | 'semantic' | null;
 }
 
 export interface LoadingState {

package/src/utils/android/utils.ts

@@ -342,6 +342,9 @@ function normalizeClassName(value: unknown): string {
 
 function inferAndroidRole(className: string): string | null {
   if (/seekbar|slider/.test(className)) return 'slider'
+  if (/stepper|numberpicker/.test(className)) return 'stepper'
+  if (/spinner|dropdown/.test(className)) return 'dropdown'
+  if (/segment|tablayout/.test(className)) return 'segmented_control'
   if (/switch|toggle/.test(className)) return 'switch'
   if (/checkbox/.test(className)) return 'checkbox'
   if (/radiobutton|radio/.test(className)) return 'radio'
@@ -375,11 +378,35 @@ function buildAndroidSelector(text: string | null, contentDescription: string |
   return null
 }
 
-function buildAndroidSemantic(clickable: boolean, className: string): UIElementSemanticMetadata {
-  return {
+function buildAndroidSemantic(clickable: boolean, className: string, role: string | null): UIElementSemanticMetadata {
+  const semantic: UIElementSemanticMetadata = {
     is_clickable: clickable,
     is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
   }
+
+  if (role === 'slider') {
+    semantic.semantic_role = 'slider'
+    semantic.adjustable = true
+    semantic.supported_actions = ['adjust']
+    semantic.state_shape = 'continuous'
+  } else if (role === 'stepper') {
+    semantic.semantic_role = 'stepper'
+    semantic.adjustable = true
+    semantic.supported_actions = ['increment', 'decrement']
+    semantic.state_shape = 'discrete'
+  } else if (role === 'dropdown') {
+    semantic.semantic_role = 'dropdown'
+    semantic.supported_actions = ['tap', 'expand']
+    semantic.state_shape = 'semantic'
+  } else if (role === 'segmented_control') {
+    semantic.semantic_role = 'segmented_control'
+    semantic.supported_actions = ['tap']
+    semantic.state_shape = 'discrete'
+  } else if (clickable) {
+    semantic.supported_actions = ['tap']
+  }
+
+  return semantic
 }
 
 function isSliderLikeAndroid(node: any): boolean {
@@ -459,7 +486,7 @@ export function traverseNode(node: any, elements: UIElement[], parentIndex: numb
     const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null)
     const testTag = stableId
     const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className))
-    const semantic = buildAndroidSemantic(clickable, normalizeClassName(className))
+    const semantic = buildAndroidSemantic(clickable, normalizeClassName(className), role)
 
     const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
 

package/test/unit/observe/find_element.test.ts

@@ -78,6 +78,52 @@ async function run() {
     assert.ok((res4.resolution?.alternates || []).length >= 1, 'Parent promotion should preserve alternates')
     process.stdout.write('Test 4: ' + (pass4 ? 'PASS' : 'FAIL') + '\n');
 
+    // Test 4b: semantic-only stepper should be discoverable by supported action
+    ;(ToolsObserve as any).getUITreeHandler = async () => ({
+      device: { platform: 'android', id: 'mock' },
+      screen: '',
+      resolution: { width: 1080, height: 1920 },
+      elements: [
+        {
+          text: null,
+          contentDescription: 'Quantity stepper',
+          type: 'android.widget.NumberPicker',
+          clickable: false,
+          enabled: true,
+          visible: true,
+          bounds: [10,10,200,80],
+          resourceId: 'picker_quantity',
+          semantic: {
+            is_clickable: false,
+            is_container: true,
+            semantic_role: 'stepper',
+            supported_actions: ['increment', 'decrement'],
+            adjustable: true,
+            state_shape: 'discrete'
+          }
+        }
+      ]
+    })
+
+    const res4b: any = await ToolsInteract.findElementHandler({ query: 'increment', exact: false, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res4b ' + JSON.stringify(res4b, null, 2) + '\n');
+    const pass4b = res4b.found === true && res4b.element && res4b.element.resourceId === 'picker_quantity' && res4b.element.semantic?.semantic_role === 'stepper'
+    assert.ok(pass4b, 'Semantic-only steppers should be discoverable by supported actions')
+    assert.strictEqual(res4b.resolution?.reason, 'semantic_action_match')
+    process.stdout.write('Test 4b: ' + (pass4b ? 'PASS' : 'FAIL') + '\n');
+
+    const res4bb: any = await ToolsInteract.findElementHandler({ query: 'increment', exact: true, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res4bb ' + JSON.stringify(res4bb, null, 2) + '\n');
+    const pass4bb = res4bb.found === true && res4bb.element && res4bb.element.resourceId === 'picker_quantity' && res4bb.resolution?.reason === 'semantic_action_match'
+    assert.ok(pass4bb, 'Exact searches should still match exact semantic actions')
+    process.stdout.write('Test 4bb: ' + (pass4bb ? 'PASS' : 'FAIL') + '\n');
+
+    const res4c: any = await ToolsInteract.findElementHandler({ query: 'control', exact: true, platform: 'android', timeoutMs: 300 })
+    process.stdout.write('res4c ' + JSON.stringify(res4c, null, 2) + '\n');
+    const pass4c = res4c.found === false
+    assert.ok(pass4c, 'Exact searches should not fall back to broad semantic keywords')
+    process.stdout.write('Test 4c: ' + (pass4c ? 'PASS' : 'FAIL') + '\n');
+
     // Test 5: duration label should resolve to the nearby slider control
     ;(ToolsObserve as any).getUITreeHandler = async () => ({
       device: { platform: 'android', id: 'mock' },

package/test/unit/observe/state_extraction.test.ts

@@ -29,7 +29,14 @@ async function run() {
     value: 'com.example:id/duration',
     confidence: { score: 1, reason: 'resource_id' }
   })
-  assert.deepStrictEqual(androidElements[0].semantic, { is_clickable: true, is_container: false })
+  assert.deepStrictEqual(androidElements[0].semantic, {
+    is_clickable: true,
+    is_container: false,
+    semantic_role: 'slider',
+    supported_actions: ['adjust'],
+    adjustable: true,
+    state_shape: 'continuous'
+  })
 
   const androidProgressElements: any[] = []
   traverseNode({
@@ -44,6 +51,21 @@ async function run() {
 
   assert.notStrictEqual(androidProgressElements[0]?.role, 'slider')
   assert.notStrictEqual(androidProgressElements[0]?.state?.value, 40)
+  assert.notStrictEqual(androidProgressElements[0]?.semantic?.adjustable, true)
+
+  const androidStepperElements: any[] = []
+  traverseNode({
+    '@_class': 'android.widget.NumberPicker',
+    '@_text': 'Quantity',
+    '@_content-desc': 'Quantity stepper',
+    '@_clickable': 'false',
+    '@_enabled': 'true',
+    '@_bounds': '[0,0][200,80]'
+  }, androidStepperElements)
+  assert.strictEqual(androidStepperElements[0].role, 'stepper')
+  assert.deepStrictEqual(androidStepperElements[0].semantic?.semantic_role, 'stepper')
+  assert.deepStrictEqual(androidStepperElements[0].semantic?.supported_actions, ['increment', 'decrement'])
+  assert.strictEqual(androidStepperElements[0].semantic?.adjustable, true)
 
   const androidFallbackElements: any[] = []
   traverseNode({
@@ -82,7 +104,14 @@ async function run() {
     value: 'playback_speed_slider',
     confidence: { score: 1, reason: 'accessibility_identifier' }
   })
-  assert.deepStrictEqual(iosElements[0].semantic, { is_clickable: true, is_container: false })
+  assert.deepStrictEqual(iosElements[0].semantic, {
+    is_clickable: true,
+    is_container: false,
+    semantic_role: 'slider',
+    supported_actions: ['adjust'],
+    adjustable: true,
+    state_shape: 'continuous'
+  })
 
   const iosProgressElements: any[] = []
   traverseIDBNode({
@@ -94,6 +123,40 @@ async function run() {
 
   assert.notStrictEqual(iosProgressElements[0]?.role, 'slider')
 
+  const iosStepperElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Stepper',
+    AXLabel: 'Quantity',
+    AXValue: '1',
+    AXTraits: ['UIAccessibilityTraitAdjustable']
+  }, iosStepperElements)
+  assert.strictEqual(iosStepperElements[0].role, 'stepper')
+  assert.strictEqual(iosStepperElements[0].semantic?.semantic_role, 'stepper')
+  assert.deepStrictEqual(iosStepperElements[0].semantic?.supported_actions, ['increment', 'decrement'])
+  assert.strictEqual(iosStepperElements[0].semantic?.state_shape, 'discrete')
+
+  const iosSegmentedElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'Segmented Control',
+    AXLabel: 'Playback mode',
+    AXTraits: ['UIAccessibilityTraitButton']
+  }, iosSegmentedElements)
+  assert.strictEqual(iosSegmentedElements[0].role, 'segmented_control')
+  assert.strictEqual(iosSegmentedElements[0].semantic?.semantic_role, 'segmented_control')
+  assert.deepStrictEqual(iosSegmentedElements[0].semantic?.supported_actions, ['tap'])
+
+  const iosCustomAdjustableElements: any[] = []
+  traverseIDBNode({
+    AXElementType: 'CustomControl',
+    AXLabel: 'Intensity',
+    AXValue: '0.25',
+    AXTraits: ['UIAccessibilityTraitAdjustable']
+  }, iosCustomAdjustableElements)
+  assert.strictEqual(iosCustomAdjustableElements[0].semantic?.semantic_role, 'custom_adjustable')
+  assert.strictEqual(iosCustomAdjustableElements[0].semantic?.adjustable, true)
+  assert.deepStrictEqual(iosCustomAdjustableElements[0].semantic?.supported_actions, ['adjust'])
+  assert.strictEqual(iosCustomAdjustableElements[0].semantic?.state_shape, 'continuous')
+
   const iosFallbackElements: any[] = []
   traverseIDBNode({
     AXElementType: 'Button',