mobile-debug-mcp 0.26.3 → 0.26.5

package/dist/server/tool-definitions.js

@@ -555,6 +555,62 @@ Failure Handling:
             required: ['property', 'expected']
         }
     },
+    {
+        name: 'adjust_control',
+        description: `Purpose:
+Adjust a numeric control value with verification.
+
+This is the initial adjustable-control surface for slider-like controls and other controls that expose a numeric value or value_range.
+
+Inputs:
+- selector or element_id
+- property (defaults to "value")
+- targetValue
+- tolerance (optional)
+- maxAttempts (optional)
+- platform/deviceId (optional)
+
+Output Structure:
+- action_id, timestamp (ISO 8601), action_type
+- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
+- source_module: runtime source of the action envelope
+- target_state / actual_state / within_tolerance / converged / attempts / adjustment_mode
+- target.selector = original selector or element handle
+- success = true when the control converges within tolerance
+
+Verification Guidance:
+- Prefer direct target placement when value_range is available; fall back to a drag only if the direct tap does not converge
+- Use expect_state for the control value readback
+- Treat coordinate fallback as degraded mode
+
+Failure Handling:
+- ELEMENT_NOT_FOUND → re-resolve the control
+- ELEMENT_NOT_INTERACTABLE → the control cannot be adjusted through the current runtime
+- TIMEOUT → the control did not converge within bounded retries
+- UNKNOWN → capture a snapshot and stop`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                selector: {
+                    type: 'object',
+                    properties: {
+                        text: { type: 'string' },
+                        resource_id: { type: 'string' },
+                        accessibility_id: { type: 'string' },
+                        contains: { type: 'boolean', default: false }
+                    }
+                },
+                element_id: { type: 'string', description: 'Optional previously resolved element identifier.' },
+                property: { type: 'string', description: 'Readable numeric state property to adjust.', default: 'value' },
+                targetValue: { type: 'number', description: 'Target numeric value.' },
+                tolerance: { type: 'number', description: 'Accepted numeric tolerance around the target value.', default: 0 },
+                maxAttempts: { type: 'number', description: 'Maximum adjustment attempts.', default: 3 },
+                platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override' },
+                deviceId: { type: 'string', description: 'Optional device serial/udid' }
+            },
+            required: ['targetValue']
+        }
+    },
     {
         name: 'wait_for_ui',
         description: `Purpose:

package/dist/server/tool-handlers.js

@@ -226,6 +226,30 @@ async function handleExpectState(args) {
     const res = await ToolsInteract.expectStateHandler({ selector: selector ?? undefined, element_id: element_id ?? undefined, property, expected, platform, deviceId });
     return wrapResponse(res);
 }
+async function handleAdjustControl(args) {
+    const selector = getObjectArg(args, 'selector');
+    const element_id = getStringArg(args, 'element_id');
+    const property = getStringArg(args, 'property') ?? 'value';
+    const targetValue = requireNumberArg(args, 'targetValue');
+    const tolerance = getNumberArg(args, 'tolerance') ?? 0;
+    const maxAttempts = getNumberArg(args, 'maxAttempts') ?? 3;
+    const platform = getStringArg(args, 'platform');
+    const deviceId = getStringArg(args, 'deviceId');
+    if (!selector && !element_id) {
+        throw new Error('Missing selector or element_id argument');
+    }
+    const res = await ToolsInteract.adjustControlHandler({
+        selector: selector ?? undefined,
+        element_id: element_id ?? undefined,
+        property,
+        targetValue,
+        tolerance,
+        maxAttempts,
+        platform,
+        deviceId
+    });
+    return wrapResponse(res);
+}
 async function handleWaitForUI(args) {
     const selector = getObjectArg(args, 'selector');
     const condition = getStringArg(args, 'condition') ?? 'exists';
@@ -431,6 +455,7 @@ export const toolHandlers = {
     expect_screen: handleExpectScreen,
     expect_element_visible: handleExpectElementVisible,
     expect_state: handleExpectState,
+    adjust_control: handleAdjustControl,
     wait_for_ui: handleWaitForUI,
     find_element: handleFindElement,
     tap: handleTap,

package/dist/server-core.js

@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
 export { wrapResponse, toolDefinitions, handleToolCall };
 export const serverInfo = {
     name: 'mobile-debug-mcp',
-    version: '0.26.3'
+    version: '0.26.5'
 };
 export function createServer() {
     const server = new Server(serverInfo, {

package/dist/utils/android/utils.js

@@ -360,8 +360,14 @@ function normalizeClassName(value) {
     return typeof value === 'string' ? value.trim().toLowerCase() : '';
 }
 function inferAndroidRole(className) {
-    if (/seekbar|slider|progress/.test(className))
+    if (/seekbar|slider/.test(className))
         return 'slider';
+    if (/stepper|numberpicker/.test(className))
+        return 'stepper';
+    if (/spinner|dropdown/.test(className))
+        return 'dropdown';
+    if (/segment|tablayout/.test(className))
+        return 'segmented_control';
     if (/switch|toggle/.test(className))
         return 'switch';
     if (/checkbox/.test(className))
@@ -403,15 +409,41 @@ function buildAndroidSelector(text, contentDescription, resourceId, className) {
         return { value: className, confidence: buildAndroidSelectorConfidence('class') };
     return null;
 }
-function buildAndroidSemantic(clickable, className) {
-    return {
+function buildAndroidSemantic(clickable, className, role) {
+    const semantic = {
         is_clickable: clickable,
         is_container: /recyclerview|scroll|layout|viewgroup|frame/.test(className)
     };
+    if (role === 'slider') {
+        semantic.semantic_role = 'slider';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['adjust'];
+        semantic.state_shape = 'continuous';
+    }
+    else if (role === 'stepper') {
+        semantic.semantic_role = 'stepper';
+        semantic.adjustable = true;
+        semantic.supported_actions = ['increment', 'decrement'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (role === 'dropdown') {
+        semantic.semantic_role = 'dropdown';
+        semantic.supported_actions = ['tap', 'expand'];
+        semantic.state_shape = 'semantic';
+    }
+    else if (role === 'segmented_control') {
+        semantic.semantic_role = 'segmented_control';
+        semantic.supported_actions = ['tap'];
+        semantic.state_shape = 'discrete';
+    }
+    else if (clickable) {
+        semantic.supported_actions = ['tap'];
+    }
+    return semantic;
 }
 function isSliderLikeAndroid(node) {
     const className = String(node['@_class'] || '').toLowerCase();
-    return /seekbar|slider|range|progress/i.test(className);
+    return /seekbar|slider|range/i.test(className);
 }
 function extractAndroidState(node) {
     const checked = parseBooleanAttr(node['@_checked']);
@@ -487,7 +519,7 @@ export function traverseNode(node, elements, parentIndex = -1, depth = 0) {
         const stableId = resourceId ?? (typeof contentDescription === 'string' && contentDescription.trim().length > 0 ? contentDescription : null);
         const testTag = stableId;
         const selector = buildAndroidSelector(text, contentDescription, resourceId, normalizeClassName(className));
-        const semantic = buildAndroidSemantic(clickable, normalizeClassName(className));
+        const semantic = buildAndroidSemantic(clickable, normalizeClassName(className), role);
         const isUseful = clickable || (text && text.length > 0) || (contentDescription && contentDescription.length > 0);
         if (isUseful) {
             const element = {

package/docs/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to the **Mobile Debug MCP** project will be documented in this file.
 
+## [0.26.5]
+- Introduces a semantic control model to improve the identification and interaction with custom and composite UI controls. 
+
+## [0.26.4]
+- Improved slider accuracy
+
 ## [0.26.3]
 - updates the `find_element` tool to return detailed resolution metadata, including confidence scores,
 

package/docs/ROADMAP.md

@@ -44,6 +44,7 @@ Higher task success with fewer retries.
 
 - Stronger State Verification — Complete (Foundational verification layer shipped)
 - Richer Element Identity — Complete (Identity and selector confidence foundations shipped)
+- Better Compose / Custom Control Semantics — Complete (Semantic role enrichment and custom-adjustable inference shipped)
 
 ## Current Focus
 
@@ -52,10 +53,61 @@ Higher task success with fewer retries.
 
 ## Upcoming Work
 
+- Environment Auto-Configuration and Toolchain Discovery
 - Adjustable Control Support
-- Better Compose / Custom Control Semantics
 - Signal-Oriented Diagnostic Filtering
 - Long Press Gesture
+# Stronger State Verification
+# Richer Element Identity
+# Wait and Synchronization Reliability
+# Environment Auto-Configuration and Toolchain Discovery
+
+## Rationale
+Reduce onboarding friction and improve developer experience by minimizing manual setup dependencies.
+
+**Status:** Planned
+
+Addresses friction around:
+- manual idb installation
+- manual adb path configuration
+- manual xcrun path configuration
+- environment drift across machines
+- setup failures blocking first use
+
+## Scope
+- Automatic discovery of adb
+- Automatic discovery of xcrun
+- idb detection and guided bootstrap support
+- Startup toolchain validation
+- Environment health diagnostics / doctor-style checks
+- Minimal-manual-configuration defaults
+
+## Expected Impact
+High.
+
+## Exit Criteria
+- adb and xcrun auto-discovery implemented
+- Missing dependencies surfaced with guided remediation
+- Startup environment validation available
+- Manual path configuration eliminated or minimized for standard setups
+- First-run setup validated on representative developer environments
+
+## Success Metrics
+- Reduced setup friction during onboarding
+- Lower environment configuration failures
+- Faster time-to-first-successful-session
+- Reduced support/debugging caused by local setup issues
+
+## Dependencies
+Depends on:
+- Stronger State Verification
+- Richer Element Identity
+
+Strengthens:
+- Actionability Resolution
+- Broader user adoption readiness
+
+---
 
 ## Later Horizon
 
@@ -383,7 +435,7 @@ Strengthens:
 ## Rationale
 Higher priority after agent feedback exposed custom control semantics as a core reliability gap, not a later optimization.
 
-**Status:** Spec Ready
+**Status:** Completed
 
 Semantics become more useful once:
 - identity is stronger
@@ -394,8 +446,8 @@ Semantics become more useful once:
 
 ## Scope
 - Composite control traits
-- Control role enrichment (adjustable, expandable, selectable_group)
-- Interaction contracts metadata
+- Control role enrichment (`slider`, `stepper`, `dropdown`, `segmented_control`, `custom_adjustable`)
+- Interaction contract metadata (`supported_actions`, `adjustable`, `state_shape`)
 - Custom widget gesture affordance hints
 - Semantic confidence annotations
 - Compose-aware selectors for waits (merged semantics and element relationships)
@@ -405,7 +457,7 @@ High.
 
 ## Exit Criteria
 - Semantic traits implemented for major custom control classes
-- Interaction contracts surfaced in snapshot model
+- Interaction contracts surfaced in observation and resolution paths
 - Confidence model defined for derived semantics
 - Custom control manipulation success validated in benchmark flows
 
@@ -516,6 +568,7 @@ Foundation
 - Richer Element Identity
 
 Synchronization & Actionability
+- Environment Auto-Configuration and Toolchain Discovery
 - Wait and Synchronization Reliability
 - Actionability Resolution
 
@@ -536,9 +589,10 @@ Deep Observability
 - Richer Element Identity
 - Wait and Synchronization Reliability
 - Actionability Resolution
+- Environment Auto-Configuration and Toolchain Discovery
 
 Focus:
-Make core loop more reliable.
+Make core loop reliable and reduce onboarding friction.
 
 ---
 
@@ -575,16 +629,17 @@ Roadmap Ordering:
 1. Stronger State Verification
 2. Richer Element Identity
 3. Wait and Synchronization Reliability
-4. Actionability Resolution
-5. Adjustable Control Support
-6. Better Compose / Custom Control Semantics
-7. Signal-Oriented Diagnostic Filtering
-8. Long Press Gesture
-9. Pinch to Zoom
-10. Action Trace Correlation
+4. Environment Auto-Configuration and Toolchain Discovery
+5. Actionability Resolution
+6. Adjustable Control Support
+7. Better Compose / Custom Control Semantics
+8. Signal-Oriented Diagnostic Filtering
+9. Long Press Gesture
+10. Pinch to Zoom
+11. Action Trace Correlation
 
 Rationale:
-- Early roadmap items harden state, targeting, synchronization, action execution.
+- Early roadmap items harden state, targeting, synchronization, environment readiness, and action execution.
 - Mid roadmap items improve control precision and signal observability.
 - Later interaction-focused items expand interaction coverage.
 - Final observability work deepens debugging observability.

package/docs/rfcs/008-adjustable-control-support-and-semantic-value-manipulation.md

@@ -0,0 +1,273 @@
+
+
+
+# RFC 008 — Adjustable Control Support and Semantic Value Manipulation
+
+## 1. Summary
+
+This RFC defines semantic interaction support for adjustable controls whose primary interaction changes a value rather than triggering a discrete action.
+
+Examples include:
+- sliders
+- steppers
+- seek bars
+- drag-based range controls
+- quantized parameter selectors
+
+Goal:
+Enable reliable value-setting interactions with verification, minimizing coordinate guessing and brittle gesture calibration.
+
+Builds on:
+- RFC 005 — correctness model
+- RFC 006 — runtime execution binding
+- RFC 007 — target resolution
+
+---
+
+## 2. Problem Statement
+
+Current control adjustment often degrades into coordinate heuristics.
+
+Observed failure modes:
+- slider handles not semantically surfaced
+- coordinate calibration guesswork
+- snapping or quantized values behaving unexpectedly
+- weak confirmation of resulting value
+- retries caused by partial adjustment success
+
+This is not primarily a gesture problem.
+
+It is:
+- a control semantics problem
+- a value verification problem
+- an adjustment convergence problem
+
+---
+
+## 3. Design Goals
+
+Support MUST:
+- Prefer semantic adjustment over coordinate manipulation
+- Support deterministic value targeting
+- Verify resulting value after adjustment
+- Handle quantized or snapping controls
+- Support bounded tolerance when exact values are impossible
+- Use coordinate fallback only as degraded mode
+
+---
+
+## 4. Adjustable Control Model
+
+Treat adjustable controls as a combination of:
+- target control
+- value model
+- adjustment mechanism
+- verification loop
+
+Control should expose where possible:
+- current value
+- minimum and maximum range
+- step granularity (if known)
+- adjustable role metadata
+
+---
+
+## 5. Primary Primitive
+
+Illustrative adjustment primitive (conceptual, not yet a committed tool surface):
+
+set_slider_value(target, value, tolerance?)
+
+This denotes an adjustment capability, not a required standalone tool API. Implementations may realize it as:
+- an extension of existing gesture tools
+- an internal adjustment helper
+- a future dedicated control-adjustment tool surface
+
+This RFC does not mandate which mechanism is used.
+
+Semantics:
+- resolve control target
+- perform adjustment
+- read back resulting value
+- converge or fail explicitly
+
+This is not:
+blind drag gestures
+
+It is:
+set and verify
+
+## 5.1 Tool Surface Boundary
+
+This RFC specifies adjustment semantics, not a committed tool API.
+
+It does not assume a new public tool is introduced in this RFC.
+Implementation may extend existing gesture/runtime surfaces before introducing any dedicated adjustment command.
+
+---
+
+## 6. Adjustment Modes
+
+### Semantic mode (preferred)
+Use control semantics to set value directly or intelligently drive adjustment.
+
+---
+
+### Gesture-assisted mode
+Use controlled drag informed by:
+- bounds
+- target percentage
+- snapping model
+
+---
+
+### Coordinate fallback (last resort)
+Allowed only when semantic control support is absent.
+
+Must be explicit degraded mode.
+
+---
+
+## 7. Verification Loop
+
+Adjustment is incomplete until verified.
+
+Loop:
+- adjust
+- read back
+- compare to target or tolerance
+- converge or fail after bounded retries
+
+Possible outcomes:
+- Verified
+- Tolerance satisfied
+- Failed to converge
+
+Aligns with RFC 005.
+Verification may initially be realized through explicit expect_state mappings for control value assertions. Dedicated value verifiers are a possible future extension, not a prerequisite of this RFC.
+
+---
+
+## 8. Quantized / Snapping Controls
+
+Support:
+- discrete step controls
+- snapping values
+- non-linear scales (where detectable)
+
+Tolerance model required.
+
+Example:
+Target 30
+Actual 29.8
+Within tolerance acceptable
+Discrete or non-numeric controls may satisfy convergence through semantic state equivalence rather than numeric tolerance alone.
+
+---
+
+## 9. Control Resolution Dependency
+
+Uses RFC 007 target resolution for:
+- locating actual adjustable control
+- avoiding fake slider containers
+- resolving executable adjustable target
+
+RFC 007 resolves what to adjust.
+RFC 008 defines how to adjust it.
+
+---
+
+## 10. Compose / Custom Control Support
+
+Support derived adjustable semantics for:
+- Compose sliders
+- custom parameter widgets
+- composite adjustable controls
+
+Strengthens Better Compose / Custom Control Semantics.
+
+---
+
+## 11. Output / Result Model (Illustrative)
+
+Illustrative result shape:
+
+{
+  "target_state": 30,
+  "actual_state": 30,
+  "converged": true,
+  "adjustment_mode": "semantic"
+}
+
+State may be numeric, discrete, or semantic depending on control type. This model is illustrative and current implementations may expose only a subset through existing verification surfaces.
+
+---
+
+## 12. Success Metrics
+
+Track:
+- reduction in coordinate fallback usage
+- reduced retries adjusting controls
+- improved first-pass value convergence
+- improved custom control adjustment success
+
+---
+
+## 13. Dependencies
+
+Depends on:
+- Stronger State Verification
+- Actionability Resolution (RFC 007)
+
+Strengthens:
+- Better Compose / Custom Control Semantics
+- Pinch to Zoom (future)
+
+---
+
+## 14. Relationship to Prior RFCs
+
+RFC 005
+Defines successful adjustment verification.
+
+RFC 006
+Defines runtime execution interpretation.
+
+RFC 007
+Defines which adjustable target gets selected.
+
+RFC 008
+Defines how value-changing controls are manipulated reliably.
+
+Together:
+- RFC 005 — correctness
+- RFC 006 — runtime binding
+- RFC 007 — target resolution
+- RFC 008 — value manipulation
+
+---
+
+## 15. Summary
+
+This RFC moves adjustable controls from:
+- gesture guesswork
+
+to:
+- semantic value manipulation with verification
+
+It reduces one of the largest remaining sources of interaction brittleness.
+
+---
+
+## 16. Non-Goals / Scope Boundary
+
+This RFC defines adjustment semantics and convergence behavior.
+
+It does not commit in this RFC to:
+- a specific runtime tool API
+- full adjustable control support across all control types
+- generalized gesture framework support
+- arbitrary drag or canvas manipulation
+- pinch-to-zoom or broader gesture semantics
+
+This RFC specifies the behavioral model adjustable-control support should satisfy as implementations mature.