mobile-debug-mcp 0.29.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -0
- package/README.md +44 -21
- package/dist/interact/index.js +359 -46
- package/dist/observe/index.js +1 -0
- package/dist/observe/snapshot-metadata.js +62 -1
- package/dist/server/tool-definitions.js +8 -3
- package/dist/server/tool-handlers.js +4 -2
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +11 -0
- package/docs/ROADMAP.md +18 -7
- package/docs/rfcs/013-wait-and-synchronization-reliability.md +870 -0
- package/docs/rfcs/014-actionability-resolution.md +394 -0
- package/docs/specs/mcp-tooling-spec-v1.md +28 -0
- package/docs/tools/interact.md +6 -0
- package/package.json +1 -1
- package/src/interact/index.ts +444 -45
- package/src/observe/index.ts +1 -0
- package/src/observe/snapshot-metadata.ts +69 -2
- package/src/server/tool-definitions.ts +8 -3
- package/src/server/tool-handlers.ts +4 -2
- package/src/server-core.ts +1 -1
- package/src/types.ts +24 -0
- package/test/unit/interact/adjust_control.test.ts +104 -0
- package/test/unit/interact/subtree_collection.test.ts +24 -0
- package/test/unit/interact/tap_element.test.ts +71 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +189 -45
- package/test/unit/observe/snapshot_metadata.test.ts +67 -0
|
@@ -30,6 +30,58 @@ function stableElementSignature(element) {
|
|
|
30
30
|
bounds: normalizeBounds(element.bounds)
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
+
function stableElementIdentity(element, index) {
|
|
34
|
+
const stableId = normalize(element.stable_id);
|
|
35
|
+
if (stableId)
|
|
36
|
+
return `stable:${stableId}`;
|
|
37
|
+
return `fallback:${crypto.createHash('sha1').update(JSON.stringify({
|
|
38
|
+
text: normalize(element.text),
|
|
39
|
+
contentDescription: normalize(element.contentDescription),
|
|
40
|
+
resourceId: normalize(element.resourceId),
|
|
41
|
+
type: normalize(element.type),
|
|
42
|
+
bounds: normalizeBounds(element.bounds),
|
|
43
|
+
index
|
|
44
|
+
})).digest('hex')}`;
|
|
45
|
+
}
|
|
46
|
+
function buildElementSignatures(tree) {
|
|
47
|
+
const signatures = new Map();
|
|
48
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
49
|
+
for (let index = 0; index < elements.length; index++) {
|
|
50
|
+
const element = elements[index];
|
|
51
|
+
if (!element)
|
|
52
|
+
continue;
|
|
53
|
+
const identity = stableElementIdentity(element, index);
|
|
54
|
+
signatures.set(identity, crypto.createHash('sha1').update(JSON.stringify(stableElementSignature(element))).digest('hex'));
|
|
55
|
+
}
|
|
56
|
+
return signatures;
|
|
57
|
+
}
|
|
58
|
+
function summarizeSnapshotDelta(previous, currentElements) {
|
|
59
|
+
if (!previous)
|
|
60
|
+
return null;
|
|
61
|
+
let added = 0;
|
|
62
|
+
let removed = 0;
|
|
63
|
+
let mutated = 0;
|
|
64
|
+
for (const [identity, signature] of currentElements.entries()) {
|
|
65
|
+
const previousSignature = previous.elementSignatures.get(identity);
|
|
66
|
+
if (previousSignature === undefined) {
|
|
67
|
+
added++;
|
|
68
|
+
}
|
|
69
|
+
else if (previousSignature !== signature) {
|
|
70
|
+
mutated++;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const identity of previous.elementSignatures.keys()) {
|
|
74
|
+
if (!currentElements.has(identity))
|
|
75
|
+
removed++;
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
previous_snapshot_revision: previous.revision,
|
|
79
|
+
added_elements: added,
|
|
80
|
+
removed_elements: removed,
|
|
81
|
+
mutated_elements: mutated,
|
|
82
|
+
total_elements: currentElements.size
|
|
83
|
+
};
|
|
84
|
+
}
|
|
33
85
|
export function computeSnapshotSignature(tree) {
|
|
34
86
|
if (!tree || tree.error)
|
|
35
87
|
return null;
|
|
@@ -67,6 +119,10 @@ export function detectLoadingState(tree, source) {
|
|
|
67
119
|
export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverride) {
|
|
68
120
|
const signature = signatureOverride ?? computeSnapshotSignature(tree);
|
|
69
121
|
const previous = snapshotStateByDevice.get(deviceKey);
|
|
122
|
+
const hasValidTree = !!tree && !tree.error;
|
|
123
|
+
const currentElementSignatures = hasValidTree
|
|
124
|
+
? buildElementSignatures(tree)
|
|
125
|
+
: previous?.elementSignatures ?? new Map();
|
|
70
126
|
let revision = 1;
|
|
71
127
|
if (previous) {
|
|
72
128
|
if (signature === null) {
|
|
@@ -76,10 +132,15 @@ export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverrid
|
|
|
76
132
|
revision = previous.signature === signature ? previous.revision : previous.revision + 1;
|
|
77
133
|
}
|
|
78
134
|
}
|
|
79
|
-
snapshotStateByDevice.set(deviceKey, {
|
|
135
|
+
snapshotStateByDevice.set(deviceKey, {
|
|
136
|
+
revision,
|
|
137
|
+
signature,
|
|
138
|
+
elementSignatures: currentElementSignatures
|
|
139
|
+
});
|
|
80
140
|
return {
|
|
81
141
|
snapshot_revision: revision,
|
|
82
142
|
captured_at_ms: Date.now(),
|
|
143
|
+
snapshot_delta: hasValidTree ? summarizeSnapshotDelta(previous, currentElementSignatures) : null,
|
|
83
144
|
loading_state: detectLoadingState(tree, source)
|
|
84
145
|
};
|
|
85
146
|
}
|
|
@@ -244,7 +244,7 @@ Failure Handling:
|
|
|
244
244
|
},
|
|
245
245
|
{
|
|
246
246
|
name: 'capture_debug_snapshot',
|
|
247
|
-
description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON with snapshot_revision, captured_at_ms, and loading_state when detectable.',
|
|
247
|
+
description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON with snapshot_revision, captured_at_ms, snapshot_delta, and loading_state when detectable.',
|
|
248
248
|
inputSchema: {
|
|
249
249
|
type: 'object',
|
|
250
250
|
properties: {
|
|
@@ -295,7 +295,7 @@ Failure Handling:
|
|
|
295
295
|
},
|
|
296
296
|
{
|
|
297
297
|
name: 'get_ui_tree',
|
|
298
|
-
description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content with snapshot metadata when available.',
|
|
298
|
+
description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content with snapshot metadata and incremental delta signals when available.',
|
|
299
299
|
inputSchema: {
|
|
300
300
|
type: 'object',
|
|
301
301
|
properties: {
|
|
@@ -376,11 +376,14 @@ Inputs:
|
|
|
376
376
|
- expected_change (optional): hierarchy_diff, text_change, or state_change
|
|
377
377
|
- timeout_ms (optional)
|
|
378
378
|
- stability_window_ms (optional)
|
|
379
|
+
- scope (optional): screen or subtree
|
|
380
|
+
- target (optional): element_id when scope=subtree
|
|
379
381
|
|
|
380
382
|
Guidance:
|
|
381
383
|
- Prefer wait_for_screen_change for navigation transitions.
|
|
382
384
|
- Prefer wait_for_ui_change for in-place mutations and non-navigation updates.
|
|
383
385
|
- Use the returned snapshot_revision as the observed synchronization point when available.
|
|
386
|
+
- Scoped waits return scope-aware stability metadata and a lightweight change summary.
|
|
384
387
|
|
|
385
388
|
Failure Handling:
|
|
386
389
|
- TIMEOUT means the UI did not change in a stable way within the allotted time.`,
|
|
@@ -390,8 +393,10 @@ Failure Handling:
|
|
|
390
393
|
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override (android|ios)' },
|
|
391
394
|
deviceId: { type: 'string', description: 'Optional device id/udid to target' },
|
|
392
395
|
expected_change: { type: 'string', enum: ['hierarchy_diff', 'text_change', 'state_change'], description: 'Optional type of UI change to wait for' },
|
|
396
|
+
scope: { type: 'string', enum: ['screen', 'subtree'], default: 'screen', description: 'Synchronization scope for the wait' },
|
|
397
|
+
target: { type: 'string', description: 'Target element_id when scope is subtree' },
|
|
393
398
|
timeout_ms: { type: 'number', description: 'Timeout in ms to wait for change (default 60000)', default: 60000 },
|
|
394
|
-
stability_window_ms: { type: 'number', description: 'How long the change must remain stable before success (default
|
|
399
|
+
stability_window_ms: { type: 'number', description: 'How long the change must remain stable before success (default 300)', default: 300 }
|
|
395
400
|
}
|
|
396
401
|
}
|
|
397
402
|
},
|
|
@@ -266,9 +266,11 @@ async function handleWaitForUIChange(args) {
|
|
|
266
266
|
const platform = getStringArg(args, 'platform');
|
|
267
267
|
const deviceId = getStringArg(args, 'deviceId');
|
|
268
268
|
const timeout_ms = getNumberArg(args, 'timeout_ms') ?? 60000;
|
|
269
|
-
const stability_window_ms = getNumberArg(args, 'stability_window_ms') ??
|
|
269
|
+
const stability_window_ms = getNumberArg(args, 'stability_window_ms') ?? 300;
|
|
270
270
|
const expected_change = getStringArg(args, 'expected_change');
|
|
271
|
-
const
|
|
271
|
+
const scope = getStringArg(args, 'scope');
|
|
272
|
+
const target = getStringArg(args, 'target');
|
|
273
|
+
const res = await ToolsInteract.waitForUIChangeHandler({ platform, deviceId, timeout_ms, stability_window_ms, expected_change, scope, target });
|
|
272
274
|
return wrapResponse(res);
|
|
273
275
|
}
|
|
274
276
|
async function handleFindElement(args) {
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.
|
|
9
|
+
version: '0.30.1'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the **Mobile Debug MCP** project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.30.1]
|
|
6
|
+
- Synced the server-reported version in `src/server-core.ts` with `package.json` so contract checks pass.
|
|
7
|
+
- Completed the RFC 013 wait/synchronization implementation, including scoped waits and freshness metadata.
|
|
8
|
+
- Completed the RFC 014 actionability implementation for taps and adjustable controls.
|
|
9
|
+
- Added regression coverage for subtree collection, scoped waits, snapshot deltas, and stale actionability checks.
|
|
10
|
+
|
|
11
|
+
## [0.30.0]
|
|
12
|
+
- Folded RFC 013 synchronization semantics into the main spec and aligned the interact docs with the shipped `wait_for_ui_change` behavior.
|
|
13
|
+
- Updated `wait_for_ui_change` to use a 300ms stabilization default and to reset stabilization on new in-place mutations.
|
|
14
|
+
- Validated the in-place UI mutation flow on the Modul8 emulator app, including a delayed state-change case.
|
|
15
|
+
|
|
5
16
|
## [0.29.0]
|
|
6
17
|
- Added empty resource handlers and declared the `resources` capability so Codex MCP discovery can complete the handshake against the published npm package.
|
|
7
18
|
- Moved the startup healthcheck behind an opt-in flag to keep the stdio protocol channel quiet by default.
|
package/docs/ROADMAP.md
CHANGED
|
@@ -32,6 +32,7 @@ Track roadmap impact across releases using:
|
|
|
32
32
|
- Gesture success rate
|
|
33
33
|
- Mean time to root cause during debugging
|
|
34
34
|
- Overall agent task completion rate
|
|
35
|
+
- Reduced sequencing errors in multi-step interaction flows
|
|
35
36
|
|
|
36
37
|
Primary KPI:
|
|
37
38
|
Higher task success with fewer retries.
|
|
@@ -47,10 +48,10 @@ Higher task success with fewer retries.
|
|
|
47
48
|
- Better Compose / Custom Control Semantics — Complete (Semantic role enrichment and custom-adjustable inference shipped)
|
|
48
49
|
- Verification Stabilization and Temporal Convergence — Complete (Temporal verification and convergence logic shipped)
|
|
49
50
|
- Action Trace and Execution Observability — Complete (Structured execution trace model shipped)
|
|
51
|
+
- Wait and Synchronization Reliability — Complete (RFC 013 folded into the main spec and shipped behavior verified on emulator)
|
|
50
52
|
|
|
51
53
|
## Current Focus
|
|
52
54
|
|
|
53
|
-
- Wait and Synchronization Reliability (implementation + tuning)
|
|
54
55
|
- Actionability Resolution
|
|
55
56
|
- Adjustable Control Precision Hardening
|
|
56
57
|
|
|
@@ -74,13 +75,14 @@ Addresses friction around:
|
|
|
74
75
|
- environment drift across machines
|
|
75
76
|
- setup failures blocking first use
|
|
76
77
|
|
|
77
|
-
## Scope
|
|
78
78
|
- Automatic discovery of adb
|
|
79
79
|
- Automatic discovery of xcrun
|
|
80
80
|
- idb detection and guided bootstrap support
|
|
81
81
|
- Startup toolchain validation
|
|
82
82
|
- Environment health diagnostics / doctor-style checks
|
|
83
83
|
- Minimal-manual-configuration defaults
|
|
84
|
+
- Runtime device/emulator health signals (crash detection, process lifecycle awareness)
|
|
85
|
+
- App stability monitoring during active sessions
|
|
84
86
|
|
|
85
87
|
## Expected Impact
|
|
86
88
|
High.
|
|
@@ -97,6 +99,7 @@ High.
|
|
|
97
99
|
- Lower environment configuration failures
|
|
98
100
|
- Faster time-to-first-successful-session
|
|
99
101
|
- Reduced support/debugging caused by local setup issues
|
|
102
|
+
- Reduced unknown-failure sessions caused by app or emulator instability
|
|
100
103
|
|
|
101
104
|
## Dependencies
|
|
102
105
|
Depends on:
|
|
@@ -151,7 +154,7 @@ Very high.
|
|
|
151
154
|
Blocks or strengthens:
|
|
152
155
|
- Better Compose / Custom Control Semantics
|
|
153
156
|
- Pinch to Zoom
|
|
154
|
-
-
|
|
157
|
+
- Advanced Trace Correlation and Analysis
|
|
155
158
|
|
|
156
159
|
---
|
|
157
160
|
|
|
@@ -212,8 +215,11 @@ Addresses failures where agents:
|
|
|
212
215
|
- wait_for_ui_change (hierarchy diff based waiting)
|
|
213
216
|
- Structured loading state detection
|
|
214
217
|
- Snapshot revision / staleness metadata
|
|
215
|
-
-
|
|
218
|
+
- Incremental / diff-based snapshot delivery (token-efficient)
|
|
219
|
+
- Focused snapshot scoping (subtree / target-based)
|
|
216
220
|
- Compose-aware wait robustness improvements
|
|
221
|
+
- Explicit interaction sequencing guidance (tap → wait → verify pattern)
|
|
222
|
+
- Exploration of optional action-level synchronization ergonomics (e.g. implicit stabilization or wait flags)
|
|
217
223
|
|
|
218
224
|
## Expected Impact
|
|
219
225
|
Very high.
|
|
@@ -231,6 +237,8 @@ Very high.
|
|
|
231
237
|
- Fewer retries caused by premature actions
|
|
232
238
|
- Higher wait success rate for dynamic UI flows
|
|
233
239
|
- Lower fallback usage to network/log checks
|
|
240
|
+
- Reduced need for manual sequencing by agents in stateful flows
|
|
241
|
+
- Reduced average snapshot size (tokens)
|
|
234
242
|
|
|
235
243
|
## Dependencies
|
|
236
244
|
Depends on:
|
|
@@ -239,7 +247,7 @@ Depends on:
|
|
|
239
247
|
|
|
240
248
|
Blocks or strengthens:
|
|
241
249
|
- Better Compose / Custom Control Semantics
|
|
242
|
-
-
|
|
250
|
+
- Advanced Trace Correlation and Analysis
|
|
243
251
|
|
|
244
252
|
---
|
|
245
253
|
|
|
@@ -307,6 +315,7 @@ Addresses cases where:
|
|
|
307
315
|
- Executable-target preference rules
|
|
308
316
|
- Actionability confidence metadata
|
|
309
317
|
- Post-action state verification integration
|
|
318
|
+
- Geometry-aware fallback targeting for weak semantic surfaces (e.g. sliders without accessible nodes)
|
|
310
319
|
|
|
311
320
|
## Expected Impact
|
|
312
321
|
High.
|
|
@@ -321,6 +330,7 @@ High.
|
|
|
321
330
|
- Reduced mis-targeted action failures
|
|
322
331
|
- Lower retarget retries
|
|
323
332
|
- Higher first-attempt action success
|
|
333
|
+
- Reduced need for empirical coordinate probing on custom controls
|
|
324
334
|
|
|
325
335
|
## Dependencies
|
|
326
336
|
Depends on:
|
|
@@ -403,6 +413,7 @@ Addresses friction around:
|
|
|
403
413
|
- Drag vs tap adjustment strategy heuristics
|
|
404
414
|
- Improved value snapping convergence
|
|
405
415
|
- Control-specific adjustment fallback policies
|
|
416
|
+
- Controlled search strategies for value convergence (e.g. binary / progressive adjustment)
|
|
406
417
|
|
|
407
418
|
## Expected Impact
|
|
408
419
|
High.
|
|
@@ -465,7 +476,7 @@ Depends on:
|
|
|
465
476
|
- Wait and Synchronization Reliability
|
|
466
477
|
|
|
467
478
|
Strengthens:
|
|
468
|
-
-
|
|
479
|
+
- Advanced Trace Correlation and Analysis
|
|
469
480
|
|
|
470
481
|
---
|
|
471
482
|
|
|
@@ -675,7 +686,7 @@ Interaction Expansion
|
|
|
675
686
|
- Pinch to Zoom
|
|
676
687
|
|
|
677
688
|
Deep Observability
|
|
678
|
-
-
|
|
689
|
+
- Advanced Trace Correlation and Analysis
|
|
679
690
|
|
|
680
691
|
## Wave 1 (Current Focus)
|
|
681
692
|
- Stronger State Verification
|