mobile-debug-mcp 0.26.1 → 0.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +26 -33
- package/dist/server/common.js +14 -1
- package/dist/server/tool-definitions.js +19 -3
- package/dist/server/tool-handlers.js +7 -0
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +3 -0
- package/docs/ROADMAP.md +239 -74
- package/docs/rfcs/005-unified-action-execution-and-verification-model.md +216 -0
- package/docs/rfcs/006-runtime-action-instrumentation-and-binding-layer.md +230 -0
- package/docs/specs/mcp-tooling-spec-v1.md +4 -0
- package/docs/tools/interact.md +4 -0
- package/package.json +1 -1
- package/src/interact/index.ts +27 -35
- package/src/server/common.ts +22 -1
- package/src/server/tool-definitions.ts +19 -3
- package/src/server/tool-handlers.ts +7 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +2 -0
- package/test/unit/server/response_shapes.test.ts +8 -0
package/dist/interact/index.js
CHANGED
|
@@ -5,7 +5,7 @@ export { AndroidInteract, iOSInteract };
|
|
|
5
5
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
6
6
|
import { ToolsObserve } from '../observe/index.js';
|
|
7
7
|
import { computeSnapshotSignature } from '../observe/snapshot-metadata.js';
|
|
8
|
-
import {
|
|
8
|
+
import { buildActionExecutionResult } from '../server/common.js';
|
|
9
9
|
export class ToolsInteract {
|
|
10
10
|
static _maxResolvedUiElements = 256;
|
|
11
11
|
static _uiChangeKinds = ['hierarchy_diff', 'text_change', 'state_change'];
|
|
@@ -203,18 +203,17 @@ export class ToolsInteract {
|
|
|
203
203
|
semantic: element.semantic ?? null
|
|
204
204
|
};
|
|
205
205
|
}
|
|
206
|
-
static _actionFailure(
|
|
207
|
-
return {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
target: { selector, resolved },
|
|
206
|
+
static _actionFailure(actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter, sourceModule = 'interact') {
|
|
207
|
+
return buildActionExecutionResult({
|
|
208
|
+
actionType,
|
|
209
|
+
selector,
|
|
210
|
+
resolved,
|
|
212
211
|
success: false,
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
};
|
|
212
|
+
uiFingerprintBefore,
|
|
213
|
+
uiFingerprintAfter: uiFingerprintAfter ?? null,
|
|
214
|
+
failure: { failureCode, retryable },
|
|
215
|
+
sourceModule
|
|
216
|
+
});
|
|
218
217
|
}
|
|
219
218
|
static _resetResolvedUiElementsForTests() {
|
|
220
219
|
ToolsInteract._resolvedUiElements.clear();
|
|
@@ -350,14 +349,11 @@ export class ToolsInteract {
|
|
|
350
349
|
return await interact.tap(x, y, resolved.id);
|
|
351
350
|
}
|
|
352
351
|
static async tapElementHandler({ elementId }) {
|
|
353
|
-
const timestampMs = Date.now();
|
|
354
|
-
const timestamp = new Date(timestampMs).toISOString();
|
|
355
352
|
const actionType = 'tap_element';
|
|
356
|
-
const actionId = nextActionId(actionType, timestampMs);
|
|
357
353
|
const selector = { elementId };
|
|
358
354
|
const resolved = ToolsInteract._resolvedUiElements.get(elementId);
|
|
359
355
|
if (!resolved) {
|
|
360
|
-
return ToolsInteract._actionFailure(
|
|
356
|
+
return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, null);
|
|
361
357
|
}
|
|
362
358
|
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
|
|
363
359
|
const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
|
|
@@ -366,40 +362,37 @@ export class ToolsInteract {
|
|
|
366
362
|
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
367
363
|
const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
|
|
368
364
|
if (!currentMatch) {
|
|
369
|
-
return ToolsInteract._actionFailure(
|
|
365
|
+
return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
|
|
370
366
|
}
|
|
371
367
|
const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
|
|
372
368
|
if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
|
|
373
|
-
return ToolsInteract._actionFailure(
|
|
369
|
+
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
374
370
|
}
|
|
375
371
|
if (currentMatch.el.enabled === false) {
|
|
376
|
-
return ToolsInteract._actionFailure(
|
|
372
|
+
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
377
373
|
}
|
|
378
374
|
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
|
|
379
375
|
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
|
|
380
|
-
return ToolsInteract._actionFailure(
|
|
376
|
+
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
381
377
|
}
|
|
382
378
|
const x = Math.floor((bounds[0] + bounds[2]) / 2);
|
|
383
379
|
const y = Math.floor((bounds[1] + bounds[3]) / 2);
|
|
384
380
|
const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId });
|
|
385
381
|
if (!tapResult.success) {
|
|
386
382
|
const fingerprintAfterFailure = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
|
|
387
|
-
return ToolsInteract._actionFailure(
|
|
383
|
+
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'UNKNOWN', false, fingerprintBefore, fingerprintAfterFailure);
|
|
388
384
|
}
|
|
389
385
|
const fingerprintAfter = await ToolsInteract._captureFingerprint(resolved.platform, resolved.deviceId);
|
|
390
|
-
return {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
target: {
|
|
396
|
-
selector,
|
|
397
|
-
resolved: resolvedTarget
|
|
398
|
-
},
|
|
386
|
+
return buildActionExecutionResult({
|
|
387
|
+
actionType,
|
|
388
|
+
device: tree?.device,
|
|
389
|
+
selector,
|
|
390
|
+
resolved: resolvedTarget,
|
|
399
391
|
success: true,
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
392
|
+
uiFingerprintBefore: fingerprintBefore,
|
|
393
|
+
uiFingerprintAfter: fingerprintAfter,
|
|
394
|
+
sourceModule: 'interact'
|
|
395
|
+
});
|
|
403
396
|
}
|
|
404
397
|
static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
|
|
405
398
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
package/dist/server/common.js
CHANGED
|
@@ -95,13 +95,26 @@ export function inferScrollFailure(message) {
|
|
|
95
95
|
return { failureCode: 'TIMEOUT', retryable: true };
|
|
96
96
|
return { failureCode: 'UNKNOWN', retryable: false };
|
|
97
97
|
}
|
|
98
|
-
|
|
98
|
+
const ACTION_LIFECYCLE_STATE_BY_OUTCOME = {
|
|
99
|
+
success: 'pending_verification',
|
|
100
|
+
failure: 'failed'
|
|
101
|
+
};
|
|
102
|
+
export function determineActionLifecycleState({ success, failure }) {
|
|
103
|
+
if (failure)
|
|
104
|
+
return ACTION_LIFECYCLE_STATE_BY_OUTCOME.failure;
|
|
105
|
+
if (success)
|
|
106
|
+
return ACTION_LIFECYCLE_STATE_BY_OUTCOME.success;
|
|
107
|
+
return ACTION_LIFECYCLE_STATE_BY_OUTCOME.success;
|
|
108
|
+
}
|
|
109
|
+
export function buildActionExecutionResult({ actionType, device, selector, resolved, success, uiFingerprintBefore, uiFingerprintAfter, failure, details, sourceModule }) {
|
|
99
110
|
const timestampMs = Date.now();
|
|
100
111
|
const timestamp = new Date(timestampMs).toISOString();
|
|
101
112
|
return {
|
|
102
113
|
action_id: nextActionId(actionType, timestampMs),
|
|
103
114
|
timestamp,
|
|
104
115
|
action_type: actionType,
|
|
116
|
+
lifecycle_state: determineActionLifecycleState({ success, failure }),
|
|
117
|
+
source_module: sourceModule,
|
|
105
118
|
...(device ? { device } : {}),
|
|
106
119
|
target: {
|
|
107
120
|
selector,
|
|
@@ -11,7 +11,9 @@ Inputs:
|
|
|
11
11
|
|
|
12
12
|
Output Structure:
|
|
13
13
|
- action_id, timestamp (ISO 8601), action_type
|
|
14
|
-
-
|
|
14
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
15
|
+
- source_module: runtime source of the action envelope
|
|
16
|
+
- target.selector = { appId }
|
|
15
17
|
- success = true when launch was dispatched successfully
|
|
16
18
|
- failure_code/retryable when launch dispatch fails
|
|
17
19
|
- ui_fingerprint_before/ui_fingerprint_after when available
|
|
@@ -84,7 +86,9 @@ Inputs:
|
|
|
84
86
|
|
|
85
87
|
Output Structure:
|
|
86
88
|
- action_id, timestamp (ISO 8601), action_type
|
|
87
|
-
-
|
|
89
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
90
|
+
- source_module: runtime source of the action envelope
|
|
91
|
+
- target.selector = { appId }
|
|
88
92
|
- success = true when the restart command completed
|
|
89
93
|
- failure_code/retryable when restart dispatch fails
|
|
90
94
|
- ui_fingerprint_before/ui_fingerprint_after when available
|
|
@@ -617,7 +621,9 @@ Inputs:
|
|
|
617
621
|
|
|
618
622
|
Output Structure:
|
|
619
623
|
- action_id, timestamp (ISO 8601), action_type
|
|
620
|
-
-
|
|
624
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
625
|
+
- source_module: runtime source of the action envelope
|
|
626
|
+
- target.selector = { x, y }
|
|
621
627
|
- success = true when the tap was dispatched
|
|
622
628
|
- failure_code/retryable when dispatch fails
|
|
623
629
|
- ui_fingerprint_before/ui_fingerprint_after when available
|
|
@@ -673,6 +679,8 @@ Output Structure:
|
|
|
673
679
|
- action_id: unique timestamp-based action identifier
|
|
674
680
|
- timestamp: ISO 8601 timestamp for the action attempt
|
|
675
681
|
- action_type: "tap_element"
|
|
682
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
683
|
+
- source_module: runtime source of the action envelope
|
|
676
684
|
- target.selector: original target handle ({ elementId })
|
|
677
685
|
- target.resolved: minimal resolved element info used for the tap
|
|
678
686
|
- success: true when the tap was dispatched
|
|
@@ -725,6 +733,8 @@ Inputs:
|
|
|
725
733
|
|
|
726
734
|
Output Structure:
|
|
727
735
|
- action_id, timestamp (ISO 8601), action_type
|
|
736
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
737
|
+
- source_module: runtime source of the action envelope
|
|
728
738
|
- target.selector = { x1, y1, x2, y2, duration }
|
|
729
739
|
- success = true when the swipe was dispatched
|
|
730
740
|
- failure_code/retryable when dispatch fails
|
|
@@ -777,6 +787,8 @@ Inputs:
|
|
|
777
787
|
|
|
778
788
|
Output Structure:
|
|
779
789
|
- action_id, timestamp (ISO 8601), action_type
|
|
790
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
791
|
+
- source_module: runtime source of the action envelope
|
|
780
792
|
- target.selector = original selector
|
|
781
793
|
- target.resolved = minimal resolved element info when found
|
|
782
794
|
- success = true when scrolling produced a visible target element
|
|
@@ -831,6 +843,8 @@ Inputs:
|
|
|
831
843
|
|
|
832
844
|
Output Structure:
|
|
833
845
|
- action_id, timestamp (ISO 8601), action_type
|
|
846
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
847
|
+
- source_module: runtime source of the action envelope
|
|
834
848
|
- target.selector = { text }
|
|
835
849
|
- success = true when text input was dispatched
|
|
836
850
|
- failure_code/retryable when dispatch fails
|
|
@@ -880,6 +894,8 @@ Inputs:
|
|
|
880
894
|
|
|
881
895
|
Output Structure:
|
|
882
896
|
- action_id, timestamp (ISO 8601), action_type
|
|
897
|
+
- lifecycle_state: post-dispatch lifecycle state (pending_verification or failed)
|
|
898
|
+
- source_module: runtime source of the action envelope
|
|
883
899
|
- target.selector = { key: "back" }
|
|
884
900
|
- success = true when the back action was dispatched
|
|
885
901
|
- failure_code/retryable when dispatch fails
|
|
@@ -15,6 +15,7 @@ async function handleStartApp(args) {
|
|
|
15
15
|
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
16
16
|
return wrapResponse(buildActionExecutionResult({
|
|
17
17
|
actionType: 'start_app',
|
|
18
|
+
sourceModule: 'server',
|
|
18
19
|
device: res.device,
|
|
19
20
|
selector: { appId },
|
|
20
21
|
success: !!res.appStarted,
|
|
@@ -48,6 +49,7 @@ async function handleRestartApp(args) {
|
|
|
48
49
|
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
49
50
|
return wrapResponse(buildActionExecutionResult({
|
|
50
51
|
actionType: 'restart_app',
|
|
52
|
+
sourceModule: 'server',
|
|
51
53
|
device: res.device,
|
|
52
54
|
selector: { appId },
|
|
53
55
|
success: !!res.appRestarted,
|
|
@@ -265,6 +267,7 @@ async function handleTap(args) {
|
|
|
265
267
|
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
266
268
|
return wrapResponse(buildActionExecutionResult({
|
|
267
269
|
actionType: 'tap',
|
|
270
|
+
sourceModule: 'server',
|
|
268
271
|
selector: { x, y },
|
|
269
272
|
success: !!res.success,
|
|
270
273
|
uiFingerprintBefore,
|
|
@@ -292,6 +295,7 @@ async function handleSwipe(args) {
|
|
|
292
295
|
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
293
296
|
return wrapResponse(buildActionExecutionResult({
|
|
294
297
|
actionType: 'swipe',
|
|
298
|
+
sourceModule: 'server',
|
|
295
299
|
selector: { x1, y1, x2, y2, duration },
|
|
296
300
|
success: !!res.success,
|
|
297
301
|
uiFingerprintBefore,
|
|
@@ -312,6 +316,7 @@ async function handleScrollToElement(args) {
|
|
|
312
316
|
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
313
317
|
return wrapResponse(buildActionExecutionResult({
|
|
314
318
|
actionType: 'scroll_to_element',
|
|
319
|
+
sourceModule: 'server',
|
|
315
320
|
selector: selector ?? null,
|
|
316
321
|
resolved: res?.success && res?.element ? {
|
|
317
322
|
elementId: null,
|
|
@@ -337,6 +342,7 @@ async function handleTypeText(args) {
|
|
|
337
342
|
const uiFingerprintAfter = await captureActionFingerprint('android', deviceId);
|
|
338
343
|
return wrapResponse(buildActionExecutionResult({
|
|
339
344
|
actionType: 'type_text',
|
|
345
|
+
sourceModule: 'server',
|
|
340
346
|
selector: { text },
|
|
341
347
|
success: !!res.success,
|
|
342
348
|
uiFingerprintBefore,
|
|
@@ -352,6 +358,7 @@ async function handlePressBack(args) {
|
|
|
352
358
|
const uiFingerprintAfter = await captureActionFingerprint('android', deviceId);
|
|
353
359
|
return wrapResponse(buildActionExecutionResult({
|
|
354
360
|
actionType: 'press_back',
|
|
361
|
+
sourceModule: 'server',
|
|
355
362
|
selector: { key: 'back' },
|
|
356
363
|
success: !!res.success,
|
|
357
364
|
uiFingerprintBefore,
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.26.
|
|
9
|
+
version: '0.26.2'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|