mobile-debug-mcp 0.31.0 → 0.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +1 -0
  2. package/dist/interact/android.js +13 -8
  3. package/dist/interact/index.js +33 -22
  4. package/dist/interact/ios.js +5 -2
  5. package/dist/manage/android.js +20 -10
  6. package/dist/manage/ios.js +19 -10
  7. package/dist/observe/android-tree-bridge.js +192 -0
  8. package/dist/observe/android.js +53 -14
  9. package/dist/observe/index.js +22 -5
  10. package/dist/observe/ios.js +50 -29
  11. package/dist/observe/performance-metrics.js +124 -0
  12. package/dist/observe/snapshot-metadata.js +94 -15
  13. package/dist/server/common.js +14 -7
  14. package/dist/server/journey-schema.js +60 -0
  15. package/dist/server/tool-definitions.js +59 -6
  16. package/dist/server/tool-handlers.js +381 -56
  17. package/dist/server-core.js +1 -1
  18. package/dist/utils/exec.js +18 -5
  19. package/dist/utils/ios/utils.js +5 -2
  20. package/dist/utils/operation-budget.js +25 -0
  21. package/dist/utils/resolve-device.js +12 -1
  22. package/dist/utils/timing.js +19 -0
  23. package/dist/utils/ui/index.js +2 -0
  24. package/docs/CHANGELOG.md +6 -0
  25. package/docs/ROADMAP.md +9 -0
  26. package/docs/benchmarks/interaction-latency/README.md +26 -0
  27. package/docs/benchmarks/interaction-latency/final.json +1810 -0
  28. package/docs/benchmarks/interaction-latency/final.md +17 -0
  29. package/docs/benchmarks/interaction-latency/ios-validation.md +11 -0
  30. package/docs/benchmarks/ui-tree-performance/README.md +51 -0
  31. package/docs/benchmarks/ui-tree-performance/baseline.json +59546 -0
  32. package/docs/benchmarks/ui-tree-performance/bridge-probe.json +28 -0
  33. package/docs/benchmarks/ui-tree-performance/candidate.json +51691 -0
  34. package/docs/benchmarks/ui-tree-performance/comparison-attempt-1.json +110679 -0
  35. package/docs/benchmarks/ui-tree-performance/comparison.json +111230 -0
  36. package/docs/benchmarks/ui-tree-performance/comparison.md +41 -0
  37. package/docs/benchmarks/ui-tree-performance/initial-baseline.json +59560 -0
  38. package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +301 -0
  39. package/docs/specs/019-ui-tree-observation-performance/spec.md +148 -0
  40. package/docs/tools/interact.md +30 -0
  41. package/docs/tools/observe.md +1 -0
  42. package/package.json +1 -1
  43. package/skills/mobile-debug-tools-usage/SKILL.md +74 -0
  44. package/src/interact/android.ts +13 -8
  45. package/src/interact/index.ts +32 -23
  46. package/src/interact/ios.ts +5 -2
  47. package/src/manage/android.ts +19 -10
  48. package/src/manage/ios.ts +18 -10
  49. package/src/observe/android-tree-bridge.ts +170 -0
  50. package/src/observe/android.ts +48 -17
  51. package/src/observe/index.ts +21 -5
  52. package/src/observe/ios.ts +49 -36
  53. package/src/observe/performance-metrics.ts +125 -0
  54. package/src/observe/snapshot-metadata.ts +94 -15
  55. package/src/server/common.ts +13 -6
  56. package/src/server/journey-schema.ts +63 -0
  57. package/src/server/tool-definitions.ts +60 -7
  58. package/src/server/tool-handlers.ts +349 -54
  59. package/src/server-core.ts +1 -1
  60. package/src/types.ts +31 -2
  61. package/src/utils/exec.ts +13 -5
  62. package/src/utils/ios/utils.ts +5 -2
  63. package/src/utils/operation-budget.ts +22 -0
  64. package/src/utils/resolve-device.ts +13 -1
  65. package/src/utils/timing.ts +15 -0
  66. package/src/utils/ui/index.ts +2 -0
  67. package/test/device/automated/latency/benchmark.ts +136 -0
  68. package/test/device/automated/latency/report.ts +6 -0
  69. package/test/device/automated/ui-tree-performance/benchmark.ts +401 -0
  70. package/test/device/automated/ui-tree-performance/bridge-probe.ts +183 -0
  71. package/test/fixtures/latency-journey-app/AndroidManifest.xml +7 -0
  72. package/test/fixtures/latency-journey-app/README.md +28 -0
  73. package/test/fixtures/latency-journey-app/build.ts +26 -0
  74. package/test/fixtures/latency-journey-app/src/MainActivity.java +104 -0
  75. package/test/fixtures/ui-tree-bridge/AndroidManifest.xml +6 -0
  76. package/test/fixtures/ui-tree-bridge/build.ts +34 -0
  77. package/test/fixtures/ui-tree-bridge/src/TreeInstrumentation.java +119 -0
  78. package/test/unit/manage/restart_budget.test.ts +37 -0
  79. package/test/unit/observe/android_tree_acquisition.test.ts +70 -0
  80. package/test/unit/observe/android_tree_bridge_cleanup.test.ts +76 -0
  81. package/test/unit/observe/capture_debug_snapshot.test.ts +6 -0
  82. package/test/unit/observe/ios_tree_acquisition.test.ts +77 -0
  83. package/test/unit/observe/performance_metrics.test.ts +40 -0
  84. package/test/unit/observe/snapshot_metadata.test.ts +20 -1
  85. package/test/unit/observe/state_delta.test.ts +15 -0
  86. package/test/unit/server/benchmark_report.test.ts +4 -0
  87. package/test/unit/server/contract.test.ts +1 -1
  88. package/test/unit/server/interaction_modes.test.ts +60 -0
  89. package/test/unit/server/operation_budget.test.ts +45 -0
  90. package/test/unit/server/response_shapes.test.ts +3 -3
  91. package/test/unit/server/run_journey.test.ts +120 -0
  92. package/test/unit/server/tap_selector.test.ts +68 -0
  93. /package/docs/{specs-deltas/001-state-verification.md → specs/001-state-verification/spec.md} +0 -0
  94. /package/docs/{specs-deltas/002-richer-element-identity → specs/002-richer-element-identity/spec.md} +0 -0
  95. /package/docs/{specs-deltas/003-wait-and-synchronization-reliability.md → specs/003-wait-and-synchronization-reliability/spec.md} +0 -0
  96. /package/docs/{specs-deltas/004-action-verification-routing.md → specs/004-action-verification-routing/spec.md} +0 -0
  97. /package/docs/{specs-deltas/005-unified-action-execution-and-verification-model.md → specs/005-unified-action-execution-and-verification-model/spec.md} +0 -0
  98. /package/docs/{specs-deltas/006-runtime-action-instrumentation-and-binding-layer.md → specs/006-runtime-action-instrumentation-and-binding-layer/spec.md} +0 -0
  99. /package/docs/{specs-deltas/007-actionability-resolution-and-executable-target-selection.md → specs/007-actionability-resolution-and-executable-target-selection/spec.md} +0 -0
  100. /package/docs/{specs-deltas/008-adjustable-control-support-and-semantic-value-manipulation.md → specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md} +0 -0
  101. /package/docs/{specs-deltas/009-semantic-control-modeling-for-custom-and-composite-controls.md → specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md} +0 -0
  102. /package/docs/{specs-deltas/010-verification-stabilization-and-temporal-convergence.md → specs/010-verification-stabilization-and-temporal-convergence/spec.md} +0 -0
  103. /package/docs/{specs-deltas/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows.md → specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md} +0 -0
  104. /package/docs/{specs-deltas/011.1-recovery-contract-types-and-runtime-wiring-spec.md → specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md} +0 -0
  105. /package/docs/{specs-deltas/012-action-trace-and-xecution-observability.md → specs/012-action-trace-and-xecution-observability/spec.md} +0 -0
  106. /package/docs/{specs-deltas/013-wait-and-synchronization-reliability.md → specs/013-wait-and-synchronization-reliability/spec.md} +0 -0
  107. /package/docs/{specs-deltas/014-actionability-resolution.md → specs/014-actionability-resolution/spec.md} +0 -0
  108. /package/docs/{specs-deltas/015-adjustable-control-support.md → specs/015-adjustable-control-support/spec.md} +0 -0
  109. /package/docs/{specs-deltas/016-environment-tools.md → specs/016-environment-tools/spec.md} +0 -0
@@ -1,19 +1,140 @@
1
+ import { measure, withTiming } from '../utils/timing.js';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { runWithBudget } from '../utils/operation-budget.js';
4
+ import { journeySchema, tapElementControlsSchema } from './journey-schema.js';
5
+ import { resolveTargetDevice, withResolvedDevice } from '../utils/resolve-device.js';
1
6
  import { AndroidManage, iOSManage, ToolsManage } from '../manage/index.js';
2
7
  import { ToolsInteract } from '../interact/index.js';
3
8
  import { ToolsObserve } from '../observe/index.js';
4
9
  import { classifyActionOutcome } from '../interact/classify.js';
5
10
  import { ToolsNetwork } from '../network/index.js';
6
11
  import { getSystemStatus } from '../system/index.js';
12
+ import { recordObservationConsumer, recordObservationEvent, withObservationContext } from '../observe/performance-metrics.js';
7
13
  import { buildActionExecutionResult, captureActionFingerprint, getArrayArg, getBooleanArg, getNumberArg, getObjectArg, getStringArg, inferGenericFailure, inferScrollFailure, requireBooleanArg, requireNumberArg, requireObjectArg, requireStringArg, wrapResponse, wrapToolError } from './common.js';
8
- async function handleStartApp(args) {
14
+ function interactionControls(args) {
15
+ const verificationMode = getStringArg(args, 'verificationMode') ?? 'full';
16
+ const responseMode = getStringArg(args, 'responseMode') ?? 'debug';
17
+ if (!['none', 'light', 'full'].includes(verificationMode) || !['compact', 'debug'].includes(responseMode))
18
+ throw new Error('INVALID_ARGUMENT');
19
+ for (const key of ['verificationMode', 'responseMode'])
20
+ if (args[key] !== undefined && typeof args[key] !== 'string')
21
+ throw new Error('INVALID_ARGUMENT');
22
+ for (const key of ['actionTimeoutMs', 'verificationTimeoutMs'])
23
+ if (args[key] !== undefined && (typeof args[key] !== 'number' || !Number.isInteger(args[key])))
24
+ throw new Error('INVALID_ARGUMENT');
25
+ const actionTimeoutMs = getNumberArg(args, 'actionTimeoutMs');
26
+ const verificationTimeoutMs = getNumberArg(args, 'verificationTimeoutMs');
27
+ return { verificationMode, responseMode,
28
+ actionTimeoutMs: actionTimeoutMs === undefined ? undefined : Math.max(100, Math.min(30000, actionTimeoutMs)),
29
+ verificationTimeoutMs: verificationTimeoutMs === undefined ? 3000 : Math.max(100, Math.min(60000, verificationTimeoutMs)) };
30
+ }
31
+ function compactAction(result, verificationMode, started = true, stateDelta, totalMs = 0) {
32
+ return {
33
+ action_id: result.action_id,
34
+ action_type: result.action_type,
35
+ success: !!result.success,
36
+ lifecycle_state: result.success ? (verificationMode === 'full' ? result.lifecycle_state : 'pending_verification') : 'failed',
37
+ ...(result.failure_code ? { failure_code: result.failure_code } : {}),
38
+ ...(typeof result.retryable === 'boolean' ? { retryable: result.retryable } : {}),
39
+ delivery_status: result.success ? 'delivered' : (started ? 'unknown' : 'not_delivered'),
40
+ dispatch_started: started,
41
+ verification: { mode: verificationMode, status: result.success ? (verificationMode === 'none' ? 'not_requested' : verificationMode === 'light' ? 'observed' : (result.lifecycle_state === 'verified' ? 'verified' : 'observed')) : 'failed' },
42
+ ...(stateDelta ? { state_delta: stateDelta } : {}),
43
+ timing: { total_ms: totalMs }
44
+ };
45
+ }
46
+ function actionHandler(type, execute) {
47
+ return async (args) => {
48
+ const timing = {};
49
+ return withTiming(timing, async () => {
50
+ const started = performance.now();
51
+ const controls = interactionControls(args);
52
+ const budgetMs = controls.actionTimeoutMs ?? ({ start_app: 10000, restart_app: 15000, scroll_to_element: 10000 }[type] ?? 5000);
53
+ let dispatchStarted = false;
54
+ let result;
55
+ const dispatchWithBudget = (timeoutMs) => runWithBudget(timeoutMs, async (budget) => {
56
+ try {
57
+ return payloadOf(await execute(args));
58
+ }
59
+ finally {
60
+ dispatchStarted = budget.dispatchStarted;
61
+ }
62
+ });
63
+ try {
64
+ // Omitted controls retain the existing adapter budgets.
65
+ result = args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined
66
+ ? await dispatchWithBudget(Infinity) : await dispatchWithBudget(budgetMs);
67
+ if (result.success)
68
+ dispatchStarted = true;
69
+ }
70
+ catch (error) {
71
+ if (controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined && !/TIMEOUT|timed out/i.test(String(error)))
72
+ return wrapToolError(type, error);
73
+ result = { success: false, failure_code: /TIMEOUT|timed out/i.test(String(error)) ? 'ACTION_TIMEOUT' : 'UNKNOWN', error: String(error) };
74
+ }
75
+ if (result.failure_code === 'TIMEOUT' || /ACTION_TIMEOUT|timeout/i.test(result.error ?? result.reason ?? result.details?.error ?? ''))
76
+ result.failure_code = 'ACTION_TIMEOUT';
77
+ if (result.failure_code === 'ACTION_TIMEOUT')
78
+ result.retryable = !dispatchStarted;
79
+ if (!result.success && typeof result.retryable !== 'boolean')
80
+ result.retryable = !dispatchStarted;
81
+ result.action_id ??= randomUUID();
82
+ result.action_type ??= type;
83
+ recordObservationEvent({
84
+ step_id: args._stepId,
85
+ action_type: type,
86
+ dispatch_ms: timing.dispatch_ms,
87
+ verification_mode: controls.verificationMode,
88
+ dispatch_started: dispatchStarted
89
+ });
90
+ let delta;
91
+ let observationAvailable = false;
92
+ if (result.success && controls.verificationMode === 'light') {
93
+ const verificationStarted = performance.now();
94
+ const tree = await withObservationContext({ stepId: args._stepId, purpose: 'verification' }, () => measure('post_observation_ms', () => runWithBudget(controls.verificationTimeoutMs, () => ToolsObserve.getUITreeHandler({ platform: args.platform ?? result.device?.platform, deviceId: args.deviceId ?? result.device?.id })))).catch(() => null);
95
+ observationAvailable = !!tree && !tree.error;
96
+ recordObservationEvent({
97
+ verification_budget_ms: controls.verificationTimeoutMs,
98
+ verification_elapsed_ms: performance.now() - verificationStarted,
99
+ verification_outcome: observationAvailable ? 'observed' : 'unavailable',
100
+ verification_budget_expired: !observationAvailable && performance.now() - verificationStarted >= controls.verificationTimeoutMs
101
+ });
102
+ if (observationAvailable && tree.device)
103
+ delta = ToolsObserve.getLatestStateDelta(tree.device.platform, tree.device.id) ?? undefined;
104
+ }
105
+ const compact = compactAction(result, controls.verificationMode, dispatchStarted, delta, performance.now() - started);
106
+ if (result.success && controls.verificationMode === 'light' && !observationAvailable)
107
+ compact.verification.status = 'unavailable';
108
+ const evidence = {
109
+ ...compact,
110
+ ...(controls.verificationMode === 'light' ? { state_delta_available: !!delta } : {}),
111
+ ...(result.success && controls.verificationMode === 'light' && !observationAvailable ? { verification_diagnostic: { code: 'VERIFICATION_UNAVAILABLE', retryable: true } } : {})
112
+ };
113
+ return wrapResponse(controls.responseMode === 'compact' ? evidence : {
114
+ ...result, ...evidence, timing: { ...timing, total_ms: performance.now() - started },
115
+ effective_controls: { ...controls, actionTimeoutMs: args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined ? null : budgetMs, timeout_bound: !(args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined) }
116
+ }, controls.responseMode === 'debug');
117
+ });
118
+ };
119
+ }
120
+ const handleStartApp = actionHandler('start_app', executeStartApp);
121
+ const handleRestartApp = actionHandler('restart_app', executeRestartApp);
122
+ const handleTap = actionHandler('tap', executeTap);
123
+ const handleTapElement = actionHandler('tap_element', executeTapElement);
124
+ const handleSwipe = actionHandler('swipe', executeSwipe);
125
+ const handleScrollToElement = actionHandler('scroll_to_element', executeScrollToElement);
126
+ const handleTypeText = actionHandler('type_text', executeTypeText);
127
+ const handlePressBack = actionHandler('press_back', executePressBack);
128
+ async function executeStartApp(args) {
9
129
  const platform = requireStringArg(args, 'platform');
10
130
  const appId = requireStringArg(args, 'appId');
11
131
  const deviceId = getStringArg(args, 'deviceId');
12
- const uiFingerprintBefore = await captureActionFingerprint(platform, deviceId);
132
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
133
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
13
134
  ToolsNetwork.notifyActionStart();
14
- const res = await (platform === 'android' ? new AndroidManage().startApp(appId, deviceId) : new iOSManage().startApp(appId, deviceId));
15
- const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
16
- return wrapResponse(buildActionExecutionResult({
135
+ const res = await measure('dispatch_ms', () => (platform === 'android' ? new AndroidManage().startApp(appId, deviceId, actionTimeoutMs ?? (verificationMode === 'full' && getStringArg(args, 'responseMode') === undefined && args._journey !== true ? undefined : 10000)) : new iOSManage().startApp(appId, deviceId, verificationMode === 'full')));
136
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
137
+ const result = buildActionExecutionResult({
17
138
  actionType: 'start_app',
18
139
  sourceModule: 'server',
19
140
  device: res.device,
@@ -27,9 +148,11 @@ async function handleStartApp(args) {
27
148
  ...(typeof res.output === 'string' ? { output: res.output } : {}),
28
149
  ...(res.device ? { device_id: res.device.id } : {}),
29
150
  ...(typeof res.error === 'string' ? { error: res.error } : {}),
151
+ ...(res.diagnostics ? { diagnostics: res.diagnostics } : {}),
30
152
  ...(res.observedApp ? { observed_app: res.observedApp } : {})
31
153
  }
32
- }));
154
+ });
155
+ return wrapResponse(result);
33
156
  }
34
157
  async function handleTerminateApp(args) {
35
158
  const platform = requireStringArg(args, 'platform');
@@ -39,15 +162,17 @@ async function handleTerminateApp(args) {
39
162
  const response = { device: res.device, appTerminated: res.appTerminated };
40
163
  return wrapResponse(response);
41
164
  }
42
- async function handleRestartApp(args) {
165
+ async function executeRestartApp(args) {
43
166
  const platform = requireStringArg(args, 'platform');
44
167
  const appId = requireStringArg(args, 'appId');
45
168
  const deviceId = getStringArg(args, 'deviceId');
46
- const uiFingerprintBefore = await captureActionFingerprint(platform, deviceId);
169
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
170
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
47
171
  ToolsNetwork.notifyActionStart();
48
- const res = await (platform === 'android' ? new AndroidManage().restartApp(appId, deviceId) : new iOSManage().restartApp(appId, deviceId));
49
- const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
50
- return wrapResponse(buildActionExecutionResult({
172
+ const bounded = args._journey === true || verificationMode !== 'full' || args.responseMode !== undefined || actionTimeoutMs !== undefined;
173
+ const res = await measure('dispatch_ms', () => (platform === 'android' ? new AndroidManage().restartApp(appId, deviceId, bounded, actionTimeoutMs ?? 15000) : new iOSManage().restartApp(appId, deviceId, verificationMode === 'full', bounded, actionTimeoutMs ?? 15000)));
174
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
175
+ const result = buildActionExecutionResult({
51
176
  actionType: 'restart_app',
52
177
  sourceModule: 'server',
53
178
  device: res.device,
@@ -62,9 +187,11 @@ async function handleRestartApp(args) {
62
187
  ...(typeof res.terminatedBeforeRestart === 'boolean' ? { terminated_before_restart: res.terminatedBeforeRestart } : {}),
63
188
  ...(typeof res.terminateError === 'string' ? { terminate_error: res.terminateError } : {}),
64
189
  ...(typeof res.error === 'string' ? { error: res.error } : {}),
190
+ ...(res.diagnostics ? { diagnostics: res.diagnostics } : {}),
65
191
  ...(res.observedApp ? { observed_app: res.observedApp } : {})
66
192
  }
67
- }));
193
+ });
194
+ return wrapResponse(result);
68
195
  }
69
196
  async function handleResetAppData(args) {
70
197
  const platform = requireStringArg(args, 'platform');
@@ -143,10 +270,12 @@ async function handleGetSystemStatus() {
143
270
  async function handleCaptureScreenshot(args) {
144
271
  const platform = requireStringArg(args, 'platform');
145
272
  const deviceId = getStringArg(args, 'deviceId');
273
+ const started = performance.now();
146
274
  const res = await ToolsObserve.captureScreenshotHandler({ platform, deviceId });
275
+ const screenshotMs = performance.now() - started;
147
276
  const mime = res.screenshot_mime || 'image/png';
148
277
  const content = [
149
- { type: 'text', text: JSON.stringify({ device: res.device, result: { resolution: res.resolution, mimeType: mime } }, null, 2) },
278
+ { type: 'text', text: JSON.stringify({ device: res.device, result: { resolution: res.resolution, mimeType: mime }, timing: { total_ms: screenshotMs, screenshot_ms: screenshotMs } }, null, 2) },
150
279
  { type: 'image', data: res.screenshot, mimeType: mime }
151
280
  ];
152
281
  if (res.screenshot_fallback) {
@@ -163,13 +292,17 @@ async function handleCaptureDebugSnapshot(args) {
163
292
  const appId = getStringArg(args, 'appId');
164
293
  const deviceId = getStringArg(args, 'deviceId');
165
294
  const sessionId = getStringArg(args, 'sessionId');
166
- const res = await ToolsObserve.captureDebugSnapshotHandler({ reason, includeLogs, logLines, platform, appId, deviceId, sessionId });
295
+ const includeScreenshot = getBooleanArg(args, 'includeScreenshot');
296
+ const res = await ToolsObserve.captureDebugSnapshotHandler({ reason, includeLogs, includeScreenshot, logLines, platform, appId, deviceId, sessionId });
167
297
  return wrapResponse(res);
168
298
  }
169
299
  async function handleGetUITree(args) {
170
300
  const platform = requireStringArg(args, 'platform');
171
301
  const deviceId = getStringArg(args, 'deviceId');
172
- const res = await ToolsObserve.getUITreeHandler({ platform, deviceId });
302
+ const responseMode = getStringArg(args, 'responseMode') ?? 'debug';
303
+ const sinceSnapshotRevision = getNumberArg(args, 'sinceSnapshotRevision');
304
+ const metricsStepId = process.env.MOBILE_DEBUG_MCP_UI_TREE_METRICS === '1' ? getStringArg(args, '_metricsStepId') : undefined;
305
+ const res = await withObservationContext({ stepId: metricsStepId, purpose: metricsStepId?.includes('post-response') ? 'assertion' : 'debug' }, () => ToolsObserve.getUITreeHandler({ platform, deviceId, responseMode, sinceSnapshotRevision }));
173
306
  return wrapResponse(res);
174
307
  }
175
308
  async function handleGetCurrentScreen(args) {
@@ -279,19 +412,22 @@ async function handleFindElement(args) {
279
412
  const timeoutMs = getNumberArg(args, 'timeoutMs') ?? 3000;
280
413
  const platform = getStringArg(args, 'platform');
281
414
  const deviceId = getStringArg(args, 'deviceId');
282
- const res = await ToolsInteract.findElementHandler({ query, exact, timeoutMs, platform, deviceId });
415
+ const targetStarted = performance.now();
416
+ const res = await withObservationContext({ purpose: 'target' }, () => ToolsInteract.findElementHandler({ query, exact, timeoutMs, platform, deviceId }));
417
+ recordObservationEvent({ target_resolution_ms: performance.now() - targetStarted, target_resolution_outcome: res.found ? 'found' : 'not_found', operation: 'find_element' });
283
418
  return wrapResponse(res);
284
419
  }
285
- async function handleTap(args) {
420
+ async function executeTap(args) {
286
421
  const platform = getStringArg(args, 'platform');
287
422
  const x = requireNumberArg(args, 'x');
288
423
  const y = requireNumberArg(args, 'y');
289
424
  const deviceId = getStringArg(args, 'deviceId');
290
- const uiFingerprintBefore = await captureActionFingerprint(platform, deviceId);
425
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
426
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
291
427
  ToolsNetwork.notifyActionStart();
292
- const res = await ToolsInteract.tapHandler({ platform, x, y, deviceId });
293
- const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
294
- return wrapResponse(buildActionExecutionResult({
428
+ const res = await measure('dispatch_ms', () => ToolsInteract.tapHandler({ platform, x, y, deviceId, timeoutMs: actionTimeoutMs }));
429
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
430
+ const result = buildActionExecutionResult({
295
431
  actionType: 'tap',
296
432
  sourceModule: 'server',
297
433
  selector: { x, y },
@@ -299,15 +435,177 @@ async function handleTap(args) {
299
435
  uiFingerprintBefore,
300
436
  uiFingerprintAfter,
301
437
  failure: res.success ? undefined : inferGenericFailure(res.error)
302
- }));
438
+ });
439
+ return wrapResponse(result);
303
440
  }
304
- async function handleTapElement(args) {
305
- const elementId = requireStringArg(args, 'elementId');
441
+ async function executeTapElement(args) {
442
+ const controls = tapElementControlsSchema.safeParse({ elementId: args.elementId, selector: args.selector, waitFor: args.waitFor });
443
+ if (!controls.success)
444
+ throw new Error(`INVALID_ARGUMENT: ${controls.error.message}`);
445
+ const elementId = controls.data.elementId;
446
+ const selector = controls.data.selector;
447
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
448
+ let resolvedId = elementId;
449
+ let freshTree;
450
+ if (selector) {
451
+ const waitFor = getObjectArg(args, 'waitFor');
452
+ const targetStarted = performance.now();
453
+ const resolution = await withObservationContext({ stepId: args._stepId, purpose: 'target' }, () => measure('target_resolution_ms', () => runWithBudget(waitFor?.timeoutMs ?? 1500, () => ToolsInteract.waitForUIHandler({ selector, condition: waitFor?.condition ?? 'clickable', timeout_ms: waitFor?.timeoutMs ?? 1500, poll_interval_ms: waitFor?.pollIntervalMs ?? 100, match: waitFor?.match, singleObservation: true, singleAttempt: !waitFor, rejectAmbiguous: !waitFor, platform: getStringArg(args, 'platform'), deviceId: getStringArg(args, 'deviceId') }))));
454
+ recordObservationEvent({ step_id: args._stepId, purpose: 'target', target_resolution_ms: performance.now() - targetStarted, target_resolution_outcome: resolution.status });
455
+ if (resolution.status !== 'success' || !resolution.element?.elementId)
456
+ return wrapResponse({ success: false, action_type: 'tap_element', failure_code: resolution.error?.code ?? 'ELEMENT_NOT_FOUND', retryable: true });
457
+ resolvedId = resolution.element.elementId;
458
+ freshTree = resolution._tree;
459
+ }
306
460
  ToolsNetwork.notifyActionStart();
307
- const res = await ToolsInteract.tapElementHandler({ elementId });
461
+ if (freshTree)
462
+ recordObservationConsumer(freshTree, 'target', args._stepId);
463
+ const res = await measure('dispatch_ms', () => ToolsInteract.tapElementHandler({ elementId: resolvedId, freshTree, verificationMode, timeoutMs: actionTimeoutMs, platform: getStringArg(args, 'platform'), deviceId: getStringArg(args, 'deviceId') }));
308
464
  return wrapResponse(res);
309
465
  }
310
- async function handleSwipe(args) {
466
+ function payloadOf(response) {
467
+ const text = response.content.find((item) => item.type === 'text')?.text;
468
+ return text ? JSON.parse(text) : null;
469
+ }
470
+ async function expectElementAbsent({ selector, timeoutMs, pollIntervalMs, platform, deviceId }) {
471
+ const deadline = performance.now() + timeoutMs;
472
+ let consecutive = 0;
473
+ let observed;
474
+ while (performance.now() < deadline) {
475
+ const result = await ToolsInteract.waitForUIHandler({ selector, condition: 'not_exists', timeout_ms: 0, singleObservation: true, singleAttempt: true, platform, deviceId });
476
+ observed = result.observed;
477
+ consecutive = result.status === 'success' ? consecutive + 1 : 0;
478
+ if (consecutive === 2 && performance.now() <= deadline)
479
+ return { success: true, observed };
480
+ const delay = Math.max(100, pollIntervalMs);
481
+ if (performance.now() + delay >= deadline)
482
+ break;
483
+ await new Promise(resolve => setTimeout(resolve, delay));
484
+ }
485
+ return { success: false, failure_code: 'ASSERTION_TIMEOUT', observed };
486
+ }
487
+ async function expectFingerprint({ fingerprint, timeoutMs, platform, deviceId }) {
488
+ const deadline = performance.now() + timeoutMs;
489
+ let consecutive = 0;
490
+ let observed = null;
491
+ while (performance.now() < deadline) {
492
+ const result = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
493
+ observed = result?.fingerprint ?? null;
494
+ consecutive = observed === fingerprint ? consecutive + 1 : 0;
495
+ if (consecutive === 2 && performance.now() <= deadline)
496
+ return { success: true, observed_fingerprint: observed };
497
+ if (performance.now() + 100 >= deadline)
498
+ break;
499
+ await new Promise(resolve => setTimeout(resolve, 100));
500
+ }
501
+ return { success: false, failure_code: 'ASSERTION_TIMEOUT', observed_fingerprint: observed };
502
+ }
503
+ async function handleRunJourney(args) {
504
+ const parsed = journeySchema.safeParse(args);
505
+ if (!parsed.success)
506
+ throw new Error(`INVALID_ARGUMENT: ${parsed.error.message}`);
507
+ const { platform, responseMode, defaults, captureOnFailure } = parsed.data;
508
+ const steps = parsed.data.steps;
509
+ const device = await resolveTargetDevice({ platform, deviceId: parsed.data.deviceId });
510
+ const deviceId = device.id;
511
+ return withResolvedDevice(device, async () => {
512
+ const started = performance.now();
513
+ const results = [];
514
+ let stoppedAt = null;
515
+ let pendingLightStep = null;
516
+ for (let index = 0; index < steps.length; index++) {
517
+ const step = steps[index];
518
+ const requestedVerification = step.verificationMode ?? defaults?.verificationMode ?? 'full';
519
+ const next = steps[index + 1];
520
+ const lightCoveredByAssertion = step.type === 'tap' && responseMode === 'compact' && requestedVerification === 'light' && next?.type === 'assert' &&
521
+ next.assertion?.kind === 'element_visible';
522
+ const stepStart = performance.now();
523
+ let result;
524
+ if (stoppedAt) {
525
+ results.push({ id: step.id, type: step.type, status: 'not_run', success: false, reason: 'PREVIOUS_STEP_FAILED', result: null, timing: { total_ms: 0 } });
526
+ continue;
527
+ }
528
+ try {
529
+ if (step.type === 'tap') {
530
+ result = payloadOf(await handleTapElement({ ...defaults, ...step, platform, deviceId, responseMode, _journey: true, _stepId: step.id, verificationMode: lightCoveredByAssertion ? 'none' : requestedVerification }));
531
+ }
532
+ else if (step.type === 'wait') {
533
+ result = await withObservationContext({ stepId: step.id, purpose: 'target' }, () => ToolsInteract.waitForUIHandler({ selector: step.selector, condition: step.condition ?? 'exists', timeout_ms: step.timeoutMs ?? 5000, poll_interval_ms: step.pollIntervalMs ?? 100, platform, deviceId }));
534
+ result = { success: result.status === 'success', ...result };
535
+ }
536
+ else if (step.type === 'assert') {
537
+ result = await withObservationContext({ stepId: step.id, purpose: 'assertion' }, () => runWithBudget(step.timeoutMs ?? 5000, async () => {
538
+ const assertion = step.assertion;
539
+ if (assertion?.kind === 'element_visible')
540
+ result = await ToolsInteract.expectElementVisibleHandler({ selector: assertion.selector, timeout_ms: step.timeoutMs ?? 5000, poll_interval_ms: step.pollIntervalMs ?? 100, platform, deviceId });
541
+ else if (assertion?.kind === 'element_absent')
542
+ result = await expectElementAbsent({ selector: assertion.selector, timeoutMs: step.timeoutMs ?? 5000, pollIntervalMs: step.pollIntervalMs ?? 100, platform, deviceId });
543
+ else if (assertion?.kind === 'screen_fingerprint')
544
+ result = await expectFingerprint({ fingerprint: assertion.fingerprint, timeoutMs: step.timeoutMs ?? 5000, platform, deviceId });
545
+ else if (assertion?.kind === 'state_equals')
546
+ result = await ToolsInteract.expectStateHandler({ selector: assertion.selector, element_id: assertion.elementId, property: assertion.property, expected: assertion.expected, platform, deviceId, stabilization_window_ms: step.timeoutMs ?? 5000, poll_interval_ms: step.pollIntervalMs ?? 100, exact_budget: true });
547
+ else
548
+ result = { success: false, failure_code: 'INVALID_ASSERTION' };
549
+ return result;
550
+ }));
551
+ }
552
+ else if (step.type === 'start_app') {
553
+ result = payloadOf(await handleStartApp({ ...defaults, platform, deviceId, appId: step.appId, responseMode, _journey: true, _stepId: step.id, verificationMode: step.verificationMode ?? defaults?.verificationMode ?? 'full' }));
554
+ }
555
+ else {
556
+ result = { success: false, failure_code: 'UNSUPPORTED_JOURNEY_STEP' };
557
+ }
558
+ }
559
+ catch (error) {
560
+ result = { success: false, failure_code: step.type === 'assert' ? 'ASSERTION_TIMEOUT' : 'STEP_FAILED', error: error instanceof Error ? error.message : String(error) };
561
+ }
562
+ if (step.type === 'assert' && !result?.success && /TIMEOUT/.test(result?.failure_code ?? ''))
563
+ result.failure_code = 'ASSERTION_TIMEOUT';
564
+ const success = !!result?.success;
565
+ if (lightCoveredByAssertion) {
566
+ result.verification = { mode: 'light', status: success ? 'unavailable' : 'failed' };
567
+ if (success) {
568
+ result.verification_diagnostic = { code: 'VERIFICATION_UNAVAILABLE', retryable: true };
569
+ pendingLightStep = results.length;
570
+ }
571
+ }
572
+ if (step.type === 'assert' && pendingLightStep !== null) {
573
+ const pending = results[pendingLightStep].result;
574
+ if (typeof result?.observed?.matched_count === 'number') {
575
+ pending.verification.status = 'observed';
576
+ delete pending.verification_diagnostic;
577
+ const delta = ToolsObserve.getLatestStateDelta(platform, deviceId);
578
+ if (delta) {
579
+ pending.state_delta = delta;
580
+ pending.state_delta_available = true;
581
+ }
582
+ }
583
+ pendingLightStep = null;
584
+ }
585
+ if (responseMode === 'compact' && ['wait', 'assert'].includes(step.type))
586
+ result = {
587
+ success, ...(result.failure_code ? { failure_code: result.failure_code } : {}),
588
+ ...(result.error?.code ? { failure_code: result.error.code } : {}),
589
+ ...(result.observed?.matched_count !== undefined ? { matched_count: result.observed.matched_count } : {}),
590
+ ...(result.observed_fingerprint !== undefined ? { matched: success } : {}),
591
+ ...(result.actual_state ? { observed: result.actual_state } : {})
592
+ };
593
+ results.push({ id: step.id, type: step.type, status: success ? 'passed' : 'failed', success, result, timing: { total_ms: performance.now() - stepStart } });
594
+ if (!success)
595
+ stoppedAt = step.id;
596
+ }
597
+ const failureSnapshot = stoppedAt && captureOnFailure && responseMode === 'debug'
598
+ ? await ToolsObserve.captureDebugSnapshotHandler({ platform, deviceId, includeScreenshot: false }).catch(() => null) : undefined;
599
+ const componentTiming = {};
600
+ for (const step of results)
601
+ for (const [key, value] of Object.entries(step.result?.timing ?? {})) {
602
+ if (key !== 'total_ms' && typeof value === 'number')
603
+ componentTiming[key] = (componentTiming[key] ?? 0) + value;
604
+ }
605
+ return wrapResponse({ success: stoppedAt === null, stopped_at_step_id: stoppedAt, steps: results, ...(failureSnapshot ? { failure_snapshot: failureSnapshot } : {}), timing: responseMode === 'compact' ? { total_ms: performance.now() - started } : { ...componentTiming, total_ms: performance.now() - started } }, responseMode === 'debug');
606
+ });
607
+ }
608
+ async function executeSwipe(args) {
311
609
  const platform = getStringArg(args, 'platform') ?? 'android';
312
610
  const x1 = requireNumberArg(args, 'x1');
313
611
  const y1 = requireNumberArg(args, 'y1');
@@ -315,11 +613,12 @@ async function handleSwipe(args) {
315
613
  const y2 = requireNumberArg(args, 'y2');
316
614
  const duration = requireNumberArg(args, 'duration');
317
615
  const deviceId = getStringArg(args, 'deviceId');
318
- const uiFingerprintBefore = await captureActionFingerprint(platform, deviceId);
616
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
617
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
319
618
  ToolsNetwork.notifyActionStart();
320
- const res = await ToolsInteract.swipeHandler({ platform, x1, y1, x2, y2, duration, deviceId });
321
- const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
322
- return wrapResponse(buildActionExecutionResult({
619
+ const res = await measure('dispatch_ms', () => ToolsInteract.swipeHandler({ platform, x1, y1, x2, y2, duration, deviceId, timeoutMs: actionTimeoutMs }));
620
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
621
+ const result = buildActionExecutionResult({
323
622
  actionType: 'swipe',
324
623
  sourceModule: 'server',
325
624
  selector: { x1, y1, x2, y2, duration },
@@ -327,20 +626,22 @@ async function handleSwipe(args) {
327
626
  uiFingerprintBefore,
328
627
  uiFingerprintAfter,
329
628
  failure: res.success ? undefined : inferGenericFailure(res.error)
330
- }));
629
+ });
630
+ return wrapResponse(result);
331
631
  }
332
- async function handleScrollToElement(args) {
632
+ async function executeScrollToElement(args) {
333
633
  const platform = requireStringArg(args, 'platform');
334
634
  const selector = requireObjectArg(args, 'selector');
335
635
  const direction = getStringArg(args, 'direction');
336
636
  const maxScrolls = getNumberArg(args, 'maxScrolls');
337
637
  const scrollAmount = getNumberArg(args, 'scrollAmount');
338
638
  const deviceId = getStringArg(args, 'deviceId');
339
- const uiFingerprintBefore = await captureActionFingerprint(platform, deviceId);
639
+ const { verificationMode } = interactionControls(args);
640
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
340
641
  ToolsNetwork.notifyActionStart();
341
- const res = await ToolsInteract.scrollToElementHandler({ platform, selector, direction, maxScrolls, scrollAmount, deviceId });
342
- const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
343
- return wrapResponse(buildActionExecutionResult({
642
+ const res = await measure('dispatch_ms', () => ToolsInteract.scrollToElementHandler({ platform, selector, direction, maxScrolls, scrollAmount, deviceId }));
643
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
644
+ const result = buildActionExecutionResult({
344
645
  actionType: 'scroll_to_element',
345
646
  sourceModule: 'server',
346
647
  selector: selector ?? null,
@@ -357,16 +658,18 @@ async function handleScrollToElement(args) {
357
658
  uiFingerprintBefore,
358
659
  uiFingerprintAfter,
359
660
  failure: res.success ? undefined : inferScrollFailure(res.reason)
360
- }));
661
+ });
662
+ return wrapResponse(result);
361
663
  }
362
- async function handleTypeText(args) {
664
+ async function executeTypeText(args) {
363
665
  const text = requireStringArg(args, 'text');
364
666
  const deviceId = getStringArg(args, 'deviceId');
365
- const uiFingerprintBefore = await captureActionFingerprint('android', deviceId);
667
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
668
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
366
669
  ToolsNetwork.notifyActionStart();
367
- const res = await ToolsInteract.typeTextHandler({ text, deviceId });
368
- const uiFingerprintAfter = await captureActionFingerprint('android', deviceId);
369
- return wrapResponse(buildActionExecutionResult({
670
+ const res = await measure('dispatch_ms', () => ToolsInteract.typeTextHandler({ text, deviceId, timeoutMs: actionTimeoutMs }));
671
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
672
+ const result = buildActionExecutionResult({
370
673
  actionType: 'type_text',
371
674
  sourceModule: 'server',
372
675
  selector: { text },
@@ -374,15 +677,17 @@ async function handleTypeText(args) {
374
677
  uiFingerprintBefore,
375
678
  uiFingerprintAfter,
376
679
  failure: res.success ? undefined : inferGenericFailure(res.error)
377
- }));
680
+ });
681
+ return wrapResponse(result);
378
682
  }
379
- async function handlePressBack(args) {
683
+ async function executePressBack(args) {
380
684
  const deviceId = getStringArg(args, 'deviceId');
381
- const uiFingerprintBefore = await captureActionFingerprint('android', deviceId);
685
+ const { verificationMode, actionTimeoutMs } = interactionControls(args);
686
+ const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
382
687
  ToolsNetwork.notifyActionStart();
383
- const res = await ToolsInteract.pressBackHandler({ deviceId });
384
- const uiFingerprintAfter = await captureActionFingerprint('android', deviceId);
385
- return wrapResponse(buildActionExecutionResult({
688
+ const res = await measure('dispatch_ms', () => ToolsInteract.pressBackHandler({ deviceId, timeoutMs: actionTimeoutMs }));
689
+ const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
690
+ const result = buildActionExecutionResult({
386
691
  actionType: 'press_back',
387
692
  sourceModule: 'server',
388
693
  selector: { key: 'back' },
@@ -390,7 +695,8 @@ async function handlePressBack(args) {
390
695
  uiFingerprintBefore,
391
696
  uiFingerprintAfter,
392
697
  failure: res.success ? undefined : inferGenericFailure(res.error)
393
- }));
698
+ });
699
+ return wrapResponse(result);
394
700
  }
395
701
  async function handleStartLogStream(args) {
396
702
  const platform = getStringArg(args, 'platform') ?? 'android';
@@ -462,6 +768,7 @@ export const toolHandlers = {
462
768
  find_element: handleFindElement,
463
769
  tap: handleTap,
464
770
  tap_element: handleTapElement,
771
+ run_journey: handleRunJourney,
465
772
  swipe: handleSwipe,
466
773
  scroll_to_element: handleScrollToElement,
467
774
  type_text: handleTypeText,
@@ -476,11 +783,29 @@ export async function handleToolCall(name, args = {}) {
476
783
  const handler = toolHandlers[name];
477
784
  if (!handler)
478
785
  throw new Error(`Unknown tool: ${name}`);
479
- try {
480
- return await handler(args);
481
- }
482
- catch (error) {
483
- console.error(`Error executing tool ${name}:`, error);
484
- return wrapToolError(name, error);
485
- }
786
+ const started = performance.now();
787
+ const measured = {};
788
+ const addTiming = (response) => {
789
+ if (!['start_app', 'restart_app', 'tap', 'tap_element', 'swipe', 'scroll_to_element', 'type_text', 'press_back', 'wait_for_ui', 'get_ui_tree', 'capture_screenshot', 'capture_debug_snapshot'].includes(name))
790
+ return response;
791
+ const item = response.content.find(item => item.type === 'text');
792
+ if (item?.text) {
793
+ try {
794
+ const value = JSON.parse(item.text);
795
+ value.timing = { ...measured, ...value.timing, total_ms: performance.now() - started };
796
+ item.text = JSON.stringify(value);
797
+ }
798
+ catch { /* Non-JSON text responses retain their original content. */ }
799
+ }
800
+ return response;
801
+ };
802
+ return withTiming(measured, async () => {
803
+ try {
804
+ return addTiming(await handler(args));
805
+ }
806
+ catch (error) {
807
+ console.error(`Error executing tool ${name}:`, error);
808
+ return addTiming(wrapToolError(name, error));
809
+ }
810
+ });
486
811
  }
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
6
6
  export { wrapResponse, toolDefinitions, handleToolCall };
7
7
  export const serverInfo = {
8
8
  name: 'mobile-debug-mcp',
9
- version: '0.31.0'
9
+ version: '0.32.1'
10
10
  };
11
11
  export function createServer() {
12
12
  const server = new Server(serverInfo, {