mobile-debug-mcp 0.30.2 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -5
- package/dist/interact/android.js +13 -8
- package/dist/interact/index.js +33 -22
- package/dist/interact/ios.js +5 -2
- package/dist/manage/android.js +20 -10
- package/dist/manage/ios.js +20 -11
- package/dist/observe/android.js +6 -1
- package/dist/observe/index.js +22 -5
- package/dist/observe/ios.js +7 -2
- package/dist/observe/snapshot-metadata.js +94 -15
- package/dist/server/common.js +14 -7
- package/dist/server/journey-schema.js +60 -0
- package/dist/server/tool-definitions.js +59 -6
- package/dist/server/tool-handlers.js +358 -55
- package/dist/server-core.js +1 -1
- package/dist/system/index.js +485 -48
- package/dist/system/ios.js +15 -7
- package/dist/utils/android/utils.js +4 -4
- package/dist/utils/cli/idb/check-idb.js +10 -3
- package/dist/utils/cli/idb/idb-helper.js +47 -13
- package/dist/utils/exec.js +4 -2
- package/dist/utils/ios/utils.js +10 -11
- package/dist/utils/operation-budget.js +25 -0
- package/dist/utils/resolve-device.js +12 -1
- package/dist/utils/timing.js +19 -0
- package/dist/utils/ui/index.js +2 -0
- package/docs/CHANGELOG.md +6 -0
- package/docs/ROADMAP.md +4 -4
- package/docs/benchmarks/interaction-latency/README.md +26 -0
- package/docs/benchmarks/interaction-latency/final.json +1810 -0
- package/docs/benchmarks/interaction-latency/final.md +17 -0
- package/docs/benchmarks/interaction-latency/ios-validation.md +11 -0
- package/docs/specs/baseline-spec-v0.md +1 -1
- package/docs/specs-deltas/016-environment-tools.md +533 -0
- package/docs/specs-deltas/018-interaction-latency-and-batched-journeys.md +301 -0
- package/docs/tools/TOOLS.md +2 -0
- package/docs/tools/interact.md +30 -0
- package/docs/tools/manage.md +4 -2
- package/docs/tools/system.md +9 -2
- package/package.json +1 -1
- package/src/interact/android.ts +13 -8
- package/src/interact/index.ts +32 -23
- package/src/interact/ios.ts +5 -2
- package/src/manage/android.ts +19 -10
- package/src/manage/ios.ts +19 -11
- package/src/observe/android.ts +5 -1
- package/src/observe/index.ts +21 -5
- package/src/observe/ios.ts +6 -2
- package/src/observe/snapshot-metadata.ts +94 -15
- package/src/server/common.ts +13 -6
- package/src/server/journey-schema.ts +63 -0
- package/src/server/tool-definitions.ts +60 -7
- package/src/server/tool-handlers.ts +324 -53
- package/src/server-core.ts +1 -1
- package/src/system/index.ts +572 -50
- package/src/system/ios.ts +14 -7
- package/src/types.ts +31 -2
- package/src/utils/android/utils.ts +3 -3
- package/src/utils/cli/idb/check-idb.ts +11 -2
- package/src/utils/cli/idb/idb-helper.ts +49 -12
- package/src/utils/exec.ts +4 -2
- package/src/utils/ios/utils.ts +9 -10
- package/src/utils/operation-budget.ts +22 -0
- package/src/utils/resolve-device.ts +13 -1
- package/src/utils/timing.ts +15 -0
- package/src/utils/ui/index.ts +2 -0
- package/test/device/automated/latency/benchmark.ts +136 -0
- package/test/device/automated/latency/report.ts +6 -0
- package/test/fixtures/latency-journey-app/AndroidManifest.xml +6 -0
- package/test/fixtures/latency-journey-app/README.md +10 -0
- package/test/fixtures/latency-journey-app/build.ts +26 -0
- package/test/fixtures/latency-journey-app/src/MainActivity.java +48 -0
- package/test/unit/interact/tap_element.test.ts +1 -1
- package/test/unit/manage/restart_budget.test.ts +37 -0
- package/test/unit/observe/capture_debug_snapshot.test.ts +6 -0
- package/test/unit/observe/snapshot_metadata.test.ts +20 -1
- package/test/unit/observe/state_delta.test.ts +15 -0
- package/test/unit/server/benchmark_report.test.ts +4 -0
- package/test/unit/server/contract.test.ts +1 -1
- package/test/unit/server/interaction_modes.test.ts +60 -0
- package/test/unit/server/operation_budget.test.ts +32 -0
- package/test/unit/server/response_shapes.test.ts +3 -3
- package/test/unit/server/run_journey.test.ts +90 -0
- package/test/unit/server/tap_selector.test.ts +68 -0
- package/test/unit/system/get_system_status.test.ts +3 -0
- package/test/unit/system/system_status.test.ts +40 -0
- package/test/unit/utils/idb_health.test.ts +74 -0
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import { measure, withTiming } from '../utils/timing.js';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { runWithBudget } from '../utils/operation-budget.js';
|
|
4
|
+
import { journeySchema, tapElementControlsSchema } from './journey-schema.js';
|
|
5
|
+
import { resolveTargetDevice, withResolvedDevice } from '../utils/resolve-device.js';
|
|
1
6
|
import { AndroidManage, iOSManage, ToolsManage } from '../manage/index.js';
|
|
2
7
|
import { ToolsInteract } from '../interact/index.js';
|
|
3
8
|
import { ToolsObserve } from '../observe/index.js';
|
|
@@ -5,15 +10,116 @@ import { classifyActionOutcome } from '../interact/classify.js';
|
|
|
5
10
|
import { ToolsNetwork } from '../network/index.js';
|
|
6
11
|
import { getSystemStatus } from '../system/index.js';
|
|
7
12
|
import { buildActionExecutionResult, captureActionFingerprint, getArrayArg, getBooleanArg, getNumberArg, getObjectArg, getStringArg, inferGenericFailure, inferScrollFailure, requireBooleanArg, requireNumberArg, requireObjectArg, requireStringArg, wrapResponse, wrapToolError } from './common.js';
|
|
8
|
-
|
|
13
|
+
function interactionControls(args) {
|
|
14
|
+
const verificationMode = getStringArg(args, 'verificationMode') ?? 'full';
|
|
15
|
+
const responseMode = getStringArg(args, 'responseMode') ?? 'debug';
|
|
16
|
+
if (!['none', 'light', 'full'].includes(verificationMode) || !['compact', 'debug'].includes(responseMode))
|
|
17
|
+
throw new Error('INVALID_ARGUMENT');
|
|
18
|
+
for (const key of ['verificationMode', 'responseMode'])
|
|
19
|
+
if (args[key] !== undefined && typeof args[key] !== 'string')
|
|
20
|
+
throw new Error('INVALID_ARGUMENT');
|
|
21
|
+
for (const key of ['actionTimeoutMs', 'verificationTimeoutMs'])
|
|
22
|
+
if (args[key] !== undefined && (typeof args[key] !== 'number' || !Number.isInteger(args[key])))
|
|
23
|
+
throw new Error('INVALID_ARGUMENT');
|
|
24
|
+
const actionTimeoutMs = getNumberArg(args, 'actionTimeoutMs');
|
|
25
|
+
const verificationTimeoutMs = getNumberArg(args, 'verificationTimeoutMs');
|
|
26
|
+
return { verificationMode, responseMode,
|
|
27
|
+
actionTimeoutMs: actionTimeoutMs === undefined ? undefined : Math.max(100, Math.min(30000, actionTimeoutMs)),
|
|
28
|
+
verificationTimeoutMs: verificationTimeoutMs === undefined ? 3000 : Math.max(100, Math.min(60000, verificationTimeoutMs)) };
|
|
29
|
+
}
|
|
30
|
+
function compactAction(result, verificationMode, started = true, stateDelta, totalMs = 0) {
|
|
31
|
+
return {
|
|
32
|
+
action_id: result.action_id,
|
|
33
|
+
action_type: result.action_type,
|
|
34
|
+
success: !!result.success,
|
|
35
|
+
lifecycle_state: result.success ? (verificationMode === 'full' ? result.lifecycle_state : 'pending_verification') : 'failed',
|
|
36
|
+
...(result.failure_code ? { failure_code: result.failure_code } : {}),
|
|
37
|
+
...(typeof result.retryable === 'boolean' ? { retryable: result.retryable } : {}),
|
|
38
|
+
delivery_status: result.success ? 'delivered' : (started ? 'unknown' : 'not_delivered'),
|
|
39
|
+
dispatch_started: started,
|
|
40
|
+
verification: { mode: verificationMode, status: result.success ? (verificationMode === 'none' ? 'not_requested' : verificationMode === 'light' ? 'observed' : (result.lifecycle_state === 'verified' ? 'verified' : 'observed')) : 'failed' },
|
|
41
|
+
...(stateDelta ? { state_delta: stateDelta } : {}),
|
|
42
|
+
timing: { total_ms: totalMs }
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function actionHandler(type, execute) {
|
|
46
|
+
return async (args) => {
|
|
47
|
+
const timing = {};
|
|
48
|
+
return withTiming(timing, async () => {
|
|
49
|
+
const started = performance.now();
|
|
50
|
+
const controls = interactionControls(args);
|
|
51
|
+
const budgetMs = controls.actionTimeoutMs ?? ({ start_app: 10000, restart_app: 15000, scroll_to_element: 10000 }[type] ?? 5000);
|
|
52
|
+
let dispatchStarted = false;
|
|
53
|
+
let result;
|
|
54
|
+
const dispatchWithBudget = (timeoutMs) => runWithBudget(timeoutMs, async (budget) => {
|
|
55
|
+
try {
|
|
56
|
+
return payloadOf(await execute(args));
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
dispatchStarted = budget.dispatchStarted;
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
try {
|
|
63
|
+
// Omitted controls retain the existing adapter budgets.
|
|
64
|
+
result = args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined
|
|
65
|
+
? await dispatchWithBudget(Infinity) : await dispatchWithBudget(budgetMs);
|
|
66
|
+
if (result.success)
|
|
67
|
+
dispatchStarted = true;
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
if (controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined && !/TIMEOUT|timed out/i.test(String(error)))
|
|
71
|
+
return wrapToolError(type, error);
|
|
72
|
+
result = { success: false, failure_code: /TIMEOUT|timed out/i.test(String(error)) ? 'ACTION_TIMEOUT' : 'UNKNOWN', error: String(error) };
|
|
73
|
+
}
|
|
74
|
+
if (result.failure_code === 'TIMEOUT' || /ACTION_TIMEOUT|timeout/i.test(result.error ?? result.reason ?? result.details?.error ?? ''))
|
|
75
|
+
result.failure_code = 'ACTION_TIMEOUT';
|
|
76
|
+
if (result.failure_code === 'ACTION_TIMEOUT')
|
|
77
|
+
result.retryable = !dispatchStarted;
|
|
78
|
+
if (!result.success && typeof result.retryable !== 'boolean')
|
|
79
|
+
result.retryable = !dispatchStarted;
|
|
80
|
+
result.action_id ??= randomUUID();
|
|
81
|
+
result.action_type ??= type;
|
|
82
|
+
let delta;
|
|
83
|
+
let observationAvailable = false;
|
|
84
|
+
if (result.success && controls.verificationMode === 'light') {
|
|
85
|
+
const tree = await measure('post_observation_ms', () => runWithBudget(controls.verificationTimeoutMs, () => ToolsObserve.getUITreeHandler({ platform: args.platform ?? result.device?.platform, deviceId: args.deviceId ?? result.device?.id }))).catch(() => null);
|
|
86
|
+
observationAvailable = !!tree && !tree.error;
|
|
87
|
+
if (observationAvailable && tree.device)
|
|
88
|
+
delta = ToolsObserve.getLatestStateDelta(tree.device.platform, tree.device.id) ?? undefined;
|
|
89
|
+
}
|
|
90
|
+
const compact = compactAction(result, controls.verificationMode, dispatchStarted, delta, performance.now() - started);
|
|
91
|
+
if (result.success && controls.verificationMode === 'light' && !observationAvailable)
|
|
92
|
+
compact.verification.status = 'unavailable';
|
|
93
|
+
const evidence = {
|
|
94
|
+
...compact,
|
|
95
|
+
...(controls.verificationMode === 'light' ? { state_delta_available: !!delta } : {}),
|
|
96
|
+
...(result.success && controls.verificationMode === 'light' && !observationAvailable ? { verification_diagnostic: { code: 'VERIFICATION_UNAVAILABLE', retryable: true } } : {})
|
|
97
|
+
};
|
|
98
|
+
return wrapResponse(controls.responseMode === 'compact' ? evidence : {
|
|
99
|
+
...result, ...evidence, timing: { ...timing, total_ms: performance.now() - started },
|
|
100
|
+
effective_controls: { ...controls, actionTimeoutMs: args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined ? null : budgetMs, timeout_bound: !(args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined) }
|
|
101
|
+
}, controls.responseMode === 'debug');
|
|
102
|
+
});
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
const handleStartApp = actionHandler('start_app', executeStartApp);
|
|
106
|
+
const handleRestartApp = actionHandler('restart_app', executeRestartApp);
|
|
107
|
+
const handleTap = actionHandler('tap', executeTap);
|
|
108
|
+
const handleTapElement = actionHandler('tap_element', executeTapElement);
|
|
109
|
+
const handleSwipe = actionHandler('swipe', executeSwipe);
|
|
110
|
+
const handleScrollToElement = actionHandler('scroll_to_element', executeScrollToElement);
|
|
111
|
+
const handleTypeText = actionHandler('type_text', executeTypeText);
|
|
112
|
+
const handlePressBack = actionHandler('press_back', executePressBack);
|
|
113
|
+
async function executeStartApp(args) {
|
|
9
114
|
const platform = requireStringArg(args, 'platform');
|
|
10
115
|
const appId = requireStringArg(args, 'appId');
|
|
11
116
|
const deviceId = getStringArg(args, 'deviceId');
|
|
12
|
-
const
|
|
117
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
118
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
13
119
|
ToolsNetwork.notifyActionStart();
|
|
14
|
-
const res = await (platform === 'android' ? new AndroidManage().startApp(appId, deviceId) : new iOSManage().startApp(appId, deviceId));
|
|
15
|
-
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
16
|
-
|
|
120
|
+
const res = await measure('dispatch_ms', () => (platform === 'android' ? new AndroidManage().startApp(appId, deviceId, actionTimeoutMs ?? (verificationMode === 'full' && getStringArg(args, 'responseMode') === undefined && args._journey !== true ? undefined : 10000)) : new iOSManage().startApp(appId, deviceId, verificationMode === 'full')));
|
|
121
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
122
|
+
const result = buildActionExecutionResult({
|
|
17
123
|
actionType: 'start_app',
|
|
18
124
|
sourceModule: 'server',
|
|
19
125
|
device: res.device,
|
|
@@ -27,9 +133,11 @@ async function handleStartApp(args) {
|
|
|
27
133
|
...(typeof res.output === 'string' ? { output: res.output } : {}),
|
|
28
134
|
...(res.device ? { device_id: res.device.id } : {}),
|
|
29
135
|
...(typeof res.error === 'string' ? { error: res.error } : {}),
|
|
136
|
+
...(res.diagnostics ? { diagnostics: res.diagnostics } : {}),
|
|
30
137
|
...(res.observedApp ? { observed_app: res.observedApp } : {})
|
|
31
138
|
}
|
|
32
|
-
})
|
|
139
|
+
});
|
|
140
|
+
return wrapResponse(result);
|
|
33
141
|
}
|
|
34
142
|
async function handleTerminateApp(args) {
|
|
35
143
|
const platform = requireStringArg(args, 'platform');
|
|
@@ -39,15 +147,17 @@ async function handleTerminateApp(args) {
|
|
|
39
147
|
const response = { device: res.device, appTerminated: res.appTerminated };
|
|
40
148
|
return wrapResponse(response);
|
|
41
149
|
}
|
|
42
|
-
async function
|
|
150
|
+
async function executeRestartApp(args) {
|
|
43
151
|
const platform = requireStringArg(args, 'platform');
|
|
44
152
|
const appId = requireStringArg(args, 'appId');
|
|
45
153
|
const deviceId = getStringArg(args, 'deviceId');
|
|
46
|
-
const
|
|
154
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
155
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
47
156
|
ToolsNetwork.notifyActionStart();
|
|
48
|
-
const
|
|
49
|
-
const
|
|
50
|
-
|
|
157
|
+
const bounded = args._journey === true || verificationMode !== 'full' || args.responseMode !== undefined || actionTimeoutMs !== undefined;
|
|
158
|
+
const res = await measure('dispatch_ms', () => (platform === 'android' ? new AndroidManage().restartApp(appId, deviceId, bounded, actionTimeoutMs ?? 15000) : new iOSManage().restartApp(appId, deviceId, verificationMode === 'full', bounded, actionTimeoutMs ?? 15000)));
|
|
159
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
160
|
+
const result = buildActionExecutionResult({
|
|
51
161
|
actionType: 'restart_app',
|
|
52
162
|
sourceModule: 'server',
|
|
53
163
|
device: res.device,
|
|
@@ -62,9 +172,11 @@ async function handleRestartApp(args) {
|
|
|
62
172
|
...(typeof res.terminatedBeforeRestart === 'boolean' ? { terminated_before_restart: res.terminatedBeforeRestart } : {}),
|
|
63
173
|
...(typeof res.terminateError === 'string' ? { terminate_error: res.terminateError } : {}),
|
|
64
174
|
...(typeof res.error === 'string' ? { error: res.error } : {}),
|
|
175
|
+
...(res.diagnostics ? { diagnostics: res.diagnostics } : {}),
|
|
65
176
|
...(res.observedApp ? { observed_app: res.observedApp } : {})
|
|
66
177
|
}
|
|
67
|
-
})
|
|
178
|
+
});
|
|
179
|
+
return wrapResponse(result);
|
|
68
180
|
}
|
|
69
181
|
async function handleResetAppData(args) {
|
|
70
182
|
const platform = requireStringArg(args, 'platform');
|
|
@@ -143,10 +255,12 @@ async function handleGetSystemStatus() {
|
|
|
143
255
|
async function handleCaptureScreenshot(args) {
|
|
144
256
|
const platform = requireStringArg(args, 'platform');
|
|
145
257
|
const deviceId = getStringArg(args, 'deviceId');
|
|
258
|
+
const started = performance.now();
|
|
146
259
|
const res = await ToolsObserve.captureScreenshotHandler({ platform, deviceId });
|
|
260
|
+
const screenshotMs = performance.now() - started;
|
|
147
261
|
const mime = res.screenshot_mime || 'image/png';
|
|
148
262
|
const content = [
|
|
149
|
-
{ type: 'text', text: JSON.stringify({ device: res.device, result: { resolution: res.resolution, mimeType: mime } }, null, 2) },
|
|
263
|
+
{ type: 'text', text: JSON.stringify({ device: res.device, result: { resolution: res.resolution, mimeType: mime }, timing: { total_ms: screenshotMs, screenshot_ms: screenshotMs } }, null, 2) },
|
|
150
264
|
{ type: 'image', data: res.screenshot, mimeType: mime }
|
|
151
265
|
];
|
|
152
266
|
if (res.screenshot_fallback) {
|
|
@@ -163,13 +277,16 @@ async function handleCaptureDebugSnapshot(args) {
|
|
|
163
277
|
const appId = getStringArg(args, 'appId');
|
|
164
278
|
const deviceId = getStringArg(args, 'deviceId');
|
|
165
279
|
const sessionId = getStringArg(args, 'sessionId');
|
|
166
|
-
const
|
|
280
|
+
const includeScreenshot = getBooleanArg(args, 'includeScreenshot');
|
|
281
|
+
const res = await ToolsObserve.captureDebugSnapshotHandler({ reason, includeLogs, includeScreenshot, logLines, platform, appId, deviceId, sessionId });
|
|
167
282
|
return wrapResponse(res);
|
|
168
283
|
}
|
|
169
284
|
async function handleGetUITree(args) {
|
|
170
285
|
const platform = requireStringArg(args, 'platform');
|
|
171
286
|
const deviceId = getStringArg(args, 'deviceId');
|
|
172
|
-
const
|
|
287
|
+
const responseMode = getStringArg(args, 'responseMode') ?? 'debug';
|
|
288
|
+
const sinceSnapshotRevision = getNumberArg(args, 'sinceSnapshotRevision');
|
|
289
|
+
const res = await ToolsObserve.getUITreeHandler({ platform, deviceId, responseMode, sinceSnapshotRevision });
|
|
173
290
|
return wrapResponse(res);
|
|
174
291
|
}
|
|
175
292
|
async function handleGetCurrentScreen(args) {
|
|
@@ -282,16 +399,17 @@ async function handleFindElement(args) {
|
|
|
282
399
|
const res = await ToolsInteract.findElementHandler({ query, exact, timeoutMs, platform, deviceId });
|
|
283
400
|
return wrapResponse(res);
|
|
284
401
|
}
|
|
285
|
-
async function
|
|
402
|
+
async function executeTap(args) {
|
|
286
403
|
const platform = getStringArg(args, 'platform');
|
|
287
404
|
const x = requireNumberArg(args, 'x');
|
|
288
405
|
const y = requireNumberArg(args, 'y');
|
|
289
406
|
const deviceId = getStringArg(args, 'deviceId');
|
|
290
|
-
const
|
|
407
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
408
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
291
409
|
ToolsNetwork.notifyActionStart();
|
|
292
|
-
const res = await ToolsInteract.tapHandler({ platform, x, y, deviceId });
|
|
293
|
-
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
294
|
-
|
|
410
|
+
const res = await measure('dispatch_ms', () => ToolsInteract.tapHandler({ platform, x, y, deviceId, timeoutMs: actionTimeoutMs }));
|
|
411
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
412
|
+
const result = buildActionExecutionResult({
|
|
295
413
|
actionType: 'tap',
|
|
296
414
|
sourceModule: 'server',
|
|
297
415
|
selector: { x, y },
|
|
@@ -299,15 +417,173 @@ async function handleTap(args) {
|
|
|
299
417
|
uiFingerprintBefore,
|
|
300
418
|
uiFingerprintAfter,
|
|
301
419
|
failure: res.success ? undefined : inferGenericFailure(res.error)
|
|
302
|
-
})
|
|
420
|
+
});
|
|
421
|
+
return wrapResponse(result);
|
|
303
422
|
}
|
|
304
|
-
async function
|
|
305
|
-
const
|
|
423
|
+
async function executeTapElement(args) {
|
|
424
|
+
const controls = tapElementControlsSchema.safeParse({ elementId: args.elementId, selector: args.selector, waitFor: args.waitFor });
|
|
425
|
+
if (!controls.success)
|
|
426
|
+
throw new Error(`INVALID_ARGUMENT: ${controls.error.message}`);
|
|
427
|
+
const elementId = controls.data.elementId;
|
|
428
|
+
const selector = controls.data.selector;
|
|
429
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
430
|
+
let resolvedId = elementId;
|
|
431
|
+
let freshTree;
|
|
432
|
+
if (selector) {
|
|
433
|
+
const waitFor = getObjectArg(args, 'waitFor');
|
|
434
|
+
const resolution = await measure('target_resolution_ms', () => runWithBudget(waitFor?.timeoutMs ?? 1500, () => ToolsInteract.waitForUIHandler({ selector, condition: waitFor?.condition ?? 'clickable', timeout_ms: waitFor?.timeoutMs ?? 1500, poll_interval_ms: waitFor?.pollIntervalMs ?? 100, match: waitFor?.match, singleObservation: true, singleAttempt: !waitFor, rejectAmbiguous: !waitFor, platform: getStringArg(args, 'platform'), deviceId: getStringArg(args, 'deviceId') })));
|
|
435
|
+
if (resolution.status !== 'success' || !resolution.element?.elementId)
|
|
436
|
+
return wrapResponse({ success: false, action_type: 'tap_element', failure_code: resolution.error?.code ?? 'ELEMENT_NOT_FOUND', retryable: true });
|
|
437
|
+
resolvedId = resolution.element.elementId;
|
|
438
|
+
freshTree = resolution._tree;
|
|
439
|
+
}
|
|
306
440
|
ToolsNetwork.notifyActionStart();
|
|
307
|
-
const res = await ToolsInteract.tapElementHandler({ elementId });
|
|
441
|
+
const res = await measure('dispatch_ms', () => ToolsInteract.tapElementHandler({ elementId: resolvedId, freshTree, verificationMode, timeoutMs: actionTimeoutMs, platform: getStringArg(args, 'platform'), deviceId: getStringArg(args, 'deviceId') }));
|
|
308
442
|
return wrapResponse(res);
|
|
309
443
|
}
|
|
310
|
-
|
|
444
|
+
function payloadOf(response) {
|
|
445
|
+
const text = response.content.find((item) => item.type === 'text')?.text;
|
|
446
|
+
return text ? JSON.parse(text) : null;
|
|
447
|
+
}
|
|
448
|
+
async function expectElementAbsent({ selector, timeoutMs, pollIntervalMs, platform, deviceId }) {
|
|
449
|
+
const deadline = performance.now() + timeoutMs;
|
|
450
|
+
let consecutive = 0;
|
|
451
|
+
let observed;
|
|
452
|
+
while (performance.now() < deadline) {
|
|
453
|
+
const result = await ToolsInteract.waitForUIHandler({ selector, condition: 'not_exists', timeout_ms: 0, singleObservation: true, singleAttempt: true, platform, deviceId });
|
|
454
|
+
observed = result.observed;
|
|
455
|
+
consecutive = result.status === 'success' ? consecutive + 1 : 0;
|
|
456
|
+
if (consecutive === 2 && performance.now() <= deadline)
|
|
457
|
+
return { success: true, observed };
|
|
458
|
+
const delay = Math.max(100, pollIntervalMs);
|
|
459
|
+
if (performance.now() + delay >= deadline)
|
|
460
|
+
break;
|
|
461
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
462
|
+
}
|
|
463
|
+
return { success: false, failure_code: 'ASSERTION_TIMEOUT', observed };
|
|
464
|
+
}
|
|
465
|
+
async function expectFingerprint({ fingerprint, timeoutMs, platform, deviceId }) {
|
|
466
|
+
const deadline = performance.now() + timeoutMs;
|
|
467
|
+
let consecutive = 0;
|
|
468
|
+
let observed = null;
|
|
469
|
+
while (performance.now() < deadline) {
|
|
470
|
+
const result = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
471
|
+
observed = result?.fingerprint ?? null;
|
|
472
|
+
consecutive = observed === fingerprint ? consecutive + 1 : 0;
|
|
473
|
+
if (consecutive === 2 && performance.now() <= deadline)
|
|
474
|
+
return { success: true, observed_fingerprint: observed };
|
|
475
|
+
if (performance.now() + 100 >= deadline)
|
|
476
|
+
break;
|
|
477
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
478
|
+
}
|
|
479
|
+
return { success: false, failure_code: 'ASSERTION_TIMEOUT', observed_fingerprint: observed };
|
|
480
|
+
}
|
|
481
|
+
async function handleRunJourney(args) {
|
|
482
|
+
const parsed = journeySchema.safeParse(args);
|
|
483
|
+
if (!parsed.success)
|
|
484
|
+
throw new Error(`INVALID_ARGUMENT: ${parsed.error.message}`);
|
|
485
|
+
const { platform, responseMode, defaults, captureOnFailure } = parsed.data;
|
|
486
|
+
const steps = parsed.data.steps;
|
|
487
|
+
const device = await resolveTargetDevice({ platform, deviceId: parsed.data.deviceId });
|
|
488
|
+
const deviceId = device.id;
|
|
489
|
+
return withResolvedDevice(device, async () => {
|
|
490
|
+
const started = performance.now();
|
|
491
|
+
const results = [];
|
|
492
|
+
let stoppedAt = null;
|
|
493
|
+
let pendingLightStep = null;
|
|
494
|
+
for (let index = 0; index < steps.length; index++) {
|
|
495
|
+
const step = steps[index];
|
|
496
|
+
const requestedVerification = step.verificationMode ?? defaults?.verificationMode ?? 'full';
|
|
497
|
+
const next = steps[index + 1];
|
|
498
|
+
const lightCoveredByAssertion = step.type === 'tap' && responseMode === 'compact' && requestedVerification === 'light' && next?.type === 'assert' &&
|
|
499
|
+
next.assertion?.kind === 'element_visible';
|
|
500
|
+
const stepStart = performance.now();
|
|
501
|
+
let result;
|
|
502
|
+
if (stoppedAt) {
|
|
503
|
+
results.push({ id: step.id, type: step.type, status: 'not_run', success: false, reason: 'PREVIOUS_STEP_FAILED', result: null, timing: { total_ms: 0 } });
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
try {
|
|
507
|
+
if (step.type === 'tap') {
|
|
508
|
+
result = payloadOf(await handleTapElement({ ...defaults, ...step, platform, deviceId, responseMode, _journey: true, verificationMode: lightCoveredByAssertion ? 'none' : requestedVerification }));
|
|
509
|
+
}
|
|
510
|
+
else if (step.type === 'wait') {
|
|
511
|
+
result = await ToolsInteract.waitForUIHandler({ selector: step.selector, condition: step.condition ?? 'exists', timeout_ms: step.timeoutMs ?? 5000, poll_interval_ms: step.pollIntervalMs ?? 100, platform, deviceId });
|
|
512
|
+
result = { success: result.status === 'success', ...result };
|
|
513
|
+
}
|
|
514
|
+
else if (step.type === 'assert') {
|
|
515
|
+
result = await runWithBudget(step.timeoutMs ?? 5000, async () => {
|
|
516
|
+
const assertion = step.assertion;
|
|
517
|
+
if (assertion?.kind === 'element_visible')
|
|
518
|
+
result = await ToolsInteract.expectElementVisibleHandler({ selector: assertion.selector, timeout_ms: step.timeoutMs ?? 5000, poll_interval_ms: step.pollIntervalMs ?? 100, platform, deviceId });
|
|
519
|
+
else if (assertion?.kind === 'element_absent')
|
|
520
|
+
result = await expectElementAbsent({ selector: assertion.selector, timeoutMs: step.timeoutMs ?? 5000, pollIntervalMs: step.pollIntervalMs ?? 100, platform, deviceId });
|
|
521
|
+
else if (assertion?.kind === 'screen_fingerprint')
|
|
522
|
+
result = await expectFingerprint({ fingerprint: assertion.fingerprint, timeoutMs: step.timeoutMs ?? 5000, platform, deviceId });
|
|
523
|
+
else if (assertion?.kind === 'state_equals')
|
|
524
|
+
result = await ToolsInteract.expectStateHandler({ selector: assertion.selector, element_id: assertion.elementId, property: assertion.property, expected: assertion.expected, platform, deviceId, stabilization_window_ms: step.timeoutMs ?? 5000, poll_interval_ms: step.pollIntervalMs ?? 100, exact_budget: true });
|
|
525
|
+
else
|
|
526
|
+
result = { success: false, failure_code: 'INVALID_ASSERTION' };
|
|
527
|
+
return result;
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
else if (step.type === 'start_app') {
|
|
531
|
+
result = payloadOf(await handleStartApp({ ...defaults, platform, deviceId, appId: step.appId, responseMode, _journey: true, verificationMode: step.verificationMode ?? defaults?.verificationMode ?? 'full' }));
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
result = { success: false, failure_code: 'UNSUPPORTED_JOURNEY_STEP' };
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
catch (error) {
|
|
538
|
+
result = { success: false, failure_code: step.type === 'assert' ? 'ASSERTION_TIMEOUT' : 'STEP_FAILED', error: error instanceof Error ? error.message : String(error) };
|
|
539
|
+
}
|
|
540
|
+
if (step.type === 'assert' && !result?.success && /TIMEOUT/.test(result?.failure_code ?? ''))
|
|
541
|
+
result.failure_code = 'ASSERTION_TIMEOUT';
|
|
542
|
+
const success = !!result?.success;
|
|
543
|
+
if (lightCoveredByAssertion) {
|
|
544
|
+
result.verification = { mode: 'light', status: success ? 'unavailable' : 'failed' };
|
|
545
|
+
if (success) {
|
|
546
|
+
result.verification_diagnostic = { code: 'VERIFICATION_UNAVAILABLE', retryable: true };
|
|
547
|
+
pendingLightStep = results.length;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
if (step.type === 'assert' && pendingLightStep !== null) {
|
|
551
|
+
const pending = results[pendingLightStep].result;
|
|
552
|
+
if (typeof result?.observed?.matched_count === 'number') {
|
|
553
|
+
pending.verification.status = 'observed';
|
|
554
|
+
delete pending.verification_diagnostic;
|
|
555
|
+
const delta = ToolsObserve.getLatestStateDelta(platform, deviceId);
|
|
556
|
+
if (delta) {
|
|
557
|
+
pending.state_delta = delta;
|
|
558
|
+
pending.state_delta_available = true;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
pendingLightStep = null;
|
|
562
|
+
}
|
|
563
|
+
if (responseMode === 'compact' && ['wait', 'assert'].includes(step.type))
|
|
564
|
+
result = {
|
|
565
|
+
success, ...(result.failure_code ? { failure_code: result.failure_code } : {}),
|
|
566
|
+
...(result.error?.code ? { failure_code: result.error.code } : {}),
|
|
567
|
+
...(result.observed?.matched_count !== undefined ? { matched_count: result.observed.matched_count } : {}),
|
|
568
|
+
...(result.observed_fingerprint !== undefined ? { matched: success } : {}),
|
|
569
|
+
...(result.actual_state ? { observed: result.actual_state } : {})
|
|
570
|
+
};
|
|
571
|
+
results.push({ id: step.id, type: step.type, status: success ? 'passed' : 'failed', success, result, timing: { total_ms: performance.now() - stepStart } });
|
|
572
|
+
if (!success)
|
|
573
|
+
stoppedAt = step.id;
|
|
574
|
+
}
|
|
575
|
+
const failureSnapshot = stoppedAt && captureOnFailure && responseMode === 'debug'
|
|
576
|
+
? await ToolsObserve.captureDebugSnapshotHandler({ platform, deviceId, includeScreenshot: false }).catch(() => null) : undefined;
|
|
577
|
+
const componentTiming = {};
|
|
578
|
+
for (const step of results)
|
|
579
|
+
for (const [key, value] of Object.entries(step.result?.timing ?? {})) {
|
|
580
|
+
if (key !== 'total_ms' && typeof value === 'number')
|
|
581
|
+
componentTiming[key] = (componentTiming[key] ?? 0) + value;
|
|
582
|
+
}
|
|
583
|
+
return wrapResponse({ success: stoppedAt === null, stopped_at_step_id: stoppedAt, steps: results, ...(failureSnapshot ? { failure_snapshot: failureSnapshot } : {}), timing: responseMode === 'compact' ? { total_ms: performance.now() - started } : { ...componentTiming, total_ms: performance.now() - started } }, responseMode === 'debug');
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
async function executeSwipe(args) {
|
|
311
587
|
const platform = getStringArg(args, 'platform') ?? 'android';
|
|
312
588
|
const x1 = requireNumberArg(args, 'x1');
|
|
313
589
|
const y1 = requireNumberArg(args, 'y1');
|
|
@@ -315,11 +591,12 @@ async function handleSwipe(args) {
|
|
|
315
591
|
const y2 = requireNumberArg(args, 'y2');
|
|
316
592
|
const duration = requireNumberArg(args, 'duration');
|
|
317
593
|
const deviceId = getStringArg(args, 'deviceId');
|
|
318
|
-
const
|
|
594
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
595
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
319
596
|
ToolsNetwork.notifyActionStart();
|
|
320
|
-
const res = await ToolsInteract.swipeHandler({ platform, x1, y1, x2, y2, duration, deviceId });
|
|
321
|
-
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
322
|
-
|
|
597
|
+
const res = await measure('dispatch_ms', () => ToolsInteract.swipeHandler({ platform, x1, y1, x2, y2, duration, deviceId, timeoutMs: actionTimeoutMs }));
|
|
598
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
599
|
+
const result = buildActionExecutionResult({
|
|
323
600
|
actionType: 'swipe',
|
|
324
601
|
sourceModule: 'server',
|
|
325
602
|
selector: { x1, y1, x2, y2, duration },
|
|
@@ -327,20 +604,22 @@ async function handleSwipe(args) {
|
|
|
327
604
|
uiFingerprintBefore,
|
|
328
605
|
uiFingerprintAfter,
|
|
329
606
|
failure: res.success ? undefined : inferGenericFailure(res.error)
|
|
330
|
-
})
|
|
607
|
+
});
|
|
608
|
+
return wrapResponse(result);
|
|
331
609
|
}
|
|
332
|
-
async function
|
|
610
|
+
async function executeScrollToElement(args) {
|
|
333
611
|
const platform = requireStringArg(args, 'platform');
|
|
334
612
|
const selector = requireObjectArg(args, 'selector');
|
|
335
613
|
const direction = getStringArg(args, 'direction');
|
|
336
614
|
const maxScrolls = getNumberArg(args, 'maxScrolls');
|
|
337
615
|
const scrollAmount = getNumberArg(args, 'scrollAmount');
|
|
338
616
|
const deviceId = getStringArg(args, 'deviceId');
|
|
339
|
-
const
|
|
617
|
+
const { verificationMode } = interactionControls(args);
|
|
618
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
340
619
|
ToolsNetwork.notifyActionStart();
|
|
341
|
-
const res = await ToolsInteract.scrollToElementHandler({ platform, selector, direction, maxScrolls, scrollAmount, deviceId });
|
|
342
|
-
const uiFingerprintAfter = await captureActionFingerprint(platform, deviceId);
|
|
343
|
-
|
|
620
|
+
const res = await measure('dispatch_ms', () => ToolsInteract.scrollToElementHandler({ platform, selector, direction, maxScrolls, scrollAmount, deviceId }));
|
|
621
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null;
|
|
622
|
+
const result = buildActionExecutionResult({
|
|
344
623
|
actionType: 'scroll_to_element',
|
|
345
624
|
sourceModule: 'server',
|
|
346
625
|
selector: selector ?? null,
|
|
@@ -357,16 +636,18 @@ async function handleScrollToElement(args) {
|
|
|
357
636
|
uiFingerprintBefore,
|
|
358
637
|
uiFingerprintAfter,
|
|
359
638
|
failure: res.success ? undefined : inferScrollFailure(res.reason)
|
|
360
|
-
})
|
|
639
|
+
});
|
|
640
|
+
return wrapResponse(result);
|
|
361
641
|
}
|
|
362
|
-
async function
|
|
642
|
+
async function executeTypeText(args) {
|
|
363
643
|
const text = requireStringArg(args, 'text');
|
|
364
644
|
const deviceId = getStringArg(args, 'deviceId');
|
|
365
|
-
const
|
|
645
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
646
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
|
|
366
647
|
ToolsNetwork.notifyActionStart();
|
|
367
|
-
const res = await ToolsInteract.typeTextHandler({ text, deviceId });
|
|
368
|
-
const uiFingerprintAfter = await captureActionFingerprint('android', deviceId);
|
|
369
|
-
|
|
648
|
+
const res = await measure('dispatch_ms', () => ToolsInteract.typeTextHandler({ text, deviceId, timeoutMs: actionTimeoutMs }));
|
|
649
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
|
|
650
|
+
const result = buildActionExecutionResult({
|
|
370
651
|
actionType: 'type_text',
|
|
371
652
|
sourceModule: 'server',
|
|
372
653
|
selector: { text },
|
|
@@ -374,15 +655,17 @@ async function handleTypeText(args) {
|
|
|
374
655
|
uiFingerprintBefore,
|
|
375
656
|
uiFingerprintAfter,
|
|
376
657
|
failure: res.success ? undefined : inferGenericFailure(res.error)
|
|
377
|
-
})
|
|
658
|
+
});
|
|
659
|
+
return wrapResponse(result);
|
|
378
660
|
}
|
|
379
|
-
async function
|
|
661
|
+
async function executePressBack(args) {
|
|
380
662
|
const deviceId = getStringArg(args, 'deviceId');
|
|
381
|
-
const
|
|
663
|
+
const { verificationMode, actionTimeoutMs } = interactionControls(args);
|
|
664
|
+
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
|
|
382
665
|
ToolsNetwork.notifyActionStart();
|
|
383
|
-
const res = await ToolsInteract.pressBackHandler({ deviceId });
|
|
384
|
-
const uiFingerprintAfter = await captureActionFingerprint('android', deviceId);
|
|
385
|
-
|
|
666
|
+
const res = await measure('dispatch_ms', () => ToolsInteract.pressBackHandler({ deviceId, timeoutMs: actionTimeoutMs }));
|
|
667
|
+
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint('android', deviceId)) : null;
|
|
668
|
+
const result = buildActionExecutionResult({
|
|
386
669
|
actionType: 'press_back',
|
|
387
670
|
sourceModule: 'server',
|
|
388
671
|
selector: { key: 'back' },
|
|
@@ -390,7 +673,8 @@ async function handlePressBack(args) {
|
|
|
390
673
|
uiFingerprintBefore,
|
|
391
674
|
uiFingerprintAfter,
|
|
392
675
|
failure: res.success ? undefined : inferGenericFailure(res.error)
|
|
393
|
-
})
|
|
676
|
+
});
|
|
677
|
+
return wrapResponse(result);
|
|
394
678
|
}
|
|
395
679
|
async function handleStartLogStream(args) {
|
|
396
680
|
const platform = getStringArg(args, 'platform') ?? 'android';
|
|
@@ -462,6 +746,7 @@ export const toolHandlers = {
|
|
|
462
746
|
find_element: handleFindElement,
|
|
463
747
|
tap: handleTap,
|
|
464
748
|
tap_element: handleTapElement,
|
|
749
|
+
run_journey: handleRunJourney,
|
|
465
750
|
swipe: handleSwipe,
|
|
466
751
|
scroll_to_element: handleScrollToElement,
|
|
467
752
|
type_text: handleTypeText,
|
|
@@ -476,11 +761,29 @@ export async function handleToolCall(name, args = {}) {
|
|
|
476
761
|
const handler = toolHandlers[name];
|
|
477
762
|
if (!handler)
|
|
478
763
|
throw new Error(`Unknown tool: ${name}`);
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
764
|
+
const started = performance.now();
|
|
765
|
+
const measured = {};
|
|
766
|
+
const addTiming = (response) => {
|
|
767
|
+
if (!['start_app', 'restart_app', 'tap', 'tap_element', 'swipe', 'scroll_to_element', 'type_text', 'press_back', 'wait_for_ui', 'get_ui_tree', 'capture_screenshot', 'capture_debug_snapshot'].includes(name))
|
|
768
|
+
return response;
|
|
769
|
+
const item = response.content.find(item => item.type === 'text');
|
|
770
|
+
if (item?.text) {
|
|
771
|
+
try {
|
|
772
|
+
const value = JSON.parse(item.text);
|
|
773
|
+
value.timing = { ...measured, ...value.timing, total_ms: performance.now() - started };
|
|
774
|
+
item.text = JSON.stringify(value);
|
|
775
|
+
}
|
|
776
|
+
catch { /* Non-JSON text responses retain their original content. */ }
|
|
777
|
+
}
|
|
778
|
+
return response;
|
|
779
|
+
};
|
|
780
|
+
return withTiming(measured, async () => {
|
|
781
|
+
try {
|
|
782
|
+
return addTiming(await handler(args));
|
|
783
|
+
}
|
|
784
|
+
catch (error) {
|
|
785
|
+
console.error(`Error executing tool ${name}:`, error);
|
|
786
|
+
return addTiming(wrapToolError(name, error));
|
|
787
|
+
}
|
|
788
|
+
});
|
|
486
789
|
}
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.
|
|
9
|
+
version: '0.32.0'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|