mobile-debug-mcp 0.31.0 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/interact/android.js +13 -8
- package/dist/interact/index.js +33 -22
- package/dist/interact/ios.js +5 -2
- package/dist/manage/android.js +20 -10
- package/dist/manage/ios.js +19 -10
- package/dist/observe/android-tree-bridge.js +192 -0
- package/dist/observe/android.js +53 -14
- package/dist/observe/index.js +22 -5
- package/dist/observe/ios.js +50 -29
- package/dist/observe/performance-metrics.js +124 -0
- package/dist/observe/snapshot-metadata.js +94 -15
- package/dist/server/common.js +14 -7
- package/dist/server/journey-schema.js +60 -0
- package/dist/server/tool-definitions.js +59 -6
- package/dist/server/tool-handlers.js +381 -56
- package/dist/server-core.js +1 -1
- package/dist/utils/exec.js +18 -5
- package/dist/utils/ios/utils.js +5 -2
- package/dist/utils/operation-budget.js +25 -0
- package/dist/utils/resolve-device.js +12 -1
- package/dist/utils/timing.js +19 -0
- package/dist/utils/ui/index.js +2 -0
- package/docs/CHANGELOG.md +6 -0
- package/docs/ROADMAP.md +9 -0
- package/docs/benchmarks/interaction-latency/README.md +26 -0
- package/docs/benchmarks/interaction-latency/final.json +1810 -0
- package/docs/benchmarks/interaction-latency/final.md +17 -0
- package/docs/benchmarks/interaction-latency/ios-validation.md +11 -0
- package/docs/benchmarks/ui-tree-performance/README.md +51 -0
- package/docs/benchmarks/ui-tree-performance/baseline.json +59546 -0
- package/docs/benchmarks/ui-tree-performance/bridge-probe.json +28 -0
- package/docs/benchmarks/ui-tree-performance/candidate.json +51691 -0
- package/docs/benchmarks/ui-tree-performance/comparison-attempt-1.json +110679 -0
- package/docs/benchmarks/ui-tree-performance/comparison.json +111230 -0
- package/docs/benchmarks/ui-tree-performance/comparison.md +41 -0
- package/docs/benchmarks/ui-tree-performance/initial-baseline.json +59560 -0
- package/docs/specs/018-interaction-latency-and-batched-journeys/spec.md +301 -0
- package/docs/specs/019-ui-tree-observation-performance/spec.md +148 -0
- package/docs/tools/interact.md +30 -0
- package/docs/tools/observe.md +1 -0
- package/package.json +1 -1
- package/skills/mobile-debug-tools-usage/SKILL.md +74 -0
- package/src/interact/android.ts +13 -8
- package/src/interact/index.ts +32 -23
- package/src/interact/ios.ts +5 -2
- package/src/manage/android.ts +19 -10
- package/src/manage/ios.ts +18 -10
- package/src/observe/android-tree-bridge.ts +170 -0
- package/src/observe/android.ts +48 -17
- package/src/observe/index.ts +21 -5
- package/src/observe/ios.ts +49 -36
- package/src/observe/performance-metrics.ts +125 -0
- package/src/observe/snapshot-metadata.ts +94 -15
- package/src/server/common.ts +13 -6
- package/src/server/journey-schema.ts +63 -0
- package/src/server/tool-definitions.ts +60 -7
- package/src/server/tool-handlers.ts +349 -54
- package/src/server-core.ts +1 -1
- package/src/types.ts +31 -2
- package/src/utils/exec.ts +13 -5
- package/src/utils/ios/utils.ts +5 -2
- package/src/utils/operation-budget.ts +22 -0
- package/src/utils/resolve-device.ts +13 -1
- package/src/utils/timing.ts +15 -0
- package/src/utils/ui/index.ts +2 -0
- package/test/device/automated/latency/benchmark.ts +136 -0
- package/test/device/automated/latency/report.ts +6 -0
- package/test/device/automated/ui-tree-performance/benchmark.ts +401 -0
- package/test/device/automated/ui-tree-performance/bridge-probe.ts +183 -0
- package/test/fixtures/latency-journey-app/AndroidManifest.xml +7 -0
- package/test/fixtures/latency-journey-app/README.md +28 -0
- package/test/fixtures/latency-journey-app/build.ts +26 -0
- package/test/fixtures/latency-journey-app/src/MainActivity.java +104 -0
- package/test/fixtures/ui-tree-bridge/AndroidManifest.xml +6 -0
- package/test/fixtures/ui-tree-bridge/build.ts +34 -0
- package/test/fixtures/ui-tree-bridge/src/TreeInstrumentation.java +119 -0
- package/test/unit/manage/restart_budget.test.ts +37 -0
- package/test/unit/observe/android_tree_acquisition.test.ts +70 -0
- package/test/unit/observe/android_tree_bridge_cleanup.test.ts +76 -0
- package/test/unit/observe/capture_debug_snapshot.test.ts +6 -0
- package/test/unit/observe/ios_tree_acquisition.test.ts +77 -0
- package/test/unit/observe/performance_metrics.test.ts +40 -0
- package/test/unit/observe/snapshot_metadata.test.ts +20 -1
- package/test/unit/observe/state_delta.test.ts +15 -0
- package/test/unit/server/benchmark_report.test.ts +4 -0
- package/test/unit/server/contract.test.ts +1 -1
- package/test/unit/server/interaction_modes.test.ts +60 -0
- package/test/unit/server/operation_budget.test.ts +45 -0
- package/test/unit/server/response_shapes.test.ts +3 -3
- package/test/unit/server/run_journey.test.ts +120 -0
- package/test/unit/server/tap_selector.test.ts +68 -0
- /package/docs/{specs-deltas/001-state-verification.md → specs/001-state-verification/spec.md} +0 -0
- /package/docs/{specs-deltas/002-richer-element-identity → specs/002-richer-element-identity/spec.md} +0 -0
- /package/docs/{specs-deltas/003-wait-and-synchronization-reliability.md → specs/003-wait-and-synchronization-reliability/spec.md} +0 -0
- /package/docs/{specs-deltas/004-action-verification-routing.md → specs/004-action-verification-routing/spec.md} +0 -0
- /package/docs/{specs-deltas/005-unified-action-execution-and-verification-model.md → specs/005-unified-action-execution-and-verification-model/spec.md} +0 -0
- /package/docs/{specs-deltas/006-runtime-action-instrumentation-and-binding-layer.md → specs/006-runtime-action-instrumentation-and-binding-layer/spec.md} +0 -0
- /package/docs/{specs-deltas/007-actionability-resolution-and-executable-target-selection.md → specs/007-actionability-resolution-and-executable-target-selection/spec.md} +0 -0
- /package/docs/{specs-deltas/008-adjustable-control-support-and-semantic-value-manipulation.md → specs/008-adjustable-control-support-and-semantic-value-manipulation/spec.md} +0 -0
- /package/docs/{specs-deltas/009-semantic-control-modeling-for-custom-and-composite-controls.md → specs/009-semantic-control-modeling-for-custom-and-composite-controls/spec.md} +0 -0
- /package/docs/{specs-deltas/010-verification-stabilization-and-temporal-convergence.md → specs/010-verification-stabilization-and-temporal-convergence/spec.md} +0 -0
- /package/docs/{specs-deltas/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows.md → specs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows/spec.md} +0 -0
- /package/docs/{specs-deltas/011.1-recovery-contract-types-and-runtime-wiring-spec.md → specs/011.1-recovery-contract-types-and-runtime-wiring-spec/spec.md} +0 -0
- /package/docs/{specs-deltas/012-action-trace-and-xecution-observability.md → specs/012-action-trace-and-xecution-observability/spec.md} +0 -0
- /package/docs/{specs-deltas/013-wait-and-synchronization-reliability.md → specs/013-wait-and-synchronization-reliability/spec.md} +0 -0
- /package/docs/{specs-deltas/014-actionability-resolution.md → specs/014-actionability-resolution/spec.md} +0 -0
- /package/docs/{specs-deltas/015-adjustable-control-support.md → specs/015-adjustable-control-support/spec.md} +0 -0
- /package/docs/{specs-deltas/016-environment-tools.md → specs/016-environment-tools/spec.md} +0 -0
package/dist/observe/android.js
CHANGED
|
@@ -7,17 +7,29 @@ import path from "path";
|
|
|
7
7
|
import { computeScreenFingerprint } from "../utils/ui/index.js";
|
|
8
8
|
import { parsePngSize } from "../utils/image.js";
|
|
9
9
|
import { deriveSnapshotMetadata } from "./snapshot-metadata.js";
|
|
10
|
+
import { remainingBudget } from '../utils/operation-budget.js';
|
|
11
|
+
import { createObservationMeasurement } from './performance-metrics.js';
|
|
12
|
+
import { androidTreeBridgeEnabled, readAndroidTreeFromBridge } from './android-tree-bridge.js';
|
|
10
13
|
const activeLogStreams = new Map();
|
|
11
14
|
export class AndroidObserve {
|
|
12
15
|
async getDeviceMetadata(appId, deviceId) {
|
|
13
16
|
return getAndroidDeviceMetadata(appId, deviceId);
|
|
14
17
|
}
|
|
15
18
|
async getUITree(deviceId) {
|
|
16
|
-
const
|
|
19
|
+
const measurement = createObservationMeasurement('android');
|
|
20
|
+
let metadata;
|
|
21
|
+
try {
|
|
22
|
+
metadata = await measurement.timed('metadata_ms', () => getAndroidDeviceMetadata("", deviceId));
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
measurement.setOutcome('unavailable');
|
|
26
|
+
measurement.finish();
|
|
27
|
+
throw error;
|
|
28
|
+
}
|
|
17
29
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata);
|
|
18
30
|
try {
|
|
19
31
|
// Get screen resolution first
|
|
20
|
-
const resolution = await getScreenResolution(deviceId);
|
|
32
|
+
const resolution = await measurement.timed('resolution_ms', () => getScreenResolution(deviceId));
|
|
21
33
|
if (resolution.width === 0 && resolution.height === 0) {
|
|
22
34
|
throw new Error("Failed to get screen resolution. Is the device connected and authorized?");
|
|
23
35
|
}
|
|
@@ -26,20 +38,42 @@ export class AndroidObserve {
|
|
|
26
38
|
let attempts = 0;
|
|
27
39
|
const maxAttempts = 3;
|
|
28
40
|
while (attempts < maxAttempts) {
|
|
41
|
+
remainingBudget();
|
|
29
42
|
attempts++;
|
|
30
43
|
try {
|
|
31
|
-
//
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (
|
|
39
|
-
|
|
44
|
+
// The first read starts immediately so it remains fresh for in-place UI changes.
|
|
45
|
+
// Delay only a retry after a failed or invalid acquisition.
|
|
46
|
+
if (attempts > 1) {
|
|
47
|
+
await measurement.timed('observation_wait_ms', () => delay(Math.min(remainingBudget(300 + (attempts * 100)), 300 + (attempts * 100))));
|
|
48
|
+
}
|
|
49
|
+
remainingBudget();
|
|
50
|
+
let bridgeContent = null;
|
|
51
|
+
if (androidTreeBridgeEnabled()) {
|
|
52
|
+
try {
|
|
53
|
+
bridgeContent = await measurement.physicalRead(async () => {
|
|
54
|
+
const content = await readAndroidTreeFromBridge(deviceInfo.id);
|
|
55
|
+
if (!content)
|
|
56
|
+
throw new Error('Android UI tree bridge unavailable');
|
|
57
|
+
return content;
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
if (error instanceof Error && (error.message === 'ACTION_TIMEOUT' || error.message.startsWith('BRIDGE_CLEANUP_UNCONFIRMED')))
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
40
64
|
}
|
|
65
|
+
xmlContent = bridgeContent ?? await measurement.physicalRead(async () => {
|
|
66
|
+
await execAdb(['shell', 'uiautomator', 'dump', '/sdcard/ui.xml'], deviceId);
|
|
67
|
+
const content = await execAdb(['shell', 'cat', '/sdcard/ui.xml'], deviceId);
|
|
68
|
+
if (!content || !content.trim() || content.includes('ERROR:'))
|
|
69
|
+
throw new Error('Empty or invalid UI dump');
|
|
70
|
+
return content;
|
|
71
|
+
});
|
|
72
|
+
break;
|
|
41
73
|
}
|
|
42
74
|
catch (e) {
|
|
75
|
+
if (e instanceof Error && (e.message === 'ACTION_TIMEOUT' || e.message.startsWith('BRIDGE_CLEANUP_UNCONFIRMED')))
|
|
76
|
+
throw e;
|
|
43
77
|
console.error(`Attempt ${attempts} failed: ${e}`);
|
|
44
78
|
}
|
|
45
79
|
if (attempts === maxAttempts) {
|
|
@@ -50,7 +84,7 @@ export class AndroidObserve {
|
|
|
50
84
|
ignoreAttributes: false,
|
|
51
85
|
attributeNamePrefix: "@_"
|
|
52
86
|
});
|
|
53
|
-
const result = parser.parse(xmlContent);
|
|
87
|
+
const result = await measurement.timed('parse_ms', async () => parser.parse(xmlContent));
|
|
54
88
|
const elements = [];
|
|
55
89
|
// The root is usually hierarchy -> node
|
|
56
90
|
if (result.hierarchy && result.hierarchy.node) {
|
|
@@ -67,18 +101,23 @@ export class AndroidObserve {
|
|
|
67
101
|
resolution,
|
|
68
102
|
elements
|
|
69
103
|
}, 'ui_tree');
|
|
70
|
-
|
|
104
|
+
measurement.setOutcome('observed');
|
|
105
|
+
const response = measurement.bindObservation({
|
|
71
106
|
device: deviceInfo,
|
|
72
107
|
screen: "",
|
|
73
108
|
resolution,
|
|
74
109
|
elements,
|
|
75
110
|
...snapshotMetadata
|
|
76
|
-
};
|
|
111
|
+
});
|
|
112
|
+
measurement.finish();
|
|
113
|
+
return response;
|
|
77
114
|
}
|
|
78
115
|
catch (e) {
|
|
79
116
|
const errorMessage = `Failed to get UI tree. ADB Path: '${getAdbCmd()}'. Error: ${e instanceof Error ? e.message : String(e)}`;
|
|
80
117
|
console.error(errorMessage);
|
|
81
118
|
const snapshotMetadata = deriveSnapshotMetadata(`android:${deviceInfo.id}`, null, 'ui_tree');
|
|
119
|
+
measurement.setOutcome('unavailable');
|
|
120
|
+
measurement.finish();
|
|
82
121
|
return {
|
|
83
122
|
device: deviceInfo,
|
|
84
123
|
screen: "",
|
package/dist/observe/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { measure } from '../utils/timing.js';
|
|
1
2
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
2
3
|
import { AndroidObserve } from './android.js';
|
|
3
4
|
import { iOSObserve } from './ios.js';
|
|
4
|
-
import { deriveSnapshotMetadata } from './snapshot-metadata.js';
|
|
5
|
+
import { deriveSnapshotMetadata, getLatestStateDelta, getStateDelta } from './snapshot-metadata.js';
|
|
5
6
|
export { AndroidObserve } from './android.js';
|
|
6
7
|
export { iOSObserve } from './ios.js';
|
|
7
8
|
function normalizeHint(value) {
|
|
@@ -110,6 +111,9 @@ function deriveSnapshotSemantic(raw) {
|
|
|
110
111
|
};
|
|
111
112
|
}
|
|
112
113
|
export class ToolsObserve {
|
|
114
|
+
static getLatestStateDelta(platform, deviceId) {
|
|
115
|
+
return getLatestStateDelta(`${platform}:${deviceId}`);
|
|
116
|
+
}
|
|
113
117
|
// Resolve a target device and return the appropriate observe instance and resolved info.
|
|
114
118
|
static async resolveObserve(platform, deviceId, appId) {
|
|
115
119
|
if (platform === 'android') {
|
|
@@ -130,9 +134,22 @@ export class ToolsObserve {
|
|
|
130
134
|
return { observe: new iOSObserve(), resolved };
|
|
131
135
|
}
|
|
132
136
|
}
|
|
133
|
-
static async getUITreeHandler({ platform, deviceId }) {
|
|
137
|
+
static async getUITreeHandler({ platform, deviceId, responseMode = 'debug', sinceSnapshotRevision }) {
|
|
134
138
|
const { observe, resolved } = await ToolsObserve.resolveObserve(platform, deviceId);
|
|
135
|
-
|
|
139
|
+
const tree = await observe.getUITree(resolved.id);
|
|
140
|
+
if (responseMode !== 'compact')
|
|
141
|
+
return tree;
|
|
142
|
+
const revision = tree?.snapshot_revision ?? null;
|
|
143
|
+
const baseRevision = sinceSnapshotRevision;
|
|
144
|
+
if (!Number.isInteger(baseRevision) || revision === null) {
|
|
145
|
+
return { delta_available: false, snapshot_revision: revision };
|
|
146
|
+
}
|
|
147
|
+
const delta = getStateDelta(`${resolved.platform}:${resolved.id}`, baseRevision, revision);
|
|
148
|
+
return delta ? { delta_available: true, state_delta: delta, snapshot_revision: revision } : {
|
|
149
|
+
delta_available: false,
|
|
150
|
+
snapshot_revision: revision,
|
|
151
|
+
reason: 'BASE_REVISION_UNAVAILABLE'
|
|
152
|
+
};
|
|
136
153
|
}
|
|
137
154
|
static async getCurrentScreenHandler({ deviceId }) {
|
|
138
155
|
const { observe, resolved } = await ToolsObserve.resolveObserve('android', deviceId);
|
|
@@ -199,7 +216,7 @@ export class ToolsObserve {
|
|
|
199
216
|
// Both observes implement getScreenFingerprint
|
|
200
217
|
return await observe.getScreenFingerprint(resolved.id);
|
|
201
218
|
}
|
|
202
|
-
static async captureDebugSnapshotHandler({ reason, includeLogs = true, logLines = 200, platform, appId, deviceId, sessionId } = {}) {
|
|
219
|
+
static async captureDebugSnapshotHandler({ reason, includeLogs = true, includeScreenshot = true, logLines = 200, platform, appId, deviceId, sessionId } = {}) {
|
|
203
220
|
const timestamp = Date.now();
|
|
204
221
|
const raw = {
|
|
205
222
|
timestamp,
|
|
@@ -215,7 +232,7 @@ export class ToolsObserve {
|
|
|
215
232
|
// Parallel fetches for performance: screenshot, current screen, fingerprint, ui tree, and log stream/get logs
|
|
216
233
|
const sid = sessionId || 'default';
|
|
217
234
|
const tasks = {
|
|
218
|
-
screenshot: ToolsObserve.captureScreenshotHandler({ platform, deviceId }),
|
|
235
|
+
screenshot: includeScreenshot ? measure('screenshot_ms', () => ToolsObserve.captureScreenshotHandler({ platform, deviceId })) : Promise.resolve(null),
|
|
219
236
|
currentScreen: (!platform || platform === 'android') ? ToolsObserve.getCurrentScreenHandler({ deviceId }) : Promise.resolve(null),
|
|
220
237
|
fingerprint: ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }),
|
|
221
238
|
uiTree: ToolsObserve.getUITreeHandler({ platform, deviceId }),
|
package/dist/observe/ios.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { spawn } from "child_process";
|
|
2
|
+
import { remainingBudget } from '../utils/operation-budget.js';
|
|
3
|
+
import { execCmd } from '../utils/exec.js';
|
|
4
|
+
import { createObservationMeasurement } from './performance-metrics.js';
|
|
2
5
|
import { promises as fs } from "fs";
|
|
3
6
|
import { execCommand, getIOSDeviceMetadata, validateBundleId, getIdbCmd, getXcrunCmd, isIDBInstalled } from "../utils/ios/utils.js";
|
|
4
7
|
import { createWriteStream, promises as fsPromises } from 'fs';
|
|
@@ -482,11 +485,22 @@ export class iOSObserve {
|
|
|
482
485
|
}
|
|
483
486
|
}
|
|
484
487
|
async getUITree(deviceId = "booted") {
|
|
485
|
-
const
|
|
488
|
+
const measurement = createObservationMeasurement('ios');
|
|
489
|
+
let device;
|
|
490
|
+
try {
|
|
491
|
+
device = await measurement.timed('metadata_ms', () => getIOSDeviceMetadata(deviceId));
|
|
492
|
+
}
|
|
493
|
+
catch (error) {
|
|
494
|
+
measurement.setOutcome('unavailable');
|
|
495
|
+
measurement.finish();
|
|
496
|
+
throw error;
|
|
497
|
+
}
|
|
486
498
|
const deviceKey = `ios:${device.id}`;
|
|
487
|
-
const idbExists = await isIDBInstalled();
|
|
499
|
+
const idbExists = await measurement.timed('capability_check_ms', () => isIDBInstalled());
|
|
488
500
|
if (!idbExists) {
|
|
489
501
|
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
|
|
502
|
+
measurement.setOutcome('unavailable');
|
|
503
|
+
measurement.finish();
|
|
490
504
|
return {
|
|
491
505
|
device,
|
|
492
506
|
screen: "",
|
|
@@ -501,39 +515,39 @@ export class iOSObserve {
|
|
|
501
515
|
let attempts = 0;
|
|
502
516
|
const maxAttempts = 3;
|
|
503
517
|
while (attempts < maxAttempts) {
|
|
518
|
+
remainingBudget();
|
|
504
519
|
attempts++;
|
|
505
520
|
try {
|
|
506
|
-
await delay(300 + (attempts * 100));
|
|
521
|
+
await measurement.timed('observation_wait_ms', () => delay(Math.min(remainingBudget(300 + (attempts * 100)), 300 + (attempts * 100))));
|
|
522
|
+
remainingBudget();
|
|
507
523
|
const args = ['ui', 'describe-all', '--json'];
|
|
508
524
|
if (targetUdid) {
|
|
509
525
|
args.push('--udid', targetUdid);
|
|
510
526
|
}
|
|
511
|
-
const output = await
|
|
512
|
-
const
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
reject(new Error(`idb failed (code ${code}): ${stderr.trim()}`));
|
|
521
|
-
}
|
|
522
|
-
else {
|
|
523
|
-
resolve(stdout);
|
|
524
|
-
}
|
|
525
|
-
});
|
|
527
|
+
const output = await measurement.physicalRead(async () => {
|
|
528
|
+
const result = await execCmd(getIdbCmd(), args, { timeout: remainingBudget() });
|
|
529
|
+
if (result.stderr === 'ACTION_TIMEOUT')
|
|
530
|
+
throw new Error('ACTION_TIMEOUT');
|
|
531
|
+
if (result.exitCode !== 0)
|
|
532
|
+
throw new Error(`idb failed (code ${result.exitCode}): ${result.stderr}`);
|
|
533
|
+
if (!result.stdout.trim())
|
|
534
|
+
throw new Error('Empty idb UI tree output');
|
|
535
|
+
return result.stdout;
|
|
526
536
|
});
|
|
527
537
|
if (output && output.trim().length > 0) {
|
|
528
|
-
jsonContent = JSON.parse(output);
|
|
538
|
+
jsonContent = await measurement.timed('parse_ms', async () => JSON.parse(output));
|
|
529
539
|
break; // Success
|
|
530
540
|
}
|
|
531
541
|
}
|
|
532
542
|
catch (e) {
|
|
543
|
+
if (e instanceof Error && e.message === 'ACTION_TIMEOUT')
|
|
544
|
+
throw e;
|
|
533
545
|
console.error(`Attempt ${attempts} failed: ${e}`);
|
|
534
546
|
}
|
|
535
547
|
if (attempts === maxAttempts) {
|
|
536
548
|
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
|
|
549
|
+
measurement.setOutcome('unavailable');
|
|
550
|
+
measurement.finish();
|
|
537
551
|
return {
|
|
538
552
|
device,
|
|
539
553
|
screen: "",
|
|
@@ -545,15 +559,17 @@ export class iOSObserve {
|
|
|
545
559
|
}
|
|
546
560
|
}
|
|
547
561
|
try {
|
|
548
|
-
const elements =
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
562
|
+
const elements = await measurement.timed('conversion_ms', async () => {
|
|
563
|
+
const parsed = [];
|
|
564
|
+
if (Array.isArray(jsonContent)) {
|
|
565
|
+
for (const node of jsonContent)
|
|
566
|
+
traverseIDBNode(node, parsed);
|
|
552
567
|
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
568
|
+
else {
|
|
569
|
+
traverseIDBNode(jsonContent, parsed);
|
|
570
|
+
}
|
|
571
|
+
return parsed;
|
|
572
|
+
});
|
|
557
573
|
let width = 0;
|
|
558
574
|
let height = 0;
|
|
559
575
|
if (elements.length > 0) {
|
|
@@ -566,16 +582,21 @@ export class iOSObserve {
|
|
|
566
582
|
resolution: { width, height },
|
|
567
583
|
elements
|
|
568
584
|
}, 'ui_tree');
|
|
569
|
-
|
|
585
|
+
measurement.setOutcome('observed');
|
|
586
|
+
const response = measurement.bindObservation({
|
|
570
587
|
device,
|
|
571
588
|
screen: "",
|
|
572
589
|
resolution: { width, height },
|
|
573
590
|
elements,
|
|
574
591
|
...snapshotMetadata
|
|
575
|
-
};
|
|
592
|
+
});
|
|
593
|
+
measurement.finish();
|
|
594
|
+
return response;
|
|
576
595
|
}
|
|
577
596
|
catch (e) {
|
|
578
597
|
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
|
|
598
|
+
measurement.setOutcome('unavailable');
|
|
599
|
+
measurement.finish();
|
|
579
600
|
return {
|
|
580
601
|
device,
|
|
581
602
|
screen: "",
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
|
+
const context = new AsyncLocalStorage();
|
|
4
|
+
const observationIds = new WeakMap();
|
|
5
|
+
export function withObservationContext(value, operation) {
|
|
6
|
+
if (process.env.MOBILE_DEBUG_MCP_UI_TREE_METRICS !== '1')
|
|
7
|
+
return operation();
|
|
8
|
+
return context.run({ purpose: value.purpose ?? context.getStore()?.purpose ?? 'unspecified', stepId: value.stepId ?? context.getStore()?.stepId }, operation);
|
|
9
|
+
}
|
|
10
|
+
export function recordObservationEvent(event) {
|
|
11
|
+
if (process.env.MOBILE_DEBUG_MCP_UI_TREE_METRICS !== '1')
|
|
12
|
+
return;
|
|
13
|
+
const current = context.getStore();
|
|
14
|
+
process.stderr.write(`${JSON.stringify({
|
|
15
|
+
event: 'ui_tree_metric',
|
|
16
|
+
host_monotonic_ns: process.hrtime.bigint().toString(),
|
|
17
|
+
type: 'action_observation',
|
|
18
|
+
step_id: current?.stepId,
|
|
19
|
+
purpose: current?.purpose ?? 'unspecified',
|
|
20
|
+
...event
|
|
21
|
+
})}\n`);
|
|
22
|
+
}
|
|
23
|
+
export function recordObservationConsumer(observation, purpose, stepId) {
|
|
24
|
+
if (process.env.MOBILE_DEBUG_MCP_UI_TREE_METRICS !== '1' || !observation || typeof observation !== 'object')
|
|
25
|
+
return;
|
|
26
|
+
const acquisitionId = observationIds.get(observation);
|
|
27
|
+
if (!acquisitionId)
|
|
28
|
+
return;
|
|
29
|
+
const current = context.getStore();
|
|
30
|
+
process.stderr.write(`${JSON.stringify({
|
|
31
|
+
event: 'ui_tree_metric',
|
|
32
|
+
host_monotonic_ns: process.hrtime.bigint().toString(),
|
|
33
|
+
type: 'observation_consumer',
|
|
34
|
+
acquisition_id: acquisitionId,
|
|
35
|
+
consumer_step_id: stepId ?? current?.stepId,
|
|
36
|
+
consumer_purpose: purpose,
|
|
37
|
+
reused_observations: 1
|
|
38
|
+
})}\n`);
|
|
39
|
+
}
|
|
40
|
+
export function createObservationMeasurement(platform) {
|
|
41
|
+
const enabled = process.env.MOBILE_DEBUG_MCP_UI_TREE_METRICS === '1';
|
|
42
|
+
const startedAt = enabled ? performance.now() : 0;
|
|
43
|
+
const id = enabled ? randomUUID() : undefined;
|
|
44
|
+
const current = context.getStore();
|
|
45
|
+
const durations = {};
|
|
46
|
+
let successfulOutput = false;
|
|
47
|
+
let outcome = 'unavailable';
|
|
48
|
+
let readAttempt = 0;
|
|
49
|
+
const emit = (event) => {
|
|
50
|
+
if (!enabled)
|
|
51
|
+
return;
|
|
52
|
+
process.stderr.write(`${JSON.stringify({
|
|
53
|
+
event: 'ui_tree_metric',
|
|
54
|
+
host_monotonic_ns: process.hrtime.bigint().toString(),
|
|
55
|
+
acquisition_id: id,
|
|
56
|
+
platform,
|
|
57
|
+
step_id: current?.stepId,
|
|
58
|
+
purpose: current?.purpose ?? 'unspecified',
|
|
59
|
+
...event
|
|
60
|
+
})}\n`);
|
|
61
|
+
};
|
|
62
|
+
return {
|
|
63
|
+
id,
|
|
64
|
+
bindObservation(observation) {
|
|
65
|
+
if (enabled && id)
|
|
66
|
+
observationIds.set(observation, id);
|
|
67
|
+
return observation;
|
|
68
|
+
},
|
|
69
|
+
timed: async (stage, operation) => {
|
|
70
|
+
if (!enabled)
|
|
71
|
+
return operation();
|
|
72
|
+
const start = performance.now();
|
|
73
|
+
let success = true;
|
|
74
|
+
try {
|
|
75
|
+
return await operation();
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
success = false;
|
|
79
|
+
throw error;
|
|
80
|
+
}
|
|
81
|
+
finally {
|
|
82
|
+
const elapsed = performance.now() - start;
|
|
83
|
+
durations[stage] = (durations[stage] ?? 0) + elapsed;
|
|
84
|
+
emit({ type: 'stage', stage, attempt_index: stage === 'parse_ms' ? readAttempt : undefined, elapsed_ms: elapsed, success });
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
physicalRead: async (operation) => {
|
|
88
|
+
if (!enabled)
|
|
89
|
+
return operation();
|
|
90
|
+
const start = performance.now();
|
|
91
|
+
readAttempt++;
|
|
92
|
+
emit({ type: 'physical_read_started', attempt_index: readAttempt });
|
|
93
|
+
try {
|
|
94
|
+
const result = await operation();
|
|
95
|
+
successfulOutput = true;
|
|
96
|
+
emit({ type: 'physical_read_completed', attempt_index: readAttempt, elapsed_ms: performance.now() - start, success: true });
|
|
97
|
+
return result;
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
emit({ type: 'physical_read_completed', attempt_index: readAttempt, elapsed_ms: performance.now() - start, success: false });
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
setOutcome(value) { if (enabled)
|
|
105
|
+
outcome = value; },
|
|
106
|
+
finish() {
|
|
107
|
+
if (!enabled)
|
|
108
|
+
return;
|
|
109
|
+
emit({
|
|
110
|
+
type: 'observation_consumer',
|
|
111
|
+
consumer_step_id: current?.stepId,
|
|
112
|
+
consumer_purpose: current?.purpose ?? 'unspecified',
|
|
113
|
+
reused_observations: 0
|
|
114
|
+
});
|
|
115
|
+
emit({
|
|
116
|
+
type: 'acquisition_completed',
|
|
117
|
+
elapsed_ms: performance.now() - startedAt,
|
|
118
|
+
successful_output: successfulOutput,
|
|
119
|
+
outcome,
|
|
120
|
+
durations_ms: durations
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from 'crypto';
|
|
2
|
+
let nextRevision = 1;
|
|
2
3
|
const snapshotStateByDevice = new Map();
|
|
3
4
|
function normalize(value) {
|
|
4
5
|
if (value === null || value === undefined)
|
|
@@ -43,17 +44,19 @@ function stableElementIdentity(element, index) {
|
|
|
43
44
|
index
|
|
44
45
|
})).digest('hex')}`;
|
|
45
46
|
}
|
|
46
|
-
function
|
|
47
|
+
function buildElementRecords(tree) {
|
|
47
48
|
const signatures = new Map();
|
|
49
|
+
const elementsByIdentity = new Map();
|
|
48
50
|
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
49
|
-
for (let index = 0; index < elements.length; index++) {
|
|
51
|
+
for (let index = 0; index < Math.min(elements.length, 500); index++) {
|
|
50
52
|
const element = elements[index];
|
|
51
53
|
if (!element)
|
|
52
54
|
continue;
|
|
53
55
|
const identity = stableElementIdentity(element, index);
|
|
54
56
|
signatures.set(identity, crypto.createHash('sha1').update(JSON.stringify(stableElementSignature(element))).digest('hex'));
|
|
57
|
+
elementsByIdentity.set(identity, structuredClone(element));
|
|
55
58
|
}
|
|
56
|
-
return signatures;
|
|
59
|
+
return { signatures, elementsByIdentity };
|
|
57
60
|
}
|
|
58
61
|
function summarizeSnapshotDelta(previous, currentElements) {
|
|
59
62
|
if (!previous)
|
|
@@ -118,24 +121,47 @@ export function detectLoadingState(tree, source) {
|
|
|
118
121
|
}
|
|
119
122
|
export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverride) {
|
|
120
123
|
const signature = signatureOverride ?? computeSnapshotSignature(tree);
|
|
121
|
-
const
|
|
124
|
+
const existing = snapshotStateByDevice.get(deviceKey);
|
|
125
|
+
const previous = existing && Date.now() - existing.updatedAt <= 10 * 60 * 1000 ? existing : undefined;
|
|
126
|
+
if (!previous && existing)
|
|
127
|
+
snapshotStateByDevice.delete(deviceKey);
|
|
122
128
|
const hasValidTree = !!tree && !tree.error;
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
129
|
+
const records = hasValidTree ? buildElementRecords(tree) : null;
|
|
130
|
+
const currentElementSignatures = records?.signatures ?? previous?.elementSignatures ?? new Map();
|
|
131
|
+
const currentElements = records?.elementsByIdentity ?? previous?.elements ?? new Map();
|
|
132
|
+
const revision = previous && (signature === null || previous.signature === signature) ? previous.revision : nextRevision++;
|
|
133
|
+
const history = previous?.history ?? new Map();
|
|
134
|
+
const historyTimes = previous?.historyTimes ?? new Map();
|
|
135
|
+
const oversized = previous?.oversized ?? new Map();
|
|
136
|
+
if (hasValidTree) {
|
|
137
|
+
history.set(revision, currentElements);
|
|
138
|
+
if (!historyTimes.has(revision))
|
|
139
|
+
historyTimes.set(revision, Date.now());
|
|
140
|
+
if ((tree.elements?.length ?? 0) > 500)
|
|
141
|
+
oversized.set(revision, (tree.elements?.length ?? 0) - 500);
|
|
142
|
+
}
|
|
143
|
+
for (const [id, timestamp] of historyTimes) {
|
|
144
|
+
if (Date.now() - timestamp > 10 * 60 * 1000) {
|
|
145
|
+
history.delete(id);
|
|
146
|
+
historyTimes.delete(id);
|
|
147
|
+
oversized.delete(id);
|
|
133
148
|
}
|
|
134
149
|
}
|
|
150
|
+
while (history.size > 8) {
|
|
151
|
+
const oldest = history.keys().next().value;
|
|
152
|
+
if (oldest === undefined)
|
|
153
|
+
break;
|
|
154
|
+
history.delete(oldest);
|
|
155
|
+
historyTimes.delete(oldest);
|
|
156
|
+
oversized.delete(oldest);
|
|
157
|
+
}
|
|
135
158
|
snapshotStateByDevice.set(deviceKey, {
|
|
136
159
|
revision,
|
|
137
160
|
signature,
|
|
138
|
-
elementSignatures: currentElementSignatures
|
|
161
|
+
elementSignatures: currentElementSignatures,
|
|
162
|
+
elements: currentElements,
|
|
163
|
+
history, historyTimes, oversized, previousRevision: previous?.revision ?? null,
|
|
164
|
+
updatedAt: Date.now()
|
|
139
165
|
});
|
|
140
166
|
return {
|
|
141
167
|
snapshot_revision: revision,
|
|
@@ -144,6 +170,59 @@ export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverrid
|
|
|
144
170
|
loading_state: detectLoadingState(tree, source)
|
|
145
171
|
};
|
|
146
172
|
}
|
|
173
|
+
export function getStateDelta(deviceKey, baseRevision, currentRevision) {
|
|
174
|
+
const state = snapshotStateByDevice.get(deviceKey);
|
|
175
|
+
if (!state || Date.now() - (state.historyTimes.get(baseRevision) ?? 0) > 10 * 60 * 1000 || Date.now() - state.updatedAt > 10 * 60 * 1000)
|
|
176
|
+
return null;
|
|
177
|
+
const base = state.history.get(baseRevision);
|
|
178
|
+
const current = state?.history.get(currentRevision);
|
|
179
|
+
if (!base || !current)
|
|
180
|
+
return null;
|
|
181
|
+
const added = [];
|
|
182
|
+
const changed = [];
|
|
183
|
+
const removed = [];
|
|
184
|
+
for (const [identity, element] of current) {
|
|
185
|
+
const before = base.get(identity);
|
|
186
|
+
if (!before)
|
|
187
|
+
added.push(element);
|
|
188
|
+
else if (JSON.stringify(stableElementSignature(before)) !== JSON.stringify(stableElementSignature(element)))
|
|
189
|
+
changed.push(element);
|
|
190
|
+
}
|
|
191
|
+
let index = 0;
|
|
192
|
+
for (const [identity, element] of base) {
|
|
193
|
+
if (!current.has(identity))
|
|
194
|
+
removed.push({ stable_id: element.stable_id ?? undefined, resourceId: element.resourceId ?? undefined, contentDescription: element.contentDescription ?? undefined, text: element.text ?? undefined, index });
|
|
195
|
+
index++;
|
|
196
|
+
}
|
|
197
|
+
const all = added.length + changed.length + removed.length;
|
|
198
|
+
const limit = 200;
|
|
199
|
+
return {
|
|
200
|
+
base_snapshot_revision: baseRevision,
|
|
201
|
+
snapshot_revision: currentRevision,
|
|
202
|
+
added: added.slice(0, limit),
|
|
203
|
+
changed: changed.slice(0, Math.max(0, limit - added.length)),
|
|
204
|
+
removed: removed.slice(0, Math.max(0, limit - added.length - changed.length)),
|
|
205
|
+
truncated: all > limit || state.oversized.has(baseRevision) || state.oversized.has(currentRevision),
|
|
206
|
+
...((all > limit || state.oversized.has(baseRevision) || state.oversized.has(currentRevision))
|
|
207
|
+
? { omitted_changes: Math.max(all - limit, state.oversized.get(baseRevision) ?? 0, state.oversized.get(currentRevision) ?? 0) } : {})
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
export function getLatestStateDelta(deviceKey) {
|
|
211
|
+
const state = snapshotStateByDevice.get(deviceKey);
|
|
212
|
+
if (!state || state.previousRevision === null)
|
|
213
|
+
return null;
|
|
214
|
+
return getStateDelta(deviceKey, state.previousRevision, state.revision);
|
|
215
|
+
}
|
|
216
|
+
export function retainConnectedDeviceSnapshots(platform, deviceIds) {
|
|
217
|
+
const keys = new Set(deviceIds.map(id => `${platform}:${id}`));
|
|
218
|
+
for (const key of snapshotStateByDevice.keys())
|
|
219
|
+
if (key.startsWith(`${platform}:`) && !keys.has(key))
|
|
220
|
+
snapshotStateByDevice.delete(key);
|
|
221
|
+
}
|
|
222
|
+
export function clearDeviceSnapshots(deviceKey) {
|
|
223
|
+
snapshotStateByDevice.delete(deviceKey);
|
|
224
|
+
}
|
|
147
225
|
export function resetSnapshotMetadataForTests() {
|
|
148
226
|
snapshotStateByDevice.clear();
|
|
227
|
+
nextRevision = 1;
|
|
149
228
|
}
|
package/dist/server/common.js
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
|
+
import { recordTiming } from '../utils/timing.js';
|
|
1
2
|
import { ToolsObserve } from '../observe/index.js';
|
|
2
3
|
export const DEFAULT_MAX_RECOVERY_ATTEMPTS = 3;
|
|
3
4
|
export const DEFAULT_MAX_RETRY_DEPTH = 3;
|
|
4
|
-
export function wrapResponse(data) {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
export function wrapResponse(data, includeSerializationTiming = true) {
|
|
6
|
+
const start = performance.now();
|
|
7
|
+
let text = JSON.stringify(data, null, 2);
|
|
8
|
+
const elapsed = performance.now() - start;
|
|
9
|
+
const value = data;
|
|
10
|
+
if (includeSerializationTiming && value && typeof value === 'object' && value.timing && typeof value.timing === 'object') {
|
|
11
|
+
value.timing.serialization_ms = (value.timing.serialization_ms ?? 0) + elapsed;
|
|
12
|
+
recordTiming('serialization_ms', elapsed);
|
|
13
|
+
text = JSON.stringify(value, null, 2);
|
|
14
|
+
}
|
|
15
|
+
return { content: [{ type: 'text', text }] };
|
|
11
16
|
}
|
|
12
17
|
export function getStringArg(args, key) {
|
|
13
18
|
const value = args[key];
|
|
@@ -230,6 +235,7 @@ function mapFailureCodeToFailureClass(code) {
|
|
|
230
235
|
case 'STALE_REFERENCE':
|
|
231
236
|
return 'TargetResolutionFailure';
|
|
232
237
|
case 'ELEMENT_NOT_INTERACTABLE':
|
|
238
|
+
case 'ACTION_TIMEOUT':
|
|
233
239
|
return 'ExecutionFailure';
|
|
234
240
|
case 'TIMEOUT':
|
|
235
241
|
case 'ACTION_REJECTED':
|
|
@@ -238,6 +244,7 @@ function mapFailureCodeToFailureClass(code) {
|
|
|
238
244
|
return 'ExecutionFailure';
|
|
239
245
|
case 'VERIFICATION_FAILED':
|
|
240
246
|
case 'EXPECT_STATE_MISMATCH':
|
|
247
|
+
case 'ASSERTION_TIMEOUT':
|
|
241
248
|
return 'VerificationFailure';
|
|
242
249
|
case 'CONTROL_CONVERGENCE_FAILED':
|
|
243
250
|
return 'ControlConvergenceFailure';
|