mobile-debug-mcp 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +143 -4
- package/dist/observe/android.js +10 -1
- package/dist/observe/index.js +19 -1
- package/dist/observe/ios.js +86 -3
- package/dist/observe/snapshot-metadata.js +88 -0
- package/dist/server/tool-definitions.js +30 -2
- package/dist/server/tool-handlers.js +10 -0
- package/dist/server-core.js +1 -1
- package/dist/utils/android/utils.js +68 -3
- package/docs/CHANGELOG.md +12 -0
- package/docs/ROADMAP.md +19 -1
- package/docs/rfcs/002-richer-element-identity +400 -0
- package/docs/rfcs/003-wait-and-synchronization-reliability.md +296 -0
- package/docs/specs/mcp-tooling-spec-v1.md +9 -0
- package/docs/tools/interact.md +21 -0
- package/docs/tools/observe.md +5 -2
- package/package.json +1 -1
- package/skills/rfc-review/SKILL.md +52 -0
- package/skills/rfc-review/references/rfc-review-checklist.md +12 -0
- package/skills/rfc-review/references/rfc-review-template.md +28 -0
- package/src/interact/index.ts +186 -4
- package/src/observe/android.ts +11 -1
- package/src/observe/index.ts +32 -1
- package/src/observe/ios.ts +97 -16
- package/src/observe/snapshot-metadata.ts +107 -0
- package/src/server/tool-definitions.ts +30 -2
- package/src/server/tool-handlers.ts +11 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +49 -1
- package/src/utils/android/utils.ts +78 -20
- package/test/unit/interact/wait_for_ui_change.test.ts +76 -0
- package/test/unit/observe/state_extraction.test.ts +47 -0
- package/test/unit/server/response_shapes.test.ts +37 -3
package/dist/interact/index.js
CHANGED
|
@@ -4,9 +4,11 @@ import { iOSInteract } from './ios.js';
|
|
|
4
4
|
export { AndroidInteract, iOSInteract };
|
|
5
5
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
6
6
|
import { ToolsObserve } from '../observe/index.js';
|
|
7
|
+
import { computeSnapshotSignature } from '../observe/snapshot-metadata.js';
|
|
7
8
|
import { nextActionId } from '../server/common.js';
|
|
8
9
|
export class ToolsInteract {
|
|
9
10
|
static _maxResolvedUiElements = 256;
|
|
11
|
+
static _uiChangeKinds = ['hierarchy_diff', 'text_change', 'state_change'];
|
|
10
12
|
static _sliderSearchLookahead = 8;
|
|
11
13
|
static _sliderNegativeGapTolerancePx = 32;
|
|
12
14
|
static _sliderPositiveGapLimitPx = 640;
|
|
@@ -34,6 +36,9 @@ export class ToolsInteract {
|
|
|
34
36
|
return null;
|
|
35
37
|
return normalized;
|
|
36
38
|
}
|
|
39
|
+
static _hash(value) {
|
|
40
|
+
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
41
|
+
}
|
|
37
42
|
static _matchesSelector(el, selector) {
|
|
38
43
|
if (!selector)
|
|
39
44
|
return false;
|
|
@@ -105,7 +110,13 @@ export class ToolsInteract {
|
|
|
105
110
|
class: el.type ?? el.class ?? null,
|
|
106
111
|
bounds,
|
|
107
112
|
index,
|
|
108
|
-
elementId
|
|
113
|
+
elementId,
|
|
114
|
+
state: el.state ?? null,
|
|
115
|
+
stable_id: el.stable_id ?? null,
|
|
116
|
+
role: el.role ?? null,
|
|
117
|
+
test_tag: el.test_tag ?? null,
|
|
118
|
+
selector: el.selector ?? null,
|
|
119
|
+
semantic: el.semantic ?? null
|
|
109
120
|
};
|
|
110
121
|
}
|
|
111
122
|
static _rememberResolvedElement(elementId, context) {
|
|
@@ -129,6 +140,52 @@ export class ToolsInteract {
|
|
|
129
140
|
return null;
|
|
130
141
|
}
|
|
131
142
|
}
|
|
143
|
+
static _buildUiChangeSignatures(tree) {
|
|
144
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
145
|
+
const textPayload = [];
|
|
146
|
+
const statePayload = [];
|
|
147
|
+
for (const el of elements) {
|
|
148
|
+
textPayload.push({
|
|
149
|
+
text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
|
|
150
|
+
contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
|
|
151
|
+
resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
|
|
152
|
+
});
|
|
153
|
+
statePayload.push({
|
|
154
|
+
checked: el?.state?.checked ?? null,
|
|
155
|
+
selected: el?.state?.selected ?? null,
|
|
156
|
+
focused: el?.state?.focused ?? null,
|
|
157
|
+
expanded: el?.state?.expanded ?? null,
|
|
158
|
+
enabled: el?.state?.enabled ?? null,
|
|
159
|
+
text_value: el?.state?.text_value ?? null,
|
|
160
|
+
value: el?.state?.value ?? null,
|
|
161
|
+
raw_value: el?.state?.raw_value ?? null,
|
|
162
|
+
value_range: el?.state?.value_range ?? null
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
return {
|
|
166
|
+
hierarchy: computeSnapshotSignature(tree),
|
|
167
|
+
text: ToolsInteract._hash({
|
|
168
|
+
screen: ToolsInteract._normalize(tree?.screen),
|
|
169
|
+
elements: textPayload
|
|
170
|
+
}),
|
|
171
|
+
state: ToolsInteract._hash({
|
|
172
|
+
screen: ToolsInteract._normalize(tree?.screen),
|
|
173
|
+
elements: statePayload
|
|
174
|
+
})
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
static _matchesUiChange(expected, initial, current) {
|
|
178
|
+
const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds;
|
|
179
|
+
for (const changeKind of candidates) {
|
|
180
|
+
if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy)
|
|
181
|
+
return changeKind;
|
|
182
|
+
if (changeKind === 'text_change' && initial.text !== current.text)
|
|
183
|
+
return changeKind;
|
|
184
|
+
if (changeKind === 'state_change' && initial.state !== current.state)
|
|
185
|
+
return changeKind;
|
|
186
|
+
}
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
132
189
|
static _resolvedTargetFromElement(elementId, element, index) {
|
|
133
190
|
return {
|
|
134
191
|
elementId,
|
|
@@ -138,7 +195,12 @@ export class ToolsInteract {
|
|
|
138
195
|
class: element.type ?? element.class ?? null,
|
|
139
196
|
bounds: ToolsInteract._normalizeBounds(element.bounds),
|
|
140
197
|
index,
|
|
141
|
-
state: element.state ?? null
|
|
198
|
+
state: element.state ?? null,
|
|
199
|
+
stable_id: element.stable_id ?? null,
|
|
200
|
+
role: element.role ?? null,
|
|
201
|
+
test_tag: element.test_tag ?? null,
|
|
202
|
+
selector: element.selector ?? null,
|
|
203
|
+
semantic: element.semantic ?? null
|
|
142
204
|
};
|
|
143
205
|
}
|
|
144
206
|
static _actionFailure(actionId, timestamp, actionType, selector, resolved, failureCode, retryable, uiFingerprintBefore, uiFingerprintAfter) {
|
|
@@ -542,6 +604,11 @@ export class ToolsInteract {
|
|
|
542
604
|
bounds: boundsObj,
|
|
543
605
|
clickable: !!best.clickable,
|
|
544
606
|
enabled: !!best.enabled,
|
|
607
|
+
stable_id: best.stable_id ?? null,
|
|
608
|
+
role: best.role ?? null,
|
|
609
|
+
test_tag: best.test_tag ?? null,
|
|
610
|
+
selector: best.selector ?? null,
|
|
611
|
+
semantic: best.semantic ?? null,
|
|
545
612
|
tapCoordinates,
|
|
546
613
|
telemetry: {
|
|
547
614
|
matchedIndex: best?._index ?? null,
|
|
@@ -860,6 +927,68 @@ export class ToolsInteract {
|
|
|
860
927
|
}
|
|
861
928
|
};
|
|
862
929
|
}
|
|
930
|
+
static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 250, expected_change }) {
|
|
931
|
+
const start = Date.now();
|
|
932
|
+
const pollIntervalMs = 300;
|
|
933
|
+
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 250);
|
|
934
|
+
let baseline = null;
|
|
935
|
+
let lastObservedRevision = null;
|
|
936
|
+
let lastLoadingState = null;
|
|
937
|
+
while (Date.now() - start < timeout_ms) {
|
|
938
|
+
try {
|
|
939
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
940
|
+
const signatures = ToolsInteract._buildUiChangeSignatures(tree);
|
|
941
|
+
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
|
|
942
|
+
lastLoadingState = tree?.loading_state ?? lastLoadingState;
|
|
943
|
+
if (!baseline) {
|
|
944
|
+
baseline = signatures;
|
|
945
|
+
}
|
|
946
|
+
else {
|
|
947
|
+
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
|
|
948
|
+
if (observedChange) {
|
|
949
|
+
if (stabilityWindow > 0) {
|
|
950
|
+
await new Promise(resolve => setTimeout(resolve, stabilityWindow));
|
|
951
|
+
const confirmTree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
952
|
+
const confirmSignatures = ToolsInteract._buildUiChangeSignatures(confirmTree);
|
|
953
|
+
const confirmChange = ToolsInteract._matchesUiChange(expected_change, baseline, confirmSignatures);
|
|
954
|
+
if (!confirmChange || confirmSignatures.hierarchy !== signatures.hierarchy || confirmSignatures.text !== signatures.text || confirmSignatures.state !== signatures.state) {
|
|
955
|
+
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision;
|
|
956
|
+
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState;
|
|
957
|
+
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
|
|
958
|
+
continue;
|
|
959
|
+
}
|
|
960
|
+
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision;
|
|
961
|
+
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState;
|
|
962
|
+
}
|
|
963
|
+
return {
|
|
964
|
+
success: true,
|
|
965
|
+
observed_change: observedChange,
|
|
966
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
967
|
+
timeout: false,
|
|
968
|
+
elapsed_ms: Date.now() - start,
|
|
969
|
+
expected_change,
|
|
970
|
+
loading_state: lastLoadingState ?? null,
|
|
971
|
+
reason: 'UI change observed'
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
catch {
|
|
977
|
+
// Keep polling until timeout; the observable surface should be best-effort.
|
|
978
|
+
}
|
|
979
|
+
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
|
|
980
|
+
}
|
|
981
|
+
return {
|
|
982
|
+
success: false,
|
|
983
|
+
observed_change: null,
|
|
984
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
985
|
+
timeout: true,
|
|
986
|
+
elapsed_ms: Date.now() - start,
|
|
987
|
+
expected_change,
|
|
988
|
+
loading_state: lastLoadingState ?? null,
|
|
989
|
+
reason: 'timeout'
|
|
990
|
+
};
|
|
991
|
+
}
|
|
863
992
|
static async expectScreenHandler({ platform, fingerprint, screen, deviceId }) {
|
|
864
993
|
const observedFingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId });
|
|
865
994
|
const observedScreen = {
|
|
@@ -940,7 +1069,12 @@ export class ToolsInteract {
|
|
|
940
1069
|
class: result.element.class ?? null,
|
|
941
1070
|
bounds: result.element.bounds ?? null,
|
|
942
1071
|
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
943
|
-
state: result.element.state ?? null
|
|
1072
|
+
state: result.element.state ?? null,
|
|
1073
|
+
stable_id: result.element.stable_id ?? null,
|
|
1074
|
+
role: result.element.role ?? null,
|
|
1075
|
+
test_tag: result.element.test_tag ?? null,
|
|
1076
|
+
selector: result.element.selector ?? null,
|
|
1077
|
+
semantic: result.element.semantic ?? null
|
|
944
1078
|
},
|
|
945
1079
|
observed: {
|
|
946
1080
|
status: result.status,
|
|
@@ -955,7 +1089,12 @@ export class ToolsInteract {
|
|
|
955
1089
|
class: result.element.class ?? null,
|
|
956
1090
|
bounds: result.element.bounds ?? null,
|
|
957
1091
|
index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
958
|
-
state: result.element.state ?? null
|
|
1092
|
+
state: result.element.state ?? null,
|
|
1093
|
+
stable_id: result.element.stable_id ?? null,
|
|
1094
|
+
role: result.element.role ?? null,
|
|
1095
|
+
test_tag: result.element.test_tag ?? null,
|
|
1096
|
+
selector: result.element.selector ?? null,
|
|
1097
|
+
semantic: result.element.semantic ?? null
|
|
959
1098
|
}
|
|
960
1099
|
},
|
|
961
1100
|
reason: 'selector is visible'
|
package/dist/observe/android.js
CHANGED
|
@@ -6,6 +6,7 @@ import { promises as fsPromises } from "fs";
|
|
|
6
6
|
import path from "path";
|
|
7
7
|
import { computeScreenFingerprint } from "../utils/ui/index.js";
|
|
8
8
|
import { parsePngSize } from "../utils/image.js";
|
|
9
|
+
import { deriveSnapshotMetadata } from "./snapshot-metadata.js";
|
|
9
10
|
const activeLogStreams = new Map();
|
|
10
11
|
export class AndroidObserve {
|
|
11
12
|
async getDeviceMetadata(appId, deviceId) {
|
|
@@ -61,21 +62,29 @@ export class AndroidObserve {
|
|
|
61
62
|
traverseNode(result.hierarchy.node, elements);
|
|
62
63
|
}
|
|
63
64
|
}
|
|
65
|
+
const snapshotMetadata = deriveSnapshotMetadata(`android:${deviceInfo.id}`, {
|
|
66
|
+
screen: "",
|
|
67
|
+
resolution,
|
|
68
|
+
elements
|
|
69
|
+
}, 'ui_tree');
|
|
64
70
|
return {
|
|
65
71
|
device: deviceInfo,
|
|
66
72
|
screen: "",
|
|
67
73
|
resolution,
|
|
68
|
-
elements
|
|
74
|
+
elements,
|
|
75
|
+
...snapshotMetadata
|
|
69
76
|
};
|
|
70
77
|
}
|
|
71
78
|
catch (e) {
|
|
72
79
|
const errorMessage = `Failed to get UI tree. ADB Path: '${getAdbCmd()}'. Error: ${e instanceof Error ? e.message : String(e)}`;
|
|
73
80
|
console.error(errorMessage);
|
|
81
|
+
const snapshotMetadata = deriveSnapshotMetadata(`android:${deviceInfo.id}`, null, 'ui_tree');
|
|
74
82
|
return {
|
|
75
83
|
device: deviceInfo,
|
|
76
84
|
screen: "",
|
|
77
85
|
resolution: { width: 0, height: 0 },
|
|
78
86
|
elements: [],
|
|
87
|
+
...snapshotMetadata,
|
|
79
88
|
error: errorMessage
|
|
80
89
|
};
|
|
81
90
|
}
|
package/dist/observe/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
2
2
|
import { AndroidObserve } from './android.js';
|
|
3
3
|
import { iOSObserve } from './ios.js';
|
|
4
|
+
import { deriveSnapshotMetadata } from './snapshot-metadata.js';
|
|
4
5
|
export { AndroidObserve } from './android.js';
|
|
5
6
|
export { iOSObserve } from './ios.js';
|
|
6
7
|
function normalizeHint(value) {
|
|
@@ -200,7 +201,17 @@ export class ToolsObserve {
|
|
|
200
201
|
}
|
|
201
202
|
static async captureDebugSnapshotHandler({ reason, includeLogs = true, logLines = 200, platform, appId, deviceId, sessionId } = {}) {
|
|
202
203
|
const timestamp = Date.now();
|
|
203
|
-
const raw = {
|
|
204
|
+
const raw = {
|
|
205
|
+
timestamp,
|
|
206
|
+
snapshot_revision: 0,
|
|
207
|
+
captured_at_ms: timestamp,
|
|
208
|
+
reason: reason || '',
|
|
209
|
+
activity: null,
|
|
210
|
+
fingerprint: null,
|
|
211
|
+
screenshot: null,
|
|
212
|
+
ui_tree: null,
|
|
213
|
+
logs: []
|
|
214
|
+
};
|
|
204
215
|
// Parallel fetches for performance: screenshot, current screen, fingerprint, ui tree, and log stream/get logs
|
|
205
216
|
const sid = sessionId || 'default';
|
|
206
217
|
const tasks = {
|
|
@@ -308,6 +319,13 @@ export class ToolsObserve {
|
|
|
308
319
|
raw.logs_error = e instanceof Error ? e.message : String(e);
|
|
309
320
|
}
|
|
310
321
|
}
|
|
322
|
+
const snapshotDeviceKey = raw.ui_tree?.device
|
|
323
|
+
? `${raw.ui_tree.device.platform}:${raw.ui_tree.device.id}`
|
|
324
|
+
: `${platform || 'unknown'}:${deviceId || 'default'}`;
|
|
325
|
+
const snapshotMetadata = deriveSnapshotMetadata(snapshotDeviceKey, raw.ui_tree, 'snapshot', raw.ui_tree?.snapshot_revision ? null : (raw.fingerprint || raw.activity || null));
|
|
326
|
+
raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision;
|
|
327
|
+
raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms;
|
|
328
|
+
raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state;
|
|
311
329
|
const semantic = deriveSnapshotSemantic(raw);
|
|
312
330
|
return semantic ? { raw, semantic } : { raw };
|
|
313
331
|
}
|
package/dist/observe/ios.js
CHANGED
|
@@ -6,6 +6,7 @@ import path from 'path';
|
|
|
6
6
|
import { parseLogLine } from '../utils/android/utils.js';
|
|
7
7
|
import { computeScreenFingerprint } from '../utils/ui/index.js';
|
|
8
8
|
import { parsePngSize } from '../utils/image.js';
|
|
9
|
+
import { deriveSnapshotMetadata } from './snapshot-metadata.js';
|
|
9
10
|
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
10
11
|
let iosExecCommand = execCommand;
|
|
11
12
|
export function _setIOSExecCommandForTests(fn) {
|
|
@@ -45,6 +46,65 @@ function parseIOSNumber(value) {
|
|
|
45
46
|
const parsed = Number(value);
|
|
46
47
|
return Number.isFinite(parsed) ? parsed : null;
|
|
47
48
|
}
|
|
49
|
+
function normalizeIOSType(value) {
|
|
50
|
+
return typeof value === 'string' ? value.trim().toLowerCase() : '';
|
|
51
|
+
}
|
|
52
|
+
function inferIOSRole(type, traits) {
|
|
53
|
+
if (/slider|adjustable/.test(type) || traits.some((trait) => /adjustable|slider/.test(trait)))
|
|
54
|
+
return 'slider';
|
|
55
|
+
if (/button/.test(type) || traits.some((trait) => /button/.test(trait)))
|
|
56
|
+
return 'button';
|
|
57
|
+
if (/cell/.test(type))
|
|
58
|
+
return 'cell';
|
|
59
|
+
if (/switch/.test(type))
|
|
60
|
+
return 'switch';
|
|
61
|
+
if (/text field|textfield|search field/.test(type))
|
|
62
|
+
return 'text_field';
|
|
63
|
+
if (/image/.test(type))
|
|
64
|
+
return 'image';
|
|
65
|
+
if (/window|application|group|scroll view|collection view/.test(type))
|
|
66
|
+
return 'container';
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
function getIOSStableId(node) {
|
|
70
|
+
const candidates = [node.AXIdentifier, node.accessibilityIdentifier, node.identifier, node.AXUniqueId];
|
|
71
|
+
for (const candidate of candidates) {
|
|
72
|
+
if (typeof candidate === 'string' && candidate.trim().length > 0)
|
|
73
|
+
return candidate;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
function buildIOSSelectorConfidence(source) {
|
|
78
|
+
switch (source) {
|
|
79
|
+
case 'identifier':
|
|
80
|
+
return { score: 1, reason: 'accessibility_identifier' };
|
|
81
|
+
case 'label':
|
|
82
|
+
return { score: 0.9, reason: 'label_match' };
|
|
83
|
+
case 'value':
|
|
84
|
+
return { score: 0.75, reason: 'value_match' };
|
|
85
|
+
case 'type':
|
|
86
|
+
return { score: 0.35, reason: 'type_match' };
|
|
87
|
+
default:
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function buildIOSSelector(type, label, value, stableId) {
|
|
92
|
+
if (stableId)
|
|
93
|
+
return { value: stableId, confidence: buildIOSSelectorConfidence('identifier') };
|
|
94
|
+
if (label)
|
|
95
|
+
return { value: label, confidence: buildIOSSelectorConfidence('label') };
|
|
96
|
+
if (value)
|
|
97
|
+
return { value: value, confidence: buildIOSSelectorConfidence('value') };
|
|
98
|
+
if (type)
|
|
99
|
+
return { value: type, confidence: buildIOSSelectorConfidence('type') };
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
function buildIOSSemantic(type, traits) {
|
|
103
|
+
return {
|
|
104
|
+
is_clickable: traits.includes("UIAccessibilityTraitButton") || /adjustable|slider/.test(type) || type === "Button" || type === "Cell",
|
|
105
|
+
is_container: /window|application|group|scroll view|collection view/.test(type)
|
|
106
|
+
};
|
|
107
|
+
}
|
|
48
108
|
function isIOSAdjustable(node, type, traits) {
|
|
49
109
|
return /slider|adjustable|stepper|progress/i.test(type) || traits.some((trait) => /adjustable|slider|progress/i.test(trait));
|
|
50
110
|
}
|
|
@@ -99,6 +159,11 @@ export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
|
|
|
99
159
|
const frame = node.AXFrame || node.frame;
|
|
100
160
|
const traits = node.AXTraits || [];
|
|
101
161
|
const state = extractIOSState(node, type, label, value, traits);
|
|
162
|
+
const normalizedType = normalizeIOSType(type);
|
|
163
|
+
const stableId = getIOSStableId(node);
|
|
164
|
+
const selector = buildIOSSelector(type, label, value, stableId);
|
|
165
|
+
const semantic = buildIOSSemantic(normalizedType, traits);
|
|
166
|
+
const role = inferIOSRole(normalizedType, traits);
|
|
102
167
|
const clickable = traits.includes("UIAccessibilityTraitButton") || type === "Button" || type === "Cell";
|
|
103
168
|
const isUseful = clickable || (label && label.length > 0) || (value && value.length > 0) || type === "Application" || type === "Window";
|
|
104
169
|
if (isUseful) {
|
|
@@ -107,14 +172,19 @@ export function traverseIDBNode(node, elements, parentIndex = -1, depth = 0) {
|
|
|
107
172
|
text: label,
|
|
108
173
|
contentDescription: value,
|
|
109
174
|
type: type,
|
|
110
|
-
resourceId:
|
|
175
|
+
resourceId: stableId,
|
|
111
176
|
clickable: clickable,
|
|
112
177
|
enabled: true,
|
|
113
178
|
visible: true,
|
|
114
179
|
bounds: bounds,
|
|
115
180
|
center: getCenter(bounds),
|
|
116
181
|
depth: depth,
|
|
117
|
-
state
|
|
182
|
+
state,
|
|
183
|
+
stable_id: stableId,
|
|
184
|
+
role,
|
|
185
|
+
test_tag: stableId,
|
|
186
|
+
selector,
|
|
187
|
+
semantic
|
|
118
188
|
};
|
|
119
189
|
if (parentIndex !== -1) {
|
|
120
190
|
element.parentId = parentIndex;
|
|
@@ -369,13 +439,16 @@ export class iOSObserve {
|
|
|
369
439
|
}
|
|
370
440
|
async getUITree(deviceId = "booted") {
|
|
371
441
|
const device = await getIOSDeviceMetadata(deviceId);
|
|
442
|
+
const deviceKey = `ios:${device.id}`;
|
|
372
443
|
const idbExists = await isIDBInstalled();
|
|
373
444
|
if (!idbExists) {
|
|
445
|
+
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
|
|
374
446
|
return {
|
|
375
447
|
device,
|
|
376
448
|
screen: "",
|
|
377
449
|
resolution: { width: 0, height: 0 },
|
|
378
450
|
elements: [],
|
|
451
|
+
...snapshotMetadata,
|
|
379
452
|
error: "iOS UI tree retrieval requires 'idb' (iOS Device Bridge). Please install it via Homebrew: `brew tap facebook/fb && brew install idb-companion` and `pip3 install fb-idb`."
|
|
380
453
|
};
|
|
381
454
|
}
|
|
@@ -416,11 +489,13 @@ export class iOSObserve {
|
|
|
416
489
|
console.error(`Attempt ${attempts} failed: ${e}`);
|
|
417
490
|
}
|
|
418
491
|
if (attempts === maxAttempts) {
|
|
492
|
+
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
|
|
419
493
|
return {
|
|
420
494
|
device,
|
|
421
495
|
screen: "",
|
|
422
496
|
resolution: { width: 0, height: 0 },
|
|
423
497
|
elements: [],
|
|
498
|
+
...snapshotMetadata,
|
|
424
499
|
error: `Failed to retrieve valid UI dump after ${maxAttempts} attempts.`
|
|
425
500
|
};
|
|
426
501
|
}
|
|
@@ -442,19 +517,27 @@ export class iOSObserve {
|
|
|
442
517
|
width = rootBounds[2] - rootBounds[0];
|
|
443
518
|
height = rootBounds[3] - rootBounds[1];
|
|
444
519
|
}
|
|
520
|
+
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, {
|
|
521
|
+
screen: "",
|
|
522
|
+
resolution: { width, height },
|
|
523
|
+
elements
|
|
524
|
+
}, 'ui_tree');
|
|
445
525
|
return {
|
|
446
526
|
device,
|
|
447
527
|
screen: "",
|
|
448
528
|
resolution: { width, height },
|
|
449
|
-
elements
|
|
529
|
+
elements,
|
|
530
|
+
...snapshotMetadata
|
|
450
531
|
};
|
|
451
532
|
}
|
|
452
533
|
catch (e) {
|
|
534
|
+
const snapshotMetadata = deriveSnapshotMetadata(deviceKey, null, 'ui_tree');
|
|
453
535
|
return {
|
|
454
536
|
device,
|
|
455
537
|
screen: "",
|
|
456
538
|
resolution: { width: 0, height: 0 },
|
|
457
539
|
elements: [],
|
|
540
|
+
...snapshotMetadata,
|
|
458
541
|
error: `Failed to parse idb output: ${e instanceof Error ? e.message : String(e)}`
|
|
459
542
|
};
|
|
460
543
|
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import crypto from 'crypto';
|
|
2
|
+
const snapshotStateByDevice = new Map();
|
|
3
|
+
function normalize(value) {
|
|
4
|
+
if (value === null || value === undefined)
|
|
5
|
+
return '';
|
|
6
|
+
return String(value).trim().toLowerCase();
|
|
7
|
+
}
|
|
8
|
+
function normalizeBounds(bounds) {
|
|
9
|
+
if (!Array.isArray(bounds) || bounds.length < 4)
|
|
10
|
+
return null;
|
|
11
|
+
const normalized = bounds.slice(0, 4).map((value) => Number(value));
|
|
12
|
+
if (normalized.some((value) => Number.isNaN(value)))
|
|
13
|
+
return null;
|
|
14
|
+
return normalized;
|
|
15
|
+
}
|
|
16
|
+
function stableElementSignature(element) {
|
|
17
|
+
return {
|
|
18
|
+
text: normalize(element.text),
|
|
19
|
+
contentDescription: normalize(element.contentDescription),
|
|
20
|
+
resourceId: normalize(element.resourceId),
|
|
21
|
+
type: normalize(element.type),
|
|
22
|
+
stable_id: normalize(element.stable_id),
|
|
23
|
+
role: normalize(element.role),
|
|
24
|
+
test_tag: normalize(element.test_tag),
|
|
25
|
+
selector: normalize(element.selector?.value),
|
|
26
|
+
clickable: !!element.clickable,
|
|
27
|
+
enabled: !!element.enabled,
|
|
28
|
+
visible: !!element.visible,
|
|
29
|
+
state: element.state ?? null,
|
|
30
|
+
bounds: normalizeBounds(element.bounds)
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export function computeSnapshotSignature(tree) {
|
|
34
|
+
if (!tree || tree.error)
|
|
35
|
+
return null;
|
|
36
|
+
const payload = {
|
|
37
|
+
screen: normalize(tree.screen),
|
|
38
|
+
resolution: tree.resolution || { width: 0, height: 0 },
|
|
39
|
+
elements: Array.isArray(tree.elements) ? tree.elements.map((element) => stableElementSignature(element)) : []
|
|
40
|
+
};
|
|
41
|
+
return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex');
|
|
42
|
+
}
|
|
43
|
+
export function detectLoadingState(tree, source) {
|
|
44
|
+
if (!tree || tree.error || !Array.isArray(tree.elements))
|
|
45
|
+
return null;
|
|
46
|
+
for (const element of tree.elements) {
|
|
47
|
+
if (!element?.visible)
|
|
48
|
+
continue;
|
|
49
|
+
const text = normalize(element?.text ?? element?.contentDescription ?? '');
|
|
50
|
+
const type = normalize(element?.type ?? '');
|
|
51
|
+
const combined = `${type} ${text}`;
|
|
52
|
+
if (/progress|spinner|loading|please wait|busy|loading indicator|skeleton|pending/.test(combined)) {
|
|
53
|
+
const signal = /progress/.test(combined)
|
|
54
|
+
? 'progress_indicator'
|
|
55
|
+
: /spinner/.test(combined)
|
|
56
|
+
? 'spinner'
|
|
57
|
+
: /busy/.test(combined)
|
|
58
|
+
? 'busy_indicator'
|
|
59
|
+
: /skeleton/.test(combined)
|
|
60
|
+
? 'skeleton'
|
|
61
|
+
: 'loading_indicator';
|
|
62
|
+
return { active: true, signal, source };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverride) {
|
|
68
|
+
const signature = signatureOverride ?? computeSnapshotSignature(tree);
|
|
69
|
+
const previous = snapshotStateByDevice.get(deviceKey);
|
|
70
|
+
let revision = 1;
|
|
71
|
+
if (previous) {
|
|
72
|
+
if (signature === null) {
|
|
73
|
+
revision = previous.revision;
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
revision = previous.signature === signature ? previous.revision : previous.revision + 1;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
snapshotStateByDevice.set(deviceKey, { revision, signature });
|
|
80
|
+
return {
|
|
81
|
+
snapshot_revision: revision,
|
|
82
|
+
captured_at_ms: Date.now(),
|
|
83
|
+
loading_state: detectLoadingState(tree, source)
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export function resetSnapshotMetadataForTests() {
|
|
87
|
+
snapshotStateByDevice.clear();
|
|
88
|
+
}
|
|
@@ -240,7 +240,7 @@ Failure Handling:
|
|
|
240
240
|
},
|
|
241
241
|
{
|
|
242
242
|
name: 'capture_debug_snapshot',
|
|
243
|
-
description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON.',
|
|
243
|
+
description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON with snapshot_revision, captured_at_ms, and loading_state when detectable.',
|
|
244
244
|
inputSchema: {
|
|
245
245
|
type: 'object',
|
|
246
246
|
properties: {
|
|
@@ -291,7 +291,7 @@ Failure Handling:
|
|
|
291
291
|
},
|
|
292
292
|
{
|
|
293
293
|
name: 'get_ui_tree',
|
|
294
|
-
description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content.',
|
|
294
|
+
description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content with snapshot metadata when available.',
|
|
295
295
|
inputSchema: {
|
|
296
296
|
type: 'object',
|
|
297
297
|
properties: {
|
|
@@ -363,6 +363,34 @@ Recommended Usage:
|
|
|
363
363
|
required: ['previousFingerprint']
|
|
364
364
|
}
|
|
365
365
|
},
|
|
366
|
+
{
|
|
367
|
+
name: 'wait_for_ui_change',
|
|
368
|
+
description: `Purpose:
|
|
369
|
+
Wait for a non-navigation UI mutation or in-place update to become stable.
|
|
370
|
+
|
|
371
|
+
Inputs:
|
|
372
|
+
- expected_change (optional): hierarchy_diff, text_change, or state_change
|
|
373
|
+
- timeout_ms (optional)
|
|
374
|
+
- stability_window_ms (optional)
|
|
375
|
+
|
|
376
|
+
Guidance:
|
|
377
|
+
- Prefer wait_for_screen_change for navigation transitions.
|
|
378
|
+
- Prefer wait_for_ui_change for in-place mutations and non-navigation updates.
|
|
379
|
+
- Use the returned snapshot_revision as the observed synchronization point when available.
|
|
380
|
+
|
|
381
|
+
Failure Handling:
|
|
382
|
+
- TIMEOUT means the UI did not change in a stable way within the allotted time.`,
|
|
383
|
+
inputSchema: {
|
|
384
|
+
type: 'object',
|
|
385
|
+
properties: {
|
|
386
|
+
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override (android|ios)' },
|
|
387
|
+
deviceId: { type: 'string', description: 'Optional device id/udid to target' },
|
|
388
|
+
expected_change: { type: 'string', enum: ['hierarchy_diff', 'text_change', 'state_change'], description: 'Optional type of UI change to wait for' },
|
|
389
|
+
timeout_ms: { type: 'number', description: 'Timeout in ms to wait for change (default 60000)', default: 60000 },
|
|
390
|
+
stability_window_ms: { type: 'number', description: 'How long the change must remain stable before success (default 250)', default: 250 }
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
},
|
|
366
394
|
{
|
|
367
395
|
name: 'expect_screen',
|
|
368
396
|
description: `Purpose:
|
|
@@ -236,6 +236,15 @@ async function handleWaitForUI(args) {
|
|
|
236
236
|
const res = await ToolsInteract.waitForUIHandler({ selector, condition, timeout_ms, poll_interval_ms, match, retry, platform, deviceId });
|
|
237
237
|
return wrapResponse(res);
|
|
238
238
|
}
|
|
239
|
+
async function handleWaitForUIChange(args) {
|
|
240
|
+
const platform = getStringArg(args, 'platform');
|
|
241
|
+
const deviceId = getStringArg(args, 'deviceId');
|
|
242
|
+
const timeout_ms = getNumberArg(args, 'timeout_ms') ?? 60000;
|
|
243
|
+
const stability_window_ms = getNumberArg(args, 'stability_window_ms') ?? 250;
|
|
244
|
+
const expected_change = getStringArg(args, 'expected_change');
|
|
245
|
+
const res = await ToolsInteract.waitForUIChangeHandler({ platform, deviceId, timeout_ms, stability_window_ms, expected_change });
|
|
246
|
+
return wrapResponse(res);
|
|
247
|
+
}
|
|
239
248
|
async function handleFindElement(args) {
|
|
240
249
|
const query = requireStringArg(args, 'query');
|
|
241
250
|
const exact = getBooleanArg(args, 'exact') ?? false;
|
|
@@ -409,6 +418,7 @@ export const toolHandlers = {
|
|
|
409
418
|
get_current_screen: handleGetCurrentScreen,
|
|
410
419
|
get_screen_fingerprint: handleGetScreenFingerprint,
|
|
411
420
|
wait_for_screen_change: handleWaitForScreenChange,
|
|
421
|
+
wait_for_ui_change: handleWaitForUIChange,
|
|
412
422
|
expect_screen: handleExpectScreen,
|
|
413
423
|
expect_element_visible: handleExpectElementVisible,
|
|
414
424
|
expect_state: handleExpectState,
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.
|
|
9
|
+
version: '0.26.0'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|