mobile-debug-mcp 0.30.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +330 -22
- package/dist/observe/index.js +1 -0
- package/dist/observe/snapshot-metadata.js +62 -1
- package/dist/server/tool-definitions.js +7 -2
- package/dist/server/tool-handlers.js +3 -1
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +6 -0
- package/docs/rfcs/013-wait-and-synchronization-reliability.md +15 -35
- package/docs/rfcs/014-actionability-resolution.md +32 -30
- package/package.json +1 -1
- package/src/interact/index.ts +427 -33
- package/src/observe/index.ts +1 -0
- package/src/observe/snapshot-metadata.ts +69 -2
- package/src/server/tool-definitions.ts +7 -2
- package/src/server/tool-handlers.ts +3 -1
- package/src/server-core.ts +1 -1
- package/src/types.ts +24 -0
- package/test/unit/interact/adjust_control.test.ts +104 -0
- package/test/unit/interact/subtree_collection.test.ts +24 -0
- package/test/unit/interact/tap_element.test.ts +71 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +92 -1
- package/test/unit/observe/snapshot_metadata.test.ts +67 -0
package/dist/interact/index.js
CHANGED
|
@@ -95,10 +95,68 @@ export class ToolsInteract {
|
|
|
95
95
|
}
|
|
96
96
|
return null;
|
|
97
97
|
}
|
|
98
|
+
static _resolveParentIndex(elements, parentId) {
|
|
99
|
+
if (parentId === undefined || parentId === null)
|
|
100
|
+
return null;
|
|
101
|
+
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
102
|
+
return parentId;
|
|
103
|
+
}
|
|
104
|
+
if (typeof parentId === 'string') {
|
|
105
|
+
const normalized = ToolsInteract._normalize(parentId);
|
|
106
|
+
if (!normalized)
|
|
107
|
+
return null;
|
|
108
|
+
if (/^\d+$/.test(normalized)) {
|
|
109
|
+
const index = Number(normalized);
|
|
110
|
+
if (index >= 0 && index < elements.length)
|
|
111
|
+
return index;
|
|
112
|
+
}
|
|
113
|
+
const foundIndex = elements.findIndex((el) => {
|
|
114
|
+
if (!el)
|
|
115
|
+
return false;
|
|
116
|
+
return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
117
|
+
ToolsInteract._normalize(el.stable_id ?? '') === normalized;
|
|
118
|
+
});
|
|
119
|
+
return foundIndex >= 0 ? foundIndex : null;
|
|
120
|
+
}
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
98
123
|
static _isVisibleElement(el) {
|
|
99
124
|
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
100
125
|
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
|
|
101
126
|
}
|
|
127
|
+
static _isTapActionable(el, storedStableId, platform) {
|
|
128
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
129
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
130
|
+
}
|
|
131
|
+
if (el.enabled === false) {
|
|
132
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
133
|
+
}
|
|
134
|
+
const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
|
|
135
|
+
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap')));
|
|
136
|
+
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
137
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
|
|
138
|
+
}
|
|
139
|
+
if (storedStableId) {
|
|
140
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
141
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return { actionable: true };
|
|
145
|
+
}
|
|
146
|
+
static _isAdjustableActionable(el, storedStableId) {
|
|
147
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
148
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
149
|
+
}
|
|
150
|
+
if (el.enabled === false) {
|
|
151
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
152
|
+
}
|
|
153
|
+
if (storedStableId) {
|
|
154
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
155
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return { actionable: true };
|
|
159
|
+
}
|
|
102
160
|
static _computeElementId(platform, deviceId, el, index) {
|
|
103
161
|
const identity = {
|
|
104
162
|
platform,
|
|
@@ -120,7 +178,8 @@ export class ToolsInteract {
|
|
|
120
178
|
platform,
|
|
121
179
|
deviceId,
|
|
122
180
|
bounds,
|
|
123
|
-
index
|
|
181
|
+
index,
|
|
182
|
+
stable_id: el.stable_id ?? null
|
|
124
183
|
});
|
|
125
184
|
return {
|
|
126
185
|
text: el.text ?? null,
|
|
@@ -138,6 +197,214 @@ export class ToolsInteract {
|
|
|
138
197
|
semantic: el.semantic ?? null
|
|
139
198
|
};
|
|
140
199
|
}
|
|
200
|
+
static _resolveUiChangeScope(tree, scope, target) {
|
|
201
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
202
|
+
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
|
|
203
|
+
if (normalizedScope === 'screen') {
|
|
204
|
+
return {
|
|
205
|
+
elements,
|
|
206
|
+
resolution: {
|
|
207
|
+
scope: 'screen',
|
|
208
|
+
target: null,
|
|
209
|
+
resolved: true,
|
|
210
|
+
resolvedIndex: null,
|
|
211
|
+
resolvedStableId: null,
|
|
212
|
+
reason: 'screen scope'
|
|
213
|
+
}
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
|
|
217
|
+
if (!requestedTarget) {
|
|
218
|
+
return {
|
|
219
|
+
elements: [],
|
|
220
|
+
resolution: {
|
|
221
|
+
scope: 'subtree',
|
|
222
|
+
target: null,
|
|
223
|
+
resolved: false,
|
|
224
|
+
resolvedIndex: null,
|
|
225
|
+
resolvedStableId: null,
|
|
226
|
+
reason: 'subtree scope requires a target element id'
|
|
227
|
+
},
|
|
228
|
+
error: {
|
|
229
|
+
code: 'INVALID_SCOPE',
|
|
230
|
+
message: 'scope=subtree requires a target element_id'
|
|
231
|
+
}
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const resolved = ToolsInteract._findScopedElement(tree, requestedTarget);
|
|
235
|
+
if (!resolved) {
|
|
236
|
+
return {
|
|
237
|
+
elements: [],
|
|
238
|
+
resolution: {
|
|
239
|
+
scope: 'subtree',
|
|
240
|
+
target: requestedTarget,
|
|
241
|
+
resolved: false,
|
|
242
|
+
resolvedIndex: null,
|
|
243
|
+
resolvedStableId: null,
|
|
244
|
+
reason: 'target element could not be resolved'
|
|
245
|
+
},
|
|
246
|
+
error: {
|
|
247
|
+
code: 'ELEMENT_NOT_FOUND',
|
|
248
|
+
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index);
|
|
253
|
+
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
|
|
254
|
+
return {
|
|
255
|
+
elements: scopedElements,
|
|
256
|
+
resolution: {
|
|
257
|
+
scope: 'subtree',
|
|
258
|
+
target: requestedTarget,
|
|
259
|
+
resolved: true,
|
|
260
|
+
resolvedIndex: resolved.index,
|
|
261
|
+
resolvedStableId: resolved.stableId,
|
|
262
|
+
reason: resolved.reason
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
static _findScopedElement(tree, targetElementId) {
|
|
267
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
268
|
+
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
|
|
269
|
+
const deviceId = tree?.device?.id ?? undefined;
|
|
270
|
+
const normalizedTarget = ToolsInteract._normalize(targetElementId);
|
|
271
|
+
for (let i = 0; i < elements.length; i++) {
|
|
272
|
+
const el = elements[i];
|
|
273
|
+
if (!el)
|
|
274
|
+
continue;
|
|
275
|
+
const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i);
|
|
276
|
+
if (computedElementId === targetElementId) {
|
|
277
|
+
return {
|
|
278
|
+
index: i,
|
|
279
|
+
stableId: el.stable_id ?? null,
|
|
280
|
+
reason: 'element_id_match'
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (let i = 0; i < elements.length; i++) {
|
|
285
|
+
const el = elements[i];
|
|
286
|
+
if (!el)
|
|
287
|
+
continue;
|
|
288
|
+
if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
|
|
289
|
+
return {
|
|
290
|
+
index: i,
|
|
291
|
+
stableId: el.stable_id,
|
|
292
|
+
reason: 'stable_id_match'
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId);
|
|
297
|
+
if (storedContext?.stable_id) {
|
|
298
|
+
const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id);
|
|
299
|
+
for (let i = 0; i < elements.length; i++) {
|
|
300
|
+
const el = elements[i];
|
|
301
|
+
if (!el?.stable_id)
|
|
302
|
+
continue;
|
|
303
|
+
if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
|
|
304
|
+
return {
|
|
305
|
+
index: i,
|
|
306
|
+
stableId: el.stable_id,
|
|
307
|
+
reason: 'stored_stable_id_match'
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return null;
|
|
313
|
+
}
|
|
314
|
+
static _collectSubtreeIndices(elements, rootIndex) {
|
|
315
|
+
if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length)
|
|
316
|
+
return [];
|
|
317
|
+
const visited = new Set();
|
|
318
|
+
const stack = [rootIndex];
|
|
319
|
+
const result = [];
|
|
320
|
+
while (stack.length > 0) {
|
|
321
|
+
const index = stack.pop();
|
|
322
|
+
if (index === undefined || visited.has(index) || index < 0 || index >= elements.length)
|
|
323
|
+
continue;
|
|
324
|
+
visited.add(index);
|
|
325
|
+
result.push(index);
|
|
326
|
+
const element = elements[index];
|
|
327
|
+
if (!element)
|
|
328
|
+
continue;
|
|
329
|
+
const directChildren = new Set();
|
|
330
|
+
if (Array.isArray(element.children)) {
|
|
331
|
+
for (const childIndex of element.children) {
|
|
332
|
+
if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
|
|
333
|
+
directChildren.add(childIndex);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
for (let i = 0; i < elements.length; i++) {
|
|
338
|
+
if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
|
|
339
|
+
directChildren.add(i);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
for (const childIndex of directChildren) {
|
|
343
|
+
if (!visited.has(childIndex))
|
|
344
|
+
stack.push(childIndex);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return result.sort((left, right) => left - right);
|
|
348
|
+
}
|
|
349
|
+
static _changeIdentityForElement(el, index) {
|
|
350
|
+
const stableId = ToolsInteract._normalize(el.stable_id);
|
|
351
|
+
if (stableId)
|
|
352
|
+
return `stable:${stableId}`;
|
|
353
|
+
return `fallback:${ToolsInteract._hash({
|
|
354
|
+
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
355
|
+
contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
|
|
356
|
+
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
357
|
+
type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
358
|
+
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
359
|
+
index
|
|
360
|
+
})}`;
|
|
361
|
+
}
|
|
362
|
+
static _summarizeUiChangeDelta(initialElements, currentElements) {
|
|
363
|
+
const buildMap = (elements) => {
|
|
364
|
+
const map = new Map();
|
|
365
|
+
for (let i = 0; i < elements.length; i++) {
|
|
366
|
+
const element = elements[i];
|
|
367
|
+
if (!element)
|
|
368
|
+
continue;
|
|
369
|
+
const key = ToolsInteract._changeIdentityForElement(element, i);
|
|
370
|
+
map.set(key, ToolsInteract._hash({
|
|
371
|
+
text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
|
|
372
|
+
contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
|
|
373
|
+
resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
|
|
374
|
+
type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
|
|
375
|
+
bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
|
|
376
|
+
state: element.state ?? null,
|
|
377
|
+
visible: !!element.visible,
|
|
378
|
+
enabled: !!element.enabled,
|
|
379
|
+
clickable: !!element.clickable
|
|
380
|
+
}));
|
|
381
|
+
}
|
|
382
|
+
return map;
|
|
383
|
+
};
|
|
384
|
+
const initialMap = buildMap(initialElements);
|
|
385
|
+
const currentMap = buildMap(currentElements);
|
|
386
|
+
let added = 0;
|
|
387
|
+
let removed = 0;
|
|
388
|
+
let mutated = 0;
|
|
389
|
+
for (const [key, value] of currentMap.entries()) {
|
|
390
|
+
if (!initialMap.has(key)) {
|
|
391
|
+
added++;
|
|
392
|
+
}
|
|
393
|
+
else if (initialMap.get(key) !== value) {
|
|
394
|
+
mutated++;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
for (const key of initialMap.keys()) {
|
|
398
|
+
if (!currentMap.has(key))
|
|
399
|
+
removed++;
|
|
400
|
+
}
|
|
401
|
+
return {
|
|
402
|
+
total_elements: currentElements.length,
|
|
403
|
+
added_elements: added,
|
|
404
|
+
removed_elements: removed,
|
|
405
|
+
mutated_elements: mutated
|
|
406
|
+
};
|
|
407
|
+
}
|
|
141
408
|
static _rememberResolvedElement(elementId, context) {
|
|
142
409
|
if (ToolsInteract._resolvedUiElements.has(elementId)) {
|
|
143
410
|
ToolsInteract._resolvedUiElements.delete(elementId);
|
|
@@ -384,25 +651,12 @@ export class ToolsInteract {
|
|
|
384
651
|
let current = chosen;
|
|
385
652
|
let safety = 0;
|
|
386
653
|
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
387
|
-
const
|
|
388
|
-
let parentIndex = null;
|
|
389
|
-
if (typeof parentId === 'number')
|
|
390
|
-
parentIndex = parentId;
|
|
391
|
-
else if (typeof parentId === 'string' && /^\d+$/.test(parentId))
|
|
392
|
-
parentIndex = Number(parentId);
|
|
654
|
+
const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId);
|
|
393
655
|
if (parentIndex !== null && elements[parentIndex]) {
|
|
394
656
|
current = { el: elements[parentIndex], idx: parentIndex };
|
|
395
657
|
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
|
|
396
658
|
return current;
|
|
397
659
|
}
|
|
398
|
-
else if (typeof parentId === 'string') {
|
|
399
|
-
const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId);
|
|
400
|
-
if (foundIndex === -1)
|
|
401
|
-
break;
|
|
402
|
-
current = { el: elements[foundIndex], idx: foundIndex };
|
|
403
|
-
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
|
|
404
|
-
return current;
|
|
405
|
-
}
|
|
406
660
|
else {
|
|
407
661
|
break;
|
|
408
662
|
}
|
|
@@ -516,11 +770,9 @@ export class ToolsInteract {
|
|
|
516
770
|
return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
|
|
517
771
|
}
|
|
518
772
|
const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
if (currentMatch.el.enabled === false) {
|
|
523
|
-
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
773
|
+
const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform);
|
|
774
|
+
if (!tapActionability.actionable) {
|
|
775
|
+
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
524
776
|
}
|
|
525
777
|
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
|
|
526
778
|
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
|
|
@@ -553,6 +805,7 @@ export class ToolsInteract {
|
|
|
553
805
|
const sourcePlatform = platform || 'android';
|
|
554
806
|
let resolvedPlatform = sourcePlatform;
|
|
555
807
|
let resolvedDeviceId = deviceId;
|
|
808
|
+
const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null;
|
|
556
809
|
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
557
810
|
let semanticFallbackElement = null;
|
|
558
811
|
const traceSteps = [];
|
|
@@ -758,6 +1011,10 @@ export class ToolsInteract {
|
|
|
758
1011
|
resolvedTarget = resolved.resolvedTarget;
|
|
759
1012
|
const currentEl = resolved.match.el;
|
|
760
1013
|
cachedResolvedMatch = resolved.match;
|
|
1014
|
+
const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id);
|
|
1015
|
+
if (!adjustableActionability.actionable) {
|
|
1016
|
+
return buildFailure(adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', adjustableActionability.reason ?? 'adjustable control is not actionable', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
|
|
1017
|
+
}
|
|
761
1018
|
const bounds = ToolsInteract._normalizeBounds(currentEl.bounds);
|
|
762
1019
|
const valueRange = currentEl.state?.value_range ?? null;
|
|
763
1020
|
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property);
|
|
@@ -1711,24 +1968,65 @@ export class ToolsInteract {
|
|
|
1711
1968
|
}
|
|
1712
1969
|
};
|
|
1713
1970
|
}
|
|
1714
|
-
static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 300, expected_change }) {
|
|
1971
|
+
static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 300, expected_change, scope = 'screen', target = null }) {
|
|
1715
1972
|
const start = Date.now();
|
|
1716
1973
|
const pollIntervalMs = 300;
|
|
1717
1974
|
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300);
|
|
1718
1975
|
let baseline = null;
|
|
1976
|
+
let baselineScope = null;
|
|
1719
1977
|
let lastObservedRevision = null;
|
|
1720
1978
|
let lastLoadingState = null;
|
|
1979
|
+
let lastSnapshotFreshnessMs = null;
|
|
1721
1980
|
let candidateSignatures = null;
|
|
1722
1981
|
let candidateObservedChange = null;
|
|
1723
1982
|
let candidateSinceMs = null;
|
|
1983
|
+
let lastChangeSummary = null;
|
|
1984
|
+
let lastScopeResolution = {
|
|
1985
|
+
scope: scope === 'subtree' ? 'subtree' : 'screen',
|
|
1986
|
+
target: target && typeof target === 'string' ? target : null,
|
|
1987
|
+
resolved: scope !== 'subtree',
|
|
1988
|
+
resolvedIndex: null,
|
|
1989
|
+
resolvedStableId: null,
|
|
1990
|
+
reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
|
|
1991
|
+
};
|
|
1724
1992
|
while (Date.now() - start < timeout_ms) {
|
|
1725
1993
|
try {
|
|
1726
1994
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
1727
|
-
const
|
|
1995
|
+
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target);
|
|
1996
|
+
if (scopedTree.error) {
|
|
1997
|
+
lastScopeResolution = scopedTree.resolution;
|
|
1998
|
+
return {
|
|
1999
|
+
success: false,
|
|
2000
|
+
observed_change: null,
|
|
2001
|
+
snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
|
|
2002
|
+
snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
|
|
2003
|
+
timeout: true,
|
|
2004
|
+
elapsed_ms: Date.now() - start,
|
|
2005
|
+
expected_change,
|
|
2006
|
+
loading_state: tree?.loading_state ?? lastLoadingState ?? null,
|
|
2007
|
+
scope: scopedTree.resolution.scope,
|
|
2008
|
+
target: scopedTree.resolution.target,
|
|
2009
|
+
stability_state: 'transient',
|
|
2010
|
+
change_summary: lastChangeSummary,
|
|
2011
|
+
reason: scopedTree.error.message,
|
|
2012
|
+
error: scopedTree.error
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
2015
|
+
const scopedElements = scopedTree.elements;
|
|
2016
|
+
const scopedSignatureTree = {
|
|
2017
|
+
...tree,
|
|
2018
|
+
elements: scopedElements
|
|
2019
|
+
};
|
|
2020
|
+
const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree);
|
|
1728
2021
|
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
|
|
1729
2022
|
lastLoadingState = tree?.loading_state ?? lastLoadingState;
|
|
2023
|
+
lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs;
|
|
2024
|
+
lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary;
|
|
2025
|
+
lastScopeResolution = scopedTree.resolution;
|
|
2026
|
+
baselineScope = baselineScope ?? scopedTree;
|
|
1730
2027
|
if (!baseline) {
|
|
1731
2028
|
baseline = signatures;
|
|
2029
|
+
baselineScope = scopedTree;
|
|
1732
2030
|
}
|
|
1733
2031
|
else {
|
|
1734
2032
|
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
|
|
@@ -1744,10 +2042,15 @@ export class ToolsInteract {
|
|
|
1744
2042
|
success: true,
|
|
1745
2043
|
observed_change: candidateObservedChange ?? observedChange,
|
|
1746
2044
|
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2045
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
1747
2046
|
timeout: false,
|
|
1748
2047
|
elapsed_ms: Date.now() - start,
|
|
1749
2048
|
expected_change,
|
|
1750
2049
|
loading_state: lastLoadingState ?? null,
|
|
2050
|
+
scope: lastScopeResolution.scope,
|
|
2051
|
+
target: lastScopeResolution.target,
|
|
2052
|
+
stability_state: 'stable',
|
|
2053
|
+
change_summary: lastChangeSummary,
|
|
1751
2054
|
reason: 'UI change observed'
|
|
1752
2055
|
};
|
|
1753
2056
|
}
|
|
@@ -1768,10 +2071,15 @@ export class ToolsInteract {
|
|
|
1768
2071
|
success: false,
|
|
1769
2072
|
observed_change: null,
|
|
1770
2073
|
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2074
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
1771
2075
|
timeout: true,
|
|
1772
2076
|
elapsed_ms: Date.now() - start,
|
|
1773
2077
|
expected_change,
|
|
1774
2078
|
loading_state: lastLoadingState ?? null,
|
|
2079
|
+
scope: lastScopeResolution.scope,
|
|
2080
|
+
target: lastScopeResolution.target,
|
|
2081
|
+
stability_state: 'transient',
|
|
2082
|
+
change_summary: lastChangeSummary,
|
|
1775
2083
|
reason: 'timeout'
|
|
1776
2084
|
};
|
|
1777
2085
|
}
|
package/dist/observe/index.js
CHANGED
|
@@ -325,6 +325,7 @@ export class ToolsObserve {
|
|
|
325
325
|
const snapshotMetadata = deriveSnapshotMetadata(snapshotDeviceKey, raw.ui_tree, 'snapshot', raw.ui_tree?.snapshot_revision ? null : (raw.fingerprint || raw.activity || null));
|
|
326
326
|
raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision;
|
|
327
327
|
raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms;
|
|
328
|
+
raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? snapshotMetadata.snapshot_delta ?? null;
|
|
328
329
|
raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state;
|
|
329
330
|
const semantic = deriveSnapshotSemantic(raw);
|
|
330
331
|
return semantic ? { raw, semantic } : { raw };
|
|
@@ -30,6 +30,58 @@ function stableElementSignature(element) {
|
|
|
30
30
|
bounds: normalizeBounds(element.bounds)
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
+
function stableElementIdentity(element, index) {
|
|
34
|
+
const stableId = normalize(element.stable_id);
|
|
35
|
+
if (stableId)
|
|
36
|
+
return `stable:${stableId}`;
|
|
37
|
+
return `fallback:${crypto.createHash('sha1').update(JSON.stringify({
|
|
38
|
+
text: normalize(element.text),
|
|
39
|
+
contentDescription: normalize(element.contentDescription),
|
|
40
|
+
resourceId: normalize(element.resourceId),
|
|
41
|
+
type: normalize(element.type),
|
|
42
|
+
bounds: normalizeBounds(element.bounds),
|
|
43
|
+
index
|
|
44
|
+
})).digest('hex')}`;
|
|
45
|
+
}
|
|
46
|
+
function buildElementSignatures(tree) {
|
|
47
|
+
const signatures = new Map();
|
|
48
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
49
|
+
for (let index = 0; index < elements.length; index++) {
|
|
50
|
+
const element = elements[index];
|
|
51
|
+
if (!element)
|
|
52
|
+
continue;
|
|
53
|
+
const identity = stableElementIdentity(element, index);
|
|
54
|
+
signatures.set(identity, crypto.createHash('sha1').update(JSON.stringify(stableElementSignature(element))).digest('hex'));
|
|
55
|
+
}
|
|
56
|
+
return signatures;
|
|
57
|
+
}
|
|
58
|
+
function summarizeSnapshotDelta(previous, currentElements) {
|
|
59
|
+
if (!previous)
|
|
60
|
+
return null;
|
|
61
|
+
let added = 0;
|
|
62
|
+
let removed = 0;
|
|
63
|
+
let mutated = 0;
|
|
64
|
+
for (const [identity, signature] of currentElements.entries()) {
|
|
65
|
+
const previousSignature = previous.elementSignatures.get(identity);
|
|
66
|
+
if (previousSignature === undefined) {
|
|
67
|
+
added++;
|
|
68
|
+
}
|
|
69
|
+
else if (previousSignature !== signature) {
|
|
70
|
+
mutated++;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const identity of previous.elementSignatures.keys()) {
|
|
74
|
+
if (!currentElements.has(identity))
|
|
75
|
+
removed++;
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
previous_snapshot_revision: previous.revision,
|
|
79
|
+
added_elements: added,
|
|
80
|
+
removed_elements: removed,
|
|
81
|
+
mutated_elements: mutated,
|
|
82
|
+
total_elements: currentElements.size
|
|
83
|
+
};
|
|
84
|
+
}
|
|
33
85
|
export function computeSnapshotSignature(tree) {
|
|
34
86
|
if (!tree || tree.error)
|
|
35
87
|
return null;
|
|
@@ -67,6 +119,10 @@ export function detectLoadingState(tree, source) {
|
|
|
67
119
|
export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverride) {
|
|
68
120
|
const signature = signatureOverride ?? computeSnapshotSignature(tree);
|
|
69
121
|
const previous = snapshotStateByDevice.get(deviceKey);
|
|
122
|
+
const hasValidTree = !!tree && !tree.error;
|
|
123
|
+
const currentElementSignatures = hasValidTree
|
|
124
|
+
? buildElementSignatures(tree)
|
|
125
|
+
: previous?.elementSignatures ?? new Map();
|
|
70
126
|
let revision = 1;
|
|
71
127
|
if (previous) {
|
|
72
128
|
if (signature === null) {
|
|
@@ -76,10 +132,15 @@ export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverrid
|
|
|
76
132
|
revision = previous.signature === signature ? previous.revision : previous.revision + 1;
|
|
77
133
|
}
|
|
78
134
|
}
|
|
79
|
-
snapshotStateByDevice.set(deviceKey, {
|
|
135
|
+
snapshotStateByDevice.set(deviceKey, {
|
|
136
|
+
revision,
|
|
137
|
+
signature,
|
|
138
|
+
elementSignatures: currentElementSignatures
|
|
139
|
+
});
|
|
80
140
|
return {
|
|
81
141
|
snapshot_revision: revision,
|
|
82
142
|
captured_at_ms: Date.now(),
|
|
143
|
+
snapshot_delta: hasValidTree ? summarizeSnapshotDelta(previous, currentElementSignatures) : null,
|
|
83
144
|
loading_state: detectLoadingState(tree, source)
|
|
84
145
|
};
|
|
85
146
|
}
|
|
@@ -244,7 +244,7 @@ Failure Handling:
|
|
|
244
244
|
},
|
|
245
245
|
{
|
|
246
246
|
name: 'capture_debug_snapshot',
|
|
247
|
-
description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON with snapshot_revision, captured_at_ms, and loading_state when detectable.',
|
|
247
|
+
description: 'Capture a complete debug snapshot (raw observation layer plus optional derived semantic layer). Returns structured JSON with snapshot_revision, captured_at_ms, snapshot_delta, and loading_state when detectable.',
|
|
248
248
|
inputSchema: {
|
|
249
249
|
type: 'object',
|
|
250
250
|
properties: {
|
|
@@ -295,7 +295,7 @@ Failure Handling:
|
|
|
295
295
|
},
|
|
296
296
|
{
|
|
297
297
|
name: 'get_ui_tree',
|
|
298
|
-
description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content with snapshot metadata when available.',
|
|
298
|
+
description: 'Get the current UI hierarchy from an Android device or iOS simulator. Returns a structured JSON representation of the screen content with snapshot metadata and incremental delta signals when available.',
|
|
299
299
|
inputSchema: {
|
|
300
300
|
type: 'object',
|
|
301
301
|
properties: {
|
|
@@ -376,11 +376,14 @@ Inputs:
|
|
|
376
376
|
- expected_change (optional): hierarchy_diff, text_change, or state_change
|
|
377
377
|
- timeout_ms (optional)
|
|
378
378
|
- stability_window_ms (optional)
|
|
379
|
+
- scope (optional): screen or subtree
|
|
380
|
+
- target (optional): element_id when scope=subtree
|
|
379
381
|
|
|
380
382
|
Guidance:
|
|
381
383
|
- Prefer wait_for_screen_change for navigation transitions.
|
|
382
384
|
- Prefer wait_for_ui_change for in-place mutations and non-navigation updates.
|
|
383
385
|
- Use the returned snapshot_revision as the observed synchronization point when available.
|
|
386
|
+
- Scoped waits return scope-aware stability metadata and a lightweight change summary.
|
|
384
387
|
|
|
385
388
|
Failure Handling:
|
|
386
389
|
- TIMEOUT means the UI did not change in a stable way within the allotted time.`,
|
|
@@ -390,6 +393,8 @@ Failure Handling:
|
|
|
390
393
|
platform: { type: 'string', enum: ['android', 'ios'], description: 'Optional platform override (android|ios)' },
|
|
391
394
|
deviceId: { type: 'string', description: 'Optional device id/udid to target' },
|
|
392
395
|
expected_change: { type: 'string', enum: ['hierarchy_diff', 'text_change', 'state_change'], description: 'Optional type of UI change to wait for' },
|
|
396
|
+
scope: { type: 'string', enum: ['screen', 'subtree'], default: 'screen', description: 'Synchronization scope for the wait' },
|
|
397
|
+
target: { type: 'string', description: 'Target element_id when scope is subtree' },
|
|
393
398
|
timeout_ms: { type: 'number', description: 'Timeout in ms to wait for change (default 60000)', default: 60000 },
|
|
394
399
|
stability_window_ms: { type: 'number', description: 'How long the change must remain stable before success (default 300)', default: 300 }
|
|
395
400
|
}
|
|
@@ -268,7 +268,9 @@ async function handleWaitForUIChange(args) {
|
|
|
268
268
|
const timeout_ms = getNumberArg(args, 'timeout_ms') ?? 60000;
|
|
269
269
|
const stability_window_ms = getNumberArg(args, 'stability_window_ms') ?? 300;
|
|
270
270
|
const expected_change = getStringArg(args, 'expected_change');
|
|
271
|
-
const
|
|
271
|
+
const scope = getStringArg(args, 'scope');
|
|
272
|
+
const target = getStringArg(args, 'target');
|
|
273
|
+
const res = await ToolsInteract.waitForUIChangeHandler({ platform, deviceId, timeout_ms, stability_window_ms, expected_change, scope, target });
|
|
272
274
|
return wrapResponse(res);
|
|
273
275
|
}
|
|
274
276
|
async function handleFindElement(args) {
|
package/dist/server-core.js
CHANGED
|
@@ -6,7 +6,7 @@ import { handleToolCall } from './server/tool-handlers.js';
|
|
|
6
6
|
export { wrapResponse, toolDefinitions, handleToolCall };
|
|
7
7
|
export const serverInfo = {
|
|
8
8
|
name: 'mobile-debug-mcp',
|
|
9
|
-
version: '0.30.
|
|
9
|
+
version: '0.30.1'
|
|
10
10
|
};
|
|
11
11
|
export function createServer() {
|
|
12
12
|
const server = new Server(serverInfo, {
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to the **Mobile Debug MCP** project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.30.1]
|
|
6
|
+
- Synced the server-reported version in `src/server-core.ts` with `package.json` so contract checks pass.
|
|
7
|
+
- Completed the RFC 013 wait/synchronization implementation, including scoped waits and freshness metadata.
|
|
8
|
+
- Completed the RFC 014 actionability implementation for taps and adjustable controls.
|
|
9
|
+
- Added regression coverage for subtree collection, scoped waits, snapshot deltas, and stale actionability checks.
|
|
10
|
+
|
|
5
11
|
## [0.30.0]
|
|
6
12
|
- Folded RFC 013 synchronization semantics into the main spec and aligned the interact docs with the shipped `wait_for_ui_change` behavior.
|
|
7
13
|
- Updated `wait_for_ui_change` to use a 300ms stabilization default and to reset stabilization on new in-place mutations.
|