mobile-debug-mcp 0.30.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +330 -22
- package/dist/observe/index.js +1 -0
- package/dist/observe/snapshot-metadata.js +62 -1
- package/dist/server/tool-definitions.js +7 -2
- package/dist/server/tool-handlers.js +3 -1
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +6 -0
- package/docs/rfcs/013-wait-and-synchronization-reliability.md +15 -35
- package/docs/rfcs/014-actionability-resolution.md +32 -30
- package/package.json +1 -1
- package/src/interact/index.ts +427 -33
- package/src/observe/index.ts +1 -0
- package/src/observe/snapshot-metadata.ts +69 -2
- package/src/server/tool-definitions.ts +7 -2
- package/src/server/tool-handlers.ts +3 -1
- package/src/server-core.ts +1 -1
- package/src/types.ts +24 -0
- package/test/unit/interact/adjust_control.test.ts +104 -0
- package/test/unit/interact/subtree_collection.test.ts +24 -0
- package/test/unit/interact/tap_element.test.ts +71 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +92 -1
- package/test/unit/observe/snapshot_metadata.test.ts +67 -0
package/src/interact/index.ts
CHANGED
|
@@ -43,6 +43,7 @@ interface UiElement {
|
|
|
43
43
|
focusable?: boolean
|
|
44
44
|
visible?: boolean
|
|
45
45
|
parentId?: number | string | null
|
|
46
|
+
children?: number[]
|
|
46
47
|
_index?: number
|
|
47
48
|
_interactable?: boolean
|
|
48
49
|
_sliderLike?: boolean
|
|
@@ -60,6 +61,7 @@ interface ResolvedUiElementContext {
|
|
|
60
61
|
deviceId?: string
|
|
61
62
|
bounds: [number, number, number, number] | null
|
|
62
63
|
index: number
|
|
64
|
+
stable_id?: string | null
|
|
63
65
|
}
|
|
64
66
|
|
|
65
67
|
interface UiResolution {
|
|
@@ -73,6 +75,24 @@ interface UiChangeSignatureSet {
|
|
|
73
75
|
state: string | null
|
|
74
76
|
}
|
|
75
77
|
|
|
78
|
+
interface UiChangeScopeResolution {
|
|
79
|
+
scope: 'screen' | 'subtree'
|
|
80
|
+
target: string | null
|
|
81
|
+
resolved: boolean
|
|
82
|
+
resolvedIndex: number | null
|
|
83
|
+
resolvedStableId: string | null
|
|
84
|
+
reason: string
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
interface UiChangeScopeResult {
|
|
88
|
+
elements: UiElement[]
|
|
89
|
+
resolution: UiChangeScopeResolution
|
|
90
|
+
error?: {
|
|
91
|
+
code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
|
|
92
|
+
message: string
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
76
96
|
interface RankedResolutionCandidate {
|
|
77
97
|
el: UiElement
|
|
78
98
|
idx: number
|
|
@@ -201,11 +221,88 @@ export class ToolsInteract {
|
|
|
201
221
|
return null
|
|
202
222
|
}
|
|
203
223
|
|
|
224
|
+
private static _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
|
|
225
|
+
if (parentId === undefined || parentId === null) return null
|
|
226
|
+
|
|
227
|
+
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
228
|
+
return parentId
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (typeof parentId === 'string') {
|
|
232
|
+
const normalized = ToolsInteract._normalize(parentId)
|
|
233
|
+
if (!normalized) return null
|
|
234
|
+
|
|
235
|
+
if (/^\d+$/.test(normalized)) {
|
|
236
|
+
const index = Number(normalized)
|
|
237
|
+
if (index >= 0 && index < elements.length) return index
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const foundIndex = elements.findIndex((el) => {
|
|
241
|
+
if (!el) return false
|
|
242
|
+
return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
243
|
+
ToolsInteract._normalize(el.stable_id ?? '') === normalized
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
return foundIndex >= 0 ? foundIndex : null
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return null
|
|
250
|
+
}
|
|
251
|
+
|
|
204
252
|
private static _isVisibleElement(el: UiElement): boolean {
|
|
205
253
|
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
206
254
|
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
|
|
207
255
|
}
|
|
208
256
|
|
|
257
|
+
private static _isTapActionable(
|
|
258
|
+
el: UiElement,
|
|
259
|
+
storedStableId?: string | null,
|
|
260
|
+
platform?: 'android' | 'ios'
|
|
261
|
+
): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
|
|
262
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
263
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if (el.enabled === false) {
|
|
267
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const semanticTapActionable = !!el.semantic && (
|
|
271
|
+
el.semantic.is_clickable ||
|
|
272
|
+
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap'))
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
276
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (storedStableId) {
|
|
280
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
281
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
return { actionable: true }
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
private static _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
|
|
289
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
290
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
if (el.enabled === false) {
|
|
294
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
if (storedStableId) {
|
|
298
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
299
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
return { actionable: true }
|
|
304
|
+
}
|
|
305
|
+
|
|
209
306
|
private static _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
|
|
210
307
|
const identity = {
|
|
211
308
|
platform,
|
|
@@ -229,7 +326,8 @@ export class ToolsInteract {
|
|
|
229
326
|
platform,
|
|
230
327
|
deviceId,
|
|
231
328
|
bounds,
|
|
232
|
-
index
|
|
329
|
+
index,
|
|
330
|
+
stable_id: el.stable_id ?? null
|
|
233
331
|
})
|
|
234
332
|
|
|
235
333
|
return {
|
|
@@ -249,6 +347,235 @@ export class ToolsInteract {
|
|
|
249
347
|
}
|
|
250
348
|
}
|
|
251
349
|
|
|
350
|
+
private static _resolveUiChangeScope(
|
|
351
|
+
tree: any,
|
|
352
|
+
scope: 'screen' | 'subtree' | undefined,
|
|
353
|
+
target: string | null | undefined
|
|
354
|
+
): UiChangeScopeResult {
|
|
355
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
356
|
+
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen'
|
|
357
|
+
|
|
358
|
+
if (normalizedScope === 'screen') {
|
|
359
|
+
return {
|
|
360
|
+
elements,
|
|
361
|
+
resolution: {
|
|
362
|
+
scope: 'screen',
|
|
363
|
+
target: null,
|
|
364
|
+
resolved: true,
|
|
365
|
+
resolvedIndex: null,
|
|
366
|
+
resolvedStableId: null,
|
|
367
|
+
reason: 'screen scope'
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null
|
|
373
|
+
if (!requestedTarget) {
|
|
374
|
+
return {
|
|
375
|
+
elements: [],
|
|
376
|
+
resolution: {
|
|
377
|
+
scope: 'subtree',
|
|
378
|
+
target: null,
|
|
379
|
+
resolved: false,
|
|
380
|
+
resolvedIndex: null,
|
|
381
|
+
resolvedStableId: null,
|
|
382
|
+
reason: 'subtree scope requires a target element id'
|
|
383
|
+
},
|
|
384
|
+
error: {
|
|
385
|
+
code: 'INVALID_SCOPE',
|
|
386
|
+
message: 'scope=subtree requires a target element_id'
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const resolved = ToolsInteract._findScopedElement(tree, requestedTarget)
|
|
392
|
+
if (!resolved) {
|
|
393
|
+
return {
|
|
394
|
+
elements: [],
|
|
395
|
+
resolution: {
|
|
396
|
+
scope: 'subtree',
|
|
397
|
+
target: requestedTarget,
|
|
398
|
+
resolved: false,
|
|
399
|
+
resolvedIndex: null,
|
|
400
|
+
resolvedStableId: null,
|
|
401
|
+
reason: 'target element could not be resolved'
|
|
402
|
+
},
|
|
403
|
+
error: {
|
|
404
|
+
code: 'ELEMENT_NOT_FOUND',
|
|
405
|
+
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index)
|
|
411
|
+
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element): element is UiElement => !!element)
|
|
412
|
+
|
|
413
|
+
return {
|
|
414
|
+
elements: scopedElements,
|
|
415
|
+
resolution: {
|
|
416
|
+
scope: 'subtree',
|
|
417
|
+
target: requestedTarget,
|
|
418
|
+
resolved: true,
|
|
419
|
+
resolvedIndex: resolved.index,
|
|
420
|
+
resolvedStableId: resolved.stableId,
|
|
421
|
+
reason: resolved.reason
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
private static _findScopedElement(tree: any, targetElementId: string): { index: number, stableId: string | null, reason: string } | null {
|
|
427
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
428
|
+
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android'
|
|
429
|
+
const deviceId = tree?.device?.id ?? undefined
|
|
430
|
+
const normalizedTarget = ToolsInteract._normalize(targetElementId)
|
|
431
|
+
|
|
432
|
+
for (let i = 0; i < elements.length; i++) {
|
|
433
|
+
const el = elements[i]
|
|
434
|
+
if (!el) continue
|
|
435
|
+
|
|
436
|
+
const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i)
|
|
437
|
+
if (computedElementId === targetElementId) {
|
|
438
|
+
return {
|
|
439
|
+
index: i,
|
|
440
|
+
stableId: el.stable_id ?? null,
|
|
441
|
+
reason: 'element_id_match'
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
for (let i = 0; i < elements.length; i++) {
|
|
447
|
+
const el = elements[i]
|
|
448
|
+
if (!el) continue
|
|
449
|
+
|
|
450
|
+
if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
|
|
451
|
+
return {
|
|
452
|
+
index: i,
|
|
453
|
+
stableId: el.stable_id,
|
|
454
|
+
reason: 'stable_id_match'
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId)
|
|
460
|
+
if (storedContext?.stable_id) {
|
|
461
|
+
const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id)
|
|
462
|
+
for (let i = 0; i < elements.length; i++) {
|
|
463
|
+
const el = elements[i]
|
|
464
|
+
if (!el?.stable_id) continue
|
|
465
|
+
if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
|
|
466
|
+
return {
|
|
467
|
+
index: i,
|
|
468
|
+
stableId: el.stable_id,
|
|
469
|
+
reason: 'stored_stable_id_match'
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
return null
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private static _collectSubtreeIndices(elements: UiElement[], rootIndex: number): number[] {
|
|
479
|
+
if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length) return []
|
|
480
|
+
|
|
481
|
+
const visited = new Set<number>()
|
|
482
|
+
const stack = [rootIndex]
|
|
483
|
+
const result: number[] = []
|
|
484
|
+
|
|
485
|
+
while (stack.length > 0) {
|
|
486
|
+
const index = stack.pop()
|
|
487
|
+
if (index === undefined || visited.has(index) || index < 0 || index >= elements.length) continue
|
|
488
|
+
visited.add(index)
|
|
489
|
+
result.push(index)
|
|
490
|
+
|
|
491
|
+
const element = elements[index]
|
|
492
|
+
if (!element) continue
|
|
493
|
+
|
|
494
|
+
const directChildren = new Set<number>()
|
|
495
|
+
if (Array.isArray(element.children)) {
|
|
496
|
+
for (const childIndex of element.children) {
|
|
497
|
+
if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
|
|
498
|
+
directChildren.add(childIndex)
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
for (let i = 0; i < elements.length; i++) {
|
|
504
|
+
if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
|
|
505
|
+
directChildren.add(i)
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
for (const childIndex of directChildren) {
|
|
510
|
+
if (!visited.has(childIndex)) stack.push(childIndex)
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
return result.sort((left, right) => left - right)
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
private static _changeIdentityForElement(el: UiElement, index: number): string {
|
|
518
|
+
const stableId = ToolsInteract._normalize(el.stable_id)
|
|
519
|
+
if (stableId) return `stable:${stableId}`
|
|
520
|
+
|
|
521
|
+
return `fallback:${ToolsInteract._hash({
|
|
522
|
+
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
523
|
+
contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
|
|
524
|
+
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
525
|
+
type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
526
|
+
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
527
|
+
index
|
|
528
|
+
})}`
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
private static _summarizeUiChangeDelta(initialElements: UiElement[], currentElements: UiElement[]) {
|
|
532
|
+
const buildMap = (elements: UiElement[]) => {
|
|
533
|
+
const map = new Map<string, string>()
|
|
534
|
+
for (let i = 0; i < elements.length; i++) {
|
|
535
|
+
const element = elements[i]
|
|
536
|
+
if (!element) continue
|
|
537
|
+
const key = ToolsInteract._changeIdentityForElement(element, i)
|
|
538
|
+
map.set(key, ToolsInteract._hash({
|
|
539
|
+
text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
|
|
540
|
+
contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
|
|
541
|
+
resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
|
|
542
|
+
type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
|
|
543
|
+
bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
|
|
544
|
+
state: element.state ?? null,
|
|
545
|
+
visible: !!element.visible,
|
|
546
|
+
enabled: !!element.enabled,
|
|
547
|
+
clickable: !!element.clickable
|
|
548
|
+
}))
|
|
549
|
+
}
|
|
550
|
+
return map
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
const initialMap = buildMap(initialElements)
|
|
554
|
+
const currentMap = buildMap(currentElements)
|
|
555
|
+
let added = 0
|
|
556
|
+
let removed = 0
|
|
557
|
+
let mutated = 0
|
|
558
|
+
|
|
559
|
+
for (const [key, value] of currentMap.entries()) {
|
|
560
|
+
if (!initialMap.has(key)) {
|
|
561
|
+
added++
|
|
562
|
+
} else if (initialMap.get(key) !== value) {
|
|
563
|
+
mutated++
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
for (const key of initialMap.keys()) {
|
|
568
|
+
if (!currentMap.has(key)) removed++
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
return {
|
|
572
|
+
total_elements: currentElements.length,
|
|
573
|
+
added_elements: added,
|
|
574
|
+
removed_elements: removed,
|
|
575
|
+
mutated_elements: mutated
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
252
579
|
private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
|
|
253
580
|
if (ToolsInteract._resolvedUiElements.has(elementId)) {
|
|
254
581
|
ToolsInteract._resolvedUiElements.delete(elementId)
|
|
@@ -565,20 +892,11 @@ export class ToolsInteract {
|
|
|
565
892
|
let safety = 0
|
|
566
893
|
|
|
567
894
|
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
568
|
-
const
|
|
569
|
-
let parentIndex: number | null = null
|
|
570
|
-
|
|
571
|
-
if (typeof parentId === 'number') parentIndex = parentId
|
|
572
|
-
else if (typeof parentId === 'string' && /^\d+$/.test(parentId)) parentIndex = Number(parentId)
|
|
895
|
+
const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId)
|
|
573
896
|
|
|
574
897
|
if (parentIndex !== null && elements[parentIndex]) {
|
|
575
898
|
current = { el: elements[parentIndex], idx: parentIndex }
|
|
576
899
|
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
|
|
577
|
-
} else if (typeof parentId === 'string') {
|
|
578
|
-
const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId)
|
|
579
|
-
if (foundIndex === -1) break
|
|
580
|
-
current = { el: elements[foundIndex], idx: foundIndex }
|
|
581
|
-
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
|
|
582
900
|
} else {
|
|
583
901
|
break
|
|
584
902
|
}
|
|
@@ -718,12 +1036,16 @@ export class ToolsInteract {
|
|
|
718
1036
|
|
|
719
1037
|
const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
|
|
720
1038
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
1039
|
+
const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform)
|
|
1040
|
+
if (!tapActionability.actionable) {
|
|
1041
|
+
return ToolsInteract._actionFailure(
|
|
1042
|
+
actionType,
|
|
1043
|
+
selector,
|
|
1044
|
+
resolvedTarget,
|
|
1045
|
+
tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
|
|
1046
|
+
true,
|
|
1047
|
+
fingerprintBefore
|
|
1048
|
+
)
|
|
727
1049
|
}
|
|
728
1050
|
|
|
729
1051
|
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
|
|
@@ -779,6 +1101,7 @@ export class ToolsInteract {
|
|
|
779
1101
|
const sourcePlatform: 'android' | 'ios' = platform || 'android'
|
|
780
1102
|
let resolvedPlatform = sourcePlatform
|
|
781
1103
|
let resolvedDeviceId = deviceId
|
|
1104
|
+
const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null
|
|
782
1105
|
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
783
1106
|
let semanticFallbackElement: FindElementResponse['element'] | null = null
|
|
784
1107
|
const traceSteps: TraceStep[] = []
|
|
@@ -1062,6 +1385,21 @@ export class ToolsInteract {
|
|
|
1062
1385
|
resolvedTarget = resolved.resolvedTarget
|
|
1063
1386
|
const currentEl: UiElement = resolved.match.el
|
|
1064
1387
|
cachedResolvedMatch = resolved.match
|
|
1388
|
+
|
|
1389
|
+
const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id)
|
|
1390
|
+
if (!adjustableActionability.actionable) {
|
|
1391
|
+
return buildFailure(
|
|
1392
|
+
adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
|
|
1393
|
+
adjustableActionability.reason ?? 'adjustable control is not actionable',
|
|
1394
|
+
resolvedTarget,
|
|
1395
|
+
currentDevice,
|
|
1396
|
+
lastObservedState,
|
|
1397
|
+
attemptCount,
|
|
1398
|
+
lastAdjustmentMode,
|
|
1399
|
+
true
|
|
1400
|
+
)
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1065
1403
|
const bounds = ToolsInteract._normalizeBounds(currentEl.bounds)
|
|
1066
1404
|
const valueRange = currentEl.state?.value_range ?? null
|
|
1067
1405
|
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property)
|
|
@@ -2021,33 +2359,79 @@ export class ToolsInteract {
|
|
|
2021
2359
|
deviceId,
|
|
2022
2360
|
timeout_ms = 60000,
|
|
2023
2361
|
stability_window_ms = 300,
|
|
2024
|
-
expected_change
|
|
2362
|
+
expected_change,
|
|
2363
|
+
scope = 'screen',
|
|
2364
|
+
target = null
|
|
2025
2365
|
}: {
|
|
2026
2366
|
platform?: 'android' | 'ios',
|
|
2027
2367
|
deviceId?: string,
|
|
2028
2368
|
timeout_ms?: number,
|
|
2029
2369
|
stability_window_ms?: number,
|
|
2030
|
-
expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change'
|
|
2370
|
+
expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change',
|
|
2371
|
+
scope?: 'screen' | 'subtree',
|
|
2372
|
+
target?: string | null
|
|
2031
2373
|
}): Promise<WaitForUIChangeResponse> {
|
|
2032
2374
|
const start = Date.now()
|
|
2033
2375
|
const pollIntervalMs = 300
|
|
2034
2376
|
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
|
|
2035
2377
|
let baseline: UiChangeSignatureSet | null = null
|
|
2378
|
+
let baselineScope: UiChangeScopeResult | null = null
|
|
2036
2379
|
let lastObservedRevision: number | null = null
|
|
2037
2380
|
let lastLoadingState: any = null
|
|
2381
|
+
let lastSnapshotFreshnessMs: number | null = null
|
|
2038
2382
|
let candidateSignatures: UiChangeSignatureSet | null = null
|
|
2039
2383
|
let candidateObservedChange: 'hierarchy_diff' | 'text_change' | 'state_change' | null = null
|
|
2040
2384
|
let candidateSinceMs: number | null = null
|
|
2385
|
+
let lastChangeSummary: ReturnType<typeof ToolsInteract._summarizeUiChangeDelta> | null = null
|
|
2386
|
+
let lastScopeResolution: UiChangeScopeResolution = {
|
|
2387
|
+
scope: scope === 'subtree' ? 'subtree' : 'screen',
|
|
2388
|
+
target: target && typeof target === 'string' ? target : null,
|
|
2389
|
+
resolved: scope !== 'subtree',
|
|
2390
|
+
resolvedIndex: null,
|
|
2391
|
+
resolvedStableId: null,
|
|
2392
|
+
reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
|
|
2393
|
+
}
|
|
2041
2394
|
|
|
2042
2395
|
while (Date.now() - start < timeout_ms) {
|
|
2043
2396
|
try {
|
|
2044
2397
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
2045
|
-
const
|
|
2398
|
+
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
|
|
2399
|
+
if (scopedTree.error) {
|
|
2400
|
+
lastScopeResolution = scopedTree.resolution
|
|
2401
|
+
return {
|
|
2402
|
+
success: false,
|
|
2403
|
+
observed_change: null,
|
|
2404
|
+
snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
|
|
2405
|
+
snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
|
|
2406
|
+
timeout: true,
|
|
2407
|
+
elapsed_ms: Date.now() - start,
|
|
2408
|
+
expected_change,
|
|
2409
|
+
loading_state: tree?.loading_state ?? lastLoadingState ?? null,
|
|
2410
|
+
scope: scopedTree.resolution.scope,
|
|
2411
|
+
target: scopedTree.resolution.target,
|
|
2412
|
+
stability_state: 'transient',
|
|
2413
|
+
change_summary: lastChangeSummary,
|
|
2414
|
+
reason: scopedTree.error.message,
|
|
2415
|
+
error: scopedTree.error
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
const scopedElements = scopedTree.elements
|
|
2420
|
+
const scopedSignatureTree = {
|
|
2421
|
+
...tree,
|
|
2422
|
+
elements: scopedElements
|
|
2423
|
+
}
|
|
2424
|
+
const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree)
|
|
2046
2425
|
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
|
|
2047
2426
|
lastLoadingState = tree?.loading_state ?? lastLoadingState
|
|
2427
|
+
lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs
|
|
2428
|
+
lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary
|
|
2429
|
+
lastScopeResolution = scopedTree.resolution
|
|
2430
|
+
baselineScope = baselineScope ?? scopedTree
|
|
2048
2431
|
|
|
2049
2432
|
if (!baseline) {
|
|
2050
2433
|
baseline = signatures
|
|
2434
|
+
baselineScope = scopedTree
|
|
2051
2435
|
} else {
|
|
2052
2436
|
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
|
|
2053
2437
|
if (observedChange) {
|
|
@@ -2059,20 +2443,25 @@ export class ToolsInteract {
|
|
|
2059
2443
|
|
|
2060
2444
|
const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
|
|
2061
2445
|
if (stabilityWindow === 0 || stableForMs >= stabilityWindow) {
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2446
|
+
return {
|
|
2447
|
+
success: true,
|
|
2448
|
+
observed_change: candidateObservedChange ?? observedChange,
|
|
2449
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2450
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2451
|
+
timeout: false,
|
|
2452
|
+
elapsed_ms: Date.now() - start,
|
|
2453
|
+
expected_change,
|
|
2454
|
+
loading_state: lastLoadingState ?? null,
|
|
2455
|
+
scope: lastScopeResolution.scope,
|
|
2456
|
+
target: lastScopeResolution.target,
|
|
2457
|
+
stability_state: 'stable',
|
|
2458
|
+
change_summary: lastChangeSummary,
|
|
2459
|
+
reason: 'UI change observed'
|
|
2460
|
+
}
|
|
2071
2461
|
}
|
|
2072
|
-
}
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
candidateObservedChange = null
|
|
2462
|
+
} else {
|
|
2463
|
+
candidateSignatures = null
|
|
2464
|
+
candidateObservedChange = null
|
|
2076
2465
|
candidateSinceMs = null
|
|
2077
2466
|
}
|
|
2078
2467
|
}
|
|
@@ -2087,10 +2476,15 @@ export class ToolsInteract {
|
|
|
2087
2476
|
success: false,
|
|
2088
2477
|
observed_change: null,
|
|
2089
2478
|
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2479
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2090
2480
|
timeout: true,
|
|
2091
2481
|
elapsed_ms: Date.now() - start,
|
|
2092
2482
|
expected_change,
|
|
2093
2483
|
loading_state: lastLoadingState ?? null,
|
|
2484
|
+
scope: lastScopeResolution.scope,
|
|
2485
|
+
target: lastScopeResolution.target,
|
|
2486
|
+
stability_state: 'transient',
|
|
2487
|
+
change_summary: lastChangeSummary,
|
|
2094
2488
|
reason: 'timeout'
|
|
2095
2489
|
}
|
|
2096
2490
|
}
|
package/src/observe/index.ts
CHANGED
|
@@ -358,6 +358,7 @@ export class ToolsObserve {
|
|
|
358
358
|
|
|
359
359
|
raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision
|
|
360
360
|
raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms
|
|
361
|
+
raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? snapshotMetadata.snapshot_delta ?? null
|
|
361
362
|
raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state
|
|
362
363
|
|
|
363
364
|
const semantic = deriveSnapshotSemantic(raw)
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import crypto from 'crypto'
|
|
2
|
-
import type { GetUITreeResponse, LoadingState, UIElement } from '../types.js'
|
|
2
|
+
import type { GetUITreeResponse, LoadingState, SnapshotDelta, UIElement } from '../types.js'
|
|
3
3
|
|
|
4
4
|
interface SnapshotState {
|
|
5
5
|
revision: number
|
|
6
6
|
signature: string | null
|
|
7
|
+
elementSignatures: Map<string, string>
|
|
7
8
|
}
|
|
8
9
|
|
|
9
10
|
const snapshotStateByDevice = new Map<string, SnapshotState>()
|
|
@@ -38,6 +39,63 @@ function stableElementSignature(element: UIElement) {
|
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
41
|
|
|
42
|
+
function stableElementIdentity(element: UIElement, index: number) {
|
|
43
|
+
const stableId = normalize(element.stable_id)
|
|
44
|
+
if (stableId) return `stable:${stableId}`
|
|
45
|
+
|
|
46
|
+
return `fallback:${crypto.createHash('sha1').update(JSON.stringify({
|
|
47
|
+
text: normalize(element.text),
|
|
48
|
+
contentDescription: normalize(element.contentDescription),
|
|
49
|
+
resourceId: normalize(element.resourceId),
|
|
50
|
+
type: normalize(element.type),
|
|
51
|
+
bounds: normalizeBounds(element.bounds),
|
|
52
|
+
index
|
|
53
|
+
})).digest('hex')}`
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function buildElementSignatures(tree: Pick<GetUITreeResponse, 'elements'> | null | undefined) {
|
|
57
|
+
const signatures = new Map<string, string>()
|
|
58
|
+
const elements = Array.isArray(tree?.elements) ? tree!.elements! : []
|
|
59
|
+
|
|
60
|
+
for (let index = 0; index < elements.length; index++) {
|
|
61
|
+
const element = elements[index]
|
|
62
|
+
if (!element) continue
|
|
63
|
+
const identity = stableElementIdentity(element, index)
|
|
64
|
+
signatures.set(identity, crypto.createHash('sha1').update(JSON.stringify(stableElementSignature(element))).digest('hex'))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return signatures
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function summarizeSnapshotDelta(previous: SnapshotState | undefined, currentElements: Map<string, string>): SnapshotDelta | null {
|
|
71
|
+
if (!previous) return null
|
|
72
|
+
|
|
73
|
+
let added = 0
|
|
74
|
+
let removed = 0
|
|
75
|
+
let mutated = 0
|
|
76
|
+
|
|
77
|
+
for (const [identity, signature] of currentElements.entries()) {
|
|
78
|
+
const previousSignature = previous.elementSignatures.get(identity)
|
|
79
|
+
if (previousSignature === undefined) {
|
|
80
|
+
added++
|
|
81
|
+
} else if (previousSignature !== signature) {
|
|
82
|
+
mutated++
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
for (const identity of previous.elementSignatures.keys()) {
|
|
87
|
+
if (!currentElements.has(identity)) removed++
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
previous_snapshot_revision: previous.revision,
|
|
92
|
+
added_elements: added,
|
|
93
|
+
removed_elements: removed,
|
|
94
|
+
mutated_elements: mutated,
|
|
95
|
+
total_elements: currentElements.size
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
41
99
|
export function computeSnapshotSignature(tree: Pick<GetUITreeResponse, 'elements' | 'screen' | 'resolution' | 'error'> | null | undefined): string | null {
|
|
42
100
|
if (!tree || tree.error) return null
|
|
43
101
|
|
|
@@ -83,6 +141,10 @@ export function deriveSnapshotMetadata(
|
|
|
83
141
|
) {
|
|
84
142
|
const signature = signatureOverride ?? computeSnapshotSignature(tree)
|
|
85
143
|
const previous = snapshotStateByDevice.get(deviceKey)
|
|
144
|
+
const hasValidTree = !!tree && !tree.error
|
|
145
|
+
const currentElementSignatures = hasValidTree
|
|
146
|
+
? buildElementSignatures(tree)
|
|
147
|
+
: previous?.elementSignatures ?? new Map<string, string>()
|
|
86
148
|
|
|
87
149
|
let revision = 1
|
|
88
150
|
if (previous) {
|
|
@@ -93,11 +155,16 @@ export function deriveSnapshotMetadata(
|
|
|
93
155
|
}
|
|
94
156
|
}
|
|
95
157
|
|
|
96
|
-
snapshotStateByDevice.set(deviceKey, {
|
|
158
|
+
snapshotStateByDevice.set(deviceKey, {
|
|
159
|
+
revision,
|
|
160
|
+
signature,
|
|
161
|
+
elementSignatures: currentElementSignatures
|
|
162
|
+
})
|
|
97
163
|
|
|
98
164
|
return {
|
|
99
165
|
snapshot_revision: revision,
|
|
100
166
|
captured_at_ms: Date.now(),
|
|
167
|
+
snapshot_delta: hasValidTree ? summarizeSnapshotDelta(previous, currentElementSignatures) : null,
|
|
101
168
|
loading_state: detectLoadingState(tree, source)
|
|
102
169
|
}
|
|
103
170
|
}
|