mobile-debug-mcp 0.29.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -0
- package/README.md +44 -21
- package/dist/interact/index.js +359 -46
- package/dist/observe/index.js +1 -0
- package/dist/observe/snapshot-metadata.js +62 -1
- package/dist/server/tool-definitions.js +8 -3
- package/dist/server/tool-handlers.js +4 -2
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +11 -0
- package/docs/ROADMAP.md +18 -7
- package/docs/rfcs/013-wait-and-synchronization-reliability.md +870 -0
- package/docs/rfcs/014-actionability-resolution.md +394 -0
- package/docs/specs/mcp-tooling-spec-v1.md +28 -0
- package/docs/tools/interact.md +6 -0
- package/package.json +1 -1
- package/src/interact/index.ts +444 -45
- package/src/observe/index.ts +1 -0
- package/src/observe/snapshot-metadata.ts +69 -2
- package/src/server/tool-definitions.ts +8 -3
- package/src/server/tool-handlers.ts +4 -2
- package/src/server-core.ts +1 -1
- package/src/types.ts +24 -0
- package/test/unit/interact/adjust_control.test.ts +104 -0
- package/test/unit/interact/subtree_collection.test.ts +24 -0
- package/test/unit/interact/tap_element.test.ts +71 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +189 -45
- package/test/unit/observe/snapshot_metadata.test.ts +67 -0
package/src/interact/index.ts
CHANGED
|
@@ -43,6 +43,7 @@ interface UiElement {
|
|
|
43
43
|
focusable?: boolean
|
|
44
44
|
visible?: boolean
|
|
45
45
|
parentId?: number | string | null
|
|
46
|
+
children?: number[]
|
|
46
47
|
_index?: number
|
|
47
48
|
_interactable?: boolean
|
|
48
49
|
_sliderLike?: boolean
|
|
@@ -60,6 +61,7 @@ interface ResolvedUiElementContext {
|
|
|
60
61
|
deviceId?: string
|
|
61
62
|
bounds: [number, number, number, number] | null
|
|
62
63
|
index: number
|
|
64
|
+
stable_id?: string | null
|
|
63
65
|
}
|
|
64
66
|
|
|
65
67
|
interface UiResolution {
|
|
@@ -73,6 +75,24 @@ interface UiChangeSignatureSet {
|
|
|
73
75
|
state: string | null
|
|
74
76
|
}
|
|
75
77
|
|
|
78
|
+
interface UiChangeScopeResolution {
|
|
79
|
+
scope: 'screen' | 'subtree'
|
|
80
|
+
target: string | null
|
|
81
|
+
resolved: boolean
|
|
82
|
+
resolvedIndex: number | null
|
|
83
|
+
resolvedStableId: string | null
|
|
84
|
+
reason: string
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
interface UiChangeScopeResult {
|
|
88
|
+
elements: UiElement[]
|
|
89
|
+
resolution: UiChangeScopeResolution
|
|
90
|
+
error?: {
|
|
91
|
+
code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
|
|
92
|
+
message: string
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
76
96
|
interface RankedResolutionCandidate {
|
|
77
97
|
el: UiElement
|
|
78
98
|
idx: number
|
|
@@ -201,11 +221,88 @@ export class ToolsInteract {
|
|
|
201
221
|
return null
|
|
202
222
|
}
|
|
203
223
|
|
|
224
|
+
private static _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
|
|
225
|
+
if (parentId === undefined || parentId === null) return null
|
|
226
|
+
|
|
227
|
+
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
228
|
+
return parentId
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (typeof parentId === 'string') {
|
|
232
|
+
const normalized = ToolsInteract._normalize(parentId)
|
|
233
|
+
if (!normalized) return null
|
|
234
|
+
|
|
235
|
+
if (/^\d+$/.test(normalized)) {
|
|
236
|
+
const index = Number(normalized)
|
|
237
|
+
if (index >= 0 && index < elements.length) return index
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const foundIndex = elements.findIndex((el) => {
|
|
241
|
+
if (!el) return false
|
|
242
|
+
return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
243
|
+
ToolsInteract._normalize(el.stable_id ?? '') === normalized
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
return foundIndex >= 0 ? foundIndex : null
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return null
|
|
250
|
+
}
|
|
251
|
+
|
|
204
252
|
private static _isVisibleElement(el: UiElement): boolean {
|
|
205
253
|
const bounds = ToolsInteract._normalizeBounds(el.bounds)
|
|
206
254
|
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
|
|
207
255
|
}
|
|
208
256
|
|
|
257
|
+
private static _isTapActionable(
|
|
258
|
+
el: UiElement,
|
|
259
|
+
storedStableId?: string | null,
|
|
260
|
+
platform?: 'android' | 'ios'
|
|
261
|
+
): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
|
|
262
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
263
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if (el.enabled === false) {
|
|
267
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const semanticTapActionable = !!el.semantic && (
|
|
271
|
+
el.semantic.is_clickable ||
|
|
272
|
+
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap'))
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
276
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (storedStableId) {
|
|
280
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
281
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
return { actionable: true }
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
private static _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
|
|
289
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
290
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
if (el.enabled === false) {
|
|
294
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
if (storedStableId) {
|
|
298
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
299
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
return { actionable: true }
|
|
304
|
+
}
|
|
305
|
+
|
|
209
306
|
private static _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
|
|
210
307
|
const identity = {
|
|
211
308
|
platform,
|
|
@@ -229,7 +326,8 @@ export class ToolsInteract {
|
|
|
229
326
|
platform,
|
|
230
327
|
deviceId,
|
|
231
328
|
bounds,
|
|
232
|
-
index
|
|
329
|
+
index,
|
|
330
|
+
stable_id: el.stable_id ?? null
|
|
233
331
|
})
|
|
234
332
|
|
|
235
333
|
return {
|
|
@@ -249,6 +347,235 @@ export class ToolsInteract {
|
|
|
249
347
|
}
|
|
250
348
|
}
|
|
251
349
|
|
|
350
|
+
private static _resolveUiChangeScope(
|
|
351
|
+
tree: any,
|
|
352
|
+
scope: 'screen' | 'subtree' | undefined,
|
|
353
|
+
target: string | null | undefined
|
|
354
|
+
): UiChangeScopeResult {
|
|
355
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
356
|
+
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen'
|
|
357
|
+
|
|
358
|
+
if (normalizedScope === 'screen') {
|
|
359
|
+
return {
|
|
360
|
+
elements,
|
|
361
|
+
resolution: {
|
|
362
|
+
scope: 'screen',
|
|
363
|
+
target: null,
|
|
364
|
+
resolved: true,
|
|
365
|
+
resolvedIndex: null,
|
|
366
|
+
resolvedStableId: null,
|
|
367
|
+
reason: 'screen scope'
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null
|
|
373
|
+
if (!requestedTarget) {
|
|
374
|
+
return {
|
|
375
|
+
elements: [],
|
|
376
|
+
resolution: {
|
|
377
|
+
scope: 'subtree',
|
|
378
|
+
target: null,
|
|
379
|
+
resolved: false,
|
|
380
|
+
resolvedIndex: null,
|
|
381
|
+
resolvedStableId: null,
|
|
382
|
+
reason: 'subtree scope requires a target element id'
|
|
383
|
+
},
|
|
384
|
+
error: {
|
|
385
|
+
code: 'INVALID_SCOPE',
|
|
386
|
+
message: 'scope=subtree requires a target element_id'
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const resolved = ToolsInteract._findScopedElement(tree, requestedTarget)
|
|
392
|
+
if (!resolved) {
|
|
393
|
+
return {
|
|
394
|
+
elements: [],
|
|
395
|
+
resolution: {
|
|
396
|
+
scope: 'subtree',
|
|
397
|
+
target: requestedTarget,
|
|
398
|
+
resolved: false,
|
|
399
|
+
resolvedIndex: null,
|
|
400
|
+
resolvedStableId: null,
|
|
401
|
+
reason: 'target element could not be resolved'
|
|
402
|
+
},
|
|
403
|
+
error: {
|
|
404
|
+
code: 'ELEMENT_NOT_FOUND',
|
|
405
|
+
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index)
|
|
411
|
+
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element): element is UiElement => !!element)
|
|
412
|
+
|
|
413
|
+
return {
|
|
414
|
+
elements: scopedElements,
|
|
415
|
+
resolution: {
|
|
416
|
+
scope: 'subtree',
|
|
417
|
+
target: requestedTarget,
|
|
418
|
+
resolved: true,
|
|
419
|
+
resolvedIndex: resolved.index,
|
|
420
|
+
resolvedStableId: resolved.stableId,
|
|
421
|
+
reason: resolved.reason
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
private static _findScopedElement(tree: any, targetElementId: string): { index: number, stableId: string | null, reason: string } | null {
|
|
427
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
428
|
+
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android'
|
|
429
|
+
const deviceId = tree?.device?.id ?? undefined
|
|
430
|
+
const normalizedTarget = ToolsInteract._normalize(targetElementId)
|
|
431
|
+
|
|
432
|
+
for (let i = 0; i < elements.length; i++) {
|
|
433
|
+
const el = elements[i]
|
|
434
|
+
if (!el) continue
|
|
435
|
+
|
|
436
|
+
const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i)
|
|
437
|
+
if (computedElementId === targetElementId) {
|
|
438
|
+
return {
|
|
439
|
+
index: i,
|
|
440
|
+
stableId: el.stable_id ?? null,
|
|
441
|
+
reason: 'element_id_match'
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
for (let i = 0; i < elements.length; i++) {
|
|
447
|
+
const el = elements[i]
|
|
448
|
+
if (!el) continue
|
|
449
|
+
|
|
450
|
+
if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
|
|
451
|
+
return {
|
|
452
|
+
index: i,
|
|
453
|
+
stableId: el.stable_id,
|
|
454
|
+
reason: 'stable_id_match'
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId)
|
|
460
|
+
if (storedContext?.stable_id) {
|
|
461
|
+
const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id)
|
|
462
|
+
for (let i = 0; i < elements.length; i++) {
|
|
463
|
+
const el = elements[i]
|
|
464
|
+
if (!el?.stable_id) continue
|
|
465
|
+
if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
|
|
466
|
+
return {
|
|
467
|
+
index: i,
|
|
468
|
+
stableId: el.stable_id,
|
|
469
|
+
reason: 'stored_stable_id_match'
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
return null
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private static _collectSubtreeIndices(elements: UiElement[], rootIndex: number): number[] {
|
|
479
|
+
if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length) return []
|
|
480
|
+
|
|
481
|
+
const visited = new Set<number>()
|
|
482
|
+
const stack = [rootIndex]
|
|
483
|
+
const result: number[] = []
|
|
484
|
+
|
|
485
|
+
while (stack.length > 0) {
|
|
486
|
+
const index = stack.pop()
|
|
487
|
+
if (index === undefined || visited.has(index) || index < 0 || index >= elements.length) continue
|
|
488
|
+
visited.add(index)
|
|
489
|
+
result.push(index)
|
|
490
|
+
|
|
491
|
+
const element = elements[index]
|
|
492
|
+
if (!element) continue
|
|
493
|
+
|
|
494
|
+
const directChildren = new Set<number>()
|
|
495
|
+
if (Array.isArray(element.children)) {
|
|
496
|
+
for (const childIndex of element.children) {
|
|
497
|
+
if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
|
|
498
|
+
directChildren.add(childIndex)
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
for (let i = 0; i < elements.length; i++) {
|
|
504
|
+
if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
|
|
505
|
+
directChildren.add(i)
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
for (const childIndex of directChildren) {
|
|
510
|
+
if (!visited.has(childIndex)) stack.push(childIndex)
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
return result.sort((left, right) => left - right)
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
private static _changeIdentityForElement(el: UiElement, index: number): string {
|
|
518
|
+
const stableId = ToolsInteract._normalize(el.stable_id)
|
|
519
|
+
if (stableId) return `stable:${stableId}`
|
|
520
|
+
|
|
521
|
+
return `fallback:${ToolsInteract._hash({
|
|
522
|
+
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
523
|
+
contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
|
|
524
|
+
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
525
|
+
type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
526
|
+
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
527
|
+
index
|
|
528
|
+
})}`
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
private static _summarizeUiChangeDelta(initialElements: UiElement[], currentElements: UiElement[]) {
|
|
532
|
+
const buildMap = (elements: UiElement[]) => {
|
|
533
|
+
const map = new Map<string, string>()
|
|
534
|
+
for (let i = 0; i < elements.length; i++) {
|
|
535
|
+
const element = elements[i]
|
|
536
|
+
if (!element) continue
|
|
537
|
+
const key = ToolsInteract._changeIdentityForElement(element, i)
|
|
538
|
+
map.set(key, ToolsInteract._hash({
|
|
539
|
+
text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
|
|
540
|
+
contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
|
|
541
|
+
resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
|
|
542
|
+
type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
|
|
543
|
+
bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
|
|
544
|
+
state: element.state ?? null,
|
|
545
|
+
visible: !!element.visible,
|
|
546
|
+
enabled: !!element.enabled,
|
|
547
|
+
clickable: !!element.clickable
|
|
548
|
+
}))
|
|
549
|
+
}
|
|
550
|
+
return map
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
const initialMap = buildMap(initialElements)
|
|
554
|
+
const currentMap = buildMap(currentElements)
|
|
555
|
+
let added = 0
|
|
556
|
+
let removed = 0
|
|
557
|
+
let mutated = 0
|
|
558
|
+
|
|
559
|
+
for (const [key, value] of currentMap.entries()) {
|
|
560
|
+
if (!initialMap.has(key)) {
|
|
561
|
+
added++
|
|
562
|
+
} else if (initialMap.get(key) !== value) {
|
|
563
|
+
mutated++
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
for (const key of initialMap.keys()) {
|
|
568
|
+
if (!currentMap.has(key)) removed++
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
return {
|
|
572
|
+
total_elements: currentElements.length,
|
|
573
|
+
added_elements: added,
|
|
574
|
+
removed_elements: removed,
|
|
575
|
+
mutated_elements: mutated
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
252
579
|
private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
|
|
253
580
|
if (ToolsInteract._resolvedUiElements.has(elementId)) {
|
|
254
581
|
ToolsInteract._resolvedUiElements.delete(elementId)
|
|
@@ -332,6 +659,10 @@ export class ToolsInteract {
|
|
|
332
659
|
return null
|
|
333
660
|
}
|
|
334
661
|
|
|
662
|
+
private static _uiChangeSignaturesEqual(left: UiChangeSignatureSet, right: UiChangeSignatureSet): boolean {
|
|
663
|
+
return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state
|
|
664
|
+
}
|
|
665
|
+
|
|
335
666
|
private static _resolvedTargetFromElement(
|
|
336
667
|
elementId: string,
|
|
337
668
|
element: UiElement,
|
|
@@ -561,20 +892,11 @@ export class ToolsInteract {
|
|
|
561
892
|
let safety = 0
|
|
562
893
|
|
|
563
894
|
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
564
|
-
const
|
|
565
|
-
let parentIndex: number | null = null
|
|
566
|
-
|
|
567
|
-
if (typeof parentId === 'number') parentIndex = parentId
|
|
568
|
-
else if (typeof parentId === 'string' && /^\d+$/.test(parentId)) parentIndex = Number(parentId)
|
|
895
|
+
const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId)
|
|
569
896
|
|
|
570
897
|
if (parentIndex !== null && elements[parentIndex]) {
|
|
571
898
|
current = { el: elements[parentIndex], idx: parentIndex }
|
|
572
899
|
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
|
|
573
|
-
} else if (typeof parentId === 'string') {
|
|
574
|
-
const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId)
|
|
575
|
-
if (foundIndex === -1) break
|
|
576
|
-
current = { el: elements[foundIndex], idx: foundIndex }
|
|
577
|
-
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
|
|
578
900
|
} else {
|
|
579
901
|
break
|
|
580
902
|
}
|
|
@@ -714,12 +1036,16 @@ export class ToolsInteract {
|
|
|
714
1036
|
|
|
715
1037
|
const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
|
|
716
1038
|
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
1039
|
+
const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform)
|
|
1040
|
+
if (!tapActionability.actionable) {
|
|
1041
|
+
return ToolsInteract._actionFailure(
|
|
1042
|
+
actionType,
|
|
1043
|
+
selector,
|
|
1044
|
+
resolvedTarget,
|
|
1045
|
+
tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
|
|
1046
|
+
true,
|
|
1047
|
+
fingerprintBefore
|
|
1048
|
+
)
|
|
723
1049
|
}
|
|
724
1050
|
|
|
725
1051
|
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
|
|
@@ -775,6 +1101,7 @@ export class ToolsInteract {
|
|
|
775
1101
|
const sourcePlatform: 'android' | 'ios' = platform || 'android'
|
|
776
1102
|
let resolvedPlatform = sourcePlatform
|
|
777
1103
|
let resolvedDeviceId = deviceId
|
|
1104
|
+
const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null
|
|
778
1105
|
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
779
1106
|
let semanticFallbackElement: FindElementResponse['element'] | null = null
|
|
780
1107
|
const traceSteps: TraceStep[] = []
|
|
@@ -1058,6 +1385,21 @@ export class ToolsInteract {
|
|
|
1058
1385
|
resolvedTarget = resolved.resolvedTarget
|
|
1059
1386
|
const currentEl: UiElement = resolved.match.el
|
|
1060
1387
|
cachedResolvedMatch = resolved.match
|
|
1388
|
+
|
|
1389
|
+
const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id)
|
|
1390
|
+
if (!adjustableActionability.actionable) {
|
|
1391
|
+
return buildFailure(
|
|
1392
|
+
adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
|
|
1393
|
+
adjustableActionability.reason ?? 'adjustable control is not actionable',
|
|
1394
|
+
resolvedTarget,
|
|
1395
|
+
currentDevice,
|
|
1396
|
+
lastObservedState,
|
|
1397
|
+
attemptCount,
|
|
1398
|
+
lastAdjustmentMode,
|
|
1399
|
+
true
|
|
1400
|
+
)
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1061
1403
|
const bounds = ToolsInteract._normalizeBounds(currentEl.bounds)
|
|
1062
1404
|
const valueRange = currentEl.state?.value_range ?? null
|
|
1063
1405
|
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property)
|
|
@@ -2016,59 +2358,111 @@ export class ToolsInteract {
|
|
|
2016
2358
|
platform,
|
|
2017
2359
|
deviceId,
|
|
2018
2360
|
timeout_ms = 60000,
|
|
2019
|
-
stability_window_ms =
|
|
2020
|
-
expected_change
|
|
2361
|
+
stability_window_ms = 300,
|
|
2362
|
+
expected_change,
|
|
2363
|
+
scope = 'screen',
|
|
2364
|
+
target = null
|
|
2021
2365
|
}: {
|
|
2022
2366
|
platform?: 'android' | 'ios',
|
|
2023
2367
|
deviceId?: string,
|
|
2024
2368
|
timeout_ms?: number,
|
|
2025
2369
|
stability_window_ms?: number,
|
|
2026
|
-
expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change'
|
|
2370
|
+
expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change',
|
|
2371
|
+
scope?: 'screen' | 'subtree',
|
|
2372
|
+
target?: string | null
|
|
2027
2373
|
}): Promise<WaitForUIChangeResponse> {
|
|
2028
2374
|
const start = Date.now()
|
|
2029
2375
|
const pollIntervalMs = 300
|
|
2030
|
-
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms :
|
|
2376
|
+
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
|
|
2031
2377
|
let baseline: UiChangeSignatureSet | null = null
|
|
2378
|
+
let baselineScope: UiChangeScopeResult | null = null
|
|
2032
2379
|
let lastObservedRevision: number | null = null
|
|
2033
2380
|
let lastLoadingState: any = null
|
|
2381
|
+
let lastSnapshotFreshnessMs: number | null = null
|
|
2382
|
+
let candidateSignatures: UiChangeSignatureSet | null = null
|
|
2383
|
+
let candidateObservedChange: 'hierarchy_diff' | 'text_change' | 'state_change' | null = null
|
|
2384
|
+
let candidateSinceMs: number | null = null
|
|
2385
|
+
let lastChangeSummary: ReturnType<typeof ToolsInteract._summarizeUiChangeDelta> | null = null
|
|
2386
|
+
let lastScopeResolution: UiChangeScopeResolution = {
|
|
2387
|
+
scope: scope === 'subtree' ? 'subtree' : 'screen',
|
|
2388
|
+
target: target && typeof target === 'string' ? target : null,
|
|
2389
|
+
resolved: scope !== 'subtree',
|
|
2390
|
+
resolvedIndex: null,
|
|
2391
|
+
resolvedStableId: null,
|
|
2392
|
+
reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
|
|
2393
|
+
}
|
|
2034
2394
|
|
|
2035
2395
|
while (Date.now() - start < timeout_ms) {
|
|
2036
2396
|
try {
|
|
2037
2397
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
2038
|
-
const
|
|
2398
|
+
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
|
|
2399
|
+
if (scopedTree.error) {
|
|
2400
|
+
lastScopeResolution = scopedTree.resolution
|
|
2401
|
+
return {
|
|
2402
|
+
success: false,
|
|
2403
|
+
observed_change: null,
|
|
2404
|
+
snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
|
|
2405
|
+
snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
|
|
2406
|
+
timeout: true,
|
|
2407
|
+
elapsed_ms: Date.now() - start,
|
|
2408
|
+
expected_change,
|
|
2409
|
+
loading_state: tree?.loading_state ?? lastLoadingState ?? null,
|
|
2410
|
+
scope: scopedTree.resolution.scope,
|
|
2411
|
+
target: scopedTree.resolution.target,
|
|
2412
|
+
stability_state: 'transient',
|
|
2413
|
+
change_summary: lastChangeSummary,
|
|
2414
|
+
reason: scopedTree.error.message,
|
|
2415
|
+
error: scopedTree.error
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
const scopedElements = scopedTree.elements
|
|
2420
|
+
const scopedSignatureTree = {
|
|
2421
|
+
...tree,
|
|
2422
|
+
elements: scopedElements
|
|
2423
|
+
}
|
|
2424
|
+
const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree)
|
|
2039
2425
|
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
|
|
2040
2426
|
lastLoadingState = tree?.loading_state ?? lastLoadingState
|
|
2427
|
+
lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs
|
|
2428
|
+
lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary
|
|
2429
|
+
lastScopeResolution = scopedTree.resolution
|
|
2430
|
+
baselineScope = baselineScope ?? scopedTree
|
|
2041
2431
|
|
|
2042
2432
|
if (!baseline) {
|
|
2043
2433
|
baseline = signatures
|
|
2434
|
+
baselineScope = scopedTree
|
|
2044
2435
|
} else {
|
|
2045
2436
|
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
|
|
2046
2437
|
if (observedChange) {
|
|
2047
|
-
if (
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
const confirmChange = ToolsInteract._matchesUiChange(expected_change, baseline, confirmSignatures)
|
|
2052
|
-
if (!confirmChange || confirmSignatures.hierarchy !== signatures.hierarchy || confirmSignatures.text !== signatures.text || confirmSignatures.state !== signatures.state) {
|
|
2053
|
-
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision
|
|
2054
|
-
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState
|
|
2055
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
2056
|
-
continue
|
|
2057
|
-
}
|
|
2058
|
-
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision
|
|
2059
|
-
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState
|
|
2438
|
+
if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
|
|
2439
|
+
candidateSignatures = signatures
|
|
2440
|
+
candidateObservedChange = observedChange
|
|
2441
|
+
candidateSinceMs = Date.now()
|
|
2060
2442
|
}
|
|
2061
2443
|
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2444
|
+
const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
|
|
2445
|
+
if (stabilityWindow === 0 || stableForMs >= stabilityWindow) {
|
|
2446
|
+
return {
|
|
2447
|
+
success: true,
|
|
2448
|
+
observed_change: candidateObservedChange ?? observedChange,
|
|
2449
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2450
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2451
|
+
timeout: false,
|
|
2452
|
+
elapsed_ms: Date.now() - start,
|
|
2453
|
+
expected_change,
|
|
2454
|
+
loading_state: lastLoadingState ?? null,
|
|
2455
|
+
scope: lastScopeResolution.scope,
|
|
2456
|
+
target: lastScopeResolution.target,
|
|
2457
|
+
stability_state: 'stable',
|
|
2458
|
+
change_summary: lastChangeSummary,
|
|
2459
|
+
reason: 'UI change observed'
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
} else {
|
|
2463
|
+
candidateSignatures = null
|
|
2464
|
+
candidateObservedChange = null
|
|
2465
|
+
candidateSinceMs = null
|
|
2072
2466
|
}
|
|
2073
2467
|
}
|
|
2074
2468
|
} catch {
|
|
@@ -2082,10 +2476,15 @@ export class ToolsInteract {
|
|
|
2082
2476
|
success: false,
|
|
2083
2477
|
observed_change: null,
|
|
2084
2478
|
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2479
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2085
2480
|
timeout: true,
|
|
2086
2481
|
elapsed_ms: Date.now() - start,
|
|
2087
2482
|
expected_change,
|
|
2088
2483
|
loading_state: lastLoadingState ?? null,
|
|
2484
|
+
scope: lastScopeResolution.scope,
|
|
2485
|
+
target: lastScopeResolution.target,
|
|
2486
|
+
stability_state: 'transient',
|
|
2487
|
+
change_summary: lastChangeSummary,
|
|
2089
2488
|
reason: 'timeout'
|
|
2090
2489
|
}
|
|
2091
2490
|
}
|
package/src/observe/index.ts
CHANGED
|
@@ -358,6 +358,7 @@ export class ToolsObserve {
|
|
|
358
358
|
|
|
359
359
|
raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision
|
|
360
360
|
raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms
|
|
361
|
+
raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? snapshotMetadata.snapshot_delta ?? null
|
|
361
362
|
raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state
|
|
362
363
|
|
|
363
364
|
const semantic = deriveSnapshotSemantic(raw)
|