mobile-debug-mcp 0.30.0 → 0.30.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -43,6 +43,7 @@ interface UiElement {
43
43
  focusable?: boolean
44
44
  visible?: boolean
45
45
  parentId?: number | string | null
46
+ children?: number[]
46
47
  _index?: number
47
48
  _interactable?: boolean
48
49
  _sliderLike?: boolean
@@ -60,6 +61,7 @@ interface ResolvedUiElementContext {
60
61
  deviceId?: string
61
62
  bounds: [number, number, number, number] | null
62
63
  index: number
64
+ stable_id?: string | null
63
65
  }
64
66
 
65
67
  interface UiResolution {
@@ -73,6 +75,24 @@ interface UiChangeSignatureSet {
73
75
  state: string | null
74
76
  }
75
77
 
78
+ interface UiChangeScopeResolution {
79
+ scope: 'screen' | 'subtree'
80
+ target: string | null
81
+ resolved: boolean
82
+ resolvedIndex: number | null
83
+ resolvedStableId: string | null
84
+ reason: string
85
+ }
86
+
87
+ interface UiChangeScopeResult {
88
+ elements: UiElement[]
89
+ resolution: UiChangeScopeResolution
90
+ error?: {
91
+ code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND'
92
+ message: string
93
+ }
94
+ }
95
+
76
96
  interface RankedResolutionCandidate {
77
97
  el: UiElement
78
98
  idx: number
@@ -201,11 +221,88 @@ export class ToolsInteract {
201
221
  return null
202
222
  }
203
223
 
224
+ private static _resolveParentIndex(elements: UiElement[], parentId: number | string | null | undefined): number | null {
225
+ if (parentId === undefined || parentId === null) return null
226
+
227
+ if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
228
+ return parentId
229
+ }
230
+
231
+ if (typeof parentId === 'string') {
232
+ const normalized = ToolsInteract._normalize(parentId)
233
+ if (!normalized) return null
234
+
235
+ if (/^\d+$/.test(normalized)) {
236
+ const index = Number(normalized)
237
+ if (index >= 0 && index < elements.length) return index
238
+ }
239
+
240
+ const foundIndex = elements.findIndex((el) => {
241
+ if (!el) return false
242
+ return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
243
+ ToolsInteract._normalize(el.stable_id ?? '') === normalized
244
+ })
245
+
246
+ return foundIndex >= 0 ? foundIndex : null
247
+ }
248
+
249
+ return null
250
+ }
251
+
204
252
  private static _isVisibleElement(el: UiElement): boolean {
205
253
  const bounds = ToolsInteract._normalizeBounds(el.bounds)
206
254
  return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1]
207
255
  }
208
256
 
257
+ private static _isTapActionable(
258
+ el: UiElement,
259
+ storedStableId?: string | null,
260
+ platform?: 'android' | 'ios'
261
+ ): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
262
+ if (!ToolsInteract._isVisibleElement(el)) {
263
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
264
+ }
265
+
266
+ if (el.enabled === false) {
267
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
268
+ }
269
+
270
+ const semanticTapActionable = !!el.semantic && (
271
+ el.semantic.is_clickable ||
272
+ (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap'))
273
+ )
274
+
275
+ if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
276
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' }
277
+ }
278
+
279
+ if (storedStableId) {
280
+ if (!el.stable_id || el.stable_id !== storedStableId) {
281
+ return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
282
+ }
283
+ }
284
+
285
+ return { actionable: true }
286
+ }
287
+
288
+ private static _isAdjustableActionable(el: UiElement, storedStableId?: string | null): { actionable: boolean, failureCode?: ActionFailureCode, reason?: string } {
289
+ if (!ToolsInteract._isVisibleElement(el)) {
290
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' }
291
+ }
292
+
293
+ if (el.enabled === false) {
294
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' }
295
+ }
296
+
297
+ if (storedStableId) {
298
+ if (!el.stable_id || el.stable_id !== storedStableId) {
299
+ return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' }
300
+ }
301
+ }
302
+
303
+ return { actionable: true }
304
+ }
305
+
209
306
  private static _computeElementId(platform: 'android' | 'ios', deviceId: string | undefined, el: UiElement, index: number): string {
210
307
  const identity = {
211
308
  platform,
@@ -229,7 +326,8 @@ export class ToolsInteract {
229
326
  platform,
230
327
  deviceId,
231
328
  bounds,
232
- index
329
+ index,
330
+ stable_id: el.stable_id ?? null
233
331
  })
234
332
 
235
333
  return {
@@ -249,6 +347,235 @@ export class ToolsInteract {
249
347
  }
250
348
  }
251
349
 
350
+ private static _resolveUiChangeScope(
351
+ tree: any,
352
+ scope: 'screen' | 'subtree' | undefined,
353
+ target: string | null | undefined
354
+ ): UiChangeScopeResult {
355
+ const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
356
+ const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen'
357
+
358
+ if (normalizedScope === 'screen') {
359
+ return {
360
+ elements,
361
+ resolution: {
362
+ scope: 'screen',
363
+ target: null,
364
+ resolved: true,
365
+ resolvedIndex: null,
366
+ resolvedStableId: null,
367
+ reason: 'screen scope'
368
+ }
369
+ }
370
+ }
371
+
372
+ const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null
373
+ if (!requestedTarget) {
374
+ return {
375
+ elements: [],
376
+ resolution: {
377
+ scope: 'subtree',
378
+ target: null,
379
+ resolved: false,
380
+ resolvedIndex: null,
381
+ resolvedStableId: null,
382
+ reason: 'subtree scope requires a target element id'
383
+ },
384
+ error: {
385
+ code: 'INVALID_SCOPE',
386
+ message: 'scope=subtree requires a target element_id'
387
+ }
388
+ }
389
+ }
390
+
391
+ const resolved = ToolsInteract._findScopedElement(tree, requestedTarget)
392
+ if (!resolved) {
393
+ return {
394
+ elements: [],
395
+ resolution: {
396
+ scope: 'subtree',
397
+ target: requestedTarget,
398
+ resolved: false,
399
+ resolvedIndex: null,
400
+ resolvedStableId: null,
401
+ reason: 'target element could not be resolved'
402
+ },
403
+ error: {
404
+ code: 'ELEMENT_NOT_FOUND',
405
+ message: `Target element ${requestedTarget} could not be resolved for subtree scope`
406
+ }
407
+ }
408
+ }
409
+
410
+ const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index)
411
+ const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element): element is UiElement => !!element)
412
+
413
+ return {
414
+ elements: scopedElements,
415
+ resolution: {
416
+ scope: 'subtree',
417
+ target: requestedTarget,
418
+ resolved: true,
419
+ resolvedIndex: resolved.index,
420
+ resolvedStableId: resolved.stableId,
421
+ reason: resolved.reason
422
+ }
423
+ }
424
+ }
425
+
426
+ private static _findScopedElement(tree: any, targetElementId: string): { index: number, stableId: string | null, reason: string } | null {
427
+ const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
428
+ const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android'
429
+ const deviceId = tree?.device?.id ?? undefined
430
+ const normalizedTarget = ToolsInteract._normalize(targetElementId)
431
+
432
+ for (let i = 0; i < elements.length; i++) {
433
+ const el = elements[i]
434
+ if (!el) continue
435
+
436
+ const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i)
437
+ if (computedElementId === targetElementId) {
438
+ return {
439
+ index: i,
440
+ stableId: el.stable_id ?? null,
441
+ reason: 'element_id_match'
442
+ }
443
+ }
444
+ }
445
+
446
+ for (let i = 0; i < elements.length; i++) {
447
+ const el = elements[i]
448
+ if (!el) continue
449
+
450
+ if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
451
+ return {
452
+ index: i,
453
+ stableId: el.stable_id,
454
+ reason: 'stable_id_match'
455
+ }
456
+ }
457
+ }
458
+
459
+ const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId)
460
+ if (storedContext?.stable_id) {
461
+ const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id)
462
+ for (let i = 0; i < elements.length; i++) {
463
+ const el = elements[i]
464
+ if (!el?.stable_id) continue
465
+ if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
466
+ return {
467
+ index: i,
468
+ stableId: el.stable_id,
469
+ reason: 'stored_stable_id_match'
470
+ }
471
+ }
472
+ }
473
+ }
474
+
475
+ return null
476
+ }
477
+
478
+ private static _collectSubtreeIndices(elements: UiElement[], rootIndex: number): number[] {
479
+ if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length) return []
480
+
481
+ const visited = new Set<number>()
482
+ const stack = [rootIndex]
483
+ const result: number[] = []
484
+
485
+ while (stack.length > 0) {
486
+ const index = stack.pop()
487
+ if (index === undefined || visited.has(index) || index < 0 || index >= elements.length) continue
488
+ visited.add(index)
489
+ result.push(index)
490
+
491
+ const element = elements[index]
492
+ if (!element) continue
493
+
494
+ const directChildren = new Set<number>()
495
+ if (Array.isArray(element.children)) {
496
+ for (const childIndex of element.children) {
497
+ if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
498
+ directChildren.add(childIndex)
499
+ }
500
+ }
501
+ }
502
+
503
+ for (let i = 0; i < elements.length; i++) {
504
+ if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
505
+ directChildren.add(i)
506
+ }
507
+ }
508
+
509
+ for (const childIndex of directChildren) {
510
+ if (!visited.has(childIndex)) stack.push(childIndex)
511
+ }
512
+ }
513
+
514
+ return result.sort((left, right) => left - right)
515
+ }
516
+
517
+ private static _changeIdentityForElement(el: UiElement, index: number): string {
518
+ const stableId = ToolsInteract._normalize(el.stable_id)
519
+ if (stableId) return `stable:${stableId}`
520
+
521
+ return `fallback:${ToolsInteract._hash({
522
+ text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
523
+ contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
524
+ resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
525
+ type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
526
+ bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
527
+ index
528
+ })}`
529
+ }
530
+
531
+ private static _summarizeUiChangeDelta(initialElements: UiElement[], currentElements: UiElement[]) {
532
+ const buildMap = (elements: UiElement[]) => {
533
+ const map = new Map<string, string>()
534
+ for (let i = 0; i < elements.length; i++) {
535
+ const element = elements[i]
536
+ if (!element) continue
537
+ const key = ToolsInteract._changeIdentityForElement(element, i)
538
+ map.set(key, ToolsInteract._hash({
539
+ text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
540
+ contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
541
+ resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
542
+ type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
543
+ bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
544
+ state: element.state ?? null,
545
+ visible: !!element.visible,
546
+ enabled: !!element.enabled,
547
+ clickable: !!element.clickable
548
+ }))
549
+ }
550
+ return map
551
+ }
552
+
553
+ const initialMap = buildMap(initialElements)
554
+ const currentMap = buildMap(currentElements)
555
+ let added = 0
556
+ let removed = 0
557
+ let mutated = 0
558
+
559
+ for (const [key, value] of currentMap.entries()) {
560
+ if (!initialMap.has(key)) {
561
+ added++
562
+ } else if (initialMap.get(key) !== value) {
563
+ mutated++
564
+ }
565
+ }
566
+
567
+ for (const key of initialMap.keys()) {
568
+ if (!currentMap.has(key)) removed++
569
+ }
570
+
571
+ return {
572
+ total_elements: currentElements.length,
573
+ added_elements: added,
574
+ removed_elements: removed,
575
+ mutated_elements: mutated
576
+ }
577
+ }
578
+
252
579
  private static _rememberResolvedElement(elementId: string, context: ResolvedUiElementContext) {
253
580
  if (ToolsInteract._resolvedUiElements.has(elementId)) {
254
581
  ToolsInteract._resolvedUiElements.delete(elementId)
@@ -565,20 +892,11 @@ export class ToolsInteract {
565
892
  let safety = 0
566
893
 
567
894
  while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
568
- const parentId = current.el.parentId
569
- let parentIndex: number | null = null
570
-
571
- if (typeof parentId === 'number') parentIndex = parentId
572
- else if (typeof parentId === 'string' && /^\d+$/.test(parentId)) parentIndex = Number(parentId)
895
+ const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId)
573
896
 
574
897
  if (parentIndex !== null && elements[parentIndex]) {
575
898
  current = { el: elements[parentIndex], idx: parentIndex }
576
899
  if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
577
- } else if (typeof parentId === 'string') {
578
- const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId)
579
- if (foundIndex === -1) break
580
- current = { el: elements[foundIndex], idx: foundIndex }
581
- if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) return current
582
900
  } else {
583
901
  break
584
902
  }
@@ -718,12 +1036,16 @@ export class ToolsInteract {
718
1036
 
719
1037
  const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index)
720
1038
 
721
- if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
722
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
723
- }
724
-
725
- if (currentMatch.el.enabled === false) {
726
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore)
1039
+ const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform)
1040
+ if (!tapActionability.actionable) {
1041
+ return ToolsInteract._actionFailure(
1042
+ actionType,
1043
+ selector,
1044
+ resolvedTarget,
1045
+ tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
1046
+ true,
1047
+ fingerprintBefore
1048
+ )
727
1049
  }
728
1050
 
729
1051
  const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds
@@ -779,6 +1101,7 @@ export class ToolsInteract {
779
1101
  const sourcePlatform: 'android' | 'ios' = platform || 'android'
780
1102
  let resolvedPlatform = sourcePlatform
781
1103
  let resolvedDeviceId = deviceId
1104
+ const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null
782
1105
  const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
783
1106
  let semanticFallbackElement: FindElementResponse['element'] | null = null
784
1107
  const traceSteps: TraceStep[] = []
@@ -1062,6 +1385,21 @@ export class ToolsInteract {
1062
1385
  resolvedTarget = resolved.resolvedTarget
1063
1386
  const currentEl: UiElement = resolved.match.el
1064
1387
  cachedResolvedMatch = resolved.match
1388
+
1389
+ const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id)
1390
+ if (!adjustableActionability.actionable) {
1391
+ return buildFailure(
1392
+ adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE',
1393
+ adjustableActionability.reason ?? 'adjustable control is not actionable',
1394
+ resolvedTarget,
1395
+ currentDevice,
1396
+ lastObservedState,
1397
+ attemptCount,
1398
+ lastAdjustmentMode,
1399
+ true
1400
+ )
1401
+ }
1402
+
1065
1403
  const bounds = ToolsInteract._normalizeBounds(currentEl.bounds)
1066
1404
  const valueRange = currentEl.state?.value_range ?? null
1067
1405
  const currentValue = ToolsInteract._readNumericControlValue(currentEl, property)
@@ -2021,33 +2359,79 @@ export class ToolsInteract {
2021
2359
  deviceId,
2022
2360
  timeout_ms = 60000,
2023
2361
  stability_window_ms = 300,
2024
- expected_change
2362
+ expected_change,
2363
+ scope = 'screen',
2364
+ target = null
2025
2365
  }: {
2026
2366
  platform?: 'android' | 'ios',
2027
2367
  deviceId?: string,
2028
2368
  timeout_ms?: number,
2029
2369
  stability_window_ms?: number,
2030
- expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change'
2370
+ expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change',
2371
+ scope?: 'screen' | 'subtree',
2372
+ target?: string | null
2031
2373
  }): Promise<WaitForUIChangeResponse> {
2032
2374
  const start = Date.now()
2033
2375
  const pollIntervalMs = 300
2034
2376
  const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
2035
2377
  let baseline: UiChangeSignatureSet | null = null
2378
+ let baselineScope: UiChangeScopeResult | null = null
2036
2379
  let lastObservedRevision: number | null = null
2037
2380
  let lastLoadingState: any = null
2381
+ let lastSnapshotFreshnessMs: number | null = null
2038
2382
  let candidateSignatures: UiChangeSignatureSet | null = null
2039
2383
  let candidateObservedChange: 'hierarchy_diff' | 'text_change' | 'state_change' | null = null
2040
2384
  let candidateSinceMs: number | null = null
2385
+ let lastChangeSummary: ReturnType<typeof ToolsInteract._summarizeUiChangeDelta> | null = null
2386
+ let lastScopeResolution: UiChangeScopeResolution = {
2387
+ scope: scope === 'subtree' ? 'subtree' : 'screen',
2388
+ target: target && typeof target === 'string' ? target : null,
2389
+ resolved: scope !== 'subtree',
2390
+ resolvedIndex: null,
2391
+ resolvedStableId: null,
2392
+ reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
2393
+ }
2041
2394
 
2042
2395
  while (Date.now() - start < timeout_ms) {
2043
2396
  try {
2044
2397
  const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
2045
- const signatures = ToolsInteract._buildUiChangeSignatures(tree)
2398
+ const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
2399
+ if (scopedTree.error) {
2400
+ lastScopeResolution = scopedTree.resolution
2401
+ return {
2402
+ success: false,
2403
+ observed_change: null,
2404
+ snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
2405
+ snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
2406
+ timeout: true,
2407
+ elapsed_ms: Date.now() - start,
2408
+ expected_change,
2409
+ loading_state: tree?.loading_state ?? lastLoadingState ?? null,
2410
+ scope: scopedTree.resolution.scope,
2411
+ target: scopedTree.resolution.target,
2412
+ stability_state: 'transient',
2413
+ change_summary: lastChangeSummary,
2414
+ reason: scopedTree.error.message,
2415
+ error: scopedTree.error
2416
+ }
2417
+ }
2418
+
2419
+ const scopedElements = scopedTree.elements
2420
+ const scopedSignatureTree = {
2421
+ ...tree,
2422
+ elements: scopedElements
2423
+ }
2424
+ const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree)
2046
2425
  lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
2047
2426
  lastLoadingState = tree?.loading_state ?? lastLoadingState
2427
+ lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs
2428
+ lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary
2429
+ lastScopeResolution = scopedTree.resolution
2430
+ baselineScope = baselineScope ?? scopedTree
2048
2431
 
2049
2432
  if (!baseline) {
2050
2433
  baseline = signatures
2434
+ baselineScope = scopedTree
2051
2435
  } else {
2052
2436
  const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
2053
2437
  if (observedChange) {
@@ -2059,20 +2443,25 @@ export class ToolsInteract {
2059
2443
 
2060
2444
  const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
2061
2445
  if (stabilityWindow === 0 || stableForMs >= stabilityWindow) {
2062
- return {
2063
- success: true,
2064
- observed_change: candidateObservedChange ?? observedChange,
2065
- snapshot_revision: lastObservedRevision ?? undefined,
2066
- timeout: false,
2067
- elapsed_ms: Date.now() - start,
2068
- expected_change,
2069
- loading_state: lastLoadingState ?? null,
2070
- reason: 'UI change observed'
2446
+ return {
2447
+ success: true,
2448
+ observed_change: candidateObservedChange ?? observedChange,
2449
+ snapshot_revision: lastObservedRevision ?? undefined,
2450
+ snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2451
+ timeout: false,
2452
+ elapsed_ms: Date.now() - start,
2453
+ expected_change,
2454
+ loading_state: lastLoadingState ?? null,
2455
+ scope: lastScopeResolution.scope,
2456
+ target: lastScopeResolution.target,
2457
+ stability_state: 'stable',
2458
+ change_summary: lastChangeSummary,
2459
+ reason: 'UI change observed'
2460
+ }
2071
2461
  }
2072
- }
2073
- } else {
2074
- candidateSignatures = null
2075
- candidateObservedChange = null
2462
+ } else {
2463
+ candidateSignatures = null
2464
+ candidateObservedChange = null
2076
2465
  candidateSinceMs = null
2077
2466
  }
2078
2467
  }
@@ -2087,10 +2476,15 @@ export class ToolsInteract {
2087
2476
  success: false,
2088
2477
  observed_change: null,
2089
2478
  snapshot_revision: lastObservedRevision ?? undefined,
2479
+ snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2090
2480
  timeout: true,
2091
2481
  elapsed_ms: Date.now() - start,
2092
2482
  expected_change,
2093
2483
  loading_state: lastLoadingState ?? null,
2484
+ scope: lastScopeResolution.scope,
2485
+ target: lastScopeResolution.target,
2486
+ stability_state: 'transient',
2487
+ change_summary: lastChangeSummary,
2094
2488
  reason: 'timeout'
2095
2489
  }
2096
2490
  }
@@ -358,6 +358,7 @@ export class ToolsObserve {
358
358
 
359
359
  raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision
360
360
  raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms
361
+ raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? snapshotMetadata.snapshot_delta ?? null
361
362
  raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state
362
363
 
363
364
  const semantic = deriveSnapshotSemantic(raw)
@@ -1,9 +1,10 @@
1
1
  import crypto from 'crypto'
2
- import type { GetUITreeResponse, LoadingState, UIElement } from '../types.js'
2
+ import type { GetUITreeResponse, LoadingState, SnapshotDelta, UIElement } from '../types.js'
3
3
 
4
4
  interface SnapshotState {
5
5
  revision: number
6
6
  signature: string | null
7
+ elementSignatures: Map<string, string>
7
8
  }
8
9
 
9
10
  const snapshotStateByDevice = new Map<string, SnapshotState>()
@@ -38,6 +39,63 @@ function stableElementSignature(element: UIElement) {
38
39
  }
39
40
  }
40
41
 
42
+ function stableElementIdentity(element: UIElement, index: number) {
43
+ const stableId = normalize(element.stable_id)
44
+ if (stableId) return `stable:${stableId}`
45
+
46
+ return `fallback:${crypto.createHash('sha1').update(JSON.stringify({
47
+ text: normalize(element.text),
48
+ contentDescription: normalize(element.contentDescription),
49
+ resourceId: normalize(element.resourceId),
50
+ type: normalize(element.type),
51
+ bounds: normalizeBounds(element.bounds),
52
+ index
53
+ })).digest('hex')}`
54
+ }
55
+
56
+ function buildElementSignatures(tree: Pick<GetUITreeResponse, 'elements'> | null | undefined) {
57
+ const signatures = new Map<string, string>()
58
+ const elements = Array.isArray(tree?.elements) ? tree!.elements! : []
59
+
60
+ for (let index = 0; index < elements.length; index++) {
61
+ const element = elements[index]
62
+ if (!element) continue
63
+ const identity = stableElementIdentity(element, index)
64
+ signatures.set(identity, crypto.createHash('sha1').update(JSON.stringify(stableElementSignature(element))).digest('hex'))
65
+ }
66
+
67
+ return signatures
68
+ }
69
+
70
+ function summarizeSnapshotDelta(previous: SnapshotState | undefined, currentElements: Map<string, string>): SnapshotDelta | null {
71
+ if (!previous) return null
72
+
73
+ let added = 0
74
+ let removed = 0
75
+ let mutated = 0
76
+
77
+ for (const [identity, signature] of currentElements.entries()) {
78
+ const previousSignature = previous.elementSignatures.get(identity)
79
+ if (previousSignature === undefined) {
80
+ added++
81
+ } else if (previousSignature !== signature) {
82
+ mutated++
83
+ }
84
+ }
85
+
86
+ for (const identity of previous.elementSignatures.keys()) {
87
+ if (!currentElements.has(identity)) removed++
88
+ }
89
+
90
+ return {
91
+ previous_snapshot_revision: previous.revision,
92
+ added_elements: added,
93
+ removed_elements: removed,
94
+ mutated_elements: mutated,
95
+ total_elements: currentElements.size
96
+ }
97
+ }
98
+
41
99
  export function computeSnapshotSignature(tree: Pick<GetUITreeResponse, 'elements' | 'screen' | 'resolution' | 'error'> | null | undefined): string | null {
42
100
  if (!tree || tree.error) return null
43
101
 
@@ -83,6 +141,10 @@ export function deriveSnapshotMetadata(
83
141
  ) {
84
142
  const signature = signatureOverride ?? computeSnapshotSignature(tree)
85
143
  const previous = snapshotStateByDevice.get(deviceKey)
144
+ const hasValidTree = !!tree && !tree.error
145
+ const currentElementSignatures = hasValidTree
146
+ ? buildElementSignatures(tree)
147
+ : previous?.elementSignatures ?? new Map<string, string>()
86
148
 
87
149
  let revision = 1
88
150
  if (previous) {
@@ -93,11 +155,16 @@ export function deriveSnapshotMetadata(
93
155
  }
94
156
  }
95
157
 
96
- snapshotStateByDevice.set(deviceKey, { revision, signature })
158
+ snapshotStateByDevice.set(deviceKey, {
159
+ revision,
160
+ signature,
161
+ elementSignatures: currentElementSignatures
162
+ })
97
163
 
98
164
  return {
99
165
  snapshot_revision: revision,
100
166
  captured_at_ms: Date.now(),
167
+ snapshot_delta: hasValidTree ? summarizeSnapshotDelta(previous, currentElementSignatures) : null,
101
168
  loading_state: detectLoadingState(tree, source)
102
169
  }
103
170
  }