mobile-debug-mcp 0.29.0 → 0.30.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/AGENTS.md CHANGED
@@ -75,3 +75,16 @@ For test authoring details, rely on the `test-authoring` skill package rather th
75
75
  ## Notes for maintainers
76
76
 
77
77
  This file is intentionally short. Keep task-specific guidance in `skills/...` so multiple agent systems can reuse the same instructions.
78
+
79
+ ## Testing
80
+
81
+ - `npm run test:unit` runs every automated unit test under `test/unit/...`
82
+ - `npm run test:device` runs the automated device smoke checks under `test/device/automated/...`
83
+ - `npm run verify` runs the default maintainer verification sequence: lint, build, and unit tests
84
+ - Manual and debug-oriented device scripts live under `test/device/manual/...` and are not part of the default test commands
85
+
86
+ ## Utility Scripts
87
+
88
+ - `npm run healthcheck` runs the `idb`/tooling healthcheck helper from `src/utils/cli/idb/check-idb.ts`
89
+ - `npm run install-idb` runs the guided `idb` installer helper from `src/utils/cli/idb/install-idb.ts`
90
+ - `npm run preflight-ios` runs the iOS preflight helper from `src/utils/cli/ios/preflight-ios.ts`
package/README.md CHANGED
@@ -16,7 +16,11 @@ A minimal, secure MCP server for AI-assisted mobile development. Build, install,
16
16
  - Xcode command-line tools for iOS support
17
17
  - [idb](https://github.com/facebook/idb) for iOS device support
18
18
 
19
- ## Configuration example
19
+ ## Configuration
20
+
21
+ <details>
22
+
23
+ <summary>Android Studio</summary>
20
24
 
21
25
  ```json
22
26
  {
@@ -29,7 +33,45 @@ A minimal, secure MCP server for AI-assisted mobile development. Build, install,
29
33
  }
30
34
  }
31
35
  ```
32
- You will need to add ADB_PATH for Android and XCRUN_PATH and IDB_PATH for iOS.
36
+
37
+ </details>
38
+
39
+ <details>
40
+
41
+ <summary>Copilot</summary>
42
+
43
+ ```json
44
+ {
45
+ "mcpServers": {
46
+ "mobile-debug": {
47
+ "command": "npx",
48
+ "args": ["--yes","mobile-debug-mcp","server"],
49
+ "env": { "ADB_PATH": "/path/to/adb", "XCRUN_PATH": "/usr/bin/xcrun", "IDB_PATH": "/path/to/idb" }
50
+ }
51
+ }
52
+ }
53
+ ```
54
+
55
+ </details>
56
+
57
+ <details>
58
+
59
+ <summary>Codex</summary>
60
+
61
+ Use STDIO
62
+
63
+ command: npx
64
+
65
+ args:
66
+ * --yes
67
+ * mobile-debug-mcp
68
+
69
+ environment variables:
70
+ * ADB_PATH: /path/to/adb
71
+ * XCRUN_PATH: /usr/bin/xcrun
72
+ * IDC_PATH: /path/to/idb"
73
+
74
+ </details>
33
75
 
34
76
  ## Usage
35
77
 
@@ -48,25 +90,6 @@ Feature building:
48
90
  - Agents: [AGENTS.md](AGENTS.md) — cold-start guidance for autonomous agents entering the public repo
49
91
  - Skills: [skills/README.md](skills/README.md) — portable Markdown skill packages for agents such as Copilot, Codex, Claude, or custom systems
50
92
 
51
- ## Testing
52
-
53
- - `npm run test:unit` runs every automated unit test under `test/unit/...`
54
- - `npm run test:device` runs the automated device smoke checks under `test/device/automated/...`
55
- - `npm run verify` runs the default maintainer verification sequence: lint, build, and unit tests
56
- - Manual and debug-oriented device scripts live under `test/device/manual/...` and are not part of the default test commands
57
-
58
- ## Utility Scripts
59
-
60
- - `npm run healthcheck` runs the `idb`/tooling healthcheck helper from `src/utils/cli/idb/check-idb.ts`
61
- - `npm run install-idb` runs the guided `idb` installer helper from `src/utils/cli/idb/install-idb.ts`
62
- - `npm run preflight-ios` runs the iOS preflight helper from `src/utils/cli/ios/preflight-ios.ts`
63
-
64
- ## Agent skills
65
-
66
- - `skills/mcp-builder/` contains reusable build/install guidance for agents
67
- - `skills/test-authoring/` contains reusable test-creation guidance aligned to this repo's current test structure
68
- - Skills are written as plain Markdown packages so they can be consumed by different agent systems rather than one vendor-specific runtime
69
-
70
93
  ## License
71
94
 
72
95
  MIT
@@ -95,10 +95,68 @@ export class ToolsInteract {
95
95
  }
96
96
  return null;
97
97
  }
98
+ static _resolveParentIndex(elements, parentId) {
99
+ if (parentId === undefined || parentId === null)
100
+ return null;
101
+ if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
102
+ return parentId;
103
+ }
104
+ if (typeof parentId === 'string') {
105
+ const normalized = ToolsInteract._normalize(parentId);
106
+ if (!normalized)
107
+ return null;
108
+ if (/^\d+$/.test(normalized)) {
109
+ const index = Number(normalized);
110
+ if (index >= 0 && index < elements.length)
111
+ return index;
112
+ }
113
+ const foundIndex = elements.findIndex((el) => {
114
+ if (!el)
115
+ return false;
116
+ return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
117
+ ToolsInteract._normalize(el.stable_id ?? '') === normalized;
118
+ });
119
+ return foundIndex >= 0 ? foundIndex : null;
120
+ }
121
+ return null;
122
+ }
98
123
  static _isVisibleElement(el) {
99
124
  const bounds = ToolsInteract._normalizeBounds(el.bounds);
100
125
  return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
101
126
  }
127
+ static _isTapActionable(el, storedStableId, platform) {
128
+ if (!ToolsInteract._isVisibleElement(el)) {
129
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
130
+ }
131
+ if (el.enabled === false) {
132
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
133
+ }
134
+ const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
135
+ (Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap')));
136
+ if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
137
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
138
+ }
139
+ if (storedStableId) {
140
+ if (!el.stable_id || el.stable_id !== storedStableId) {
141
+ return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
142
+ }
143
+ }
144
+ return { actionable: true };
145
+ }
146
+ static _isAdjustableActionable(el, storedStableId) {
147
+ if (!ToolsInteract._isVisibleElement(el)) {
148
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
149
+ }
150
+ if (el.enabled === false) {
151
+ return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
152
+ }
153
+ if (storedStableId) {
154
+ if (!el.stable_id || el.stable_id !== storedStableId) {
155
+ return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
156
+ }
157
+ }
158
+ return { actionable: true };
159
+ }
102
160
  static _computeElementId(platform, deviceId, el, index) {
103
161
  const identity = {
104
162
  platform,
@@ -120,7 +178,8 @@ export class ToolsInteract {
120
178
  platform,
121
179
  deviceId,
122
180
  bounds,
123
- index
181
+ index,
182
+ stable_id: el.stable_id ?? null
124
183
  });
125
184
  return {
126
185
  text: el.text ?? null,
@@ -138,6 +197,214 @@ export class ToolsInteract {
138
197
  semantic: el.semantic ?? null
139
198
  };
140
199
  }
200
+ static _resolveUiChangeScope(tree, scope, target) {
201
+ const elements = Array.isArray(tree?.elements) ? tree.elements : [];
202
+ const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
203
+ if (normalizedScope === 'screen') {
204
+ return {
205
+ elements,
206
+ resolution: {
207
+ scope: 'screen',
208
+ target: null,
209
+ resolved: true,
210
+ resolvedIndex: null,
211
+ resolvedStableId: null,
212
+ reason: 'screen scope'
213
+ }
214
+ };
215
+ }
216
+ const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
217
+ if (!requestedTarget) {
218
+ return {
219
+ elements: [],
220
+ resolution: {
221
+ scope: 'subtree',
222
+ target: null,
223
+ resolved: false,
224
+ resolvedIndex: null,
225
+ resolvedStableId: null,
226
+ reason: 'subtree scope requires a target element id'
227
+ },
228
+ error: {
229
+ code: 'INVALID_SCOPE',
230
+ message: 'scope=subtree requires a target element_id'
231
+ }
232
+ };
233
+ }
234
+ const resolved = ToolsInteract._findScopedElement(tree, requestedTarget);
235
+ if (!resolved) {
236
+ return {
237
+ elements: [],
238
+ resolution: {
239
+ scope: 'subtree',
240
+ target: requestedTarget,
241
+ resolved: false,
242
+ resolvedIndex: null,
243
+ resolvedStableId: null,
244
+ reason: 'target element could not be resolved'
245
+ },
246
+ error: {
247
+ code: 'ELEMENT_NOT_FOUND',
248
+ message: `Target element ${requestedTarget} could not be resolved for subtree scope`
249
+ }
250
+ };
251
+ }
252
+ const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index);
253
+ const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
254
+ return {
255
+ elements: scopedElements,
256
+ resolution: {
257
+ scope: 'subtree',
258
+ target: requestedTarget,
259
+ resolved: true,
260
+ resolvedIndex: resolved.index,
261
+ resolvedStableId: resolved.stableId,
262
+ reason: resolved.reason
263
+ }
264
+ };
265
+ }
266
+ static _findScopedElement(tree, targetElementId) {
267
+ const elements = Array.isArray(tree?.elements) ? tree.elements : [];
268
+ const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
269
+ const deviceId = tree?.device?.id ?? undefined;
270
+ const normalizedTarget = ToolsInteract._normalize(targetElementId);
271
+ for (let i = 0; i < elements.length; i++) {
272
+ const el = elements[i];
273
+ if (!el)
274
+ continue;
275
+ const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i);
276
+ if (computedElementId === targetElementId) {
277
+ return {
278
+ index: i,
279
+ stableId: el.stable_id ?? null,
280
+ reason: 'element_id_match'
281
+ };
282
+ }
283
+ }
284
+ for (let i = 0; i < elements.length; i++) {
285
+ const el = elements[i];
286
+ if (!el)
287
+ continue;
288
+ if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
289
+ return {
290
+ index: i,
291
+ stableId: el.stable_id,
292
+ reason: 'stable_id_match'
293
+ };
294
+ }
295
+ }
296
+ const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId);
297
+ if (storedContext?.stable_id) {
298
+ const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id);
299
+ for (let i = 0; i < elements.length; i++) {
300
+ const el = elements[i];
301
+ if (!el?.stable_id)
302
+ continue;
303
+ if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
304
+ return {
305
+ index: i,
306
+ stableId: el.stable_id,
307
+ reason: 'stored_stable_id_match'
308
+ };
309
+ }
310
+ }
311
+ }
312
+ return null;
313
+ }
314
+ static _collectSubtreeIndices(elements, rootIndex) {
315
+ if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length)
316
+ return [];
317
+ const visited = new Set();
318
+ const stack = [rootIndex];
319
+ const result = [];
320
+ while (stack.length > 0) {
321
+ const index = stack.pop();
322
+ if (index === undefined || visited.has(index) || index < 0 || index >= elements.length)
323
+ continue;
324
+ visited.add(index);
325
+ result.push(index);
326
+ const element = elements[index];
327
+ if (!element)
328
+ continue;
329
+ const directChildren = new Set();
330
+ if (Array.isArray(element.children)) {
331
+ for (const childIndex of element.children) {
332
+ if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
333
+ directChildren.add(childIndex);
334
+ }
335
+ }
336
+ }
337
+ for (let i = 0; i < elements.length; i++) {
338
+ if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
339
+ directChildren.add(i);
340
+ }
341
+ }
342
+ for (const childIndex of directChildren) {
343
+ if (!visited.has(childIndex))
344
+ stack.push(childIndex);
345
+ }
346
+ }
347
+ return result.sort((left, right) => left - right);
348
+ }
349
+ static _changeIdentityForElement(el, index) {
350
+ const stableId = ToolsInteract._normalize(el.stable_id);
351
+ if (stableId)
352
+ return `stable:${stableId}`;
353
+ return `fallback:${ToolsInteract._hash({
354
+ text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
355
+ contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
356
+ resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
357
+ type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
358
+ bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
359
+ index
360
+ })}`;
361
+ }
362
+ static _summarizeUiChangeDelta(initialElements, currentElements) {
363
+ const buildMap = (elements) => {
364
+ const map = new Map();
365
+ for (let i = 0; i < elements.length; i++) {
366
+ const element = elements[i];
367
+ if (!element)
368
+ continue;
369
+ const key = ToolsInteract._changeIdentityForElement(element, i);
370
+ map.set(key, ToolsInteract._hash({
371
+ text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
372
+ contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
373
+ resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
374
+ type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
375
+ bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
376
+ state: element.state ?? null,
377
+ visible: !!element.visible,
378
+ enabled: !!element.enabled,
379
+ clickable: !!element.clickable
380
+ }));
381
+ }
382
+ return map;
383
+ };
384
+ const initialMap = buildMap(initialElements);
385
+ const currentMap = buildMap(currentElements);
386
+ let added = 0;
387
+ let removed = 0;
388
+ let mutated = 0;
389
+ for (const [key, value] of currentMap.entries()) {
390
+ if (!initialMap.has(key)) {
391
+ added++;
392
+ }
393
+ else if (initialMap.get(key) !== value) {
394
+ mutated++;
395
+ }
396
+ }
397
+ for (const key of initialMap.keys()) {
398
+ if (!currentMap.has(key))
399
+ removed++;
400
+ }
401
+ return {
402
+ total_elements: currentElements.length,
403
+ added_elements: added,
404
+ removed_elements: removed,
405
+ mutated_elements: mutated
406
+ };
407
+ }
141
408
  static _rememberResolvedElement(elementId, context) {
142
409
  if (ToolsInteract._resolvedUiElements.has(elementId)) {
143
410
  ToolsInteract._resolvedUiElements.delete(elementId);
@@ -205,6 +472,9 @@ export class ToolsInteract {
205
472
  }
206
473
  return null;
207
474
  }
475
+ static _uiChangeSignaturesEqual(left, right) {
476
+ return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state;
477
+ }
208
478
  static _resolvedTargetFromElement(elementId, element, index) {
209
479
  return {
210
480
  elementId,
@@ -381,25 +651,12 @@ export class ToolsInteract {
381
651
  let current = chosen;
382
652
  let safety = 0;
383
653
  while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
384
- const parentId = current.el.parentId;
385
- let parentIndex = null;
386
- if (typeof parentId === 'number')
387
- parentIndex = parentId;
388
- else if (typeof parentId === 'string' && /^\d+$/.test(parentId))
389
- parentIndex = Number(parentId);
654
+ const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId);
390
655
  if (parentIndex !== null && elements[parentIndex]) {
391
656
  current = { el: elements[parentIndex], idx: parentIndex };
392
657
  if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
393
658
  return current;
394
659
  }
395
- else if (typeof parentId === 'string') {
396
- const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId);
397
- if (foundIndex === -1)
398
- break;
399
- current = { el: elements[foundIndex], idx: foundIndex };
400
- if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
401
- return current;
402
- }
403
660
  else {
404
661
  break;
405
662
  }
@@ -513,11 +770,9 @@ export class ToolsInteract {
513
770
  return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
514
771
  }
515
772
  const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
516
- if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
517
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
518
- }
519
- if (currentMatch.el.enabled === false) {
520
- return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
773
+ const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform);
774
+ if (!tapActionability.actionable) {
775
+ return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
521
776
  }
522
777
  const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
523
778
  if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
@@ -550,6 +805,7 @@ export class ToolsInteract {
550
805
  const sourcePlatform = platform || 'android';
551
806
  let resolvedPlatform = sourcePlatform;
552
807
  let resolvedDeviceId = deviceId;
808
+ const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null;
553
809
  const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
554
810
  let semanticFallbackElement = null;
555
811
  const traceSteps = [];
@@ -755,6 +1011,10 @@ export class ToolsInteract {
755
1011
  resolvedTarget = resolved.resolvedTarget;
756
1012
  const currentEl = resolved.match.el;
757
1013
  cachedResolvedMatch = resolved.match;
1014
+ const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id);
1015
+ if (!adjustableActionability.actionable) {
1016
+ return buildFailure(adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', adjustableActionability.reason ?? 'adjustable control is not actionable', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
1017
+ }
758
1018
  const bounds = ToolsInteract._normalizeBounds(currentEl.bounds);
759
1019
  const valueRange = currentEl.state?.value_range ?? null;
760
1020
  const currentValue = ToolsInteract._readNumericControlValue(currentEl, property);
@@ -1708,49 +1968,97 @@ export class ToolsInteract {
1708
1968
  }
1709
1969
  };
1710
1970
  }
1711
- static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 250, expected_change }) {
1971
+ static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 300, expected_change, scope = 'screen', target = null }) {
1712
1972
  const start = Date.now();
1713
1973
  const pollIntervalMs = 300;
1714
- const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 250);
1974
+ const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300);
1715
1975
  let baseline = null;
1976
+ let baselineScope = null;
1716
1977
  let lastObservedRevision = null;
1717
1978
  let lastLoadingState = null;
1979
+ let lastSnapshotFreshnessMs = null;
1980
+ let candidateSignatures = null;
1981
+ let candidateObservedChange = null;
1982
+ let candidateSinceMs = null;
1983
+ let lastChangeSummary = null;
1984
+ let lastScopeResolution = {
1985
+ scope: scope === 'subtree' ? 'subtree' : 'screen',
1986
+ target: target && typeof target === 'string' ? target : null,
1987
+ resolved: scope !== 'subtree',
1988
+ resolvedIndex: null,
1989
+ resolvedStableId: null,
1990
+ reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
1991
+ };
1718
1992
  while (Date.now() - start < timeout_ms) {
1719
1993
  try {
1720
1994
  const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
1721
- const signatures = ToolsInteract._buildUiChangeSignatures(tree);
1995
+ const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target);
1996
+ if (scopedTree.error) {
1997
+ lastScopeResolution = scopedTree.resolution;
1998
+ return {
1999
+ success: false,
2000
+ observed_change: null,
2001
+ snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
2002
+ snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
2003
+ timeout: true,
2004
+ elapsed_ms: Date.now() - start,
2005
+ expected_change,
2006
+ loading_state: tree?.loading_state ?? lastLoadingState ?? null,
2007
+ scope: scopedTree.resolution.scope,
2008
+ target: scopedTree.resolution.target,
2009
+ stability_state: 'transient',
2010
+ change_summary: lastChangeSummary,
2011
+ reason: scopedTree.error.message,
2012
+ error: scopedTree.error
2013
+ };
2014
+ }
2015
+ const scopedElements = scopedTree.elements;
2016
+ const scopedSignatureTree = {
2017
+ ...tree,
2018
+ elements: scopedElements
2019
+ };
2020
+ const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree);
1722
2021
  lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
1723
2022
  lastLoadingState = tree?.loading_state ?? lastLoadingState;
2023
+ lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs;
2024
+ lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary;
2025
+ lastScopeResolution = scopedTree.resolution;
2026
+ baselineScope = baselineScope ?? scopedTree;
1724
2027
  if (!baseline) {
1725
2028
  baseline = signatures;
2029
+ baselineScope = scopedTree;
1726
2030
  }
1727
2031
  else {
1728
2032
  const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
1729
2033
  if (observedChange) {
1730
- if (stabilityWindow > 0) {
1731
- await new Promise(resolve => setTimeout(resolve, stabilityWindow));
1732
- const confirmTree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
1733
- const confirmSignatures = ToolsInteract._buildUiChangeSignatures(confirmTree);
1734
- const confirmChange = ToolsInteract._matchesUiChange(expected_change, baseline, confirmSignatures);
1735
- if (!confirmChange || confirmSignatures.hierarchy !== signatures.hierarchy || confirmSignatures.text !== signatures.text || confirmSignatures.state !== signatures.state) {
1736
- lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision;
1737
- lastLoadingState = confirmTree?.loading_state ?? lastLoadingState;
1738
- await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
1739
- continue;
1740
- }
1741
- lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision;
1742
- lastLoadingState = confirmTree?.loading_state ?? lastLoadingState;
2034
+ if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
2035
+ candidateSignatures = signatures;
2036
+ candidateObservedChange = observedChange;
2037
+ candidateSinceMs = Date.now();
1743
2038
  }
1744
- return {
1745
- success: true,
1746
- observed_change: observedChange,
1747
- snapshot_revision: lastObservedRevision ?? undefined,
1748
- timeout: false,
1749
- elapsed_ms: Date.now() - start,
1750
- expected_change,
1751
- loading_state: lastLoadingState ?? null,
1752
- reason: 'UI change observed'
1753
- };
2039
+ const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs;
2040
+ if (stabilityWindow === 0 || stableForMs >= stabilityWindow) {
2041
+ return {
2042
+ success: true,
2043
+ observed_change: candidateObservedChange ?? observedChange,
2044
+ snapshot_revision: lastObservedRevision ?? undefined,
2045
+ snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
2046
+ timeout: false,
2047
+ elapsed_ms: Date.now() - start,
2048
+ expected_change,
2049
+ loading_state: lastLoadingState ?? null,
2050
+ scope: lastScopeResolution.scope,
2051
+ target: lastScopeResolution.target,
2052
+ stability_state: 'stable',
2053
+ change_summary: lastChangeSummary,
2054
+ reason: 'UI change observed'
2055
+ };
2056
+ }
2057
+ }
2058
+ else {
2059
+ candidateSignatures = null;
2060
+ candidateObservedChange = null;
2061
+ candidateSinceMs = null;
1754
2062
  }
1755
2063
  }
1756
2064
  }
@@ -1763,10 +2071,15 @@ export class ToolsInteract {
1763
2071
  success: false,
1764
2072
  observed_change: null,
1765
2073
  snapshot_revision: lastObservedRevision ?? undefined,
2074
+ snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
1766
2075
  timeout: true,
1767
2076
  elapsed_ms: Date.now() - start,
1768
2077
  expected_change,
1769
2078
  loading_state: lastLoadingState ?? null,
2079
+ scope: lastScopeResolution.scope,
2080
+ target: lastScopeResolution.target,
2081
+ stability_state: 'transient',
2082
+ change_summary: lastChangeSummary,
1770
2083
  reason: 'timeout'
1771
2084
  };
1772
2085
  }
@@ -325,6 +325,7 @@ export class ToolsObserve {
325
325
  const snapshotMetadata = deriveSnapshotMetadata(snapshotDeviceKey, raw.ui_tree, 'snapshot', raw.ui_tree?.snapshot_revision ? null : (raw.fingerprint || raw.activity || null));
326
326
  raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision;
327
327
  raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms;
328
+ raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? snapshotMetadata.snapshot_delta ?? null;
328
329
  raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state;
329
330
  const semantic = deriveSnapshotSemantic(raw);
330
331
  return semantic ? { raw, semantic } : { raw };