mobile-debug-mcp 0.29.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -0
- package/README.md +44 -21
- package/dist/interact/index.js +359 -46
- package/dist/observe/index.js +1 -0
- package/dist/observe/snapshot-metadata.js +62 -1
- package/dist/server/tool-definitions.js +8 -3
- package/dist/server/tool-handlers.js +4 -2
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +11 -0
- package/docs/ROADMAP.md +18 -7
- package/docs/rfcs/013-wait-and-synchronization-reliability.md +870 -0
- package/docs/rfcs/014-actionability-resolution.md +394 -0
- package/docs/specs/mcp-tooling-spec-v1.md +28 -0
- package/docs/tools/interact.md +6 -0
- package/package.json +1 -1
- package/src/interact/index.ts +444 -45
- package/src/observe/index.ts +1 -0
- package/src/observe/snapshot-metadata.ts +69 -2
- package/src/server/tool-definitions.ts +8 -3
- package/src/server/tool-handlers.ts +4 -2
- package/src/server-core.ts +1 -1
- package/src/types.ts +24 -0
- package/test/unit/interact/adjust_control.test.ts +104 -0
- package/test/unit/interact/subtree_collection.test.ts +24 -0
- package/test/unit/interact/tap_element.test.ts +71 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +189 -45
- package/test/unit/observe/snapshot_metadata.test.ts +67 -0
package/AGENTS.md
CHANGED
|
@@ -75,3 +75,16 @@ For test authoring details, rely on the `test-authoring` skill package rather th
|
|
|
75
75
|
## Notes for maintainers
|
|
76
76
|
|
|
77
77
|
This file is intentionally short. Keep task-specific guidance in `skills/...` so multiple agent systems can reuse the same instructions.
|
|
78
|
+
|
|
79
|
+
## Testing
|
|
80
|
+
|
|
81
|
+
- `npm run test:unit` runs every automated unit test under `test/unit/...`
|
|
82
|
+
- `npm run test:device` runs the automated device smoke checks under `test/device/automated/...`
|
|
83
|
+
- `npm run verify` runs the default maintainer verification sequence: lint, build, and unit tests
|
|
84
|
+
- Manual and debug-oriented device scripts live under `test/device/manual/...` and are not part of the default test commands
|
|
85
|
+
|
|
86
|
+
## Utility Scripts
|
|
87
|
+
|
|
88
|
+
- `npm run healthcheck` runs the `idb`/tooling healthcheck helper from `src/utils/cli/idb/check-idb.ts`
|
|
89
|
+
- `npm run install-idb` runs the guided `idb` installer helper from `src/utils/cli/idb/install-idb.ts`
|
|
90
|
+
- `npm run preflight-ios` runs the iOS preflight helper from `src/utils/cli/ios/preflight-ios.ts`
|
package/README.md
CHANGED
|
@@ -16,7 +16,11 @@ A minimal, secure MCP server for AI-assisted mobile development. Build, install,
|
|
|
16
16
|
- Xcode command-line tools for iOS support
|
|
17
17
|
- [idb](https://github.com/facebook/idb) for iOS device support
|
|
18
18
|
|
|
19
|
-
## Configuration
|
|
19
|
+
## Configuration
|
|
20
|
+
|
|
21
|
+
<details>
|
|
22
|
+
|
|
23
|
+
<summary>Android Studio</summary>
|
|
20
24
|
|
|
21
25
|
```json
|
|
22
26
|
{
|
|
@@ -29,7 +33,45 @@ A minimal, secure MCP server for AI-assisted mobile development. Build, install,
|
|
|
29
33
|
}
|
|
30
34
|
}
|
|
31
35
|
```
|
|
32
|
-
|
|
36
|
+
|
|
37
|
+
</details>
|
|
38
|
+
|
|
39
|
+
<details>
|
|
40
|
+
|
|
41
|
+
<summary>Copilot</summary>
|
|
42
|
+
|
|
43
|
+
```json
|
|
44
|
+
{
|
|
45
|
+
"mcpServers": {
|
|
46
|
+
"mobile-debug": {
|
|
47
|
+
"command": "npx",
|
|
48
|
+
"args": ["--yes","mobile-debug-mcp","server"],
|
|
49
|
+
"env": { "ADB_PATH": "/path/to/adb", "XCRUN_PATH": "/usr/bin/xcrun", "IDB_PATH": "/path/to/idb" }
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
</details>
|
|
56
|
+
|
|
57
|
+
<details>
|
|
58
|
+
|
|
59
|
+
<summary>Codex</summary>
|
|
60
|
+
|
|
61
|
+
Use STDIO
|
|
62
|
+
|
|
63
|
+
command: npx
|
|
64
|
+
|
|
65
|
+
args:
|
|
66
|
+
* --yes
|
|
67
|
+
* mobile-debug-mcp
|
|
68
|
+
|
|
69
|
+
environment variables:
|
|
70
|
+
* ADB_PATH: /path/to/adb
|
|
71
|
+
* XCRUN_PATH: /usr/bin/xcrun
|
|
72
|
+
* IDC_PATH: /path/to/idb"
|
|
73
|
+
|
|
74
|
+
</details>
|
|
33
75
|
|
|
34
76
|
## Usage
|
|
35
77
|
|
|
@@ -48,25 +90,6 @@ Feature building:
|
|
|
48
90
|
- Agents: [AGENTS.md](AGENTS.md) — cold-start guidance for autonomous agents entering the public repo
|
|
49
91
|
- Skills: [skills/README.md](skills/README.md) — portable Markdown skill packages for agents such as Copilot, Codex, Claude, or custom systems
|
|
50
92
|
|
|
51
|
-
## Testing
|
|
52
|
-
|
|
53
|
-
- `npm run test:unit` runs every automated unit test under `test/unit/...`
|
|
54
|
-
- `npm run test:device` runs the automated device smoke checks under `test/device/automated/...`
|
|
55
|
-
- `npm run verify` runs the default maintainer verification sequence: lint, build, and unit tests
|
|
56
|
-
- Manual and debug-oriented device scripts live under `test/device/manual/...` and are not part of the default test commands
|
|
57
|
-
|
|
58
|
-
## Utility Scripts
|
|
59
|
-
|
|
60
|
-
- `npm run healthcheck` runs the `idb`/tooling healthcheck helper from `src/utils/cli/idb/check-idb.ts`
|
|
61
|
-
- `npm run install-idb` runs the guided `idb` installer helper from `src/utils/cli/idb/install-idb.ts`
|
|
62
|
-
- `npm run preflight-ios` runs the iOS preflight helper from `src/utils/cli/ios/preflight-ios.ts`
|
|
63
|
-
|
|
64
|
-
## Agent skills
|
|
65
|
-
|
|
66
|
-
- `skills/mcp-builder/` contains reusable build/install guidance for agents
|
|
67
|
-
- `skills/test-authoring/` contains reusable test-creation guidance aligned to this repo's current test structure
|
|
68
|
-
- Skills are written as plain Markdown packages so they can be consumed by different agent systems rather than one vendor-specific runtime
|
|
69
|
-
|
|
70
93
|
## License
|
|
71
94
|
|
|
72
95
|
MIT
|
package/dist/interact/index.js
CHANGED
|
@@ -95,10 +95,68 @@ export class ToolsInteract {
|
|
|
95
95
|
}
|
|
96
96
|
return null;
|
|
97
97
|
}
|
|
98
|
+
static _resolveParentIndex(elements, parentId) {
|
|
99
|
+
if (parentId === undefined || parentId === null)
|
|
100
|
+
return null;
|
|
101
|
+
if (typeof parentId === 'number' && Number.isInteger(parentId) && parentId >= 0 && parentId < elements.length) {
|
|
102
|
+
return parentId;
|
|
103
|
+
}
|
|
104
|
+
if (typeof parentId === 'string') {
|
|
105
|
+
const normalized = ToolsInteract._normalize(parentId);
|
|
106
|
+
if (!normalized)
|
|
107
|
+
return null;
|
|
108
|
+
if (/^\d+$/.test(normalized)) {
|
|
109
|
+
const index = Number(normalized);
|
|
110
|
+
if (index >= 0 && index < elements.length)
|
|
111
|
+
return index;
|
|
112
|
+
}
|
|
113
|
+
const foundIndex = elements.findIndex((el) => {
|
|
114
|
+
if (!el)
|
|
115
|
+
return false;
|
|
116
|
+
return ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? '') === normalized ||
|
|
117
|
+
ToolsInteract._normalize(el.stable_id ?? '') === normalized;
|
|
118
|
+
});
|
|
119
|
+
return foundIndex >= 0 ? foundIndex : null;
|
|
120
|
+
}
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
98
123
|
static _isVisibleElement(el) {
|
|
99
124
|
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
100
125
|
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
|
|
101
126
|
}
|
|
127
|
+
static _isTapActionable(el, storedStableId, platform) {
|
|
128
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
129
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
130
|
+
}
|
|
131
|
+
if (el.enabled === false) {
|
|
132
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
133
|
+
}
|
|
134
|
+
const semanticTapActionable = !!el.semantic && (el.semantic.is_clickable ||
|
|
135
|
+
(Array.isArray(el.semantic.supported_actions) && el.semantic.supported_actions.some((action) => ToolsInteract._normalize(action) === 'tap')));
|
|
136
|
+
if (!el.clickable && !(platform === 'ios' && semanticTapActionable)) {
|
|
137
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not clickable' };
|
|
138
|
+
}
|
|
139
|
+
if (storedStableId) {
|
|
140
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
141
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return { actionable: true };
|
|
145
|
+
}
|
|
146
|
+
static _isAdjustableActionable(el, storedStableId) {
|
|
147
|
+
if (!ToolsInteract._isVisibleElement(el)) {
|
|
148
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is not visible' };
|
|
149
|
+
}
|
|
150
|
+
if (el.enabled === false) {
|
|
151
|
+
return { actionable: false, failureCode: 'ELEMENT_NOT_INTERACTABLE', reason: 'element is disabled' };
|
|
152
|
+
}
|
|
153
|
+
if (storedStableId) {
|
|
154
|
+
if (!el.stable_id || el.stable_id !== storedStableId) {
|
|
155
|
+
return { actionable: false, failureCode: 'STALE_REFERENCE', reason: 'element stable_id changed' };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return { actionable: true };
|
|
159
|
+
}
|
|
102
160
|
static _computeElementId(platform, deviceId, el, index) {
|
|
103
161
|
const identity = {
|
|
104
162
|
platform,
|
|
@@ -120,7 +178,8 @@ export class ToolsInteract {
|
|
|
120
178
|
platform,
|
|
121
179
|
deviceId,
|
|
122
180
|
bounds,
|
|
123
|
-
index
|
|
181
|
+
index,
|
|
182
|
+
stable_id: el.stable_id ?? null
|
|
124
183
|
});
|
|
125
184
|
return {
|
|
126
185
|
text: el.text ?? null,
|
|
@@ -138,6 +197,214 @@ export class ToolsInteract {
|
|
|
138
197
|
semantic: el.semantic ?? null
|
|
139
198
|
};
|
|
140
199
|
}
|
|
200
|
+
static _resolveUiChangeScope(tree, scope, target) {
|
|
201
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
202
|
+
const normalizedScope = scope === 'subtree' ? 'subtree' : 'screen';
|
|
203
|
+
if (normalizedScope === 'screen') {
|
|
204
|
+
return {
|
|
205
|
+
elements,
|
|
206
|
+
resolution: {
|
|
207
|
+
scope: 'screen',
|
|
208
|
+
target: null,
|
|
209
|
+
resolved: true,
|
|
210
|
+
resolvedIndex: null,
|
|
211
|
+
resolvedStableId: null,
|
|
212
|
+
reason: 'screen scope'
|
|
213
|
+
}
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
const requestedTarget = typeof target === 'string' && target.trim().length > 0 ? target.trim() : null;
|
|
217
|
+
if (!requestedTarget) {
|
|
218
|
+
return {
|
|
219
|
+
elements: [],
|
|
220
|
+
resolution: {
|
|
221
|
+
scope: 'subtree',
|
|
222
|
+
target: null,
|
|
223
|
+
resolved: false,
|
|
224
|
+
resolvedIndex: null,
|
|
225
|
+
resolvedStableId: null,
|
|
226
|
+
reason: 'subtree scope requires a target element id'
|
|
227
|
+
},
|
|
228
|
+
error: {
|
|
229
|
+
code: 'INVALID_SCOPE',
|
|
230
|
+
message: 'scope=subtree requires a target element_id'
|
|
231
|
+
}
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const resolved = ToolsInteract._findScopedElement(tree, requestedTarget);
|
|
235
|
+
if (!resolved) {
|
|
236
|
+
return {
|
|
237
|
+
elements: [],
|
|
238
|
+
resolution: {
|
|
239
|
+
scope: 'subtree',
|
|
240
|
+
target: requestedTarget,
|
|
241
|
+
resolved: false,
|
|
242
|
+
resolvedIndex: null,
|
|
243
|
+
resolvedStableId: null,
|
|
244
|
+
reason: 'target element could not be resolved'
|
|
245
|
+
},
|
|
246
|
+
error: {
|
|
247
|
+
code: 'ELEMENT_NOT_FOUND',
|
|
248
|
+
message: `Target element ${requestedTarget} could not be resolved for subtree scope`
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
const subtreeIndices = ToolsInteract._collectSubtreeIndices(elements, resolved.index);
|
|
253
|
+
const scopedElements = subtreeIndices.map((index) => elements[index]).filter((element) => !!element);
|
|
254
|
+
return {
|
|
255
|
+
elements: scopedElements,
|
|
256
|
+
resolution: {
|
|
257
|
+
scope: 'subtree',
|
|
258
|
+
target: requestedTarget,
|
|
259
|
+
resolved: true,
|
|
260
|
+
resolvedIndex: resolved.index,
|
|
261
|
+
resolvedStableId: resolved.stableId,
|
|
262
|
+
reason: resolved.reason
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
static _findScopedElement(tree, targetElementId) {
|
|
267
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
268
|
+
const platform = tree?.device?.platform === 'ios' ? 'ios' : 'android';
|
|
269
|
+
const deviceId = tree?.device?.id ?? undefined;
|
|
270
|
+
const normalizedTarget = ToolsInteract._normalize(targetElementId);
|
|
271
|
+
for (let i = 0; i < elements.length; i++) {
|
|
272
|
+
const el = elements[i];
|
|
273
|
+
if (!el)
|
|
274
|
+
continue;
|
|
275
|
+
const computedElementId = ToolsInteract._computeElementId(platform, deviceId, el, i);
|
|
276
|
+
if (computedElementId === targetElementId) {
|
|
277
|
+
return {
|
|
278
|
+
index: i,
|
|
279
|
+
stableId: el.stable_id ?? null,
|
|
280
|
+
reason: 'element_id_match'
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (let i = 0; i < elements.length; i++) {
|
|
285
|
+
const el = elements[i];
|
|
286
|
+
if (!el)
|
|
287
|
+
continue;
|
|
288
|
+
if (el.stable_id && ToolsInteract._normalize(el.stable_id) === normalizedTarget) {
|
|
289
|
+
return {
|
|
290
|
+
index: i,
|
|
291
|
+
stableId: el.stable_id,
|
|
292
|
+
reason: 'stable_id_match'
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
const storedContext = ToolsInteract._resolvedUiElements.get(targetElementId);
|
|
297
|
+
if (storedContext?.stable_id) {
|
|
298
|
+
const normalizedStoredStableId = ToolsInteract._normalize(storedContext.stable_id);
|
|
299
|
+
for (let i = 0; i < elements.length; i++) {
|
|
300
|
+
const el = elements[i];
|
|
301
|
+
if (!el?.stable_id)
|
|
302
|
+
continue;
|
|
303
|
+
if (ToolsInteract._normalize(el.stable_id) === normalizedStoredStableId) {
|
|
304
|
+
return {
|
|
305
|
+
index: i,
|
|
306
|
+
stableId: el.stable_id,
|
|
307
|
+
reason: 'stored_stable_id_match'
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return null;
|
|
313
|
+
}
|
|
314
|
+
static _collectSubtreeIndices(elements, rootIndex) {
|
|
315
|
+
if (!Array.isArray(elements) || rootIndex < 0 || rootIndex >= elements.length)
|
|
316
|
+
return [];
|
|
317
|
+
const visited = new Set();
|
|
318
|
+
const stack = [rootIndex];
|
|
319
|
+
const result = [];
|
|
320
|
+
while (stack.length > 0) {
|
|
321
|
+
const index = stack.pop();
|
|
322
|
+
if (index === undefined || visited.has(index) || index < 0 || index >= elements.length)
|
|
323
|
+
continue;
|
|
324
|
+
visited.add(index);
|
|
325
|
+
result.push(index);
|
|
326
|
+
const element = elements[index];
|
|
327
|
+
if (!element)
|
|
328
|
+
continue;
|
|
329
|
+
const directChildren = new Set();
|
|
330
|
+
if (Array.isArray(element.children)) {
|
|
331
|
+
for (const childIndex of element.children) {
|
|
332
|
+
if (typeof childIndex === 'number' && Number.isInteger(childIndex) && childIndex >= 0 && childIndex < elements.length) {
|
|
333
|
+
directChildren.add(childIndex);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
for (let i = 0; i < elements.length; i++) {
|
|
338
|
+
if (ToolsInteract._resolveParentIndex(elements, elements[i]?.parentId) === index) {
|
|
339
|
+
directChildren.add(i);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
for (const childIndex of directChildren) {
|
|
343
|
+
if (!visited.has(childIndex))
|
|
344
|
+
stack.push(childIndex);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return result.sort((left, right) => left - right);
|
|
348
|
+
}
|
|
349
|
+
static _changeIdentityForElement(el, index) {
|
|
350
|
+
const stableId = ToolsInteract._normalize(el.stable_id);
|
|
351
|
+
if (stableId)
|
|
352
|
+
return `stable:${stableId}`;
|
|
353
|
+
return `fallback:${ToolsInteract._hash({
|
|
354
|
+
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
355
|
+
contentDescription: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? ''),
|
|
356
|
+
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
357
|
+
type: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
358
|
+
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
359
|
+
index
|
|
360
|
+
})}`;
|
|
361
|
+
}
|
|
362
|
+
static _summarizeUiChangeDelta(initialElements, currentElements) {
|
|
363
|
+
const buildMap = (elements) => {
|
|
364
|
+
const map = new Map();
|
|
365
|
+
for (let i = 0; i < elements.length; i++) {
|
|
366
|
+
const element = elements[i];
|
|
367
|
+
if (!element)
|
|
368
|
+
continue;
|
|
369
|
+
const key = ToolsInteract._changeIdentityForElement(element, i);
|
|
370
|
+
map.set(key, ToolsInteract._hash({
|
|
371
|
+
text: ToolsInteract._normalize(element.text ?? element.label ?? element.value ?? ''),
|
|
372
|
+
contentDescription: ToolsInteract._normalize(element.contentDescription ?? element.contentDesc ?? element.accessibilityLabel ?? ''),
|
|
373
|
+
resourceId: ToolsInteract._normalize(element.resourceId ?? element.resourceID ?? element.id ?? ''),
|
|
374
|
+
type: ToolsInteract._normalize(element.type ?? element.class ?? ''),
|
|
375
|
+
bounds: ToolsInteract._normalizeBounds(element.bounds) ?? [0, 0, 0, 0],
|
|
376
|
+
state: element.state ?? null,
|
|
377
|
+
visible: !!element.visible,
|
|
378
|
+
enabled: !!element.enabled,
|
|
379
|
+
clickable: !!element.clickable
|
|
380
|
+
}));
|
|
381
|
+
}
|
|
382
|
+
return map;
|
|
383
|
+
};
|
|
384
|
+
const initialMap = buildMap(initialElements);
|
|
385
|
+
const currentMap = buildMap(currentElements);
|
|
386
|
+
let added = 0;
|
|
387
|
+
let removed = 0;
|
|
388
|
+
let mutated = 0;
|
|
389
|
+
for (const [key, value] of currentMap.entries()) {
|
|
390
|
+
if (!initialMap.has(key)) {
|
|
391
|
+
added++;
|
|
392
|
+
}
|
|
393
|
+
else if (initialMap.get(key) !== value) {
|
|
394
|
+
mutated++;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
for (const key of initialMap.keys()) {
|
|
398
|
+
if (!currentMap.has(key))
|
|
399
|
+
removed++;
|
|
400
|
+
}
|
|
401
|
+
return {
|
|
402
|
+
total_elements: currentElements.length,
|
|
403
|
+
added_elements: added,
|
|
404
|
+
removed_elements: removed,
|
|
405
|
+
mutated_elements: mutated
|
|
406
|
+
};
|
|
407
|
+
}
|
|
141
408
|
static _rememberResolvedElement(elementId, context) {
|
|
142
409
|
if (ToolsInteract._resolvedUiElements.has(elementId)) {
|
|
143
410
|
ToolsInteract._resolvedUiElements.delete(elementId);
|
|
@@ -205,6 +472,9 @@ export class ToolsInteract {
|
|
|
205
472
|
}
|
|
206
473
|
return null;
|
|
207
474
|
}
|
|
475
|
+
static _uiChangeSignaturesEqual(left, right) {
|
|
476
|
+
return left.hierarchy === right.hierarchy && left.text === right.text && left.state === right.state;
|
|
477
|
+
}
|
|
208
478
|
static _resolvedTargetFromElement(elementId, element, index) {
|
|
209
479
|
return {
|
|
210
480
|
elementId,
|
|
@@ -381,25 +651,12 @@ export class ToolsInteract {
|
|
|
381
651
|
let current = chosen;
|
|
382
652
|
let safety = 0;
|
|
383
653
|
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el)) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
384
|
-
const
|
|
385
|
-
let parentIndex = null;
|
|
386
|
-
if (typeof parentId === 'number')
|
|
387
|
-
parentIndex = parentId;
|
|
388
|
-
else if (typeof parentId === 'string' && /^\d+$/.test(parentId))
|
|
389
|
-
parentIndex = Number(parentId);
|
|
654
|
+
const parentIndex = ToolsInteract._resolveParentIndex(elements, current.el.parentId);
|
|
390
655
|
if (parentIndex !== null && elements[parentIndex]) {
|
|
391
656
|
current = { el: elements[parentIndex], idx: parentIndex };
|
|
392
657
|
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
|
|
393
658
|
return current;
|
|
394
659
|
}
|
|
395
|
-
else if (typeof parentId === 'string') {
|
|
396
|
-
const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId);
|
|
397
|
-
if (foundIndex === -1)
|
|
398
|
-
break;
|
|
399
|
-
current = { el: elements[foundIndex], idx: foundIndex };
|
|
400
|
-
if (current.el.clickable || current.el.focusable || ToolsInteract._isSemanticActionable(current.el))
|
|
401
|
-
return current;
|
|
402
|
-
}
|
|
403
660
|
else {
|
|
404
661
|
break;
|
|
405
662
|
}
|
|
@@ -513,11 +770,9 @@ export class ToolsInteract {
|
|
|
513
770
|
return ToolsInteract._actionFailure(actionType, selector, null, 'STALE_REFERENCE', true, fingerprintBefore);
|
|
514
771
|
}
|
|
515
772
|
const resolvedTarget = ToolsInteract._resolvedTargetFromElement(resolved.elementId, currentMatch.el, currentMatch.index);
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
if (currentMatch.el.enabled === false) {
|
|
520
|
-
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
773
|
+
const tapActionability = ToolsInteract._isTapActionable(currentMatch.el, resolved.stable_id, resolved.platform);
|
|
774
|
+
if (!tapActionability.actionable) {
|
|
775
|
+
return ToolsInteract._actionFailure(actionType, selector, resolvedTarget, tapActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', true, fingerprintBefore);
|
|
521
776
|
}
|
|
522
777
|
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
|
|
523
778
|
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
|
|
@@ -550,6 +805,7 @@ export class ToolsInteract {
|
|
|
550
805
|
const sourcePlatform = platform || 'android';
|
|
551
806
|
let resolvedPlatform = sourcePlatform;
|
|
552
807
|
let resolvedDeviceId = deviceId;
|
|
808
|
+
const storedResolvedTarget = element_id ? ToolsInteract._resolvedUiElements.get(element_id) ?? null : null;
|
|
553
809
|
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId);
|
|
554
810
|
let semanticFallbackElement = null;
|
|
555
811
|
const traceSteps = [];
|
|
@@ -755,6 +1011,10 @@ export class ToolsInteract {
|
|
|
755
1011
|
resolvedTarget = resolved.resolvedTarget;
|
|
756
1012
|
const currentEl = resolved.match.el;
|
|
757
1013
|
cachedResolvedMatch = resolved.match;
|
|
1014
|
+
const adjustableActionability = ToolsInteract._isAdjustableActionable(currentEl, storedResolvedTarget?.stable_id);
|
|
1015
|
+
if (!adjustableActionability.actionable) {
|
|
1016
|
+
return buildFailure(adjustableActionability.failureCode ?? 'ELEMENT_NOT_INTERACTABLE', adjustableActionability.reason ?? 'adjustable control is not actionable', resolvedTarget, currentDevice, lastObservedState, attemptCount, lastAdjustmentMode, true);
|
|
1017
|
+
}
|
|
758
1018
|
const bounds = ToolsInteract._normalizeBounds(currentEl.bounds);
|
|
759
1019
|
const valueRange = currentEl.state?.value_range ?? null;
|
|
760
1020
|
const currentValue = ToolsInteract._readNumericControlValue(currentEl, property);
|
|
@@ -1708,49 +1968,97 @@ export class ToolsInteract {
|
|
|
1708
1968
|
}
|
|
1709
1969
|
};
|
|
1710
1970
|
}
|
|
1711
|
-
static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms =
|
|
1971
|
+
static async waitForUIChangeHandler({ platform, deviceId, timeout_ms = 60000, stability_window_ms = 300, expected_change, scope = 'screen', target = null }) {
|
|
1712
1972
|
const start = Date.now();
|
|
1713
1973
|
const pollIntervalMs = 300;
|
|
1714
|
-
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms :
|
|
1974
|
+
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300);
|
|
1715
1975
|
let baseline = null;
|
|
1976
|
+
let baselineScope = null;
|
|
1716
1977
|
let lastObservedRevision = null;
|
|
1717
1978
|
let lastLoadingState = null;
|
|
1979
|
+
let lastSnapshotFreshnessMs = null;
|
|
1980
|
+
let candidateSignatures = null;
|
|
1981
|
+
let candidateObservedChange = null;
|
|
1982
|
+
let candidateSinceMs = null;
|
|
1983
|
+
let lastChangeSummary = null;
|
|
1984
|
+
let lastScopeResolution = {
|
|
1985
|
+
scope: scope === 'subtree' ? 'subtree' : 'screen',
|
|
1986
|
+
target: target && typeof target === 'string' ? target : null,
|
|
1987
|
+
resolved: scope !== 'subtree',
|
|
1988
|
+
resolvedIndex: null,
|
|
1989
|
+
resolvedStableId: null,
|
|
1990
|
+
reason: scope === 'subtree' ? 'target not resolved yet' : 'screen scope'
|
|
1991
|
+
};
|
|
1718
1992
|
while (Date.now() - start < timeout_ms) {
|
|
1719
1993
|
try {
|
|
1720
1994
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId });
|
|
1721
|
-
const
|
|
1995
|
+
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target);
|
|
1996
|
+
if (scopedTree.error) {
|
|
1997
|
+
lastScopeResolution = scopedTree.resolution;
|
|
1998
|
+
return {
|
|
1999
|
+
success: false,
|
|
2000
|
+
observed_change: null,
|
|
2001
|
+
snapshot_revision: typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision ?? undefined,
|
|
2002
|
+
snapshot_freshness_ms: typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs ?? null,
|
|
2003
|
+
timeout: true,
|
|
2004
|
+
elapsed_ms: Date.now() - start,
|
|
2005
|
+
expected_change,
|
|
2006
|
+
loading_state: tree?.loading_state ?? lastLoadingState ?? null,
|
|
2007
|
+
scope: scopedTree.resolution.scope,
|
|
2008
|
+
target: scopedTree.resolution.target,
|
|
2009
|
+
stability_state: 'transient',
|
|
2010
|
+
change_summary: lastChangeSummary,
|
|
2011
|
+
reason: scopedTree.error.message,
|
|
2012
|
+
error: scopedTree.error
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
2015
|
+
const scopedElements = scopedTree.elements;
|
|
2016
|
+
const scopedSignatureTree = {
|
|
2017
|
+
...tree,
|
|
2018
|
+
elements: scopedElements
|
|
2019
|
+
};
|
|
2020
|
+
const signatures = ToolsInteract._buildUiChangeSignatures(scopedSignatureTree);
|
|
1722
2021
|
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision;
|
|
1723
2022
|
lastLoadingState = tree?.loading_state ?? lastLoadingState;
|
|
2023
|
+
lastSnapshotFreshnessMs = typeof tree?.captured_at_ms === 'number' ? Math.max(0, Date.now() - tree.captured_at_ms) : lastSnapshotFreshnessMs;
|
|
2024
|
+
lastChangeSummary = baseline ? ToolsInteract._summarizeUiChangeDelta((baselineScope?.elements ?? []), scopedElements) : lastChangeSummary;
|
|
2025
|
+
lastScopeResolution = scopedTree.resolution;
|
|
2026
|
+
baselineScope = baselineScope ?? scopedTree;
|
|
1724
2027
|
if (!baseline) {
|
|
1725
2028
|
baseline = signatures;
|
|
2029
|
+
baselineScope = scopedTree;
|
|
1726
2030
|
}
|
|
1727
2031
|
else {
|
|
1728
2032
|
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures);
|
|
1729
2033
|
if (observedChange) {
|
|
1730
|
-
if (
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
const confirmChange = ToolsInteract._matchesUiChange(expected_change, baseline, confirmSignatures);
|
|
1735
|
-
if (!confirmChange || confirmSignatures.hierarchy !== signatures.hierarchy || confirmSignatures.text !== signatures.text || confirmSignatures.state !== signatures.state) {
|
|
1736
|
-
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision;
|
|
1737
|
-
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState;
|
|
1738
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
|
|
1739
|
-
continue;
|
|
1740
|
-
}
|
|
1741
|
-
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision;
|
|
1742
|
-
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState;
|
|
2034
|
+
if (!candidateSignatures || !ToolsInteract._uiChangeSignaturesEqual(candidateSignatures, signatures) || candidateObservedChange !== observedChange) {
|
|
2035
|
+
candidateSignatures = signatures;
|
|
2036
|
+
candidateObservedChange = observedChange;
|
|
2037
|
+
candidateSinceMs = Date.now();
|
|
1743
2038
|
}
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
2039
|
+
const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs;
|
|
2040
|
+
if (stabilityWindow === 0 || stableForMs >= stabilityWindow) {
|
|
2041
|
+
return {
|
|
2042
|
+
success: true,
|
|
2043
|
+
observed_change: candidateObservedChange ?? observedChange,
|
|
2044
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2045
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
2046
|
+
timeout: false,
|
|
2047
|
+
elapsed_ms: Date.now() - start,
|
|
2048
|
+
expected_change,
|
|
2049
|
+
loading_state: lastLoadingState ?? null,
|
|
2050
|
+
scope: lastScopeResolution.scope,
|
|
2051
|
+
target: lastScopeResolution.target,
|
|
2052
|
+
stability_state: 'stable',
|
|
2053
|
+
change_summary: lastChangeSummary,
|
|
2054
|
+
reason: 'UI change observed'
|
|
2055
|
+
};
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
else {
|
|
2059
|
+
candidateSignatures = null;
|
|
2060
|
+
candidateObservedChange = null;
|
|
2061
|
+
candidateSinceMs = null;
|
|
1754
2062
|
}
|
|
1755
2063
|
}
|
|
1756
2064
|
}
|
|
@@ -1763,10 +2071,15 @@ export class ToolsInteract {
|
|
|
1763
2071
|
success: false,
|
|
1764
2072
|
observed_change: null,
|
|
1765
2073
|
snapshot_revision: lastObservedRevision ?? undefined,
|
|
2074
|
+
snapshot_freshness_ms: lastSnapshotFreshnessMs ?? null,
|
|
1766
2075
|
timeout: true,
|
|
1767
2076
|
elapsed_ms: Date.now() - start,
|
|
1768
2077
|
expected_change,
|
|
1769
2078
|
loading_state: lastLoadingState ?? null,
|
|
2079
|
+
scope: lastScopeResolution.scope,
|
|
2080
|
+
target: lastScopeResolution.target,
|
|
2081
|
+
stability_state: 'transient',
|
|
2082
|
+
change_summary: lastChangeSummary,
|
|
1770
2083
|
reason: 'timeout'
|
|
1771
2084
|
};
|
|
1772
2085
|
}
|
package/dist/observe/index.js
CHANGED
|
@@ -325,6 +325,7 @@ export class ToolsObserve {
|
|
|
325
325
|
const snapshotMetadata = deriveSnapshotMetadata(snapshotDeviceKey, raw.ui_tree, 'snapshot', raw.ui_tree?.snapshot_revision ? null : (raw.fingerprint || raw.activity || null));
|
|
326
326
|
raw.snapshot_revision = raw.ui_tree?.snapshot_revision ?? snapshotMetadata.snapshot_revision;
|
|
327
327
|
raw.captured_at_ms = raw.ui_tree?.captured_at_ms ?? snapshotMetadata.captured_at_ms;
|
|
328
|
+
raw.snapshot_delta = raw.ui_tree?.snapshot_delta ?? snapshotMetadata.snapshot_delta ?? null;
|
|
328
329
|
raw.loading_state = raw.ui_tree?.loading_state ?? snapshotMetadata.loading_state;
|
|
329
330
|
const semantic = deriveSnapshotSemantic(raw);
|
|
330
331
|
return semantic ? { raw, semantic } : { raw };
|