mobile-debug-mcp 0.24.1 → 0.24.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +124 -17
- package/dist/manage/android.js +24 -4
- package/dist/manage/ios.js +27 -3
- package/dist/server/common.js +32 -4
- package/dist/server/tool-definitions.js +8 -8
- package/dist/server/tool-handlers.js +22 -6
- package/dist/system/index.js +53 -2
- package/docs/CHANGELOG.md +8 -0
- package/docs/specs/baseline-spec-v0.md +312 -0
- package/docs/specs/mcp-tooling-spec-v1.md +272 -0
- package/docs/tools/interact.md +76 -10
- package/docs/tools/manage.md +17 -2
- package/docs/tools/system.md +23 -1
- package/package.json +1 -1
- package/src/interact/index.ts +126 -18
- package/src/manage/android.ts +24 -4
- package/src/manage/ios.ts +27 -3
- package/src/server/common.ts +36 -4
- package/src/server/tool-definitions.ts +8 -8
- package/src/server/tool-handlers.ts +23 -6
- package/src/system/index.ts +57 -2
- package/src/types.ts +37 -1
- package/test/unit/interact/expect_tools.test.ts +29 -2
- package/test/unit/interact/wait_for_screen_change.test.ts +7 -3
- package/test/unit/interact/wait_for_ui_selector_matching.test.ts +19 -0
- package/test/unit/server/response_shapes.test.ts +48 -4
- package/test/unit/system/get_system_status.test.ts +2 -0
- package/test/unit/system/system_status.test.ts +6 -0
package/docs/tools/interact.md
CHANGED
|
@@ -33,7 +33,7 @@ Example response:
|
|
|
33
33
|
```json
|
|
34
34
|
{
|
|
35
35
|
"action_id": "tap_1710000000000_1",
|
|
36
|
-
"timestamp":
|
|
36
|
+
"timestamp": "2026-04-23T08:00:00.000Z",
|
|
37
37
|
"action_type": "tap",
|
|
38
38
|
"target": { "selector": { "x": 100, "y": 200 }, "resolved": null },
|
|
39
39
|
"success": true,
|
|
@@ -111,13 +111,26 @@ Input example:
|
|
|
111
111
|
Success response example:
|
|
112
112
|
|
|
113
113
|
```json
|
|
114
|
-
{
|
|
114
|
+
{
|
|
115
|
+
"success": true,
|
|
116
|
+
"previousFingerprint": "<old-hex-fingerprint>",
|
|
117
|
+
"newFingerprint": "<hex-fingerprint>",
|
|
118
|
+
"elapsedMs": 420,
|
|
119
|
+
"observed_screen": { "fingerprint": "<hex-fingerprint>", "activity": "MainActivity" }
|
|
120
|
+
}
|
|
115
121
|
```
|
|
116
122
|
|
|
117
123
|
Failure (timeout) example:
|
|
118
124
|
|
|
119
125
|
```json
|
|
120
|
-
{
|
|
126
|
+
{
|
|
127
|
+
"success": false,
|
|
128
|
+
"reason": "timeout",
|
|
129
|
+
"previousFingerprint": "<old-hex-fingerprint>",
|
|
130
|
+
"lastFingerprint": "<hex-fingerprint>",
|
|
131
|
+
"elapsedMs": 5000,
|
|
132
|
+
"observed_screen": { "fingerprint": "<hex-fingerprint>", "activity": "HomeActivity" }
|
|
133
|
+
}
|
|
121
134
|
```
|
|
122
135
|
|
|
123
136
|
Notes:
|
|
@@ -214,7 +227,26 @@ Success response:
|
|
|
214
227
|
"index": 8,
|
|
215
228
|
"elementId": "el_..."
|
|
216
229
|
},
|
|
217
|
-
"metrics": { "latency_ms": 120, "poll_count": 1, "attempts": 1 }
|
|
230
|
+
"metrics": { "latency_ms": 120, "poll_count": 1, "attempts": 1 },
|
|
231
|
+
"requested": {
|
|
232
|
+
"selector": { "text": "Generate Session", "contains": false },
|
|
233
|
+
"condition": "clickable",
|
|
234
|
+
"match": { "index": 0 }
|
|
235
|
+
},
|
|
236
|
+
"observed": {
|
|
237
|
+
"matched_count": 1,
|
|
238
|
+
"condition_satisfied": true,
|
|
239
|
+
"selected_index": 8,
|
|
240
|
+
"last_matched_element": {
|
|
241
|
+
"text": "Generate Session",
|
|
242
|
+
"resource_id": null,
|
|
243
|
+
"accessibility_id": null,
|
|
244
|
+
"class": "android.widget.TextView",
|
|
245
|
+
"bounds": [471, 1098, 809, 1158],
|
|
246
|
+
"index": 8,
|
|
247
|
+
"elementId": "el_..."
|
|
248
|
+
}
|
|
249
|
+
}
|
|
218
250
|
}
|
|
219
251
|
```
|
|
220
252
|
|
|
@@ -223,8 +255,27 @@ Timeout response:
|
|
|
223
255
|
```json
|
|
224
256
|
{
|
|
225
257
|
"status": "timeout",
|
|
226
|
-
"error": { "code": "ELEMENT_NOT_FOUND", "message": "Condition visible not satisfied within timeout" },
|
|
227
|
-
"metrics": { "latency_ms": 5000, "poll_count": 17, "attempts": 1 }
|
|
258
|
+
"error": { "code": "ELEMENT_NOT_FOUND", "message": "Condition visible not satisfied within timeout; observed 1 match(es)" },
|
|
259
|
+
"metrics": { "latency_ms": 5000, "poll_count": 17, "attempts": 1 },
|
|
260
|
+
"requested": {
|
|
261
|
+
"selector": { "text": "Generate Session", "contains": false },
|
|
262
|
+
"condition": "visible",
|
|
263
|
+
"match": { "index": 0 }
|
|
264
|
+
},
|
|
265
|
+
"observed": {
|
|
266
|
+
"matched_count": 1,
|
|
267
|
+
"condition_satisfied": false,
|
|
268
|
+
"selected_index": 8,
|
|
269
|
+
"last_matched_element": {
|
|
270
|
+
"text": "Generate Session",
|
|
271
|
+
"resource_id": null,
|
|
272
|
+
"accessibility_id": null,
|
|
273
|
+
"class": "android.widget.TextView",
|
|
274
|
+
"bounds": [471, 1098, 809, 1158],
|
|
275
|
+
"index": 8,
|
|
276
|
+
"elementId": "el_..."
|
|
277
|
+
}
|
|
278
|
+
}
|
|
228
279
|
}
|
|
229
280
|
```
|
|
230
281
|
|
|
@@ -232,6 +283,7 @@ Notes:
|
|
|
232
283
|
|
|
233
284
|
- Use `wait_for_ui` to get a stable `elementId` for `tap_element`.
|
|
234
285
|
- Use it before an action when the target element or timing is uncertain.
|
|
286
|
+
- Use `requested` and `observed` to see exactly what condition was checked and what the last poll actually found.
|
|
235
287
|
- If the expected outcome is known after the action, follow with `expect_*`.
|
|
236
288
|
|
|
237
289
|
---
|
|
@@ -251,7 +303,7 @@ Success response:
|
|
|
251
303
|
```json
|
|
252
304
|
{
|
|
253
305
|
"action_id": "tap_element_1710000000000_1",
|
|
254
|
-
"timestamp":
|
|
306
|
+
"timestamp": "2026-04-23T08:00:00.000Z",
|
|
255
307
|
"action_type": "tap_element",
|
|
256
308
|
"target": {
|
|
257
309
|
"selector": { "elementId": "el_123" },
|
|
@@ -276,7 +328,7 @@ Failure response:
|
|
|
276
328
|
```json
|
|
277
329
|
{
|
|
278
330
|
"action_id": "tap_element_1710000000001_2",
|
|
279
|
-
"timestamp":
|
|
331
|
+
"timestamp": "2026-04-23T08:00:00.001Z",
|
|
280
332
|
"action_type": "tap_element",
|
|
281
333
|
"target": { "selector": { "elementId": "el_123" }, "resolved": null },
|
|
282
334
|
"success": false,
|
|
@@ -330,7 +382,12 @@ Response:
|
|
|
330
382
|
"success": true,
|
|
331
383
|
"observed_screen": { "fingerprint": "<actual-fingerprint>", "screen": "com.example.app.MainActivity" },
|
|
332
384
|
"expected_screen": { "fingerprint": "<expected-fingerprint>", "screen": null },
|
|
333
|
-
"confidence": 1
|
|
385
|
+
"confidence": 1,
|
|
386
|
+
"comparison": {
|
|
387
|
+
"basis": "fingerprint",
|
|
388
|
+
"matched": true,
|
|
389
|
+
"reason": "observed fingerprint matches expected fingerprint <expected-fingerprint>"
|
|
390
|
+
}
|
|
334
391
|
}
|
|
335
392
|
```
|
|
336
393
|
|
|
@@ -367,6 +424,7 @@ Response:
|
|
|
367
424
|
"success": true,
|
|
368
425
|
"selector": { "text": "Play session" },
|
|
369
426
|
"element_id": "el_123",
|
|
427
|
+
"expected_condition": "visible",
|
|
370
428
|
"element": {
|
|
371
429
|
"elementId": "el_123",
|
|
372
430
|
"text": "Play session",
|
|
@@ -375,7 +433,14 @@ Response:
|
|
|
375
433
|
"class": "android.widget.TextView",
|
|
376
434
|
"bounds": [519, 1770, 762, 1830],
|
|
377
435
|
"index": 11
|
|
378
|
-
}
|
|
436
|
+
},
|
|
437
|
+
"observed": {
|
|
438
|
+
"status": "success",
|
|
439
|
+
"matched_count": 1,
|
|
440
|
+
"condition_satisfied": true,
|
|
441
|
+
"selected_index": 11
|
|
442
|
+
},
|
|
443
|
+
"reason": "selector is visible"
|
|
379
444
|
}
|
|
380
445
|
```
|
|
381
446
|
|
|
@@ -384,4 +449,5 @@ Notes:
|
|
|
384
449
|
- Primary and authoritative verification tool for expected element visibility.
|
|
385
450
|
- `selector` is the primary input; `element_id` is optional context only.
|
|
386
451
|
- The tool resolves the selector internally when needed.
|
|
452
|
+
- On failure, `reason` and `observed` tell you whether the selector was missing entirely or present but not yet visible.
|
|
387
453
|
- Use when the screen should remain on the same destination but a specific element should appear or become visible.
|
package/docs/tools/manage.md
CHANGED
|
@@ -121,12 +121,25 @@ start_app response example:
|
|
|
121
121
|
```json
|
|
122
122
|
{
|
|
123
123
|
"action_id": "start_app_1710000000000_1",
|
|
124
|
-
"timestamp":
|
|
124
|
+
"timestamp": "2026-04-23T08:00:00.000Z",
|
|
125
125
|
"action_type": "start_app",
|
|
126
|
+
"device": { "platform": "android", "id": "emulator-5554", "osVersion": "14", "model": "Pixel", "simulator": true },
|
|
126
127
|
"target": { "selector": { "appId": "com.example.app" }, "resolved": null },
|
|
127
128
|
"success": true,
|
|
128
129
|
"ui_fingerprint_before": "fp_before",
|
|
129
|
-
"ui_fingerprint_after": "fp_after"
|
|
130
|
+
"ui_fingerprint_after": "fp_after",
|
|
131
|
+
"details": {
|
|
132
|
+
"launch_time_ms": 1000,
|
|
133
|
+
"output": "Events injected: 1",
|
|
134
|
+
"device_id": "emulator-5554",
|
|
135
|
+
"observed_app": {
|
|
136
|
+
"appId": "com.example.app",
|
|
137
|
+
"package": "com.example.app",
|
|
138
|
+
"activity": "com.example.app.MainActivity",
|
|
139
|
+
"screen": "MainActivity",
|
|
140
|
+
"matchedTarget": true
|
|
141
|
+
}
|
|
142
|
+
}
|
|
130
143
|
}
|
|
131
144
|
```
|
|
132
145
|
|
|
@@ -137,5 +150,7 @@ terminate_app and reset_app_data return operation-specific lifecycle results ins
|
|
|
137
150
|
Notes:
|
|
138
151
|
|
|
139
152
|
- `start_app` and `restart_app` report execution success, not outcome correctness.
|
|
153
|
+
- Use `details.observed_app` as the quick decision signal for what the tool actually saw after launch.
|
|
154
|
+
- Android launch feedback usually includes package/activity matching; iOS launch feedback includes launch output and PID when available.
|
|
140
155
|
- When the landing screen is known, use `expect_screen` as the final verification step.
|
|
141
156
|
- If launch timing is uncertain, insert `wait_for_screen_change` before `expect_screen`.
|
package/docs/tools/system.md
CHANGED
|
@@ -16,6 +16,7 @@ Response (example):
|
|
|
16
16
|
```json
|
|
17
17
|
{
|
|
18
18
|
"success": true,
|
|
19
|
+
"status": "ready",
|
|
19
20
|
"adbAvailable": true,
|
|
20
21
|
"adbVersion": "8.1.0",
|
|
21
22
|
"devices": 1,
|
|
@@ -25,7 +26,26 @@ Response (example):
|
|
|
25
26
|
"issues": [],
|
|
26
27
|
"appInstalled": true,
|
|
27
28
|
"iosAvailable": true,
|
|
28
|
-
"iosDevices": 1
|
|
29
|
+
"iosDevices": 1,
|
|
30
|
+
"summary": {
|
|
31
|
+
"overall": "ready",
|
|
32
|
+
"android": {
|
|
33
|
+
"ready": true,
|
|
34
|
+
"summary": "1 Android device(s) connected; log access available",
|
|
35
|
+
"blockers": []
|
|
36
|
+
},
|
|
37
|
+
"ios": {
|
|
38
|
+
"ready": true,
|
|
39
|
+
"summary": "1 iOS simulator(s) booted",
|
|
40
|
+
"blockers": []
|
|
41
|
+
},
|
|
42
|
+
"gradle": {
|
|
43
|
+
"ready": true,
|
|
44
|
+
"summary": "No explicit Gradle JDK override detected",
|
|
45
|
+
"blockers": [],
|
|
46
|
+
"suggestedFixes": []
|
|
47
|
+
}
|
|
48
|
+
}
|
|
29
49
|
}
|
|
30
50
|
```
|
|
31
51
|
|
|
@@ -39,6 +59,8 @@ Checks performed (fast, best-effort):
|
|
|
39
59
|
|
|
40
60
|
Behavior notes:
|
|
41
61
|
- Always returns structured JSON and never throws; any failures are surfaced in the `issues` array.
|
|
62
|
+
- `status` gives a quick overall gate: `ready`, `degraded`, or `blocked`.
|
|
63
|
+
- `summary.android`, `summary.ios`, and `summary.gradle` provide the fastest path to the actual blocker category.
|
|
42
64
|
- Designed to be fast (<~1s probes where possible); startup callers may prefer a `fastMode` variant that only checks existence.
|
|
43
65
|
- Useful to call at the start of an agent session to gate subsequent actions.
|
|
44
66
|
|
package/package.json
CHANGED
package/src/interact/index.ts
CHANGED
|
@@ -146,7 +146,7 @@ export class ToolsInteract {
|
|
|
146
146
|
|
|
147
147
|
private static _actionFailure(
|
|
148
148
|
actionId: string,
|
|
149
|
-
timestamp:
|
|
149
|
+
timestamp: string,
|
|
150
150
|
actionType: string,
|
|
151
151
|
selector: Record<string, unknown> | null,
|
|
152
152
|
resolved: ActionTargetResolved | null,
|
|
@@ -254,9 +254,10 @@ export class ToolsInteract {
|
|
|
254
254
|
}
|
|
255
255
|
|
|
256
256
|
static async tapElementHandler({ elementId }: { elementId: string }): Promise<TapElementResponse> {
|
|
257
|
-
const
|
|
257
|
+
const timestampMs = Date.now()
|
|
258
|
+
const timestamp = new Date(timestampMs).toISOString()
|
|
258
259
|
const actionType = 'tap_element'
|
|
259
|
-
const actionId = nextActionId(actionType,
|
|
260
|
+
const actionId = nextActionId(actionType, timestampMs)
|
|
260
261
|
const selector = { elementId }
|
|
261
262
|
const resolved = ToolsInteract._resolvedUiElements.get(elementId)
|
|
262
263
|
if (!resolved) {
|
|
@@ -304,6 +305,7 @@ export class ToolsInteract {
|
|
|
304
305
|
action_id: actionId,
|
|
305
306
|
timestamp,
|
|
306
307
|
action_type: actionType,
|
|
308
|
+
...(tree?.device ? { device: tree.device } : {}),
|
|
307
309
|
target: {
|
|
308
310
|
selector,
|
|
309
311
|
resolved: resolvedTarget
|
|
@@ -490,6 +492,12 @@ export class ToolsInteract {
|
|
|
490
492
|
|
|
491
493
|
static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string }) {
|
|
492
494
|
const overallStart = Date.now()
|
|
495
|
+
const requestedIndex = typeof match?.index === 'number' ? match.index : null
|
|
496
|
+
const requested = {
|
|
497
|
+
selector: selector ?? {},
|
|
498
|
+
condition,
|
|
499
|
+
match: requestedIndex === null ? null : { index: requestedIndex }
|
|
500
|
+
}
|
|
493
501
|
|
|
494
502
|
// Validate selector: require at least one non-empty field (text, resource_id, or accessibility_id)
|
|
495
503
|
const hasText = typeof selector?.text === 'string' && selector.text.trim().length > 0;
|
|
@@ -503,18 +511,20 @@ export class ToolsInteract {
|
|
|
503
511
|
code: 'INVALID_SELECTOR',
|
|
504
512
|
message: 'Selector must include at least one non-empty field: text, resource_id, or accessibility_id'
|
|
505
513
|
},
|
|
506
|
-
metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }
|
|
514
|
+
metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 },
|
|
515
|
+
requested,
|
|
516
|
+
observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null }
|
|
507
517
|
};
|
|
508
518
|
}
|
|
509
519
|
|
|
510
520
|
// Validate condition
|
|
511
521
|
if (!['exists','not_exists','visible','clickable'].includes(condition)) {
|
|
512
|
-
return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 } }
|
|
522
|
+
return { status: 'timeout', error: { code: 'INVALID_CONDITION', message: `Unsupported condition: ${condition}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
|
|
513
523
|
}
|
|
514
524
|
|
|
515
525
|
// Platform check
|
|
516
526
|
if (platform && !['android','ios'].includes(platform)) {
|
|
517
|
-
return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 } }
|
|
527
|
+
return { status: 'timeout', error: { code: 'PLATFORM_NOT_SUPPORTED', message: `Unsupported platform: ${platform}` }, metrics: { latency_ms: Date.now() - overallStart, poll_count: 0, attempts: 0 }, requested, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } }
|
|
518
528
|
}
|
|
519
529
|
|
|
520
530
|
const effectivePoll = Math.max(50, Math.min(poll_interval_ms || 300, 2000))
|
|
@@ -523,6 +533,9 @@ export class ToolsInteract {
|
|
|
523
533
|
|
|
524
534
|
let attempts = 0
|
|
525
535
|
let totalPollCount = 0
|
|
536
|
+
let lastMatchedCount = 0
|
|
537
|
+
let lastMatchedElement: ActionTargetResolved | null = null
|
|
538
|
+
let lastConditionSatisfied = false
|
|
526
539
|
|
|
527
540
|
// Precompute normalized selector values and helpers (constant across polls)
|
|
528
541
|
const normalize = ToolsInteract._normalize
|
|
@@ -623,19 +636,29 @@ export class ToolsInteract {
|
|
|
623
636
|
} else conditionMet = false
|
|
624
637
|
}
|
|
625
638
|
|
|
639
|
+
const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
|
|
640
|
+
const resolvedDeviceId = tree?.device?.id || deviceId
|
|
641
|
+
lastMatchedCount = matchedCount
|
|
642
|
+
lastConditionSatisfied = conditionMet
|
|
643
|
+
lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
|
|
644
|
+
|
|
626
645
|
if (conditionMet) {
|
|
627
646
|
const now = Date.now()
|
|
628
647
|
const latency_ms = now - overallStart
|
|
629
|
-
|
|
630
|
-
const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
|
|
631
|
-
const resolvedDeviceId = tree?.device?.id || deviceId
|
|
632
|
-
const outEl = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
|
|
648
|
+
const outEl = lastMatchedElement
|
|
633
649
|
|
|
634
650
|
return {
|
|
635
651
|
status: 'success',
|
|
636
652
|
matched: matchedCount,
|
|
637
653
|
element: outEl,
|
|
638
|
-
metrics: { latency_ms, poll_count: totalPollCount, attempts }
|
|
654
|
+
metrics: { latency_ms, poll_count: totalPollCount, attempts },
|
|
655
|
+
requested,
|
|
656
|
+
observed: {
|
|
657
|
+
matched_count: matchedCount,
|
|
658
|
+
condition_satisfied: true,
|
|
659
|
+
selected_index: outEl?.index ?? null,
|
|
660
|
+
last_matched_element: outEl
|
|
661
|
+
}
|
|
639
662
|
}
|
|
640
663
|
}
|
|
641
664
|
|
|
@@ -656,16 +679,38 @@ export class ToolsInteract {
|
|
|
656
679
|
|
|
657
680
|
// Final failure for this call
|
|
658
681
|
const elapsed = Date.now() - overallStart
|
|
682
|
+
const observed = {
|
|
683
|
+
matched_count: lastMatchedCount,
|
|
684
|
+
condition_satisfied: lastConditionSatisfied,
|
|
685
|
+
selected_index: lastMatchedElement?.index ?? null,
|
|
686
|
+
last_matched_element: lastMatchedElement
|
|
687
|
+
}
|
|
688
|
+
const matchNote = requestedIndex !== null && lastMatchedCount <= requestedIndex
|
|
689
|
+
? ` requested match.index=${requestedIndex} but observed ${lastMatchedCount} match(es)`
|
|
690
|
+
: ` observed ${lastMatchedCount} match(es)`
|
|
659
691
|
return {
|
|
660
692
|
status: 'timeout',
|
|
661
|
-
error: { code: 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout` },
|
|
662
|
-
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts }
|
|
693
|
+
error: { code: 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
|
|
694
|
+
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
695
|
+
requested,
|
|
696
|
+
observed
|
|
663
697
|
}
|
|
664
698
|
}
|
|
665
699
|
|
|
666
700
|
} catch (err) {
|
|
667
701
|
const elapsed = Date.now() - overallStart
|
|
668
|
-
return {
|
|
702
|
+
return {
|
|
703
|
+
status: 'timeout',
|
|
704
|
+
error: { code: 'INTERNAL_ERROR', message: err instanceof Error ? err.message : String(err) },
|
|
705
|
+
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
706
|
+
requested,
|
|
707
|
+
observed: {
|
|
708
|
+
matched_count: lastMatchedCount,
|
|
709
|
+
condition_satisfied: false,
|
|
710
|
+
selected_index: lastMatchedElement?.index ?? null,
|
|
711
|
+
last_matched_element: lastMatchedElement
|
|
712
|
+
}
|
|
713
|
+
}
|
|
669
714
|
}
|
|
670
715
|
}
|
|
671
716
|
|
|
@@ -682,11 +727,13 @@ export class ToolsInteract {
|
|
|
682
727
|
static async waitForScreenChangeHandler({ platform, previousFingerprint, timeoutMs = 5000, pollIntervalMs = 300, deviceId }: { platform?: 'android' | 'ios', previousFingerprint: string, timeoutMs?: number, pollIntervalMs?: number, deviceId?: string }) {
|
|
683
728
|
const start = Date.now()
|
|
684
729
|
let lastFingerprint: string | null = null
|
|
730
|
+
let lastActivity: string | null = null
|
|
685
731
|
|
|
686
732
|
while (Date.now() - start < timeoutMs) {
|
|
687
733
|
try {
|
|
688
734
|
const res = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
689
735
|
const fp = res?.fingerprint ?? null
|
|
736
|
+
lastActivity = (res as any)?.activity ?? lastActivity
|
|
690
737
|
if (fp === null || fp === undefined) {
|
|
691
738
|
lastFingerprint = null
|
|
692
739
|
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
@@ -701,8 +748,18 @@ export class ToolsInteract {
|
|
|
701
748
|
try {
|
|
702
749
|
const confirmRes = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
703
750
|
const confirmFp = confirmRes?.fingerprint ?? null
|
|
751
|
+
lastActivity = (confirmRes as any)?.activity ?? lastActivity
|
|
704
752
|
if (confirmFp === fp) {
|
|
705
|
-
return {
|
|
753
|
+
return {
|
|
754
|
+
success: true,
|
|
755
|
+
previousFingerprint,
|
|
756
|
+
newFingerprint: fp,
|
|
757
|
+
elapsedMs: Date.now() - start,
|
|
758
|
+
observed_screen: {
|
|
759
|
+
fingerprint: fp,
|
|
760
|
+
activity: lastActivity
|
|
761
|
+
}
|
|
762
|
+
}
|
|
706
763
|
}
|
|
707
764
|
lastFingerprint = confirmFp
|
|
708
765
|
continue
|
|
@@ -713,7 +770,17 @@ export class ToolsInteract {
|
|
|
713
770
|
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
714
771
|
}
|
|
715
772
|
|
|
716
|
-
return {
|
|
773
|
+
return {
|
|
774
|
+
success: false,
|
|
775
|
+
reason: 'timeout',
|
|
776
|
+
previousFingerprint,
|
|
777
|
+
lastFingerprint,
|
|
778
|
+
elapsedMs: Date.now() - start,
|
|
779
|
+
observed_screen: {
|
|
780
|
+
fingerprint: lastFingerprint,
|
|
781
|
+
activity: lastActivity
|
|
782
|
+
}
|
|
783
|
+
}
|
|
717
784
|
}
|
|
718
785
|
|
|
719
786
|
static async expectScreenHandler({
|
|
@@ -749,13 +816,23 @@ export class ToolsInteract {
|
|
|
749
816
|
}
|
|
750
817
|
|
|
751
818
|
let success = false
|
|
819
|
+
let basis: 'fingerprint' | 'screen' | 'none' = 'none'
|
|
820
|
+
let reason = 'No fingerprint or screen expectation provided'
|
|
752
821
|
if (fingerprint) {
|
|
822
|
+
basis = 'fingerprint'
|
|
753
823
|
success = observedScreen.fingerprint === fingerprint
|
|
824
|
+
reason = success
|
|
825
|
+
? `observed fingerprint matches expected fingerprint ${fingerprint}`
|
|
826
|
+
: `expected fingerprint ${fingerprint} but observed ${observedScreen.fingerprint ?? 'null'}`
|
|
754
827
|
} else if (screen) {
|
|
828
|
+
basis = 'screen'
|
|
755
829
|
const candidates = new Set<string>()
|
|
756
830
|
if (observedScreen.screen) candidates.add(observedScreen.screen)
|
|
757
831
|
if (observedScreenLabel) candidates.add(observedScreenLabel)
|
|
758
832
|
success = candidates.has(screen)
|
|
833
|
+
reason = success
|
|
834
|
+
? `observed screen matches expected screen ${screen}`
|
|
835
|
+
: `expected screen ${screen} but observed ${observedScreenLabel ?? observedScreen.screen ?? 'null'}`
|
|
759
836
|
}
|
|
760
837
|
|
|
761
838
|
return {
|
|
@@ -765,7 +842,12 @@ export class ToolsInteract {
|
|
|
765
842
|
screen: observedScreenLabel
|
|
766
843
|
},
|
|
767
844
|
expected_screen: expectedScreen,
|
|
768
|
-
confidence: success ? 1 : 0
|
|
845
|
+
confidence: success ? 1 : 0,
|
|
846
|
+
comparison: {
|
|
847
|
+
basis,
|
|
848
|
+
matched: success,
|
|
849
|
+
reason
|
|
850
|
+
}
|
|
769
851
|
}
|
|
770
852
|
}
|
|
771
853
|
|
|
@@ -798,6 +880,7 @@ export class ToolsInteract {
|
|
|
798
880
|
success: true,
|
|
799
881
|
selector,
|
|
800
882
|
element_id: result.element.elementId ?? element_id ?? null,
|
|
883
|
+
expected_condition: 'visible',
|
|
801
884
|
element: {
|
|
802
885
|
elementId: result.element.elementId ?? null,
|
|
803
886
|
text: result.element.text ?? null,
|
|
@@ -806,7 +889,23 @@ export class ToolsInteract {
|
|
|
806
889
|
class: result.element.class ?? null,
|
|
807
890
|
bounds: result.element.bounds ?? null,
|
|
808
891
|
index: typeof result.element.index === 'number' ? result.element.index : null
|
|
809
|
-
}
|
|
892
|
+
},
|
|
893
|
+
observed: {
|
|
894
|
+
status: result.status,
|
|
895
|
+
matched_count: typeof result.matched === 'number' ? result.matched : result?.observed?.matched_count ?? null,
|
|
896
|
+
condition_satisfied: true,
|
|
897
|
+
selected_index: typeof result.element.index === 'number' ? result.element.index : null,
|
|
898
|
+
last_matched_element: {
|
|
899
|
+
elementId: result.element.elementId ?? null,
|
|
900
|
+
text: result.element.text ?? null,
|
|
901
|
+
resource_id: result.element.resource_id ?? null,
|
|
902
|
+
accessibility_id: result.element.accessibility_id ?? null,
|
|
903
|
+
class: result.element.class ?? null,
|
|
904
|
+
bounds: result.element.bounds ?? null,
|
|
905
|
+
index: typeof result.element.index === 'number' ? result.element.index : null
|
|
906
|
+
}
|
|
907
|
+
},
|
|
908
|
+
reason: 'selector is visible'
|
|
810
909
|
}
|
|
811
910
|
}
|
|
812
911
|
|
|
@@ -815,6 +914,15 @@ export class ToolsInteract {
|
|
|
815
914
|
success: false,
|
|
816
915
|
selector,
|
|
817
916
|
element_id: element_id ?? null,
|
|
917
|
+
expected_condition: 'visible',
|
|
918
|
+
observed: {
|
|
919
|
+
status: result?.status,
|
|
920
|
+
matched_count: result?.observed?.matched_count,
|
|
921
|
+
condition_satisfied: result?.observed?.condition_satisfied ?? false,
|
|
922
|
+
selected_index: result?.observed?.selected_index ?? null,
|
|
923
|
+
last_matched_element: result?.observed?.last_matched_element ?? null
|
|
924
|
+
},
|
|
925
|
+
reason: result?.error?.message ?? 'selector is not visible',
|
|
818
926
|
failure_code: errorCode,
|
|
819
927
|
retryable: errorCode === 'TIMEOUT'
|
|
820
928
|
}
|
package/src/manage/android.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { existsSync } from 'fs'
|
|
|
5
5
|
import { execAdb, spawnAdb, getAndroidDeviceMetadata, getDeviceInfo, findApk } from '../utils/android/utils.js'
|
|
6
6
|
import { execAdbWithDiagnostics } from '../utils/diagnostics.js'
|
|
7
7
|
import { detectJavaHome } from '../utils/java.js'
|
|
8
|
+
import { AndroidObserve } from '../observe/android.js'
|
|
8
9
|
import { InstallAppResponse, StartAppResponse, TerminateAppResponse, RestartAppResponse, ResetAppDataResponse } from '../types.js'
|
|
9
10
|
|
|
10
11
|
export class AndroidManage {
|
|
@@ -141,8 +142,21 @@ export class AndroidManage {
|
|
|
141
142
|
const metadata = await getAndroidDeviceMetadata(appId, deviceId)
|
|
142
143
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
143
144
|
try {
|
|
144
|
-
await execAdb(['shell', 'monkey', '-p', appId, '-c', 'android.intent.category.LAUNCHER', '1'], deviceId)
|
|
145
|
-
|
|
145
|
+
const output = await execAdb(['shell', 'monkey', '-p', appId, '-c', 'android.intent.category.LAUNCHER', '1'], deviceId)
|
|
146
|
+
const current = await new AndroidObserve().getCurrentScreen(deviceId).catch(() => null)
|
|
147
|
+
return {
|
|
148
|
+
device: deviceInfo,
|
|
149
|
+
appStarted: true,
|
|
150
|
+
launchTimeMs: 1000,
|
|
151
|
+
output,
|
|
152
|
+
observedApp: {
|
|
153
|
+
appId,
|
|
154
|
+
package: current?.package ?? null,
|
|
155
|
+
activity: current?.activity ?? null,
|
|
156
|
+
screen: current?.shortActivity ?? current?.activity ?? null,
|
|
157
|
+
matchedTarget: current ? current.package === appId : null
|
|
158
|
+
}
|
|
159
|
+
}
|
|
146
160
|
} catch (e: unknown) {
|
|
147
161
|
const diag = execAdbWithDiagnostics(['shell', 'monkey', '-p', appId, '-c', 'android.intent.category.LAUNCHER', '1'], deviceId)
|
|
148
162
|
return { device: deviceInfo, appStarted: false, launchTimeMs: 0, error: e instanceof Error ? e.message : String(e), diagnostics: diag }
|
|
@@ -162,12 +176,18 @@ export class AndroidManage {
|
|
|
162
176
|
}
|
|
163
177
|
|
|
164
178
|
async restartApp(appId: string, deviceId?: string): Promise<RestartAppResponse> {
|
|
165
|
-
await this.terminateApp(appId, deviceId)
|
|
179
|
+
const terminateResult = await this.terminateApp(appId, deviceId)
|
|
166
180
|
const startResult = await this.startApp(appId, deviceId)
|
|
167
181
|
return {
|
|
168
182
|
device: startResult.device,
|
|
169
183
|
appRestarted: startResult.appStarted,
|
|
170
|
-
launchTimeMs: startResult.launchTimeMs
|
|
184
|
+
launchTimeMs: startResult.launchTimeMs,
|
|
185
|
+
output: startResult.output,
|
|
186
|
+
observedApp: startResult.observedApp,
|
|
187
|
+
terminatedBeforeRestart: terminateResult.appTerminated,
|
|
188
|
+
...(terminateResult.error ? { terminateError: terminateResult.error } : {}),
|
|
189
|
+
...(startResult.error ? { error: startResult.error } : {}),
|
|
190
|
+
...(startResult.diagnostics ? { diagnostics: startResult.diagnostics } : {})
|
|
171
191
|
}
|
|
172
192
|
}
|
|
173
193
|
|
package/src/manage/ios.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { promises as fs } from "fs"
|
|
|
2
2
|
import { spawn, spawnSync } from "child_process"
|
|
3
3
|
import { StartAppResponse, TerminateAppResponse, RestartAppResponse, ResetAppDataResponse, InstallAppResponse } from "../types.js"
|
|
4
4
|
import { execCommand, execCommandWithDiagnostics, getIOSDeviceMetadata, validateBundleId, getIdbCmd, findAppBundle } from "../utils/ios/utils.js"
|
|
5
|
+
import { iOSObserve } from "../observe/ios.js"
|
|
5
6
|
import path from "path"
|
|
6
7
|
|
|
7
8
|
export class iOSManage {
|
|
@@ -301,7 +302,20 @@ export class iOSManage {
|
|
|
301
302
|
try {
|
|
302
303
|
const result = await execCommand(['simctl', 'launch', deviceId, bundleId], deviceId)
|
|
303
304
|
const device = await getIOSDeviceMetadata(deviceId)
|
|
304
|
-
|
|
305
|
+
const fingerprint = await new iOSObserve().getScreenFingerprint(deviceId).catch(() => null)
|
|
306
|
+
const pidMatch = result.output.match(/:\s*(\d+)\s*$/)
|
|
307
|
+
return {
|
|
308
|
+
device,
|
|
309
|
+
appStarted: !!result.output,
|
|
310
|
+
launchTimeMs: 1000,
|
|
311
|
+
output: result.output,
|
|
312
|
+
observedApp: {
|
|
313
|
+
appId: bundleId,
|
|
314
|
+
pid: pidMatch ? Number(pidMatch[1]) : null,
|
|
315
|
+
screen: fingerprint?.activity ?? null,
|
|
316
|
+
matchedTarget: null
|
|
317
|
+
}
|
|
318
|
+
}
|
|
305
319
|
} catch (e: unknown) {
|
|
306
320
|
const diag = execCommandWithDiagnostics(['simctl', 'launch', deviceId, bundleId], deviceId)
|
|
307
321
|
const device = await getIOSDeviceMetadata(deviceId)
|
|
@@ -323,9 +337,19 @@ export class iOSManage {
|
|
|
323
337
|
}
|
|
324
338
|
|
|
325
339
|
async restartApp(bundleId: string, deviceId: string = "booted"): Promise<RestartAppResponse> {
|
|
326
|
-
await this.terminateApp(bundleId, deviceId)
|
|
340
|
+
const terminateResult = await this.terminateApp(bundleId, deviceId)
|
|
327
341
|
const startResult = await this.startApp(bundleId, deviceId)
|
|
328
|
-
return {
|
|
342
|
+
return {
|
|
343
|
+
device: startResult.device,
|
|
344
|
+
appRestarted: startResult.appStarted,
|
|
345
|
+
launchTimeMs: startResult.launchTimeMs,
|
|
346
|
+
output: startResult.output,
|
|
347
|
+
observedApp: startResult.observedApp,
|
|
348
|
+
terminatedBeforeRestart: terminateResult.appTerminated,
|
|
349
|
+
...(terminateResult.error ? { terminateError: terminateResult.error } : {}),
|
|
350
|
+
...(startResult.error ? { error: startResult.error } : {}),
|
|
351
|
+
...(startResult.diagnostics ? { diagnostics: startResult.diagnostics } : {})
|
|
352
|
+
}
|
|
329
353
|
}
|
|
330
354
|
|
|
331
355
|
async resetAppData(bundleId: string, deviceId: string = "booted"): Promise<ResetAppDataResponse> {
|