mobile-debug-mcp 0.21.5 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +74 -0
- package/README.md +24 -5
- package/dist/interact/classify.js +35 -0
- package/dist/interact/index.js +220 -13
- package/dist/network/index.js +232 -0
- package/dist/observe/ios.js +10 -3
- package/dist/server-core.js +822 -0
- package/dist/server.js +6 -693
- package/dist/utils/resolve-device.js +15 -3
- package/docs/CHANGELOG.md +10 -1
- package/docs/tools/interact.md +69 -30
- package/package.json +3 -3
- package/skills/README.md +35 -0
- package/skills/test-authoring/SKILL.md +57 -0
- package/skills/test-authoring/references/repo-test-layout.md +47 -0
- package/skills/test-authoring/references/test-authoring-workflow.md +73 -0
- package/skills/test-authoring/references/test-quality-checklist.md +39 -0
- package/src/interact/classify.ts +64 -0
- package/src/interact/index.ts +250 -13
- package/src/network/index.ts +268 -0
- package/src/observe/ios.ts +12 -3
- package/src/server-core.ts +879 -0
- package/src/server.ts +8 -754
- package/src/types.ts +10 -1
- package/src/utils/resolve-device.ts +19 -3
- package/test/device/automated/observe/capture_screenshot.android.smoke.ts +30 -0
- package/test/device/automated/observe/capture_screenshot.ios.smoke.ts +30 -0
- package/test/{observe/device → device/automated/observe}/get_logs.android.smoke.ts +1 -1
- package/test/{observe/device → device/automated/observe}/get_logs.ios.smoke.ts +1 -1
- package/test/device/automated/observe/get_ui_tree.android.smoke.ts +31 -0
- package/test/device/automated/observe/get_ui_tree.ios.smoke.ts +31 -0
- package/test/device/index.ts +52 -0
- package/test/{interact/device/smoke-test.ts → device/manual/interact/app_lifecycle.manual.ts} +5 -5
- package/test/{manage/device/run-build-install-ios.ts → device/manual/manage/build_install_ios.manual.ts} +1 -1
- package/test/{manage/device → device/manual/manage}/install.integration.ts +6 -6
- package/test/{manage/device/run-install-android.ts → device/manual/manage/install_android.manual.ts} +1 -1
- package/test/{manage/device/run-install-ios.ts → device/manual/manage/install_ios.manual.ts} +1 -1
- package/test/device/manual/observe/capture_screenshot.manual.ts +29 -0
- package/test/{helpers/run-get-logs.ts → device/manual/observe/get_logs.manual.ts} +1 -1
- package/test/device/manual/observe/get_ui_tree.manual.ts +29 -0
- package/test/{observe/device/logstream-real.ts → device/manual/observe/logstream.manual.ts} +1 -1
- package/test/{observe/device/run-screen-fingerprint.ts → device/manual/observe/screen_fingerprint.manual.ts} +1 -1
- package/test/{observe/device/run-scroll-test-android.ts → device/manual/observe/scroll_to_element_android.manual.ts} +1 -1
- package/test/{observe/device/test-ui-tree.ts → device/manual/observe/ui_tree.manual.ts} +6 -6
- package/test/unit/index.ts +47 -27
- package/test/unit/interact/classify_action_outcome.test.ts +110 -0
- package/test/unit/interact/handler_shapes.test.ts +55 -0
- package/test/unit/interact/tap_element.test.ts +170 -0
- package/test/unit/interact/wait_for_screen_change.test.ts +34 -0
- package/test/{interact/unit → unit/interact}/wait_for_ui_contract.test.ts +11 -10
- package/test/unit/interact/wait_for_ui_selector_matching.test.ts +76 -0
- package/test/unit/manage/handler_shapes.test.ts +43 -0
- package/test/unit/network/get_network_activity.test.ts +181 -0
- package/test/{observe/unit → unit/observe}/capture_debug_snapshot.test.ts +5 -1
- package/test/{observe/unit → unit/observe}/find_element.test.ts +12 -6
- package/test/unit/observe/get_screen_fingerprint.test.ts +71 -0
- package/test/unit/observe/ios-getlogs.test.ts +53 -0
- package/test/unit/observe/scroll_to_element.test.ts +127 -0
- package/test/unit/server/contract.test.ts +45 -0
- package/test/unit/server/response_shapes.test.ts +93 -0
- package/test/unit/system/adb_version.test.ts +35 -0
- package/test/unit/system/get_system_status.test.ts +20 -0
- package/test/unit/system/system_status.test.ts +141 -0
- package/test/{utils → unit/utils}/detect_java.test.ts +1 -1
- package/test/unit/utils/exec.test.ts +51 -0
- package/test/unit/utils/resolve_device.test.ts +63 -0
- package/tsconfig.json +2 -2
- package/test/interact/device/run-real-test.ts +0 -3
- package/test/interact/unit/wait_for_screen_change.test.ts +0 -32
- package/test/interact/unit/wait_for_ui.test.ts +0 -76
- package/test/interact/unit/wait_for_ui_new.test.ts +0 -57
- package/test/observe/device/wait_for_element_real.ts +0 -3
- package/test/observe/unit/get_screen_fingerprint.test.ts +0 -69
- package/test/observe/unit/ios-getlogs.test.ts +0 -67
- package/test/observe/unit/scroll_to_element.test.ts +0 -129
- package/test/observe/unit/wait_for_element_mock.ts +0 -2
- package/test/observe/unit/wait_for_ui_edge_cases.test.ts +0 -41
- package/test/observe/unit/wait_for_ui_stability.test.ts +0 -30
- package/test/system/adb_version.test.ts +0 -25
- package/test/system/get_system_status.test.ts +0 -52
- package/test/system/system_status.test.ts +0 -109
- /package/test/{manage/unit → unit/manage}/build.test.ts +0 -0
- /package/test/{manage/unit → unit/manage}/build_and_install.test.ts +0 -0
- /package/test/{manage/unit → unit/manage}/detection.test.ts +0 -0
- /package/test/{manage/unit → unit/manage}/diagnostics.test.ts +0 -0
- /package/test/{manage/unit → unit/manage}/install.test.ts +0 -0
- /package/test/{manage/unit → unit/manage}/mcp_disable_autodetect.test.ts +0 -0
- /package/test/{observe/unit → unit/observe}/get_logs.test.ts +0 -0
- /package/test/{observe/unit → unit/observe}/logparse.test.ts +0 -0
- /package/test/{observe/unit → unit/observe}/logstream.test.ts +0 -0
package/AGENTS.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
This file is the **cold-start entrypoint for autonomous agents** that arrive in this public repository without prior context.
|
|
4
|
+
|
|
5
|
+
It is intentionally vendor-neutral and should be useful to Copilot, Codex, Claude, or custom agent systems.
|
|
6
|
+
|
|
7
|
+
## What this repository is
|
|
8
|
+
|
|
9
|
+
`mobile-debug-mcp` is an MCP server for mobile app debugging on Android and iOS. The codebase is TypeScript and the built server entrypoint is `dist/server.js`.
|
|
10
|
+
|
|
11
|
+
Primary areas:
|
|
12
|
+
|
|
13
|
+
- `src/server.ts` — runtime entrypoint
|
|
14
|
+
- `src/server-core.ts` — tool registration and dispatch logic
|
|
15
|
+
- `src/interact/` — tap, swipe, typing, waits, and interaction helpers
|
|
16
|
+
- `src/observe/` — logs, screenshots, UI tree, fingerprints, snapshots
|
|
17
|
+
- `src/manage/` — build, install, start, terminate, restart, reset-app-data
|
|
18
|
+
- `src/system/` — environment and toolchain health checks
|
|
19
|
+
- `src/utils/` — shared helpers and platform-specific utility code
|
|
20
|
+
|
|
21
|
+
## First things an agent should know
|
|
22
|
+
|
|
23
|
+
1. Prefer existing helpers and conventions over introducing new frameworks or abstractions.
|
|
24
|
+
2. The repository already has a test structure; place tests in the correct tree instead of inventing new ones.
|
|
25
|
+
3. Use existing validation commands:
|
|
26
|
+
- `npm run build`
|
|
27
|
+
- `npm run lint`
|
|
28
|
+
- `npm run test:unit`
|
|
29
|
+
- `npm run test:device`
|
|
30
|
+
4. Device tests are split intentionally:
|
|
31
|
+
- `test/device/automated/...` = automated smoke/integration coverage
|
|
32
|
+
- `test/device/manual/...` = helper/debug/manual scripts
|
|
33
|
+
5. Unit tests use the current lightweight self-running `tsx` pattern rather than a dedicated framework.
|
|
34
|
+
|
|
35
|
+
## Where to look for deeper guidance
|
|
36
|
+
|
|
37
|
+
### Skills
|
|
38
|
+
|
|
39
|
+
Portable agent skills live under `skills/`.
|
|
40
|
+
|
|
41
|
+
- `skills/README.md` — repo-wide skill convention
|
|
42
|
+
- `skills/mcp-builder/` — build/install/toolchain guidance
|
|
43
|
+
- `skills/test-authoring/` — test creation and placement guidance
|
|
44
|
+
|
|
45
|
+
If the task is about **creating or updating tests**, load `skills/test-authoring/SKILL.md` first.
|
|
46
|
+
|
|
47
|
+
If the task is about **building, installing, or diagnosing native tooling**, load `skills/mcp-builder/SKILL.md` first.
|
|
48
|
+
|
|
49
|
+
### Repository docs
|
|
50
|
+
|
|
51
|
+
- `README.md` — high-level repo overview and commands
|
|
52
|
+
- `docs/tools/TOOLS.md` — tool documentation and contracts
|
|
53
|
+
- `docs/CHANGELOG.md` — recent behavioral changes
|
|
54
|
+
|
|
55
|
+
## Testing guidance
|
|
56
|
+
|
|
57
|
+
Use these defaults unless the task clearly requires something else:
|
|
58
|
+
|
|
59
|
+
- Add automated unit tests under `test/unit/...`
|
|
60
|
+
- Add automated device smoke tests under `test/device/automated/...`
|
|
61
|
+
- Add ad hoc or environment-specific helper scripts under `test/device/manual/...`
|
|
62
|
+
|
|
63
|
+
For test authoring details, rely on the `test-authoring` skill package rather than duplicating its rules here.
|
|
64
|
+
|
|
65
|
+
## Output expectations for agents
|
|
66
|
+
|
|
67
|
+
- Make focused changes
|
|
68
|
+
- Preserve existing repo behavior unless the task requires a change
|
|
69
|
+
- Keep docs aligned when structure or usage changes
|
|
70
|
+
- Prefer deterministic validation over ad hoc verification
|
|
71
|
+
|
|
72
|
+
## Notes for maintainers
|
|
73
|
+
|
|
74
|
+
This file is intentionally short. Keep task-specific guidance in `skills/...` so multiple agent systems can reuse the same instructions.
|
package/README.md
CHANGED
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
A minimal, secure MCP server for AI-assisted mobile development. Build, install, and inspect Android/iOS apps from an MCP-compatible client.
|
|
4
4
|
|
|
5
|
-
> **
|
|
6
|
-
> *
|
|
5
|
+
> **Support:**
|
|
6
|
+
> * Android support
|
|
7
|
+
> * iOS only tested on simulator
|
|
8
|
+
> * KMP support
|
|
7
9
|
> * Flutter iOS projects not fetching logs
|
|
8
10
|
> * React native not tested
|
|
9
11
|
|
|
@@ -32,15 +34,32 @@ You will need to add ADB_PATH for Android and XCRUN_PATH and IDB_PATH for iOS.
|
|
|
32
34
|
## Usage
|
|
33
35
|
|
|
34
36
|
Examples:
|
|
35
|
-
|
|
36
|
-
|
|
37
|
+
|
|
38
|
+
Crash fixing:
|
|
39
|
+
> I have a crash on the app, can you diagnose it, fix and validate using the mcp tools available
|
|
40
|
+
|
|
41
|
+
Feature building:
|
|
42
|
+
> Add a button, hook into the repository and confirm API request successful
|
|
37
43
|
|
|
38
44
|
## Docs
|
|
39
45
|
|
|
40
46
|
- Tools: [Tools](docs/tools/TOOLS.md) — full input/response examples
|
|
41
47
|
- Changelog: [Changelog](docs/CHANGELOG.md)
|
|
48
|
+
- Agents: [AGENTS.md](AGENTS.md) — cold-start guidance for autonomous agents entering the public repo
|
|
49
|
+
- Skills: [skills/README.md](skills/README.md) — portable Markdown skill packages for agents such as Copilot, Codex, Claude, or custom systems
|
|
50
|
+
|
|
51
|
+
## Testing
|
|
52
|
+
|
|
53
|
+
- `npm run test:unit` runs every automated unit test under `test/unit/...`
|
|
54
|
+
- `npm run test:device` runs the automated device smoke checks under `test/device/automated/...`
|
|
55
|
+
- Manual and debug-oriented device scripts live under `test/device/manual/...` and are not part of the default test commands
|
|
56
|
+
|
|
57
|
+
## Agent skills
|
|
58
|
+
|
|
59
|
+
- `skills/mcp-builder/` contains reusable build/install guidance for agents
|
|
60
|
+
- `skills/test-authoring/` contains reusable test-creation guidance aligned to this repo's current test structure
|
|
61
|
+
- Skills are written as plain Markdown packages so they can be consumed by different agent systems rather than one vendor-specific runtime
|
|
42
62
|
|
|
43
63
|
## License
|
|
44
64
|
|
|
45
65
|
MIT
|
|
46
|
-
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure deterministic classifier. Applies rules in fixed order.
|
|
3
|
+
* Same inputs always produce the same output.
|
|
4
|
+
*/
|
|
5
|
+
export function classifyActionOutcome(input) {
|
|
6
|
+
const { uiChanged, expectedElementVisible, networkRequests, hasLogErrors } = input;
|
|
7
|
+
// Step 1 — UI signal is positive
|
|
8
|
+
if (uiChanged || expectedElementVisible === true) {
|
|
9
|
+
return { outcome: 'success', reasoning: expectedElementVisible === true ? 'expected element is visible' : 'UI changed after action' };
|
|
10
|
+
}
|
|
11
|
+
// Step 2 — UI did not change; network signal is required
|
|
12
|
+
if (networkRequests === null || networkRequests === undefined) {
|
|
13
|
+
return {
|
|
14
|
+
outcome: 'unknown',
|
|
15
|
+
reasoning: 'UI did not change; get_network_activity must be called before classification can proceed',
|
|
16
|
+
nextAction: 'call_get_network_activity'
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
// Step 3 — any network failure
|
|
20
|
+
const failedRequest = networkRequests.find((r) => r.status === 'failure' || r.status === 'retryable');
|
|
21
|
+
if (failedRequest) {
|
|
22
|
+
return { outcome: 'backend_failure', reasoning: `network request ${failedRequest.endpoint} returned ${failedRequest.status}` };
|
|
23
|
+
}
|
|
24
|
+
// Step 4 — no network requests at all
|
|
25
|
+
if (networkRequests.length === 0) {
|
|
26
|
+
const logNote = hasLogErrors ? ' (log errors present)' : '';
|
|
27
|
+
return { outcome: 'no_op', reasoning: `no UI change and no network activity${logNote}` };
|
|
28
|
+
}
|
|
29
|
+
// Step 5 — network requests exist and all succeeded
|
|
30
|
+
if (networkRequests.every((r) => r.status === 'success')) {
|
|
31
|
+
return { outcome: 'ui_failure', reasoning: 'network requests succeeded but UI did not change' };
|
|
32
|
+
}
|
|
33
|
+
// Step 6 — fallback
|
|
34
|
+
return { outcome: 'unknown', reasoning: 'signals are inconclusive' };
|
|
35
|
+
}
|
package/dist/interact/index.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
1
2
|
import { AndroidInteract } from './android.js';
|
|
2
3
|
import { iOSInteract } from './ios.js';
|
|
3
4
|
export { AndroidInteract, iOSInteract };
|
|
4
5
|
import { resolveTargetDevice } from '../utils/resolve-device.js';
|
|
5
6
|
import { ToolsObserve } from '../observe/index.js';
|
|
6
7
|
export class ToolsInteract {
|
|
8
|
+
static _maxResolvedUiElements = 256;
|
|
9
|
+
static _resolvedUiElements = new Map();
|
|
7
10
|
static _normalize(s) {
|
|
8
11
|
if (s === null || s === undefined)
|
|
9
12
|
return '';
|
|
@@ -14,6 +17,129 @@ export class ToolsInteract {
|
|
|
14
17
|
return '';
|
|
15
18
|
}
|
|
16
19
|
}
|
|
20
|
+
static _normalizeBounds(bounds) {
|
|
21
|
+
if (!Array.isArray(bounds) || bounds.length < 4)
|
|
22
|
+
return null;
|
|
23
|
+
const normalized = bounds.slice(0, 4).map((value) => Number(value));
|
|
24
|
+
if (normalized.some((value) => Number.isNaN(value)))
|
|
25
|
+
return null;
|
|
26
|
+
return normalized;
|
|
27
|
+
}
|
|
28
|
+
static _isVisibleElement(el) {
|
|
29
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
30
|
+
return !!el.visible && !!bounds && bounds[2] > bounds[0] && bounds[3] > bounds[1];
|
|
31
|
+
}
|
|
32
|
+
static _computeElementId(platform, deviceId, el, index) {
|
|
33
|
+
const identity = {
|
|
34
|
+
platform,
|
|
35
|
+
deviceId: deviceId || '',
|
|
36
|
+
text: ToolsInteract._normalize(el.text ?? el.label ?? el.value ?? ''),
|
|
37
|
+
resourceId: ToolsInteract._normalize(el.resourceId ?? el.resourceID ?? el.id ?? ''),
|
|
38
|
+
accessibilityId: ToolsInteract._normalize(el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? ''),
|
|
39
|
+
class: ToolsInteract._normalize(el.type ?? el.class ?? ''),
|
|
40
|
+
bounds: ToolsInteract._normalizeBounds(el.bounds) ?? [0, 0, 0, 0],
|
|
41
|
+
index
|
|
42
|
+
};
|
|
43
|
+
return `el_${createHash('sha1').update(JSON.stringify(identity)).digest('hex').slice(0, 24)}`;
|
|
44
|
+
}
|
|
45
|
+
static _buildResolvedElement(platform, deviceId, el, index) {
|
|
46
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
47
|
+
const elementId = ToolsInteract._computeElementId(platform, deviceId, el, index);
|
|
48
|
+
ToolsInteract._rememberResolvedElement(elementId, {
|
|
49
|
+
elementId,
|
|
50
|
+
platform,
|
|
51
|
+
deviceId,
|
|
52
|
+
bounds,
|
|
53
|
+
index
|
|
54
|
+
});
|
|
55
|
+
return {
|
|
56
|
+
text: el.text ?? null,
|
|
57
|
+
resource_id: el.resourceId ?? el.resourceID ?? el.id ?? null,
|
|
58
|
+
accessibility_id: el.contentDescription ?? el.contentDesc ?? el.accessibilityLabel ?? el.label ?? null,
|
|
59
|
+
class: el.type ?? el.class ?? null,
|
|
60
|
+
bounds,
|
|
61
|
+
index,
|
|
62
|
+
elementId
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
static _rememberResolvedElement(elementId, context) {
|
|
66
|
+
if (ToolsInteract._resolvedUiElements.has(elementId)) {
|
|
67
|
+
ToolsInteract._resolvedUiElements.delete(elementId);
|
|
68
|
+
}
|
|
69
|
+
ToolsInteract._resolvedUiElements.set(elementId, context);
|
|
70
|
+
while (ToolsInteract._resolvedUiElements.size > ToolsInteract._maxResolvedUiElements) {
|
|
71
|
+
const oldestElementId = ToolsInteract._resolvedUiElements.keys().next().value;
|
|
72
|
+
if (!oldestElementId)
|
|
73
|
+
break;
|
|
74
|
+
ToolsInteract._resolvedUiElements.delete(oldestElementId);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
static _resetResolvedUiElementsForTests() {
|
|
78
|
+
ToolsInteract._resolvedUiElements.clear();
|
|
79
|
+
}
|
|
80
|
+
static _findCurrentResolvedElement(elements, platform, deviceId, resolved) {
|
|
81
|
+
const indexedCandidate = elements[resolved.index];
|
|
82
|
+
if (indexedCandidate && ToolsInteract._computeElementId(platform, deviceId, indexedCandidate, resolved.index) === resolved.elementId) {
|
|
83
|
+
return { el: indexedCandidate, index: resolved.index };
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
static _resolveActionableAncestor(elements, chosen) {
|
|
88
|
+
if (!chosen)
|
|
89
|
+
return null;
|
|
90
|
+
if (chosen.el.clickable || chosen.el.focusable)
|
|
91
|
+
return chosen;
|
|
92
|
+
let current = chosen;
|
|
93
|
+
let safety = 0;
|
|
94
|
+
while (safety < 20 && current.el && !(current.el.clickable || current.el.focusable) && current.el.parentId !== undefined && current.el.parentId !== null) {
|
|
95
|
+
const parentId = current.el.parentId;
|
|
96
|
+
let parentIndex = null;
|
|
97
|
+
if (typeof parentId === 'number')
|
|
98
|
+
parentIndex = parentId;
|
|
99
|
+
else if (typeof parentId === 'string' && /^\d+$/.test(parentId))
|
|
100
|
+
parentIndex = Number(parentId);
|
|
101
|
+
if (parentIndex !== null && elements[parentIndex]) {
|
|
102
|
+
current = { el: elements[parentIndex], idx: parentIndex };
|
|
103
|
+
if (current.el.clickable || current.el.focusable)
|
|
104
|
+
return current;
|
|
105
|
+
}
|
|
106
|
+
else if (typeof parentId === 'string') {
|
|
107
|
+
const foundIndex = elements.findIndex((el) => el.resourceId === parentId || el.id === parentId);
|
|
108
|
+
if (foundIndex === -1)
|
|
109
|
+
break;
|
|
110
|
+
current = { el: elements[foundIndex], idx: foundIndex };
|
|
111
|
+
if (current.el.clickable || current.el.focusable)
|
|
112
|
+
return current;
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
safety++;
|
|
118
|
+
}
|
|
119
|
+
const childBounds = ToolsInteract._normalizeBounds(chosen.el.bounds);
|
|
120
|
+
if (!childBounds)
|
|
121
|
+
return null;
|
|
122
|
+
const [cl, ct, cr, cb] = childBounds;
|
|
123
|
+
let best = null;
|
|
124
|
+
let bestArea = Infinity;
|
|
125
|
+
for (let i = 0; i < elements.length; i++) {
|
|
126
|
+
const el = elements[i];
|
|
127
|
+
if (!el || !(el.clickable || el.focusable))
|
|
128
|
+
continue;
|
|
129
|
+
const bounds = ToolsInteract._normalizeBounds(el.bounds);
|
|
130
|
+
if (!bounds)
|
|
131
|
+
continue;
|
|
132
|
+
const [pl, pt, pr, pb] = bounds;
|
|
133
|
+
if (pl <= cl && pt <= ct && pr >= cr && pb >= cb) {
|
|
134
|
+
const area = (pr - pl) * (pb - pt);
|
|
135
|
+
if (area < bestArea) {
|
|
136
|
+
bestArea = area;
|
|
137
|
+
best = { el, idx: i };
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return best;
|
|
142
|
+
}
|
|
17
143
|
static async getInteractionService(platform, deviceId) {
|
|
18
144
|
const effectivePlatform = platform || 'android';
|
|
19
145
|
const resolved = await resolveTargetDevice({ platform: effectivePlatform, deviceId });
|
|
@@ -24,6 +150,90 @@ export class ToolsInteract {
|
|
|
24
150
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
25
151
|
return await interact.tap(x, y, resolved.id);
|
|
26
152
|
}
|
|
153
|
+
static async tapElementHandler({ elementId }) {
|
|
154
|
+
const action = 'tap';
|
|
155
|
+
const resolved = ToolsInteract._resolvedUiElements.get(elementId);
|
|
156
|
+
if (!resolved) {
|
|
157
|
+
return {
|
|
158
|
+
success: false,
|
|
159
|
+
elementId,
|
|
160
|
+
action,
|
|
161
|
+
error: {
|
|
162
|
+
code: 'element_not_found',
|
|
163
|
+
message: 'Element ID was not found in the current UI context'
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform: resolved.platform, deviceId: resolved.deviceId });
|
|
168
|
+
const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : resolved.platform;
|
|
169
|
+
const treeDeviceId = tree?.device?.id || resolved.deviceId;
|
|
170
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements : [];
|
|
171
|
+
const currentMatch = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved);
|
|
172
|
+
if (!currentMatch) {
|
|
173
|
+
return {
|
|
174
|
+
success: false,
|
|
175
|
+
elementId,
|
|
176
|
+
action,
|
|
177
|
+
error: {
|
|
178
|
+
code: 'element_not_found',
|
|
179
|
+
message: 'Element ID is not present in the current UI context'
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
if (!ToolsInteract._isVisibleElement(currentMatch.el)) {
|
|
184
|
+
return {
|
|
185
|
+
success: false,
|
|
186
|
+
elementId,
|
|
187
|
+
action,
|
|
188
|
+
error: {
|
|
189
|
+
code: 'element_not_visible',
|
|
190
|
+
message: 'Element is not visible'
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
if (currentMatch.el.enabled === false) {
|
|
195
|
+
return {
|
|
196
|
+
success: false,
|
|
197
|
+
elementId,
|
|
198
|
+
action,
|
|
199
|
+
error: {
|
|
200
|
+
code: 'element_not_enabled',
|
|
201
|
+
message: 'Element is not enabled'
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
const bounds = ToolsInteract._normalizeBounds(currentMatch.el.bounds) ?? resolved.bounds;
|
|
206
|
+
if (!bounds || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) {
|
|
207
|
+
return {
|
|
208
|
+
success: false,
|
|
209
|
+
elementId,
|
|
210
|
+
action,
|
|
211
|
+
error: {
|
|
212
|
+
code: 'element_not_visible',
|
|
213
|
+
message: 'Element does not have valid visible bounds'
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
const x = Math.floor((bounds[0] + bounds[2]) / 2);
|
|
218
|
+
const y = Math.floor((bounds[1] + bounds[3]) / 2);
|
|
219
|
+
const tapResult = await ToolsInteract.tapHandler({ platform: resolved.platform, x, y, deviceId: resolved.deviceId });
|
|
220
|
+
if (!tapResult.success) {
|
|
221
|
+
return {
|
|
222
|
+
success: false,
|
|
223
|
+
elementId,
|
|
224
|
+
action,
|
|
225
|
+
error: {
|
|
226
|
+
code: 'tap_failed',
|
|
227
|
+
message: tapResult.error || 'Tap failed'
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
success: true,
|
|
233
|
+
elementId,
|
|
234
|
+
action
|
|
235
|
+
};
|
|
236
|
+
}
|
|
27
237
|
static async swipeHandler({ platform = 'android', x1, y1, x2, y2, duration, deviceId }) {
|
|
28
238
|
const { interact, resolved } = await ToolsInteract.getInteractionService(platform, deviceId);
|
|
29
239
|
return await interact.swipe(x1, y1, x2, y2, duration, resolved.id);
|
|
@@ -333,6 +543,7 @@ export class ToolsInteract {
|
|
|
333
543
|
chosen = null;
|
|
334
544
|
}
|
|
335
545
|
let conditionMet = false;
|
|
546
|
+
let matchedElement = chosen;
|
|
336
547
|
if (condition === 'exists') {
|
|
337
548
|
// when an index is specified, existence requires that specific index be present
|
|
338
549
|
conditionMet = (pickIndex !== undefined) ? (chosen !== null) : (matchedCount >= 1);
|
|
@@ -351,11 +562,12 @@ export class ToolsInteract {
|
|
|
351
562
|
conditionMet = false;
|
|
352
563
|
}
|
|
353
564
|
else if (condition === 'clickable') {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
const
|
|
357
|
-
const
|
|
358
|
-
const
|
|
565
|
+
matchedElement = chosen ? (ToolsInteract._resolveActionableAncestor(elements, chosen) || chosen) : null;
|
|
566
|
+
if (matchedElement) {
|
|
567
|
+
const b = matchedElement.el.bounds;
|
|
568
|
+
const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1]);
|
|
569
|
+
const enabled = !!matchedElement.el.enabled;
|
|
570
|
+
const clickable = !!matchedElement.el.clickable || !!matchedElement.el._interactable || !!matchedElement.el.focusable;
|
|
359
571
|
conditionMet = visibleFlag && enabled && clickable;
|
|
360
572
|
}
|
|
361
573
|
else
|
|
@@ -365,14 +577,9 @@ export class ToolsInteract {
|
|
|
365
577
|
const now = Date.now();
|
|
366
578
|
const latency_ms = now - overallStart;
|
|
367
579
|
// Build element output per spec
|
|
368
|
-
const
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
accessibility_id: chosen.el.contentDescription ?? chosen.el.contentDesc ?? chosen.el.accessibilityLabel ?? chosen.el.label ?? null,
|
|
372
|
-
class: chosen.el.type ?? chosen.el.class ?? null,
|
|
373
|
-
bounds: Array.isArray(chosen.el.bounds) && chosen.el.bounds.length >= 4 ? chosen.el.bounds : null,
|
|
374
|
-
index: chosen.idx
|
|
375
|
-
} : null;
|
|
580
|
+
const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android');
|
|
581
|
+
const resolvedDeviceId = tree?.device?.id || deviceId;
|
|
582
|
+
const outEl = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null;
|
|
376
583
|
return {
|
|
377
584
|
status: 'success',
|
|
378
585
|
matched: matchedCount,
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { execAdb, parseLogLine } from '../utils/android/utils.js';
|
|
2
|
+
import { execCommand } from '../utils/ios/utils.js';
|
|
3
|
+
// ─── Module state ─────────────────────────────────────────────────────────────
|
|
4
|
+
// lastActionTimestamp: set when an action tool fires (tap, swipe, etc.)
|
|
5
|
+
// lastConsumedTimestamp: advanced after each get_network_activity call to prevent duplicates
|
|
6
|
+
let lastActionTimestamp = 0;
|
|
7
|
+
let lastConsumedTimestamp = 0;
|
|
8
|
+
export function notifyActionStart() {
|
|
9
|
+
lastActionTimestamp = Date.now();
|
|
10
|
+
lastConsumedTimestamp = 0;
|
|
11
|
+
}
|
|
12
|
+
/** Exposed for unit tests only. */
|
|
13
|
+
export function _setTimestampsForTests(actionTs, consumedTs) {
|
|
14
|
+
lastActionTimestamp = actionTs;
|
|
15
|
+
lastConsumedTimestamp = consumedTs;
|
|
16
|
+
}
|
|
17
|
+
// ─── Parsing constants ────────────────────────────────────────────────────────
|
|
18
|
+
const URL_RE = /https?:\/\/[^\s"'\]\)><]+/;
|
|
19
|
+
const PATH_RE = /\/[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)+/;
|
|
20
|
+
const METHOD_RE = /\b(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b/;
|
|
21
|
+
const NETWORK_ERROR_PATTERNS = [
|
|
22
|
+
{ re: /timed?\s*out|timeout/i, code: 'timeout' },
|
|
23
|
+
{ re: /dns|name[\s_]resolution|host\s*not\s*found|nodename/i, code: 'dns_error' },
|
|
24
|
+
{ re: /\btls\b|\bssl\b|certificate|handshake/i, code: 'tls_error' },
|
|
25
|
+
{ re: /connection\s*refused/i, code: 'connection_refused' },
|
|
26
|
+
{ re: /connection\s*reset|reset\s*by\s*peer/i, code: 'connection_reset' },
|
|
27
|
+
];
|
|
28
|
+
const BACKGROUND_TOKENS = ['/analytics', '/metrics', '/tracking', '/log', '/events', '/telemetry', '/ping', '/beacon'];
|
|
29
|
+
const BACKGROUND_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.css', '.js', '.svg', '.ico', '.woff', '.ttf'];
|
|
30
|
+
const FILESYSTEM_PREFIXES = ['/data/', '/system/', '/apex/', '/proc/', '/dev/', '/vendor/', '/product/', '/storage/', '/sdcard/', '/mnt/', '/odm/', '/cache/', '/metadata/', '/acct/', '/sys/'];
|
|
31
|
+
const FILESYSTEM_EXTENSIONS = ['.apk', '.apex', '.odex', '.vdex', '.dex', '.so', '.jar', '.bin', '.img', '.db', '.sqlite', '.c', '.cc', '.cpp', '.cxx', '.h', '.hpp', '.m', '.mm', '.kt', '.java', '.swift'];
|
|
32
|
+
// ─── Parsing helpers ─────────────────────────────────────────────────────────
|
|
33
|
+
function extractUrl(text) {
|
|
34
|
+
const m = text.match(URL_RE);
|
|
35
|
+
return m ? m[0] : null;
|
|
36
|
+
}
|
|
37
|
+
function isPlausibleEndpointPath(path) {
|
|
38
|
+
const lower = path.toLowerCase();
|
|
39
|
+
if (!lower.startsWith('/'))
|
|
40
|
+
return false;
|
|
41
|
+
if (FILESYSTEM_PREFIXES.some((prefix) => lower.startsWith(prefix)))
|
|
42
|
+
return false;
|
|
43
|
+
if (FILESYSTEM_EXTENSIONS.some((ext) => lower.endsWith(ext)))
|
|
44
|
+
return false;
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
function extractPath(text) {
|
|
48
|
+
const m = text.match(PATH_RE);
|
|
49
|
+
if (!m)
|
|
50
|
+
return null;
|
|
51
|
+
return isPlausibleEndpointPath(m[0]) ? m[0] : null;
|
|
52
|
+
}
|
|
53
|
+
function toStatusCode(value) {
|
|
54
|
+
if (!value)
|
|
55
|
+
return null;
|
|
56
|
+
const code = Number(value);
|
|
57
|
+
return code >= 100 && code <= 599 ? code : null;
|
|
58
|
+
}
|
|
59
|
+
function escapeRegExp(value) {
|
|
60
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
61
|
+
}
|
|
62
|
+
function extractStatusCode(text, url, path, method) {
|
|
63
|
+
const directHttpMatch = text.match(/\bHTTP\/\d(?:\.\d)?\s+([1-5]\d{2})\b/i) || text.match(/\bHTTP\s+([1-5]\d{2})\b/i);
|
|
64
|
+
if (directHttpMatch)
|
|
65
|
+
return toStatusCode(directHttpMatch[1]);
|
|
66
|
+
const endpointToken = url || path;
|
|
67
|
+
const hasEndpointContext = endpointToken !== null;
|
|
68
|
+
if (!hasEndpointContext && method === null)
|
|
69
|
+
return null;
|
|
70
|
+
const labeledMatch = text.match(/\b(?:status(?:\s*code)?|response(?:\s*code)?)\s*[:=]?\s*([1-5]\d{2})\b/i);
|
|
71
|
+
if (labeledMatch && hasEndpointContext)
|
|
72
|
+
return toStatusCode(labeledMatch[1]);
|
|
73
|
+
if (endpointToken) {
|
|
74
|
+
const escapedEndpoint = escapeRegExp(endpointToken);
|
|
75
|
+
const endpointThenCode = new RegExp(`${escapedEndpoint}[^\\n]*?\\b([1-5]\\d{2})\\b`, 'i');
|
|
76
|
+
const codeThenEndpoint = new RegExp(`\\b([1-5]\\d{2})\\b[^\\n]*?${escapedEndpoint}`, 'i');
|
|
77
|
+
const contextualMatch = text.match(endpointThenCode) || text.match(codeThenEndpoint);
|
|
78
|
+
if (contextualMatch)
|
|
79
|
+
return toStatusCode(contextualMatch[1]);
|
|
80
|
+
}
|
|
81
|
+
if (method !== null && path !== null) {
|
|
82
|
+
const methodPathCodeMatch = text.match(/\b(?:GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)\b[^\n]*?\b([1-5]\d{2})\b/i);
|
|
83
|
+
if (methodPathCodeMatch)
|
|
84
|
+
return toStatusCode(methodPathCodeMatch[1]);
|
|
85
|
+
}
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
function extractMethod(text) {
|
|
89
|
+
const m = text.match(METHOD_RE);
|
|
90
|
+
return m ? m[1] : null;
|
|
91
|
+
}
|
|
92
|
+
function detectNetworkError(text) {
|
|
93
|
+
for (const { re, code } of NETWORK_ERROR_PATTERNS) {
|
|
94
|
+
if (re.test(text))
|
|
95
|
+
return code;
|
|
96
|
+
}
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
export function normalizeEndpoint(raw) {
|
|
100
|
+
try {
|
|
101
|
+
const u = new URL(raw.startsWith('/') ? `https://x${raw}` : raw);
|
|
102
|
+
const p = u.pathname.toLowerCase().replace(/\/+$/, '');
|
|
103
|
+
return p || '/';
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return raw.toLowerCase().replace(/\?.*$/, '').replace(/\/+$/, '') || '/';
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
export function classifyStatus(statusCode, networkError) {
|
|
110
|
+
if (networkError !== null)
|
|
111
|
+
return 'retryable';
|
|
112
|
+
if (statusCode === null)
|
|
113
|
+
return 'success'; // request detected, no failure signal
|
|
114
|
+
if (statusCode >= 200 && statusCode <= 299)
|
|
115
|
+
return 'success';
|
|
116
|
+
if (statusCode >= 400 && statusCode <= 499)
|
|
117
|
+
return 'failure';
|
|
118
|
+
return 'retryable'; // 5xx, 1xx, 3xx
|
|
119
|
+
}
|
|
120
|
+
function meetsEmissionCriteria(url, path, statusCode, method) {
|
|
121
|
+
if (url !== null)
|
|
122
|
+
return true; // condition 1: full http/https URL
|
|
123
|
+
if (statusCode !== null)
|
|
124
|
+
return true; // condition 2: valid HTTP status code
|
|
125
|
+
if (method !== null && path !== null)
|
|
126
|
+
return true; // condition 3: method + path
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
function classifyEventType(endpoint) {
|
|
130
|
+
const lower = endpoint.toLowerCase();
|
|
131
|
+
if (BACKGROUND_TOKENS.some(t => lower.includes(t)))
|
|
132
|
+
return 'background';
|
|
133
|
+
if (BACKGROUND_EXTENSIONS.some(e => lower.endsWith(e)))
|
|
134
|
+
return 'background';
|
|
135
|
+
return 'primary';
|
|
136
|
+
}
|
|
137
|
+
function filterToSignificantEvents(events) {
|
|
138
|
+
if (events.length === 0)
|
|
139
|
+
return events;
|
|
140
|
+
const hasPrimary = events.some(e => classifyEventType(e.endpoint) === 'primary');
|
|
141
|
+
return hasPrimary ? events.filter(e => classifyEventType(e.endpoint) === 'primary') : events;
|
|
142
|
+
}
|
|
143
|
+
/** Exported for unit testing. */
|
|
144
|
+
export function parseMessageToEvent(message) {
|
|
145
|
+
const url = extractUrl(message);
|
|
146
|
+
const path = url ? null : extractPath(message);
|
|
147
|
+
const method = extractMethod(message);
|
|
148
|
+
const statusCode = extractStatusCode(message, url, path, method);
|
|
149
|
+
const networkError = detectNetworkError(message);
|
|
150
|
+
if (!meetsEmissionCriteria(url, path, statusCode, method))
|
|
151
|
+
return null;
|
|
152
|
+
const rawEndpoint = url || path || 'unknown';
|
|
153
|
+
return {
|
|
154
|
+
endpoint: normalizeEndpoint(rawEndpoint),
|
|
155
|
+
method: method || 'unknown',
|
|
156
|
+
statusCode,
|
|
157
|
+
networkError,
|
|
158
|
+
status: classifyStatus(statusCode, networkError),
|
|
159
|
+
durationMs: 0
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
// ─── Android ─────────────────────────────────────────────────────────────────
|
|
163
|
+
async function getAndroidEvents(sinceMs, deviceId) {
|
|
164
|
+
try {
|
|
165
|
+
const stdout = await execAdb(['logcat', '-d', '-v', 'threadtime', '*:V', '-t', '2000'], deviceId);
|
|
166
|
+
const lines = stdout ? stdout.split(/\r?\n/).filter(Boolean) : [];
|
|
167
|
+
const events = [];
|
|
168
|
+
for (const line of lines) {
|
|
169
|
+
const parsed = parseLogLine(line);
|
|
170
|
+
if (parsed._iso) {
|
|
171
|
+
const ts = new Date(parsed._iso).getTime();
|
|
172
|
+
if (ts > 0 && ts <= sinceMs)
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
const event = parseMessageToEvent(parsed.message || line);
|
|
176
|
+
if (event)
|
|
177
|
+
events.push(event);
|
|
178
|
+
}
|
|
179
|
+
return events;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
return [];
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// ─── iOS ─────────────────────────────────────────────────────────────────────
|
|
186
|
+
async function getIOSEvents(sinceMs, deviceId = 'booted') {
|
|
187
|
+
try {
|
|
188
|
+
const lookbackSeconds = Math.max(15, Math.ceil((Date.now() - sinceMs) / 1000) + 5);
|
|
189
|
+
const args = [
|
|
190
|
+
'simctl', 'spawn', deviceId, 'log', 'show',
|
|
191
|
+
'--last', `${lookbackSeconds}s`,
|
|
192
|
+
'--style', 'syslog',
|
|
193
|
+
'--predicate', 'eventMessage contains "http" OR eventMessage contains "URLSession" OR eventMessage contains "Task <" OR eventMessage contains "HTTP/"'
|
|
194
|
+
];
|
|
195
|
+
const result = await execCommand(args, deviceId);
|
|
196
|
+
const lines = result.output ? result.output.split(/\r?\n/).filter(Boolean) : [];
|
|
197
|
+
const events = [];
|
|
198
|
+
for (const line of lines) {
|
|
199
|
+
const tsMatch = line.match(/^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})/);
|
|
200
|
+
if (tsMatch) {
|
|
201
|
+
const ts = new Date(tsMatch[1]).getTime();
|
|
202
|
+
if (ts > 0 && ts <= sinceMs)
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const event = parseMessageToEvent(line);
|
|
206
|
+
if (event)
|
|
207
|
+
events.push(event);
|
|
208
|
+
}
|
|
209
|
+
return events;
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
return [];
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// ─── Public API ───────────────────────────────────────────────────────────────
|
|
216
|
+
export class ToolsNetwork {
|
|
217
|
+
static notifyActionStart() {
|
|
218
|
+
notifyActionStart();
|
|
219
|
+
}
|
|
220
|
+
static async getNetworkActivity(params) {
|
|
221
|
+
const { platform, deviceId } = params;
|
|
222
|
+
const sinceMs = lastConsumedTimestamp > lastActionTimestamp
|
|
223
|
+
? lastConsumedTimestamp
|
|
224
|
+
: lastActionTimestamp > 0 ? lastActionTimestamp : Date.now() - 30000;
|
|
225
|
+
const raw = platform === 'android'
|
|
226
|
+
? await getAndroidEvents(sinceMs, deviceId)
|
|
227
|
+
: await getIOSEvents(sinceMs, deviceId);
|
|
228
|
+
const requests = filterToSignificantEvents(raw);
|
|
229
|
+
lastConsumedTimestamp = Date.now();
|
|
230
|
+
return { requests, count: requests.length };
|
|
231
|
+
}
|
|
232
|
+
}
|