@mobilenext/mobile-mcp 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +65 -11
- package/lib/android.js +147 -120
- package/lib/ios.js +79 -0
- package/lib/iphone-simulator.js +160 -27
- package/lib/robot.js +2 -0
- package/lib/server.js +111 -36
- package/lib/webdriver-agent.js +145 -0
- package/package.json +11 -9
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -3,14 +3,25 @@
|
|
|
3
3
|
This is a [Model Context Protocol (MCP) server](https://github.com/modelcontextprotocol) that enables scalable mobile automation through a platform-agnostic interface, eliminating the need for distinct iOS or Android knowledge.
|
|
4
4
|
This server allows Agents and LLMs to interact with native iOS/Android applications and devices through structured accessibility snapshots or coordinate-based taps based on screenshots.
|
|
5
5
|
|
|
6
|
+
https://github.com/user-attachments/assets/c4e89c4f-cc71-4424-8184-bdbc8c638fa1
|
|
6
7
|
|
|
7
8
|
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://www.npmjs.com/package/@mobilenext/mobile-mcp">
|
|
11
|
+
<img src="https://img.shields.io/badge/npm-@mobilenext/mobile--mcp-red" alt="npm">
|
|
12
|
+
</a>
|
|
13
|
+
<a href="https://github.com/mobile-next/mobile-mcp">
|
|
14
|
+
<img src="https://img.shields.io/badge/github-repo-black" alt="GitHub repo">
|
|
15
|
+
</a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
8
18
|
<p align="center">
|
|
9
19
|
<a href="https://github.com/mobile-next/">
|
|
10
20
|
<img alt="mobile-mcp" src="https://raw.githubusercontent.com/mobile-next/mobile-next-assets/refs/heads/main/mobile-mcp-banner.png" width="600">
|
|
11
21
|
</a>
|
|
12
22
|
</p>
|
|
13
23
|
|
|
24
|
+
|
|
14
25
|
### 🚀 Mobile MCP Roadmap: Building the Future of Mobile
|
|
15
26
|
|
|
16
27
|
Join us on our journey as we continuously enhance Mobile MCP!
|
|
@@ -39,26 +50,33 @@ How we help to scale mobile automation:
|
|
|
39
50
|
## Mobile MCP Architecture
|
|
40
51
|
|
|
41
52
|
<p align="center">
|
|
42
|
-
<a href="https://raw.githubusercontent.com/mobile-next/mobile-next-assets/refs/heads/main/mobile-mcp-arch.png">
|
|
43
|
-
<img alt="mobile-mcp" src="https://raw.githubusercontent.com/mobile-next/mobile-next-assets/refs/heads/main/mobile-mcp-arch.png" width="600">
|
|
53
|
+
<a href="https://raw.githubusercontent.com/mobile-next/mobile-next-assets/refs/heads/main/mobile-mcp-arch-1.png">
|
|
54
|
+
<img alt="mobile-mcp" src="https://raw.githubusercontent.com/mobile-next/mobile-next-assets/refs/heads/main/mobile-mcp-arch-1.png" width="600">
|
|
44
55
|
</a>
|
|
45
56
|
</p>
|
|
46
57
|
|
|
47
58
|
|
|
48
59
|
|
|
49
|
-
##
|
|
60
|
+
## Installation and configuration
|
|
50
61
|
|
|
51
|
-
|
|
62
|
+
[Detailed guide for Claude Desktop](https://modelcontextprotocol.io/quickstart/user)
|
|
63
|
+
|
|
64
|
+
```json
|
|
52
65
|
{
|
|
53
66
|
"mcpServers": {
|
|
54
|
-
"mobile-
|
|
67
|
+
"mobile-mcp": {
|
|
55
68
|
"command": "npx",
|
|
56
|
-
"args": [
|
|
57
|
-
"@mobilenext/mobile-mcp@latest"
|
|
58
|
-
]
|
|
69
|
+
"args": ["-y", "@mobilenext/mobile-mcp@latest"]
|
|
59
70
|
}
|
|
60
71
|
}
|
|
61
72
|
}
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
[Claude Code:](https://docs.anthropic.com/en/docs/agents-and-tools/claude-code/overview)
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
claude mcp add mobile -- npx -y @mobilenext/mobile-mcp@latest
|
|
62
80
|
```
|
|
63
81
|
|
|
64
82
|
## Prerequisites
|
|
@@ -101,6 +119,11 @@ The commands and tools support both accessibility-based locators (preferred) and
|
|
|
101
119
|
- **Parameters:**
|
|
102
120
|
- `appPath` (string): Path or URL to the app file (e.g., .apk for Android, .ipa/.app for iOS)
|
|
103
121
|
|
|
122
|
+
## mobile_list_apps
|
|
123
|
+
- **Description:** List all the installed apps on the device
|
|
124
|
+
- **Parameters:**
|
|
125
|
+
- `bundleId` (string): The application's unique bundle/package identifier like: com.google.android.keep or com.apple.mobilenotes )
|
|
126
|
+
|
|
104
127
|
## mobile_launch_app
|
|
105
128
|
- **Description:** Launches the specified app on the device/emulator
|
|
106
129
|
- **Parameters:**
|
|
@@ -109,7 +132,21 @@ The commands and tools support both accessibility-based locators (preferred) and
|
|
|
109
132
|
## mobile_terminate_app
|
|
110
133
|
- **Description:** Terminates a running application
|
|
111
134
|
- **Parameters:**
|
|
112
|
-
- `
|
|
135
|
+
- `packageName` (string): Based on the application's bundle/package identifier calls am force stop or kills the app based on pid.
|
|
136
|
+
|
|
137
|
+
## mobile_get_screen_size
|
|
138
|
+
- **Description:** Get the screen size of the mobile device in pixels
|
|
139
|
+
- **Parameters:** None
|
|
140
|
+
|
|
141
|
+
## mobile_click_on_screen_at_coordinates
|
|
142
|
+
- **Description:** Taps on specified screen coordinates based on coordinates.
|
|
143
|
+
- **Parameters:**
|
|
144
|
+
- `x` (number): X-coordinate
|
|
145
|
+
- `y` (number): Y-coordinate
|
|
146
|
+
|
|
147
|
+
## mobile_list_elements_on_screen
|
|
148
|
+
- **Description:** List elements on screen and their coordinates, with display text or accessibility label.
|
|
149
|
+
- **Parameters:** None
|
|
113
150
|
|
|
114
151
|
## mobile_element_tap
|
|
115
152
|
- **Description:** Taps on a UI element identified by accessibility locator
|
|
@@ -122,9 +159,18 @@ The commands and tools support both accessibility-based locators (preferred) and
|
|
|
122
159
|
- **Parameters:**
|
|
123
160
|
- `x` (number): X-coordinate
|
|
124
161
|
- `y` (number): Y-coordinate
|
|
162
|
+
|
|
163
|
+
## mobile_press_button
|
|
164
|
+
- **Description:** Press a button on device (home, back, volume, enter, power button.)
|
|
165
|
+
- **Parameters:** None
|
|
166
|
+
|
|
167
|
+
## mobile_open_url
|
|
168
|
+
- **Description:** Open a URL in browser on device
|
|
169
|
+
- **Parameters:**
|
|
170
|
+
- `url` (string): The URL to be opened (e.g., "https://example.com").
|
|
125
171
|
|
|
126
|
-
##
|
|
127
|
-
- **Description:** Types text into a UI element (e.g., TextField)
|
|
172
|
+
## mobile_type_text
|
|
173
|
+
- **Description:** Types text into a focused UI element (e.g., TextField, SearchField)
|
|
128
174
|
- **Parameters:**
|
|
129
175
|
- `element` (string): Human-readable element description
|
|
130
176
|
- `ref` (string): Accessibility/automation ID of the element
|
|
@@ -171,3 +217,11 @@ The commands and tools support both accessibility-based locators (preferred) and
|
|
|
171
217
|
- **Parameters:** None
|
|
172
218
|
|
|
173
219
|
|
|
220
|
+
# Thanks to all contributors ❤️
|
|
221
|
+
|
|
222
|
+
### We appreciate everyone who has helped improve this project.
|
|
223
|
+
|
|
224
|
+
<a href = "https://github.com/mobile-next/mobile-mcp/graphs/contributors">
|
|
225
|
+
<img src = "https://contrib.rocks/image?repo=mobile-next/mobile-mcp"/>
|
|
226
|
+
</a>
|
|
227
|
+
|
package/lib/android.js
CHANGED
|
@@ -32,140 +32,167 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
32
32
|
return result;
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
35
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.
|
|
39
|
+
exports.getConnectedDevices = exports.AndroidRobot = void 0;
|
|
37
40
|
const child_process_1 = require("child_process");
|
|
38
41
|
const xml = __importStar(require("fast-xml-parser"));
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
.
|
|
44
|
-
.filter(line => !line.startsWith("List of devices attached"))
|
|
45
|
-
.filter(line => line.trim() !== "");
|
|
46
|
-
};
|
|
47
|
-
exports.getConnectedDevices = getConnectedDevices;
|
|
48
|
-
const resolveLaunchableActivities = (packageName) => {
|
|
49
|
-
return (0, child_process_1.execSync)(`adb shell cmd package resolve-activity ${packageName}`)
|
|
50
|
-
.toString()
|
|
51
|
-
.split("\n")
|
|
52
|
-
.map(line => line.trim())
|
|
53
|
-
.filter(line => line.startsWith("name="))
|
|
54
|
-
.map(line => line.substring("name=".length));
|
|
55
|
-
};
|
|
56
|
-
exports.resolveLaunchableActivities = resolveLaunchableActivities;
|
|
57
|
-
const getScreenSize = () => {
|
|
58
|
-
const screenSize = (0, child_process_1.execSync)("adb shell wm size")
|
|
59
|
-
.toString()
|
|
60
|
-
.split(" ")
|
|
61
|
-
.pop();
|
|
62
|
-
if (!screenSize) {
|
|
63
|
-
throw new Error("Failed to get screen size");
|
|
42
|
+
const path_1 = __importDefault(require("path"));
|
|
43
|
+
class AndroidRobot {
|
|
44
|
+
deviceId;
|
|
45
|
+
constructor(deviceId) {
|
|
46
|
+
this.deviceId = deviceId;
|
|
64
47
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
const [,
|
|
74
|
-
return {
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
48
|
+
async getScreenSize() {
|
|
49
|
+
const screenSize = (0, child_process_1.execSync)(`adb -s ${this.deviceId} shell wm size`)
|
|
50
|
+
.toString()
|
|
51
|
+
.split(" ")
|
|
52
|
+
.pop();
|
|
53
|
+
if (!screenSize) {
|
|
54
|
+
throw new Error("Failed to get screen size");
|
|
55
|
+
}
|
|
56
|
+
const [width, height] = screenSize.split("x").map(Number);
|
|
57
|
+
return { width, height };
|
|
58
|
+
}
|
|
59
|
+
adb(...args) {
|
|
60
|
+
let executable = "adb";
|
|
61
|
+
if (process.env.ANDROID_HOME) {
|
|
62
|
+
executable = path_1.default.join(process.env.ANDROID_HOME, "platform-tools", "adb");
|
|
63
|
+
}
|
|
64
|
+
return (0, child_process_1.execFileSync)(executable, ["-s", this.deviceId, ...args], {
|
|
65
|
+
maxBuffer: 1024 * 1024 * 4,
|
|
66
|
+
timeout: 30000,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
async listApps() {
|
|
70
|
+
const result = this.adb("shell", "cmd", "package", "query-activities", "-a", "android.intent.action.MAIN", "-c", "android.intent.category.LAUNCHER")
|
|
71
|
+
.toString()
|
|
72
|
+
.split("\n")
|
|
73
|
+
.map(line => line.trim())
|
|
74
|
+
.filter(line => line.startsWith("packageName="))
|
|
75
|
+
.map(line => line.substring("packageName=".length))
|
|
76
|
+
.filter((value, index, self) => self.indexOf(value) === index);
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
async launchApp(packageName) {
|
|
80
|
+
this.adb("shell", "monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1");
|
|
81
|
+
}
|
|
82
|
+
async swipe(direction) {
|
|
83
|
+
const screenSize = await this.getScreenSize();
|
|
84
|
+
const centerX = screenSize.width >> 1;
|
|
85
|
+
// const centerY = screenSize[1] >> 1;
|
|
86
|
+
let x0, y0, x1, y1;
|
|
87
|
+
switch (direction) {
|
|
88
|
+
case "up":
|
|
89
|
+
x0 = x1 = centerX;
|
|
90
|
+
y0 = Math.floor(screenSize.height * 0.80);
|
|
91
|
+
y1 = Math.floor(screenSize.height * 0.20);
|
|
92
|
+
break;
|
|
93
|
+
case "down":
|
|
94
|
+
x0 = x1 = centerX;
|
|
95
|
+
y0 = Math.floor(screenSize.height * 0.20);
|
|
96
|
+
y1 = Math.floor(screenSize.height * 0.80);
|
|
97
|
+
break;
|
|
98
|
+
default:
|
|
99
|
+
throw new Error(`Swipe direction "${direction}" is not supported`);
|
|
100
|
+
}
|
|
101
|
+
this.adb("shell", "input", "swipe", `${x0}`, `${y0}`, `${x1}`, `${y1}`, "1000");
|
|
102
|
+
}
|
|
103
|
+
async getScreenshot() {
|
|
104
|
+
return this.adb("shell", "screencap", "-p");
|
|
105
|
+
}
|
|
106
|
+
collectElements(node, screenSize) {
|
|
107
|
+
const elements = [];
|
|
108
|
+
const getCoordinates = (element) => {
|
|
109
|
+
const bounds = String(element.bounds);
|
|
110
|
+
const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
|
|
111
|
+
return { left, top, right, bottom };
|
|
80
112
|
};
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
113
|
+
const getCenter = (coordinates) => {
|
|
114
|
+
return {
|
|
115
|
+
x: Math.floor((coordinates.left + coordinates.right) / 2),
|
|
116
|
+
y: Math.floor((coordinates.top + coordinates.bottom) / 2),
|
|
117
|
+
};
|
|
86
118
|
};
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
119
|
+
const normalizeCoordinates = (coordinates, screenSize) => {
|
|
120
|
+
return {
|
|
121
|
+
x: Number((coordinates.x / screenSize.width).toFixed(3)),
|
|
122
|
+
y: Number((coordinates.y / screenSize.height).toFixed(3)),
|
|
123
|
+
};
|
|
124
|
+
};
|
|
125
|
+
if (node.node) {
|
|
126
|
+
if (Array.isArray(node.node)) {
|
|
127
|
+
for (const childNode of node.node) {
|
|
128
|
+
elements.push(...this.collectElements(childNode, screenSize));
|
|
129
|
+
}
|
|
92
130
|
}
|
|
131
|
+
else {
|
|
132
|
+
elements.push(...this.collectElements(node.node, screenSize));
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (node.text) {
|
|
136
|
+
elements.push({
|
|
137
|
+
"text": node.text,
|
|
138
|
+
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
|
139
|
+
});
|
|
93
140
|
}
|
|
94
|
-
|
|
95
|
-
elements.push(
|
|
141
|
+
if (node["content-desc"]) {
|
|
142
|
+
elements.push({
|
|
143
|
+
"text": node["content-desc"],
|
|
144
|
+
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
|
145
|
+
});
|
|
96
146
|
}
|
|
147
|
+
return elements;
|
|
97
148
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
149
|
+
async getElementsOnScreen() {
|
|
150
|
+
const dump = this.adb("exec-out", "uiautomator", "dump", "/dev/tty");
|
|
151
|
+
const parser = new xml.XMLParser({
|
|
152
|
+
ignoreAttributes: false,
|
|
153
|
+
attributeNamePrefix: ""
|
|
102
154
|
});
|
|
155
|
+
const parsedXml = parser.parse(dump);
|
|
156
|
+
const hierarchy = parsedXml.hierarchy;
|
|
157
|
+
const screenSize = await this.getScreenSize();
|
|
158
|
+
const elements = this.collectElements(hierarchy.node, screenSize);
|
|
159
|
+
return elements;
|
|
103
160
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
"text": node["content-desc"],
|
|
107
|
-
"coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
|
|
108
|
-
});
|
|
161
|
+
async terminateApp(packageName) {
|
|
162
|
+
this.adb("shell", "am", "force-stop", packageName);
|
|
109
163
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
const getElementsOnScreen = () => {
|
|
113
|
-
const dump = (0, child_process_1.execSync)(`adb exec-out uiautomator dump /dev/tty`);
|
|
114
|
-
const parser = new xml.XMLParser({
|
|
115
|
-
ignoreAttributes: false,
|
|
116
|
-
attributeNamePrefix: ""
|
|
117
|
-
});
|
|
118
|
-
const parsedXml = parser.parse(dump);
|
|
119
|
-
const hierarchy = parsedXml.hierarchy;
|
|
120
|
-
const screenSize = (0, exports.getScreenSize)();
|
|
121
|
-
const elements = collectElements(hierarchy, screenSize);
|
|
122
|
-
return elements;
|
|
123
|
-
};
|
|
124
|
-
exports.getElementsOnScreen = getElementsOnScreen;
|
|
125
|
-
const swipe = (direction) => {
|
|
126
|
-
const screenSize = (0, exports.getScreenSize)();
|
|
127
|
-
const centerX = screenSize[0] >> 1;
|
|
128
|
-
// const centerY = screenSize[1] >> 1;
|
|
129
|
-
let x0, y0, x1, y1;
|
|
130
|
-
switch (direction) {
|
|
131
|
-
case "down":
|
|
132
|
-
x0 = x1 = centerX;
|
|
133
|
-
y0 = Math.floor(screenSize[1] * 0.80);
|
|
134
|
-
y1 = Math.floor(screenSize[1] * 0.20);
|
|
135
|
-
break;
|
|
136
|
-
case "up":
|
|
137
|
-
x0 = x1 = centerX;
|
|
138
|
-
y0 = Math.floor(screenSize[1] * 0.20);
|
|
139
|
-
y1 = Math.floor(screenSize[1] * 0.80);
|
|
140
|
-
break;
|
|
141
|
-
default:
|
|
142
|
-
throw new Error(`Swipe direction "${direction}" is not supported`);
|
|
164
|
+
async openUrl(url) {
|
|
165
|
+
this.adb("shell", "am", "start", "-a", "android.intent.action.VIEW", "-d", url);
|
|
143
166
|
}
|
|
144
|
-
(
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
167
|
+
async sendKeys(text) {
|
|
168
|
+
// adb shell requires some escaping
|
|
169
|
+
const _text = text.replace(/ /g, "\\ ");
|
|
170
|
+
this.adb("shell", "input", "text", _text);
|
|
171
|
+
}
|
|
172
|
+
async pressButton(button) {
|
|
173
|
+
const _map = {
|
|
174
|
+
"BACK": "KEYCODE_BACK",
|
|
175
|
+
"HOME": "KEYCODE_HOME",
|
|
176
|
+
"VOLUME_UP": "KEYCODE_VOLUME_UP",
|
|
177
|
+
"VOLUME_DOWN": "KEYCODE_VOLUME_DOWN",
|
|
178
|
+
"ENTER": "KEYCODE_ENTER",
|
|
179
|
+
};
|
|
180
|
+
if (!_map[button]) {
|
|
181
|
+
throw new Error(`Button "${button}" is not supported`);
|
|
182
|
+
}
|
|
183
|
+
this.adb("shell", "input", "keyevent", _map[button]);
|
|
184
|
+
}
|
|
185
|
+
async tap(x, y) {
|
|
186
|
+
this.adb("shell", "input", "tap", `${x}`, `${y}`);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
exports.AndroidRobot = AndroidRobot;
|
|
190
|
+
const getConnectedDevices = () => {
|
|
191
|
+
return (0, child_process_1.execSync)(`adb devices`)
|
|
163
192
|
.toString()
|
|
164
193
|
.split("\n")
|
|
165
|
-
.
|
|
166
|
-
.filter(line => line.
|
|
167
|
-
.map(line => line.
|
|
168
|
-
.filter((value, index, self) => self.indexOf(value) === index);
|
|
169
|
-
return result;
|
|
194
|
+
.filter(line => !line.startsWith("List of devices attached"))
|
|
195
|
+
.filter(line => line.trim() !== "")
|
|
196
|
+
.map(line => line.split("\t")[0]);
|
|
170
197
|
};
|
|
171
|
-
exports.
|
|
198
|
+
exports.getConnectedDevices = getConnectedDevices;
|
package/lib/ios.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.IosManager = exports.IosRobot = void 0;
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const child_process_1 = require("child_process");
|
|
6
|
+
const webdriver_agent_1 = require("./webdriver-agent");
|
|
7
|
+
class IosRobot {
|
|
8
|
+
deviceId;
|
|
9
|
+
wda;
|
|
10
|
+
constructor(deviceId) {
|
|
11
|
+
this.deviceId = deviceId;
|
|
12
|
+
this.wda = new webdriver_agent_1.WebDriverAgent("localhost", 8100);
|
|
13
|
+
}
|
|
14
|
+
async ios(...args) {
|
|
15
|
+
return (0, child_process_1.execFileSync)("ios", ["--udid", this.deviceId, ...args], {}).toString();
|
|
16
|
+
}
|
|
17
|
+
async getScreenSize() {
|
|
18
|
+
return await this.wda.getScreenSize();
|
|
19
|
+
}
|
|
20
|
+
swipe(direction) {
|
|
21
|
+
return Promise.resolve();
|
|
22
|
+
}
|
|
23
|
+
async listApps() {
|
|
24
|
+
const output = await this.ios("apps", "--all", "--list");
|
|
25
|
+
return output
|
|
26
|
+
.split("\n")
|
|
27
|
+
.map(line => line.split(" ")[0]);
|
|
28
|
+
}
|
|
29
|
+
async launchApp(packageName) {
|
|
30
|
+
await this.ios("launch", packageName);
|
|
31
|
+
}
|
|
32
|
+
async terminateApp(packageName) {
|
|
33
|
+
await this.ios("kill", packageName);
|
|
34
|
+
}
|
|
35
|
+
async openUrl(url) {
|
|
36
|
+
await this.wda.withinSession(async (sessionUrl) => {
|
|
37
|
+
await fetch(`${sessionUrl}/url`, {
|
|
38
|
+
method: "POST",
|
|
39
|
+
body: JSON.stringify({ url }),
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
async sendKeys(text) {
|
|
44
|
+
await this.wda.sendKeys(text);
|
|
45
|
+
}
|
|
46
|
+
async pressButton(button) {
|
|
47
|
+
await this.wda.pressButton(button);
|
|
48
|
+
}
|
|
49
|
+
async tap(x, y) {
|
|
50
|
+
await this.wda.tap(x, y);
|
|
51
|
+
}
|
|
52
|
+
async getElementsOnScreen() {
|
|
53
|
+
return await this.wda.getElementsOnScreen();
|
|
54
|
+
}
|
|
55
|
+
async getScreenshot() {
|
|
56
|
+
await this.ios("screenshot", "--output", "screenshot.png");
|
|
57
|
+
const buffer = (0, fs_1.readFileSync)("screenshot.png");
|
|
58
|
+
(0, fs_1.unlinkSync)("screenshot.png");
|
|
59
|
+
return buffer;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
exports.IosRobot = IosRobot;
|
|
63
|
+
class IosManager {
|
|
64
|
+
async listDevices() {
|
|
65
|
+
const output = (0, child_process_1.execSync)("ios list").toString();
|
|
66
|
+
const json = JSON.parse(output);
|
|
67
|
+
return json.deviceList;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
exports.IosManager = IosManager;
|
|
71
|
+
async function main() {
|
|
72
|
+
const ios = new IosRobot("4C07ED7E-AE81-412E-8AA9-1061EED59DFA");
|
|
73
|
+
const before = +new Date();
|
|
74
|
+
console.dir(await ios.getElementsOnScreen(), { depth: null });
|
|
75
|
+
const after = +new Date();
|
|
76
|
+
console.log(`Time taken: ${after - before}ms`);
|
|
77
|
+
// await ios.pressButton("VOLUME_UP");
|
|
78
|
+
}
|
|
79
|
+
main().then();
|
package/lib/iphone-simulator.js
CHANGED
|
@@ -1,40 +1,173 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.SimctlManager = exports.Simctl = void 0;
|
|
4
4
|
const child_process_1 = require("child_process");
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
const webdriver_agent_1 = require("./webdriver-agent");
|
|
6
|
+
class Simctl {
|
|
7
|
+
simulatorUuid;
|
|
8
|
+
webDriverAgent;
|
|
9
|
+
constructor(simulatorUuid) {
|
|
10
|
+
this.simulatorUuid = simulatorUuid;
|
|
11
|
+
this.webDriverAgent = new webdriver_agent_1.WebDriverAgent("localhost", 8100);
|
|
12
|
+
}
|
|
13
|
+
simctl(...args) {
|
|
14
|
+
return (0, child_process_1.execFileSync)("xcrun", ["simctl", ...args], { maxBuffer: 1024 * 1024 * 4 });
|
|
15
|
+
}
|
|
16
|
+
async getScreenshot() {
|
|
17
|
+
return this.simctl("io", this.simulatorUuid, "screenshot", "-");
|
|
18
|
+
}
|
|
19
|
+
async openUrl(url) {
|
|
20
|
+
this.simctl("openurl", this.simulatorUuid, url);
|
|
21
|
+
}
|
|
22
|
+
async launchApp(packageName) {
|
|
23
|
+
this.simctl("launch", this.simulatorUuid, packageName);
|
|
24
|
+
}
|
|
25
|
+
async terminateApp(packageName) {
|
|
26
|
+
this.simctl("terminate", this.simulatorUuid, packageName);
|
|
27
|
+
}
|
|
28
|
+
parseIOSAppData(inputText) {
|
|
29
|
+
const result = [];
|
|
30
|
+
// Remove leading and trailing characters if needed
|
|
31
|
+
const cleanText = inputText.trim();
|
|
32
|
+
// Extract each app section
|
|
33
|
+
const appRegex = /"([^"]+)"\s+=\s+\{([^}]+)\};/g;
|
|
34
|
+
let appMatch;
|
|
35
|
+
while ((appMatch = appRegex.exec(cleanText)) !== null) {
|
|
36
|
+
// const bundleId = appMatch[1];
|
|
37
|
+
const appContent = appMatch[2];
|
|
38
|
+
const appInfo = {
|
|
39
|
+
GroupContainers: {},
|
|
40
|
+
SBAppTags: []
|
|
41
|
+
};
|
|
42
|
+
// parse simple key-value pairs
|
|
43
|
+
const keyValueRegex = /\s+(\w+)\s+=\s+([^;]+);/g;
|
|
44
|
+
let keyValueMatch;
|
|
45
|
+
while ((keyValueMatch = keyValueRegex.exec(appContent)) !== null) {
|
|
46
|
+
const key = keyValueMatch[1];
|
|
47
|
+
let value = keyValueMatch[2].trim();
|
|
48
|
+
// Handle quoted string values
|
|
49
|
+
if (value.startsWith('"') && value.endsWith('"')) {
|
|
50
|
+
value = value.substring(1, value.length - 1);
|
|
51
|
+
}
|
|
52
|
+
if (key !== "GroupContainers" && key !== "SBAppTags") {
|
|
53
|
+
appInfo[key] = value;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
// parse GroupContainers
|
|
57
|
+
const groupContainersMatch = appContent.match(/GroupContainers\s+=\s+\{([^}]+)\};/);
|
|
58
|
+
if (groupContainersMatch) {
|
|
59
|
+
const groupContainersContent = groupContainersMatch[1];
|
|
60
|
+
const groupRegex = /"([^"]+)"\s+=\s+"([^"]+)"/g;
|
|
61
|
+
let groupMatch;
|
|
62
|
+
while ((groupMatch = groupRegex.exec(groupContainersContent)) !== null) {
|
|
63
|
+
const groupId = groupMatch[1];
|
|
64
|
+
const groupPath = groupMatch[2];
|
|
65
|
+
appInfo.GroupContainers[groupId] = groupPath;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
// parse SBAppTags
|
|
69
|
+
const sbAppTagsMatch = appContent.match(/SBAppTags\s+=\s+\(\s*(.*?)\s*\);/);
|
|
70
|
+
if (sbAppTagsMatch) {
|
|
71
|
+
const tagsContent = sbAppTagsMatch[1].trim();
|
|
72
|
+
if (tagsContent) {
|
|
73
|
+
const tagRegex = /"([^"]+)"/g;
|
|
74
|
+
let tagMatch;
|
|
75
|
+
while ((tagMatch = tagRegex.exec(tagsContent)) !== null) {
|
|
76
|
+
appInfo.SBAppTags.push(tagMatch[1]);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
result.push(appInfo);
|
|
81
|
+
}
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
async listApps() {
|
|
85
|
+
const text = this.simctl("listapps", this.simulatorUuid).toString();
|
|
86
|
+
const apps = this.parseIOSAppData(text);
|
|
87
|
+
return apps.map(app => app.CFBundleIdentifier);
|
|
88
|
+
}
|
|
89
|
+
async getScreenSize() {
|
|
90
|
+
return this.webDriverAgent.getScreenSize();
|
|
91
|
+
}
|
|
92
|
+
async sendKeys(keys) {
|
|
93
|
+
return this.webDriverAgent.sendKeys(keys);
|
|
94
|
+
}
|
|
95
|
+
async swipe(direction) {
|
|
96
|
+
await this.webDriverAgent.withinSession(async (sessionUrl) => {
|
|
97
|
+
const x0 = 200;
|
|
98
|
+
let y0 = 600;
|
|
99
|
+
const x1 = 200;
|
|
100
|
+
let y1 = 200;
|
|
101
|
+
if (direction === "up") {
|
|
102
|
+
const tmp = y0;
|
|
103
|
+
y0 = y1;
|
|
104
|
+
y1 = tmp;
|
|
105
|
+
}
|
|
106
|
+
const url = `${sessionUrl}/actions`;
|
|
107
|
+
await fetch(url, {
|
|
108
|
+
method: "POST",
|
|
109
|
+
headers: {
|
|
110
|
+
"Content-Type": "application/json",
|
|
111
|
+
},
|
|
112
|
+
body: JSON.stringify({
|
|
113
|
+
actions: [
|
|
114
|
+
{
|
|
115
|
+
type: "pointer",
|
|
116
|
+
id: "finger1",
|
|
117
|
+
parameters: { pointerType: "touch" },
|
|
118
|
+
actions: [
|
|
119
|
+
{ type: "pointerMove", duration: 0, x: x0, y: y0 },
|
|
120
|
+
{ type: "pointerDown", button: 0 },
|
|
121
|
+
{ type: "pointerMove", duration: 0, x: x1, y: y1 },
|
|
122
|
+
{ type: "pause", duration: 1000 },
|
|
123
|
+
{ type: "pointerUp", button: 0 }
|
|
124
|
+
]
|
|
125
|
+
}
|
|
126
|
+
]
|
|
127
|
+
}),
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
async tap(x, y) {
|
|
132
|
+
await this.webDriverAgent.tap(x, y);
|
|
133
|
+
}
|
|
134
|
+
async pressButton(button) {
|
|
135
|
+
await this.webDriverAgent.pressButton(button);
|
|
136
|
+
}
|
|
137
|
+
async getElementsOnScreen() {
|
|
138
|
+
return await this.webDriverAgent.getElementsOnScreen();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
exports.Simctl = Simctl;
|
|
142
|
+
class SimctlManager {
|
|
143
|
+
parseSimulator(line) {
|
|
10
144
|
// extract device name and UUID from the line
|
|
11
|
-
const match = line.match(/(.*?)\s+\(([\w-]+)\)\s+\(
|
|
145
|
+
const match = line.match(/(.*?)\s+\(([\w-]+)\)\s+\((\w+)\)/);
|
|
12
146
|
if (!match) {
|
|
13
147
|
return null;
|
|
14
148
|
}
|
|
15
149
|
const deviceName = match[1].trim();
|
|
16
150
|
const deviceUuid = match[2];
|
|
151
|
+
const deviceState = match[3];
|
|
17
152
|
return {
|
|
18
153
|
name: deviceName,
|
|
19
154
|
uuid: deviceUuid,
|
|
155
|
+
state: deviceState,
|
|
20
156
|
};
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
return (0, child_process_1.execSync)(`xcrun simctl list apps "${simulatorUuid}"`);
|
|
39
|
-
};
|
|
40
|
-
exports.listApps = listApps;
|
|
157
|
+
}
|
|
158
|
+
listSimulators() {
|
|
159
|
+
return (0, child_process_1.execSync)(`xcrun simctl list devices`)
|
|
160
|
+
.toString()
|
|
161
|
+
.split("\n")
|
|
162
|
+
.map(line => this.parseSimulator(line))
|
|
163
|
+
.filter(simulator => simulator !== null);
|
|
164
|
+
}
|
|
165
|
+
listBootedSimulators() {
|
|
166
|
+
return this.listSimulators()
|
|
167
|
+
.filter(simulator => simulator.state === "Booted");
|
|
168
|
+
}
|
|
169
|
+
getSimulator(uuid) {
|
|
170
|
+
return new Simctl(uuid);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
exports.SimctlManager = SimctlManager;
|
package/lib/robot.js
ADDED
package/lib/server.js
CHANGED
|
@@ -5,11 +5,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.createMcpServer = void 0;
|
|
7
7
|
const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
8
|
-
const child_process_1 = require("child_process");
|
|
9
|
-
const logger_1 = require("./logger");
|
|
10
8
|
const zod_1 = require("zod");
|
|
11
|
-
const android_1 = require("./android");
|
|
12
9
|
const sharp_1 = __importDefault(require("sharp"));
|
|
10
|
+
const logger_1 = require("./logger");
|
|
11
|
+
const android_1 = require("./android");
|
|
12
|
+
const iphone_simulator_1 = require("./iphone-simulator");
|
|
13
|
+
const ios_1 = require("./ios");
|
|
13
14
|
const getAgentVersion = () => {
|
|
14
15
|
const json = require("../package.json");
|
|
15
16
|
return json.version;
|
|
@@ -43,69 +44,143 @@ const createMcpServer = () => {
|
|
|
43
44
|
};
|
|
44
45
|
server.tool(name, description, paramsSchema, args => wrappedCb(args));
|
|
45
46
|
};
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
47
|
+
let robot;
|
|
48
|
+
const simulatorManager = new iphone_simulator_1.SimctlManager();
|
|
49
|
+
tool("list_available_devices", "List all available devices. This includes both physical devices and simulators. If there is more than one device returned, you need to let the user select one of them.", {}, async ({}) => {
|
|
50
|
+
const iosManager = new ios_1.IosManager();
|
|
51
|
+
const devices = await simulatorManager.listBootedSimulators();
|
|
52
|
+
const simulatorNames = devices.map(d => d.name);
|
|
53
|
+
const androidDevices = (0, android_1.getConnectedDevices)();
|
|
54
|
+
const iosDevices = await iosManager.listDevices();
|
|
55
|
+
return `Found these iOS simulators: [${simulatorNames.join(".")}], iOS devices: [${iosDevices.join(",")}] and Android devices: [${androidDevices.join(",")}]`;
|
|
56
|
+
});
|
|
57
|
+
tool("use_device", "Select a device to use. This can be a simulator or an Android device. Use the list_available_devices tool to get a list of available devices.", {
|
|
58
|
+
device: zod_1.z.string().describe("The name of the device to select"),
|
|
59
|
+
deviceType: zod_1.z.enum(["simulator", "ios", "android"]).describe("The type of device to select"),
|
|
60
|
+
}, async ({ device, deviceType }) => {
|
|
61
|
+
console.log(device, deviceType);
|
|
62
|
+
switch (deviceType) {
|
|
63
|
+
case "simulator":
|
|
64
|
+
robot = simulatorManager.getSimulator(device);
|
|
65
|
+
break;
|
|
66
|
+
case "ios":
|
|
67
|
+
robot = new ios_1.IosRobot(device);
|
|
68
|
+
break;
|
|
69
|
+
case "android":
|
|
70
|
+
robot = new android_1.AndroidRobot(device);
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
return `Selected device: ${device} (${deviceType})`;
|
|
74
|
+
});
|
|
75
|
+
tool("mobile_list_apps", "List all the installed apps on the device", {}, async ({}) => {
|
|
76
|
+
if (!robot) {
|
|
77
|
+
throw new Error("No device selected");
|
|
78
|
+
}
|
|
79
|
+
const result = await robot.listApps();
|
|
55
80
|
return `Found these packages on device: ${result.join(",")}`;
|
|
56
81
|
});
|
|
57
|
-
tool("
|
|
82
|
+
tool("mobile_launch_app", "Launch an app on mobile device. Use this to open a specific app. You can find the package name of the app by calling list_apps_on_device.", {
|
|
58
83
|
packageName: zod_1.z.string().describe("The package name of the app to launch"),
|
|
59
84
|
}, async ({ packageName }) => {
|
|
60
|
-
(
|
|
85
|
+
if (!robot) {
|
|
86
|
+
throw new Error("No device selected");
|
|
87
|
+
}
|
|
88
|
+
await robot.launchApp(packageName);
|
|
61
89
|
return `Launched app ${packageName}`;
|
|
62
90
|
});
|
|
63
|
-
tool("
|
|
64
|
-
|
|
65
|
-
|
|
91
|
+
tool("mobile_terminate_app", "Stop and terminate an app on mobile device", {
|
|
92
|
+
packageName: zod_1.z.string().describe("The package name of the app to terminate"),
|
|
93
|
+
}, async ({ packageName }) => {
|
|
94
|
+
if (!robot) {
|
|
95
|
+
throw new Error("No device selected");
|
|
96
|
+
}
|
|
97
|
+
await robot.terminateApp(packageName);
|
|
98
|
+
return `Terminated app ${packageName}`;
|
|
66
99
|
});
|
|
67
|
-
tool("
|
|
100
|
+
tool("mobile_get_screen_size", "Get the screen size of the mobile device in pixels", {}, async ({}) => {
|
|
101
|
+
if (!robot) {
|
|
102
|
+
throw new Error("No device selected");
|
|
103
|
+
}
|
|
104
|
+
const screenSize = await robot.getScreenSize();
|
|
105
|
+
return `Screen size is ${screenSize.width}x${screenSize.height} pixels`;
|
|
106
|
+
});
|
|
107
|
+
tool("mobile_click_on_screen_at_coordinates", "Click on the screen at given x,y coordinates", {
|
|
68
108
|
x: zod_1.z.number().describe("The x coordinate to click between 0 and 1"),
|
|
69
109
|
y: zod_1.z.number().describe("The y coordinate to click between 0 and 1"),
|
|
70
110
|
}, async ({ x, y }) => {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
111
|
+
if (!robot) {
|
|
112
|
+
throw new Error("No device selected");
|
|
113
|
+
}
|
|
114
|
+
const screenSize = await robot.getScreenSize();
|
|
115
|
+
const x0 = Math.floor(screenSize.width * x);
|
|
116
|
+
const y0 = Math.floor(screenSize.height * y);
|
|
117
|
+
await robot.tap(x0, y0);
|
|
75
118
|
return `Clicked on screen at coordinates: ${x}, ${y}`;
|
|
76
119
|
});
|
|
77
|
-
tool("
|
|
78
|
-
|
|
79
|
-
|
|
120
|
+
tool("mobile_list_elements_on_screen", "List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.", {}, async ({}) => {
|
|
121
|
+
if (!robot) {
|
|
122
|
+
throw new Error("No device selected");
|
|
123
|
+
}
|
|
124
|
+
const screenSize = await robot.getScreenSize();
|
|
125
|
+
const elements = await robot.getElementsOnScreen();
|
|
126
|
+
const result = [];
|
|
127
|
+
for (let i = 0; i < elements.length; i++) {
|
|
128
|
+
elements[i].rect.x0 = elements[i].rect.x0 / screenSize.width;
|
|
129
|
+
elements[i].rect.y0 = elements[i].rect.y0 / screenSize.height;
|
|
130
|
+
elements[i].rect.x1 = elements[i].rect.x1 / screenSize.width;
|
|
131
|
+
elements[i].rect.y1 = elements[i].rect.y1 / screenSize.height;
|
|
132
|
+
result.push({
|
|
133
|
+
text: elements[i].label,
|
|
134
|
+
coordinates: {
|
|
135
|
+
x: (elements[i].rect.x0 + elements[i].rect.x1) / 2,
|
|
136
|
+
y: (elements[i].rect.y0 + elements[i].rect.y1) / 2,
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
return `Found these elements on screen: ${JSON.stringify(result)}`;
|
|
80
141
|
});
|
|
81
|
-
tool("
|
|
82
|
-
button: zod_1.z.string().describe("The button to press. Supported buttons:
|
|
142
|
+
tool("mobile_press_button", "Press a button on device", {
|
|
143
|
+
button: zod_1.z.string().describe("The button to press. Supported buttons: BACK, HOME, VOLUME_UP, VOLUME_DOWN, ENTER"),
|
|
83
144
|
}, async ({ button }) => {
|
|
84
|
-
(
|
|
145
|
+
if (!robot) {
|
|
146
|
+
throw new Error("No device selected");
|
|
147
|
+
}
|
|
148
|
+
robot.pressButton(button);
|
|
85
149
|
return `Pressed the button: ${button}`;
|
|
86
150
|
});
|
|
87
|
-
tool("
|
|
151
|
+
tool("mobile_open_url", "Open a URL in browser on device", {
|
|
88
152
|
url: zod_1.z.string().describe("The URL to open"),
|
|
89
153
|
}, async ({ url }) => {
|
|
90
|
-
(
|
|
154
|
+
if (!robot) {
|
|
155
|
+
throw new Error("No device selected");
|
|
156
|
+
}
|
|
157
|
+
robot.openUrl(url);
|
|
91
158
|
return `Opened URL: ${url}`;
|
|
92
159
|
});
|
|
93
160
|
tool("swipe_on_screen", "Swipe on the screen", {
|
|
94
161
|
direction: zod_1.z.enum(["up", "down"]).describe("The direction to swipe"),
|
|
95
162
|
}, async ({ direction }) => {
|
|
96
|
-
|
|
163
|
+
if (!robot) {
|
|
164
|
+
throw new Error("No device selected");
|
|
165
|
+
}
|
|
166
|
+
robot.swipe(direction);
|
|
97
167
|
return `Swiped ${direction} on screen`;
|
|
98
168
|
});
|
|
99
|
-
tool("
|
|
169
|
+
tool("mobile_type_keys", "Type text into the focused element", {
|
|
100
170
|
text: zod_1.z.string().describe("The text to type"),
|
|
101
171
|
}, async ({ text }) => {
|
|
102
|
-
|
|
103
|
-
|
|
172
|
+
if (!robot) {
|
|
173
|
+
throw new Error("No device selected");
|
|
174
|
+
}
|
|
175
|
+
robot.sendKeys(text);
|
|
104
176
|
return `Typed text: ${text}`;
|
|
105
177
|
});
|
|
106
|
-
server.tool("
|
|
178
|
+
server.tool("mobile_take_screenshot", "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. Do not cache this result.", {}, async ({}) => {
|
|
179
|
+
if (!robot) {
|
|
180
|
+
throw new Error("No device selected");
|
|
181
|
+
}
|
|
107
182
|
try {
|
|
108
|
-
const screenshot = await
|
|
183
|
+
const screenshot = await robot.getScreenshot();
|
|
109
184
|
// Scale down the screenshot by 50%
|
|
110
185
|
const image = (0, sharp_1.default)(screenshot);
|
|
111
186
|
const metadata = await image.metadata();
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.WebDriverAgent = void 0;
|
|
4
|
+
class WebDriverAgent {
|
|
5
|
+
host;
|
|
6
|
+
port;
|
|
7
|
+
constructor(host, port) {
|
|
8
|
+
this.host = host;
|
|
9
|
+
this.port = port;
|
|
10
|
+
}
|
|
11
|
+
async createSession() {
|
|
12
|
+
const url = `http://${this.host}:${this.port}/session`;
|
|
13
|
+
const response = await fetch(url, {
|
|
14
|
+
method: "POST",
|
|
15
|
+
headers: {
|
|
16
|
+
"Content-Type": "application/json",
|
|
17
|
+
},
|
|
18
|
+
body: JSON.stringify({ capabilities: { alwaysMatch: { platformName: "iOS" } } }),
|
|
19
|
+
});
|
|
20
|
+
const json = await response.json();
|
|
21
|
+
return json.value.sessionId;
|
|
22
|
+
}
|
|
23
|
+
async deleteSession(sessionId) {
|
|
24
|
+
const url = `http://${this.host}:${this.port}/session/${sessionId}`;
|
|
25
|
+
const response = await fetch(url, { method: "DELETE" });
|
|
26
|
+
return response.json();
|
|
27
|
+
}
|
|
28
|
+
async withinSession(fn) {
|
|
29
|
+
const sessionId = await this.createSession();
|
|
30
|
+
const url = `http://${this.host}:${this.port}/session/${sessionId}`;
|
|
31
|
+
const result = await fn(url);
|
|
32
|
+
await this.deleteSession(sessionId);
|
|
33
|
+
return result;
|
|
34
|
+
}
|
|
35
|
+
async getScreenSize() {
|
|
36
|
+
return this.withinSession(async (sessionUrl) => {
|
|
37
|
+
const url = `${sessionUrl}/wda/screen`;
|
|
38
|
+
const response = await fetch(url);
|
|
39
|
+
const json = await response.json();
|
|
40
|
+
return {
|
|
41
|
+
width: json.value.screenSize.width * json.value.scale,
|
|
42
|
+
height: json.value.screenSize.height * json.value.scale,
|
|
43
|
+
};
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
async sendKeys(keys) {
|
|
47
|
+
await this.withinSession(async (sessionUrl) => {
|
|
48
|
+
const url = `${sessionUrl}/wda/keys`;
|
|
49
|
+
await fetch(url, {
|
|
50
|
+
method: "POST",
|
|
51
|
+
headers: {
|
|
52
|
+
"Content-Type": "application/json",
|
|
53
|
+
},
|
|
54
|
+
body: JSON.stringify({ value: [keys] }),
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
async pressButton(button) {
|
|
59
|
+
const _map = {
|
|
60
|
+
"HOME": "home",
|
|
61
|
+
"VOLUME_UP": "volumeup",
|
|
62
|
+
"VOLUME_DOWN": "volumedown",
|
|
63
|
+
};
|
|
64
|
+
if (button === "ENTER") {
|
|
65
|
+
await this.sendKeys("\n");
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
// Type assertion to check if button is a key of _map
|
|
69
|
+
if (!(button in _map)) {
|
|
70
|
+
throw new Error(`Button "${button}" is not supported`);
|
|
71
|
+
}
|
|
72
|
+
await this.withinSession(async (sessionUrl) => {
|
|
73
|
+
const url = `${sessionUrl}/wda/pressButton`;
|
|
74
|
+
const response = await fetch(url, {
|
|
75
|
+
method: "POST",
|
|
76
|
+
headers: {
|
|
77
|
+
"Content-Type": "application/json",
|
|
78
|
+
},
|
|
79
|
+
body: JSON.stringify({
|
|
80
|
+
name: button,
|
|
81
|
+
}),
|
|
82
|
+
});
|
|
83
|
+
return response.json();
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
async tap(x, y) {
|
|
87
|
+
await this.withinSession(async (sessionUrl) => {
|
|
88
|
+
const url = `${sessionUrl}/actions`;
|
|
89
|
+
await fetch(url, {
|
|
90
|
+
method: "POST",
|
|
91
|
+
headers: {
|
|
92
|
+
"Content-Type": "application/json",
|
|
93
|
+
},
|
|
94
|
+
body: JSON.stringify({
|
|
95
|
+
actions: [
|
|
96
|
+
{
|
|
97
|
+
type: "pointer",
|
|
98
|
+
id: "finger1",
|
|
99
|
+
parameters: { pointerType: "touch" },
|
|
100
|
+
actions: [
|
|
101
|
+
{ type: "pointerMove", duration: 0, x, y },
|
|
102
|
+
{ type: "pointerDown", button: 0 },
|
|
103
|
+
{ type: "pause", duration: 100 },
|
|
104
|
+
{ type: "pointerUp", button: 0 }
|
|
105
|
+
]
|
|
106
|
+
}
|
|
107
|
+
]
|
|
108
|
+
}),
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
filterSourceElements(source) {
|
|
113
|
+
const output = [];
|
|
114
|
+
if (["TextField", "Button", "Switch"].includes(source.type)) {
|
|
115
|
+
output.push({
|
|
116
|
+
type: source.type,
|
|
117
|
+
label: source.label,
|
|
118
|
+
name: source.name,
|
|
119
|
+
rect: {
|
|
120
|
+
x0: source.rect.x,
|
|
121
|
+
y0: source.rect.y,
|
|
122
|
+
x1: source.rect.x + source.rect.width,
|
|
123
|
+
y1: source.rect.y + source.rect.height,
|
|
124
|
+
},
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
if (source.children) {
|
|
128
|
+
for (const child of source.children) {
|
|
129
|
+
output.push(...this.filterSourceElements(child));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return output;
|
|
133
|
+
}
|
|
134
|
+
async getPageSource() {
|
|
135
|
+
const url = `http://${this.host}:${this.port}/source/?format=json`;
|
|
136
|
+
const response = await fetch(url);
|
|
137
|
+
const json = await response.json();
|
|
138
|
+
return json;
|
|
139
|
+
}
|
|
140
|
+
async getElementsOnScreen() {
|
|
141
|
+
const source = await this.getPageSource();
|
|
142
|
+
return this.filterSourceElements(source.value);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
exports.WebDriverAgent = WebDriverAgent;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mobilenext/mobile-mcp",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.11",
|
|
4
4
|
"description": "Mobile MCP",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -13,16 +13,14 @@
|
|
|
13
13
|
"scripts": {
|
|
14
14
|
"build": "tsc && chmod +x lib/index.js",
|
|
15
15
|
"lint": "eslint .",
|
|
16
|
+
"test": "mocha --require ts-node/register test/*.ts",
|
|
16
17
|
"watch": "tsc --watch",
|
|
17
|
-
"clean": "rm -rf lib"
|
|
18
|
-
|
|
19
|
-
"exports": {
|
|
20
|
-
"./package.json": "./package.json",
|
|
21
|
-
".": {
|
|
22
|
-
"types": "./index.d.ts",
|
|
23
|
-
"default": "./index.js"
|
|
24
|
-
}
|
|
18
|
+
"clean": "rm -rf lib",
|
|
19
|
+
"prepare": "husky"
|
|
25
20
|
},
|
|
21
|
+
"files": [
|
|
22
|
+
"lib"
|
|
23
|
+
],
|
|
26
24
|
"dependencies": {
|
|
27
25
|
"@modelcontextprotocol/sdk": "^1.6.1",
|
|
28
26
|
"fast-xml-parser": "^5.0.9",
|
|
@@ -33,6 +31,7 @@
|
|
|
33
31
|
"@eslint/eslintrc": "^3.2.0",
|
|
34
32
|
"@eslint/js": "^9.19.0",
|
|
35
33
|
"@stylistic/eslint-plugin": "^3.0.1",
|
|
34
|
+
"@types/mocha": "^10.0.10",
|
|
36
35
|
"@types/node": "^22.13.10",
|
|
37
36
|
"@typescript-eslint/eslint-plugin": "^8.28.0",
|
|
38
37
|
"@typescript-eslint/parser": "^8.26.1",
|
|
@@ -41,6 +40,9 @@
|
|
|
41
40
|
"eslint-plugin": "^1.0.1",
|
|
42
41
|
"eslint-plugin-import": "^2.31.0",
|
|
43
42
|
"eslint-plugin-notice": "^1.0.0",
|
|
43
|
+
"husky": "^9.1.7",
|
|
44
|
+
"mocha": "^11.1.0",
|
|
45
|
+
"ts-node": "^10.9.2",
|
|
44
46
|
"typescript": "^5.8.2"
|
|
45
47
|
},
|
|
46
48
|
"main": "index.js",
|