@mobilenext/mobile-mcp 0.0.31 → 0.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/android.js +54 -16
- package/lib/ios.js +4 -0
- package/lib/iphone-simulator.js +4 -0
- package/lib/server.js +10 -1
- package/lib/webdriver-agent.js +30 -0
- package/package.json +1 -1
package/lib/android.js
CHANGED
|
@@ -120,7 +120,7 @@ class AndroidRobot {
|
|
|
120
120
|
.map(line => line.substring("package:".length));
|
|
121
121
|
}
|
|
122
122
|
async launchApp(packageName) {
|
|
123
|
-
this.adb("shell", "monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1");
|
|
123
|
+
this.adb("shell", "monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1", "1>/dev/null", "2>/dev/null");
|
|
124
124
|
}
|
|
125
125
|
async listRunningProcesses() {
|
|
126
126
|
return this.adb("shell", "ps", "-e")
|
|
@@ -202,24 +202,52 @@ class AndroidRobot {
|
|
|
202
202
|
.length;
|
|
203
203
|
}
|
|
204
204
|
getFirstDisplayId() {
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
.
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
if (
|
|
216
|
-
const
|
|
217
|
-
if (
|
|
218
|
-
|
|
205
|
+
try {
|
|
206
|
+
// Try using cmd display get-displays (Android 11+)
|
|
207
|
+
const displays = this.adb("shell", "cmd", "display", "get-displays")
|
|
208
|
+
.toString()
|
|
209
|
+
.split("\n")
|
|
210
|
+
.filter(s => s.startsWith("Display id "))
|
|
211
|
+
// filter for state ON even though get-displays only returns turned on displays
|
|
212
|
+
.filter(s => s.indexOf(", state ON,") >= 0)
|
|
213
|
+
// another paranoia check
|
|
214
|
+
.filter(s => s.indexOf(", uniqueId ") >= 0);
|
|
215
|
+
if (displays.length > 0) {
|
|
216
|
+
const m = displays[0].match(/uniqueId \"([^\"]+)\"/);
|
|
217
|
+
if (m !== null) {
|
|
218
|
+
let displayId = m[1];
|
|
219
|
+
if (displayId.startsWith("local:")) {
|
|
220
|
+
displayId = displayId.substring("local:".length);
|
|
221
|
+
}
|
|
222
|
+
return displayId;
|
|
219
223
|
}
|
|
220
|
-
return displayId;
|
|
221
224
|
}
|
|
222
225
|
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
// cmd display get-displays not available on this device
|
|
228
|
+
}
|
|
229
|
+
// fallback: parse dumpsys display for display info (compatible with older Android versions)
|
|
230
|
+
try {
|
|
231
|
+
const dumpsys = this.adb("shell", "dumpsys", "display")
|
|
232
|
+
.toString();
|
|
233
|
+
// look for DisplayViewport entries with isActive=true and type=INTERNAL
|
|
234
|
+
const viewportMatch = dumpsys.match(/DisplayViewport\{type=INTERNAL[^}]*isActive=true[^}]*uniqueId='([^']+)'/);
|
|
235
|
+
if (viewportMatch) {
|
|
236
|
+
let uniqueId = viewportMatch[1];
|
|
237
|
+
if (uniqueId.startsWith("local:")) {
|
|
238
|
+
uniqueId = uniqueId.substring("local:".length);
|
|
239
|
+
}
|
|
240
|
+
return uniqueId;
|
|
241
|
+
}
|
|
242
|
+
// fallback: look for active display with state ON
|
|
243
|
+
const displayStateMatch = dumpsys.match(/Display Id=(\d+)[\s\S]*?Display State=ON/);
|
|
244
|
+
if (displayStateMatch) {
|
|
245
|
+
return displayStateMatch[1];
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
catch (error) {
|
|
249
|
+
// dumpsys display also failed
|
|
250
|
+
}
|
|
223
251
|
return null;
|
|
224
252
|
}
|
|
225
253
|
async getScreenshot() {
|
|
@@ -229,6 +257,11 @@ class AndroidRobot {
|
|
|
229
257
|
}
|
|
230
258
|
// find the first display that is turned on, and capture that one
|
|
231
259
|
const displayId = this.getFirstDisplayId();
|
|
260
|
+
if (displayId === null) {
|
|
261
|
+
// no idea why, but we have displayCount >= 2, yet we failed to parse
|
|
262
|
+
// let's go with screencap's defaults and hope for the best
|
|
263
|
+
return this.adb("exec-out", "screencap", "-p");
|
|
264
|
+
}
|
|
232
265
|
return this.adb("exec-out", "screencap", "-p", "-d", `${displayId}`);
|
|
233
266
|
}
|
|
234
267
|
collectElements(node) {
|
|
@@ -348,6 +381,11 @@ class AndroidRobot {
|
|
|
348
381
|
// a long press is a swipe with no movement and a long duration
|
|
349
382
|
this.adb("shell", "input", "swipe", `${x}`, `${y}`, `${x}`, `${y}`, "500");
|
|
350
383
|
}
|
|
384
|
+
async doubleTap(x, y) {
|
|
385
|
+
await this.tap(x, y);
|
|
386
|
+
await new Promise(r => setTimeout(r, 100)); // short delay
|
|
387
|
+
await this.tap(x, y);
|
|
388
|
+
}
|
|
351
389
|
async setOrientation(orientation) {
|
|
352
390
|
const value = orientation === "portrait" ? 0 : 1;
|
|
353
391
|
// disable auto-rotation prior to setting the orientation
|
package/lib/ios.js
CHANGED
|
@@ -141,6 +141,10 @@ class IosRobot {
|
|
|
141
141
|
const wda = await this.wda();
|
|
142
142
|
await wda.tap(x, y);
|
|
143
143
|
}
|
|
144
|
+
async doubleTap(x, y) {
|
|
145
|
+
const wda = await this.wda();
|
|
146
|
+
await wda.doubleTap(x, y);
|
|
147
|
+
}
|
|
144
148
|
async longPress(x, y) {
|
|
145
149
|
const wda = await this.wda();
|
|
146
150
|
await wda.longPress(x, y);
|
package/lib/iphone-simulator.js
CHANGED
|
@@ -188,6 +188,10 @@ class Simctl {
|
|
|
188
188
|
const wda = await this.wda();
|
|
189
189
|
return wda.tap(x, y);
|
|
190
190
|
}
|
|
191
|
+
async doubleTap(x, y) {
|
|
192
|
+
const wda = await this.wda();
|
|
193
|
+
await wda.doubleTap(x, y);
|
|
194
|
+
}
|
|
191
195
|
async longPress(x, y) {
|
|
192
196
|
const wda = await this.wda();
|
|
193
197
|
return wda.longPress(x, y);
|
package/lib/server.js
CHANGED
|
@@ -162,7 +162,7 @@ const createMcpServer = () => {
|
|
|
162
162
|
const androidMobileDevices = androidDevices.filter(d => d.deviceType === "mobile").map(d => d.deviceId);
|
|
163
163
|
const resp = ["Found these devices:"];
|
|
164
164
|
if (simulatorNames.length > 0) {
|
|
165
|
-
resp.push(`iOS simulators: [${simulatorNames.join("
|
|
165
|
+
resp.push(`iOS simulators: [${simulatorNames.join(",")}]`);
|
|
166
166
|
}
|
|
167
167
|
if (iosDevices.length > 0) {
|
|
168
168
|
resp.push(`iOS devices: [${iosDeviceNames.join(",")}]`);
|
|
@@ -230,6 +230,15 @@ const createMcpServer = () => {
|
|
|
230
230
|
await robot.tap(x, y);
|
|
231
231
|
return `Clicked on screen at coordinates: ${x}, ${y}`;
|
|
232
232
|
});
|
|
233
|
+
tool("mobile_double_tap_on_screen", "Double-tap on the screen at given x,y coordinates.", {
|
|
234
|
+
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
235
|
+
x: zod_1.z.number().describe("The x coordinate to double-tap, in pixels"),
|
|
236
|
+
y: zod_1.z.number().describe("The y coordinate to double-tap, in pixels"),
|
|
237
|
+
}, async ({ device, x, y }) => {
|
|
238
|
+
const robot = getRobotFromDevice(device);
|
|
239
|
+
await robot.doubleTap(x, y);
|
|
240
|
+
return `Double-tapped on screen at coordinates: ${x}, ${y}`;
|
|
241
|
+
});
|
|
233
242
|
tool("mobile_long_press_on_screen_at_coordinates", "Long press on the screen at given x,y coordinates. If long pressing on an element, use the list_elements_on_screen tool to find the coordinates.", {
|
|
234
243
|
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
235
244
|
x: zod_1.z.number().describe("The x coordinate to long press on the screen, in pixels"),
|
package/lib/webdriver-agent.js
CHANGED
|
@@ -142,6 +142,36 @@ class WebDriverAgent {
|
|
|
142
142
|
});
|
|
143
143
|
});
|
|
144
144
|
}
|
|
145
|
+
async doubleTap(x, y) {
|
|
146
|
+
await this.withinSession(async (sessionUrl) => {
|
|
147
|
+
const url = `${sessionUrl}/actions`;
|
|
148
|
+
await fetch(url, {
|
|
149
|
+
method: "POST",
|
|
150
|
+
headers: {
|
|
151
|
+
"Content-Type": "application/json",
|
|
152
|
+
},
|
|
153
|
+
body: JSON.stringify({
|
|
154
|
+
actions: [
|
|
155
|
+
{
|
|
156
|
+
type: "pointer",
|
|
157
|
+
id: "finger1",
|
|
158
|
+
parameters: { pointerType: "touch" },
|
|
159
|
+
actions: [
|
|
160
|
+
{ type: "pointerMove", duration: 0, x, y },
|
|
161
|
+
{ type: "pointerDown", button: 0 },
|
|
162
|
+
{ type: "pause", duration: 50 },
|
|
163
|
+
{ type: "pointerUp", button: 0 },
|
|
164
|
+
{ type: "pause", duration: 100 },
|
|
165
|
+
{ type: "pointerDown", button: 0 },
|
|
166
|
+
{ type: "pause", duration: 50 },
|
|
167
|
+
{ type: "pointerUp", button: 0 }
|
|
168
|
+
]
|
|
169
|
+
}
|
|
170
|
+
]
|
|
171
|
+
}),
|
|
172
|
+
});
|
|
173
|
+
});
|
|
174
|
+
}
|
|
145
175
|
async longPress(x, y) {
|
|
146
176
|
await this.withinSession(async (sessionUrl) => {
|
|
147
177
|
const url = `${sessionUrl}/actions`;
|