@mobilenext/mobile-mcp 1.0.17 → 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/lib/android.js +1 -1
- package/lib/coordinate-mapping.js +17 -4
- package/lib/mobile-device.js +20 -10
- package/lib/server.js +149 -78
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -121,12 +121,13 @@ How we help to scale mobile automation:
|
|
|
121
121
|
- **`mobile_clear_app_data`** - Clear an app's data so it starts fresh (Android and iOS Simulator)
|
|
122
122
|
|
|
123
123
|
### Screen Interaction
|
|
124
|
-
- **`mobile_take_screenshot`** - Take a screenshot to understand what's on screen
|
|
124
|
+
- **`mobile_take_screenshot`** - Take a screenshot to understand what's on screen, optionally clipped to a region
|
|
125
125
|
- **`mobile_save_screenshot`** - Save a screenshot to a file
|
|
126
126
|
- **`mobile_list_elements_on_screen`** - List UI elements with their coordinates and properties
|
|
127
127
|
- **`mobile_click_on_screen_at_coordinates`** - Click at specific x,y coordinates or on an element by its ref
|
|
128
128
|
- **`mobile_double_tap_on_screen`** - Double-tap at specific coordinates or on an element by its ref
|
|
129
129
|
- **`mobile_long_press_on_screen_at_coordinates`** - Long press at specific coordinates or on an element by its ref
|
|
130
|
+
- **`mobile_drag_on_screen`** - Drag from one point or element ref to another, with an optional hold before moving
|
|
130
131
|
- **`mobile_swipe_on_screen`** - Swipe in any direction (up, down, left, right)
|
|
131
132
|
- **`mobile_start_screen_recording`** - Start recording the device screen to a video file
|
|
132
133
|
- **`mobile_stop_screen_recording`** - Stop the active screen recording and save the video
|
|
@@ -140,7 +141,7 @@ How we help to scale mobile automation:
|
|
|
140
141
|
- **`mobile_get_device_logs`** - Collect live device logs (logcat on Android, unified log on iOS), optionally saved to a file
|
|
141
142
|
- **`mobile_list_crashes`** - List crash reports available on the device
|
|
142
143
|
- **`mobile_get_crash`** - Get the full content of a crash report by its ID
|
|
143
|
-
- **`mobile_batch_commands`** - Run multiple tools in sequence in a single call (e.g. click, type, click), optionally listing screen elements at the end
|
|
144
|
+
- **`mobile_batch_commands`** - Run multiple tools in sequence in a single call (e.g. click, type, click), with optional `delay` steps in between, optionally listing screen elements at the end
|
|
144
145
|
|
|
145
146
|
## 🏗️ Mobile MCP Architecture
|
|
146
147
|
|
|
@@ -378,13 +379,15 @@ Follow the MCP Servers documentation. For example in `~/.config/opencode/opencod
|
|
|
378
379
|
{
|
|
379
380
|
"$schema": "https://opencode.ai/config.json",
|
|
380
381
|
"mcp": {
|
|
381
|
-
"
|
|
382
|
-
"
|
|
383
|
-
|
|
384
|
-
"
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
382
|
+
"servers": {
|
|
383
|
+
"mobile-mcp": {
|
|
384
|
+
"type": "local",
|
|
385
|
+
"command": [
|
|
386
|
+
"npx",
|
|
387
|
+
"-y",
|
|
388
|
+
"@mobilenext/mobile-mcp@latest"
|
|
389
|
+
]
|
|
390
|
+
}
|
|
388
391
|
}
|
|
389
392
|
}
|
|
390
393
|
}
|
package/lib/android.js
CHANGED
|
@@ -44,7 +44,7 @@ const xml = __importStar(require("fast-xml-parser"));
|
|
|
44
44
|
const robot_1 = require("./robot");
|
|
45
45
|
const utils_1 = require("./utils");
|
|
46
46
|
const getAdbPath = () => {
|
|
47
|
-
const exeName = process.
|
|
47
|
+
const exeName = process.platform === "win32" ? "adb.exe" : "adb";
|
|
48
48
|
if (process.env.ANDROID_HOME) {
|
|
49
49
|
return node_path_1.default.join(process.env.ANDROID_HOME, "platform-tools", exeName);
|
|
50
50
|
}
|
|
@@ -23,11 +23,24 @@ const orientLike = (screen, screenshot) => {
|
|
|
23
23
|
* downscaled, and on iOS the screen is measured in points while a full-size
|
|
24
24
|
* screenshot is in pixels, so the two spaces rarely agree.
|
|
25
25
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
26
|
+
* When the screenshot was clipped to a region, the region's offset is added on
|
|
27
|
+
* top of the scaling.
|
|
28
|
+
*
|
|
29
|
+
* Returns null when a size it needs is unknown, in which case no instruction
|
|
30
|
+
* is better than a wrong one.
|
|
28
31
|
*/
|
|
29
|
-
const describeCoordinateMapping = (screenshot, reportedScreen) => {
|
|
30
|
-
if (!isPositive(screenshot)
|
|
32
|
+
const describeCoordinateMapping = (screenshot, reportedScreen, clip) => {
|
|
33
|
+
if (!isPositive(screenshot)) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
// a clipped screenshot shows only a region, so positions in it are offset from the screen's origin.
|
|
37
|
+
// the region and the screenshot are enough to map it, so the screen size is not needed here
|
|
38
|
+
if (clip) {
|
|
39
|
+
const clipX = formatRatio(clip.width / screenshot.width);
|
|
40
|
+
const clipY = formatRatio(clip.height / screenshot.height);
|
|
41
|
+
return `Screenshot is ${screenshot.width}x${screenshot.height} and shows the screen region at ${clip.x},${clip.y} sized ${clip.width}x${clip.height}. To tap something you see in this screenshot, multiply its x by ${clipX} and add ${clip.x}, multiply its y by ${clipY} and add ${clip.y}.`;
|
|
42
|
+
}
|
|
43
|
+
if (!isPositive(reportedScreen)) {
|
|
31
44
|
return null;
|
|
32
45
|
}
|
|
33
46
|
const screen = orientLike(reportedScreen, screenshot);
|
package/lib/mobile-device.js
CHANGED
|
@@ -23,6 +23,13 @@ const flattenUIElement = (element) => {
|
|
|
23
23
|
}
|
|
24
24
|
return [screenElement, ...element.children.flatMap(child => flattenUIElement(child))];
|
|
25
25
|
};
|
|
26
|
+
// mobilecli rejects fractional coordinates ("x and y must be integers")
|
|
27
|
+
const formatGesturePoint = (point) => {
|
|
28
|
+
if ("ref" in point) {
|
|
29
|
+
return point.ref;
|
|
30
|
+
}
|
|
31
|
+
return `${Math.round(point.x)},${Math.round(point.y)}`;
|
|
32
|
+
};
|
|
26
33
|
class MobileDevice {
|
|
27
34
|
deviceId;
|
|
28
35
|
mobilecli;
|
|
@@ -105,6 +112,10 @@ class MobileDevice {
|
|
|
105
112
|
if (format === "jpeg" && options?.quality !== undefined) {
|
|
106
113
|
fullArgs.push("--quality", `${options.quality}`);
|
|
107
114
|
}
|
|
115
|
+
if (options?.clip !== undefined) {
|
|
116
|
+
const { x, y, width, height } = options.clip;
|
|
117
|
+
fullArgs.push("--clip", `${Math.round(x)},${Math.round(y)},${Math.round(width)},${Math.round(height)}`);
|
|
118
|
+
}
|
|
108
119
|
if (options?.maxSize !== undefined) {
|
|
109
120
|
fullArgs.push("--max-size", `${options.maxSize}`);
|
|
110
121
|
}
|
|
@@ -167,19 +178,18 @@ class MobileDevice {
|
|
|
167
178
|
this.runCommand(["io", "tap", ref]);
|
|
168
179
|
}
|
|
169
180
|
async doubleTap(x, y) {
|
|
170
|
-
//
|
|
171
|
-
|
|
172
|
-
await this.tap(x, y);
|
|
181
|
+
// parse the response, older mobilecli versions print help and exit 0 on unknown commands
|
|
182
|
+
this.runJsonCommand(["io", "doubletap", `${Math.round(x)},${Math.round(y)}`]);
|
|
173
183
|
}
|
|
174
184
|
async doubleTapByRef(ref) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
185
|
+
this.runJsonCommand(["io", "doubletap", ref]);
|
|
186
|
+
}
|
|
187
|
+
async drag(from, to, hold) {
|
|
188
|
+
const args = ["io", "drag", `${formatGesturePoint(from)},${formatGesturePoint(to)}`];
|
|
189
|
+
if (hold) {
|
|
190
|
+
args.push("--hold", `${Math.round(hold)}`);
|
|
180
191
|
}
|
|
181
|
-
|
|
182
|
-
await this.doubleTap(x + width / 2, y + height / 2);
|
|
192
|
+
this.runJsonCommand(args);
|
|
183
193
|
}
|
|
184
194
|
async longPress(x, y, duration) {
|
|
185
195
|
// mobilecli rejects fractional coordinates and durations, both are parsed as integers
|
package/lib/server.js
CHANGED
|
@@ -10,6 +10,7 @@ const node_fs_1 = __importDefault(require("node:fs"));
|
|
|
10
10
|
const node_os_1 = __importDefault(require("node:os"));
|
|
11
11
|
const node_path_1 = __importDefault(require("node:path"));
|
|
12
12
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
13
|
+
const promises_1 = require("node:timers/promises");
|
|
13
14
|
const logger_1 = require("./logger");
|
|
14
15
|
const android_1 = require("./android");
|
|
15
16
|
const robot_1 = require("./robot");
|
|
@@ -29,6 +30,8 @@ const DEVICE_LOG_TIMEOUT_MS = 30000;
|
|
|
29
30
|
const DEVICE_LOG_FILTER_PATTERN = /^(pid|process|tag|level|subsystem|category|message)!?=.+$/;
|
|
30
31
|
const ALLOWED_SCREENSHOT_EXTENSIONS = [".png", ".jpg", ".jpeg"];
|
|
31
32
|
const DEFAULT_SCREENSHOT_MAX_SIZE = 1024;
|
|
33
|
+
const BATCH_DELAY_STEP = "delay";
|
|
34
|
+
const MAX_BATCH_DELAY_MS = 10000;
|
|
32
35
|
const ALLOWED_RECORDING_EXTENSIONS = [".mp4"];
|
|
33
36
|
const ALLOWED_APP_EXTENSIONS = [".apk", ".ipa", ".zip", ".app"];
|
|
34
37
|
const LOGIN_PROMPT_TIMEOUT_MS = 15000;
|
|
@@ -80,6 +83,34 @@ Prefer mobile_launch_app to open an app, and mobile_open_url to open a web page,
|
|
|
80
83
|
over navigating through the UI. mobile_open_url only accepts http:// and
|
|
81
84
|
https:// URLs unless the server was started with MOBILEMCP_ALLOW_UNSAFE_URLS=1,
|
|
82
85
|
so do not rely on it for app deep links.`;
|
|
86
|
+
// the text of a tool response, for places that can only carry text
|
|
87
|
+
const textOf = (response) => {
|
|
88
|
+
if (typeof response === "string") {
|
|
89
|
+
return response;
|
|
90
|
+
}
|
|
91
|
+
return response.flatMap(block => block.type === "text" ? [block.text] : []).join("\n");
|
|
92
|
+
};
|
|
93
|
+
const imagesOf = (response) => {
|
|
94
|
+
if (typeof response === "string") {
|
|
95
|
+
return [];
|
|
96
|
+
}
|
|
97
|
+
return response.filter(block => block.type === "image");
|
|
98
|
+
};
|
|
99
|
+
const toGesturePoint = (ref, x, y) => {
|
|
100
|
+
if (ref !== undefined) {
|
|
101
|
+
return { ref };
|
|
102
|
+
}
|
|
103
|
+
if (x === undefined || y === undefined) {
|
|
104
|
+
return undefined;
|
|
105
|
+
}
|
|
106
|
+
return { x, y };
|
|
107
|
+
};
|
|
108
|
+
const describeGesturePoint = (point) => {
|
|
109
|
+
if ("ref" in point) {
|
|
110
|
+
return `element ${point.ref}`;
|
|
111
|
+
}
|
|
112
|
+
return `coordinates ${point.x}, ${point.y}`;
|
|
113
|
+
};
|
|
83
114
|
const createMcpServer = () => {
|
|
84
115
|
const server = new server_1.McpServer({
|
|
85
116
|
name: "mobile-mcp",
|
|
@@ -114,8 +145,12 @@ const createMcpServer = () => {
|
|
|
114
145
|
const telemetry = {};
|
|
115
146
|
const response = await cb(args, telemetry);
|
|
116
147
|
const duration = +new Date() - start;
|
|
117
|
-
(0, logger_1.trace)(`=> ${response}`);
|
|
118
148
|
posthog("tool_invoked", { "ToolName": name, "Duration": duration, ...telemetry }).then();
|
|
149
|
+
if (typeof response !== "string") {
|
|
150
|
+
(0, logger_1.trace)(`=> ${response.length} content blocks`);
|
|
151
|
+
return { content: response };
|
|
152
|
+
}
|
|
153
|
+
(0, logger_1.trace)(`=> ${response}`);
|
|
119
154
|
return {
|
|
120
155
|
content: [{ type: "text", text: response }],
|
|
121
156
|
};
|
|
@@ -556,6 +591,31 @@ const createMcpServer = () => {
|
|
|
556
591
|
await robot.doubleTap(x, y);
|
|
557
592
|
return `Double-tapped on screen at coordinates: ${x}, ${y}`;
|
|
558
593
|
});
|
|
594
|
+
tool("mobile_drag_on_screen", "Drag On Screen", "Drag one finger from one point to another, e.g. to move a slider, reorder a list or drag and drop. Each end is either x,y coordinates or an element by its ref (e.g. \"@e5\") from the latest mobile_list_elements_on_screen result. Prefer ref when the element is listed.", {
|
|
595
|
+
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
596
|
+
x1: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to start the drag from, in pixels. Required unless fromRef is given"),
|
|
597
|
+
y1: zod_1.z.coerce.number().min(0).optional().describe("The y coordinate to start the drag from, in pixels. Required unless fromRef is given"),
|
|
598
|
+
x2: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to end the drag at, in pixels. Required unless toRef is given"),
|
|
599
|
+
y2: zod_1.z.coerce.number().min(0).optional().describe("The y coordinate to end the drag at, in pixels. Required unless toRef is given"),
|
|
600
|
+
fromRef: zod_1.z.string().optional().describe("Element ref to start the drag from, e.g. \"@e5\". Takes precedence over x1,y1"),
|
|
601
|
+
toRef: zod_1.z.string().optional().describe("Element ref to end the drag at, e.g. \"@e7\". Takes precedence over x2,y2"),
|
|
602
|
+
hold: zod_1.z.coerce.number().min(0).max(10000).optional().describe("How long the finger stays down before it moves, in milliseconds. Use it (e.g. 500) to pick up an item that would otherwise scroll. Defaults to 0"),
|
|
603
|
+
}, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, x1, y1, x2, y2, fromRef, toRef, hold }) => {
|
|
604
|
+
const from = toGesturePoint(fromRef, x1, y1);
|
|
605
|
+
if (!from) {
|
|
606
|
+
throw new robot_1.ActionableError("Either fromRef or both x1 and y1 must be provided");
|
|
607
|
+
}
|
|
608
|
+
const to = toGesturePoint(toRef, x2, y2);
|
|
609
|
+
if (!to) {
|
|
610
|
+
throw new robot_1.ActionableError("Either toRef or both x2 and y2 must be provided");
|
|
611
|
+
}
|
|
612
|
+
const robot = getRobotFromDevice(device);
|
|
613
|
+
if (!robot.drag) {
|
|
614
|
+
throw new robot_1.ActionableError("Dragging is not supported in legacy robot mode");
|
|
615
|
+
}
|
|
616
|
+
await robot.drag(from, to, hold);
|
|
617
|
+
return `Dragged from ${describeGesturePoint(from)} to ${describeGesturePoint(to)}`;
|
|
618
|
+
});
|
|
559
619
|
tool("mobile_long_press_on_screen_at_coordinates", "Long Press Screen", "Long press on the screen, either at x,y coordinates or on an element by its ref (e.g. \"@e5\") from the latest mobile_list_elements_on_screen result. Prefer ref when the element is listed.", {
|
|
560
620
|
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
561
621
|
x: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to long press on the screen, in pixels. Required unless ref is given"),
|
|
@@ -578,7 +638,7 @@ const createMcpServer = () => {
|
|
|
578
638
|
await robot.longPress(x, y, pressDuration);
|
|
579
639
|
return `Long pressed on screen at coordinates: ${x}, ${y} for ${pressDuration}ms`;
|
|
580
640
|
});
|
|
581
|
-
tool("mobile_list_elements_on_screen", "List Screen Elements", "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates, mobile_double_tap_on_screen or
|
|
641
|
+
tool("mobile_list_elements_on_screen", "List Screen Elements", "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates, mobile_double_tap_on_screen, mobile_long_press_on_screen_at_coordinates or mobile_drag_on_screen. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.", {
|
|
582
642
|
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
583
643
|
format: zod_1.z.enum(["text", "json", "raw"]).default("text").describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array, \"raw\" is the unfiltered platform-specific hierarchy as JSON (mobilecli only). Raw nodes may be offscreen and do not have element refs"),
|
|
584
644
|
}, { readOnlyHint: true, openWorldHint: true }, async ({ device, format }) => {
|
|
@@ -672,74 +732,66 @@ const createMcpServer = () => {
|
|
|
672
732
|
(0, utils_1.writeOutputFile)(saveTo, screenshot);
|
|
673
733
|
return `Screenshot saved to: ${saveTo}`;
|
|
674
734
|
});
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
const robot = getRobotFromDevice(device);
|
|
690
|
-
// mobilecli scales and converts natively; legacy robots ignore the options and return a full-size png
|
|
691
|
-
// maxSize wins over scale (same precedence as mobilecli); the default applies only when neither is given
|
|
692
|
-
const options = { format: "jpeg", quality: 75 };
|
|
693
|
-
if (maxSize !== undefined) {
|
|
694
|
-
options.maxSize = maxSize;
|
|
695
|
-
}
|
|
696
|
-
else if (scale !== undefined) {
|
|
697
|
-
options.scale = scale;
|
|
698
|
-
}
|
|
699
|
-
else {
|
|
700
|
-
options.maxSize = DEFAULT_SCREENSHOT_MAX_SIZE;
|
|
701
|
-
}
|
|
702
|
-
const screenshot = await robot.getScreenshot(options);
|
|
703
|
-
const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;
|
|
704
|
-
let mimeType = "image/jpeg";
|
|
705
|
-
let screenshotSize;
|
|
706
|
-
if (isJpeg) {
|
|
707
|
-
screenshotSize = (0, jpeg_1.getJpegDimensions)(screenshot);
|
|
708
|
-
}
|
|
709
|
-
else {
|
|
710
|
-
mimeType = "image/png";
|
|
711
|
-
// validate we received a png, will throw exception otherwise
|
|
712
|
-
const image = new png_1.PNG(screenshot);
|
|
713
|
-
screenshotSize = image.getDimensions();
|
|
714
|
-
}
|
|
715
|
-
if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {
|
|
716
|
-
throw new robot_1.ActionableError("Screenshot is invalid. Please try again.");
|
|
717
|
-
}
|
|
718
|
-
// the screenshot is downscaled (and on ios the screen is in points while pixels are not),
|
|
719
|
-
// so tell the model how to map what it sees onto tap coordinates. see #29.
|
|
720
|
-
const screenSize = await robot.getScreenSize();
|
|
721
|
-
const mapping = (0, coordinate_mapping_1.describeCoordinateMapping)(screenshotSize, screenSize);
|
|
722
|
-
const screenshot64 = screenshot.toString("base64");
|
|
723
|
-
(0, logger_1.trace)(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);
|
|
724
|
-
posthog("tool_invoked", {
|
|
725
|
-
"ToolName": "mobile_take_screenshot",
|
|
726
|
-
"ScreenshotFilesize": screenshot64.length,
|
|
727
|
-
"ScreenshotMimeType": mimeType,
|
|
728
|
-
}).then();
|
|
729
|
-
const content = [];
|
|
730
|
-
if (mapping !== null) {
|
|
731
|
-
content.push({ type: "text", text: mapping });
|
|
732
|
-
}
|
|
733
|
-
content.push({ type: "image", data: screenshot64, mimeType });
|
|
734
|
-
return { content };
|
|
735
|
+
tool("mobile_take_screenshot", "Take Screenshot", "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so when the device reports its screen size the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.", {
|
|
736
|
+
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
737
|
+
maxSize: zod_1.z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),
|
|
738
|
+
scale: zod_1.z.number().gt(0).max(1).optional().describe("Scale factor (0.0-1.0). Ignored if maxSize is provided."),
|
|
739
|
+
clip: zod_1.z.object({
|
|
740
|
+
x: zod_1.z.number().min(0).describe("Left edge of the region"),
|
|
741
|
+
y: zod_1.z.number().min(0).describe("Top edge of the region"),
|
|
742
|
+
width: zod_1.z.number().positive().describe("Width of the region"),
|
|
743
|
+
height: zod_1.z.number().positive().describe("Height of the region"),
|
|
744
|
+
}).optional().describe("Only capture this region of the screen, in screen coordinates (the ones used for tapping). Applied before maxSize/scale. Use it to look at one part of the screen in more detail or at a lower cost."),
|
|
745
|
+
}, { readOnlyHint: true, openWorldHint: true }, async ({ device, maxSize, scale, clip }, telemetry) => {
|
|
746
|
+
const robot = getRobotFromDevice(device);
|
|
747
|
+
if (clip !== undefined && !(robot instanceof mobile_device_1.MobileDevice)) {
|
|
748
|
+
throw new robot_1.ActionableError("Clipping a screenshot is not supported in legacy robot mode");
|
|
735
749
|
}
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
750
|
+
// mobilecli scales and converts natively; legacy robots ignore the options and return a full-size png
|
|
751
|
+
// maxSize wins over scale (same precedence as mobilecli); the default applies only when neither is given
|
|
752
|
+
const options = { format: "jpeg", quality: 75 };
|
|
753
|
+
if (maxSize !== undefined) {
|
|
754
|
+
options.maxSize = maxSize;
|
|
755
|
+
}
|
|
756
|
+
else if (scale !== undefined) {
|
|
757
|
+
options.scale = scale;
|
|
758
|
+
}
|
|
759
|
+
else {
|
|
760
|
+
options.maxSize = DEFAULT_SCREENSHOT_MAX_SIZE;
|
|
761
|
+
}
|
|
762
|
+
if (clip !== undefined) {
|
|
763
|
+
options.clip = clip;
|
|
764
|
+
}
|
|
765
|
+
const screenshot = await robot.getScreenshot(options);
|
|
766
|
+
const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;
|
|
767
|
+
let mimeType = "image/jpeg";
|
|
768
|
+
let screenshotSize;
|
|
769
|
+
if (isJpeg) {
|
|
770
|
+
screenshotSize = (0, jpeg_1.getJpegDimensions)(screenshot);
|
|
771
|
+
}
|
|
772
|
+
else {
|
|
773
|
+
mimeType = "image/png";
|
|
774
|
+
// validate we received a png, will throw exception otherwise
|
|
775
|
+
const image = new png_1.PNG(screenshot);
|
|
776
|
+
screenshotSize = image.getDimensions();
|
|
742
777
|
}
|
|
778
|
+
if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {
|
|
779
|
+
throw new robot_1.ActionableError("Screenshot is invalid");
|
|
780
|
+
}
|
|
781
|
+
// the screenshot is downscaled (and on ios the screen is in points while pixels are not),
|
|
782
|
+
// so tell the model how to map what it sees onto tap coordinates. see #29.
|
|
783
|
+
const screenSize = await robot.getScreenSize();
|
|
784
|
+
const mapping = (0, coordinate_mapping_1.describeCoordinateMapping)(screenshotSize, screenSize, clip);
|
|
785
|
+
const screenshot64 = screenshot.toString("base64");
|
|
786
|
+
(0, logger_1.trace)(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);
|
|
787
|
+
telemetry.ScreenshotFilesize = screenshot64.length;
|
|
788
|
+
telemetry.ScreenshotMimeType = mimeType;
|
|
789
|
+
const content = [];
|
|
790
|
+
if (mapping !== null) {
|
|
791
|
+
content.push({ type: "text", text: mapping });
|
|
792
|
+
}
|
|
793
|
+
content.push({ type: "image", data: screenshot64, mimeType });
|
|
794
|
+
return content;
|
|
743
795
|
});
|
|
744
796
|
tool("mobile_set_orientation", "Set Orientation", "Change the screen orientation of the device", {
|
|
745
797
|
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
@@ -921,12 +973,21 @@ const createMcpServer = () => {
|
|
|
921
973
|
const response = mobilecli.crashesGet(device, id);
|
|
922
974
|
return response.data.content;
|
|
923
975
|
});
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
976
|
+
const batchableToolNames = [...toolCallbacks.keys()];
|
|
977
|
+
// a step that only exists inside a batch: waiting is of no use as a tool call of its own
|
|
978
|
+
toolCallbacks.set(BATCH_DELAY_STEP, {
|
|
979
|
+
paramsSchema: {
|
|
980
|
+
milliseconds: zod_1.z.coerce.number().int().min(1).max(MAX_BATCH_DELAY_MS).describe("How long to wait, in milliseconds"),
|
|
981
|
+
},
|
|
982
|
+
callback: async ({ milliseconds }) => {
|
|
983
|
+
await (0, promises_1.setTimeout)(milliseconds);
|
|
984
|
+
return `Waited ${milliseconds}ms`;
|
|
985
|
+
},
|
|
986
|
+
});
|
|
987
|
+
tool("mobile_batch_commands", "Batch Commands", `Run multiple tools in sequence in a single call, e.g. click, type, click, type. Use this to fill forms or perform multi-step flows without round-trips. The device argument is applied to every step unless a step provides its own. Tools allowed as steps: ${batchableToolNames.join(", ")}. A step named "${BATCH_DELAY_STEP}" with arguments {"milliseconds": N} (up to ${MAX_BATCH_DELAY_MS}) waits before the next step, e.g. for an animation or a page load to finish before looking at the screen.`, {
|
|
927
988
|
device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
|
|
928
989
|
steps: zod_1.z.array(zod_1.z.object({
|
|
929
|
-
name: zod_1.z.string().describe(
|
|
990
|
+
name: zod_1.z.string().describe(`Name of a tool allowed as a step, e.g. mobile_click_on_screen_at_coordinates, or "${BATCH_DELAY_STEP}"`),
|
|
930
991
|
arguments: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).describe("Arguments for the tool, same as calling it directly"),
|
|
931
992
|
})).min(1).describe("Tools to run, in order"),
|
|
932
993
|
stopOnError: zod_1.z.boolean().optional().describe("Stop at the first failing step. Defaults to true"),
|
|
@@ -934,15 +995,13 @@ const createMcpServer = () => {
|
|
|
934
995
|
}, { readOnlyHint: false, destructiveHint: true, openWorldHint: true }, async ({ device, steps, stopOnError, listElementsAtEnd }, telemetry) => {
|
|
935
996
|
const shouldStopOnError = stopOnError ?? true;
|
|
936
997
|
const results = [];
|
|
998
|
+
const images = [];
|
|
937
999
|
telemetry.StepCount = steps.length;
|
|
938
1000
|
for (let i = 0; i < steps.length; i++) {
|
|
939
1001
|
const step = steps[i];
|
|
940
1002
|
if (step.name === "mobile_batch_commands") {
|
|
941
1003
|
throw new robot_1.ActionableError("mobile_batch_commands cannot be nested");
|
|
942
1004
|
}
|
|
943
|
-
if (step.name === "mobile_take_screenshot") {
|
|
944
|
-
throw new robot_1.ActionableError("mobile_take_screenshot returns an image and cannot be used in a batch, use mobile_save_screenshot instead");
|
|
945
|
-
}
|
|
946
1005
|
const entry = toolCallbacks.get(step.name);
|
|
947
1006
|
if (!entry) {
|
|
948
1007
|
throw new robot_1.ActionableError(`Unknown tool in step ${i + 1}: ${step.name}`);
|
|
@@ -950,7 +1009,8 @@ const createMcpServer = () => {
|
|
|
950
1009
|
try {
|
|
951
1010
|
const args = zod_1.z.object(entry.paramsSchema).parse({ device, ...step.arguments });
|
|
952
1011
|
const output = await entry.callback(args, {});
|
|
953
|
-
results.push(`Step ${i + 1} (${step.name}): ${output}`);
|
|
1012
|
+
results.push(`Step ${i + 1} (${step.name}): ${textOf(output)}`);
|
|
1013
|
+
images.push(...imagesOf(output));
|
|
954
1014
|
}
|
|
955
1015
|
catch (error) {
|
|
956
1016
|
results.push(`Step ${i + 1} (${step.name}) failed: ${error.message}`);
|
|
@@ -960,9 +1020,20 @@ const createMcpServer = () => {
|
|
|
960
1020
|
}
|
|
961
1021
|
}
|
|
962
1022
|
if (listElementsAtEnd) {
|
|
963
|
-
|
|
1023
|
+
// the steps already ran, so a failed listing must not throw their results away
|
|
1024
|
+
try {
|
|
1025
|
+
results.push(textOf(await toolCallbacks.get("mobile_list_elements_on_screen").callback({ device }, {})));
|
|
1026
|
+
}
|
|
1027
|
+
catch (error) {
|
|
1028
|
+
results.push(`Listing elements at the end failed: ${error.message}`);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
const text = results.join("\n");
|
|
1032
|
+
if (images.length === 0) {
|
|
1033
|
+
return text;
|
|
964
1034
|
}
|
|
965
|
-
|
|
1035
|
+
// images follow the text, in the order their steps ran
|
|
1036
|
+
return [{ type: "text", text }, ...images];
|
|
966
1037
|
});
|
|
967
1038
|
return server;
|
|
968
1039
|
};
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mobilenext/mobile-mcp",
|
|
3
3
|
"mcpName": "io.github.mobile-next/mobile-mcp",
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.18",
|
|
5
5
|
"description": "Mobile MCP",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"commander": "14.0.0",
|
|
35
35
|
"express": "5.1.0",
|
|
36
36
|
"fast-xml-parser": "5.8.0",
|
|
37
|
-
"mobilewright": "0.0.
|
|
37
|
+
"mobilewright": "0.0.70",
|
|
38
38
|
"qs": "^6.15.0",
|
|
39
39
|
"zod": "^4.2.0",
|
|
40
40
|
"zod-to-json-schema": "3.25.0"
|