@mobilenext/mobile-mcp 1.0.17 → 1.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -121,12 +121,13 @@ How we help to scale mobile automation:
121
121
  - **`mobile_clear_app_data`** - Clear an app's data so it starts fresh (Android and iOS Simulator)
122
122
 
123
123
  ### Screen Interaction
124
- - **`mobile_take_screenshot`** - Take a screenshot to understand what's on screen
124
+ - **`mobile_take_screenshot`** - Take a screenshot to understand what's on screen, optionally clipped to a region
125
125
  - **`mobile_save_screenshot`** - Save a screenshot to a file
126
126
  - **`mobile_list_elements_on_screen`** - List UI elements with their coordinates and properties
127
127
  - **`mobile_click_on_screen_at_coordinates`** - Click at specific x,y coordinates or on an element by its ref
128
128
  - **`mobile_double_tap_on_screen`** - Double-tap at specific coordinates or on an element by its ref
129
129
  - **`mobile_long_press_on_screen_at_coordinates`** - Long press at specific coordinates or on an element by its ref
130
+ - **`mobile_drag_on_screen`** - Drag from one point or element ref to another, with an optional hold before moving
130
131
  - **`mobile_swipe_on_screen`** - Swipe in any direction (up, down, left, right)
131
132
  - **`mobile_start_screen_recording`** - Start recording the device screen to a video file
132
133
  - **`mobile_stop_screen_recording`** - Stop the active screen recording and save the video
@@ -140,7 +141,7 @@ How we help to scale mobile automation:
140
141
  - **`mobile_get_device_logs`** - Collect live device logs (logcat on Android, unified log on iOS), optionally saved to a file
141
142
  - **`mobile_list_crashes`** - List crash reports available on the device
142
143
  - **`mobile_get_crash`** - Get the full content of a crash report by its ID
143
- - **`mobile_batch_commands`** - Run multiple tools in sequence in a single call (e.g. click, type, click), optionally listing screen elements at the end
144
+ - **`mobile_batch_commands`** - Run multiple tools in sequence in a single call (e.g. click, type, click), with optional `delay` steps in between, optionally listing screen elements at the end
144
145
 
145
146
  ## 🏗️ Mobile MCP Architecture
146
147
 
@@ -378,13 +379,15 @@ Follow the MCP Servers documentation. For example in `~/.config/opencode/opencod
378
379
  {
379
380
  "$schema": "https://opencode.ai/config.json",
380
381
  "mcp": {
381
- "mobile-mcp": {
382
- "type": "local",
383
- "command": [
384
- "npx",
385
- "@mobilenext/mobile-mcp@latest"
386
- ],
387
- "enabled": true
382
+ "servers": {
383
+ "mobile-mcp": {
384
+ "type": "local",
385
+ "command": [
386
+ "npx",
387
+ "-y",
388
+ "@mobilenext/mobile-mcp@latest"
389
+ ]
390
+ }
388
391
  }
389
392
  }
390
393
  }
package/lib/android.js CHANGED
@@ -44,7 +44,7 @@ const xml = __importStar(require("fast-xml-parser"));
44
44
  const robot_1 = require("./robot");
45
45
  const utils_1 = require("./utils");
46
46
  const getAdbPath = () => {
47
- const exeName = process.env.platform === "win32" ? "adb.exe" : "adb";
47
+ const exeName = process.platform === "win32" ? "adb.exe" : "adb";
48
48
  if (process.env.ANDROID_HOME) {
49
49
  return node_path_1.default.join(process.env.ANDROID_HOME, "platform-tools", exeName);
50
50
  }
@@ -23,11 +23,24 @@ const orientLike = (screen, screenshot) => {
23
23
  * downscaled, and on iOS the screen is measured in points while a full-size
24
24
  * screenshot is in pixels, so the two spaces rarely agree.
25
25
  *
26
- * Returns null when either size is unknown, in which case no instruction is
27
- * better than a wrong one.
26
+ * When the screenshot was clipped to a region, the region's offset is added on
27
+ * top of the scaling.
28
+ *
29
+ * Returns null when a size it needs is unknown, in which case no instruction
30
+ * is better than a wrong one.
28
31
  */
29
- const describeCoordinateMapping = (screenshot, reportedScreen) => {
30
- if (!isPositive(screenshot) || !isPositive(reportedScreen)) {
32
+ const describeCoordinateMapping = (screenshot, reportedScreen, clip) => {
33
+ if (!isPositive(screenshot)) {
34
+ return null;
35
+ }
36
+ // a clipped screenshot shows only a region, so positions in it are offset from the screen's origin.
37
+ // the region and the screenshot are enough to map it, so the screen size is not needed here
38
+ if (clip) {
39
+ const clipX = formatRatio(clip.width / screenshot.width);
40
+ const clipY = formatRatio(clip.height / screenshot.height);
41
+ return `Screenshot is ${screenshot.width}x${screenshot.height} and shows the screen region at ${clip.x},${clip.y} sized ${clip.width}x${clip.height}. To tap something you see in this screenshot, multiply its x by ${clipX} and add ${clip.x}, multiply its y by ${clipY} and add ${clip.y}.`;
42
+ }
43
+ if (!isPositive(reportedScreen)) {
31
44
  return null;
32
45
  }
33
46
  const screen = orientLike(reportedScreen, screenshot);
@@ -23,6 +23,13 @@ const flattenUIElement = (element) => {
23
23
  }
24
24
  return [screenElement, ...element.children.flatMap(child => flattenUIElement(child))];
25
25
  };
26
+ // mobilecli rejects fractional coordinates ("x and y must be integers")
27
+ const formatGesturePoint = (point) => {
28
+ if ("ref" in point) {
29
+ return point.ref;
30
+ }
31
+ return `${Math.round(point.x)},${Math.round(point.y)}`;
32
+ };
26
33
  class MobileDevice {
27
34
  deviceId;
28
35
  mobilecli;
@@ -105,6 +112,10 @@ class MobileDevice {
105
112
  if (format === "jpeg" && options?.quality !== undefined) {
106
113
  fullArgs.push("--quality", `${options.quality}`);
107
114
  }
115
+ if (options?.clip !== undefined) {
116
+ const { x, y, width, height } = options.clip;
117
+ fullArgs.push("--clip", `${Math.round(x)},${Math.round(y)},${Math.round(width)},${Math.round(height)}`);
118
+ }
108
119
  if (options?.maxSize !== undefined) {
109
120
  fullArgs.push("--max-size", `${options.maxSize}`);
110
121
  }
@@ -167,19 +178,18 @@ class MobileDevice {
167
178
  this.runCommand(["io", "tap", ref]);
168
179
  }
169
180
  async doubleTap(x, y) {
170
- // TODO: should move into mobilecli itself as "io doubletap"
171
- await this.tap(x, y);
172
- await this.tap(x, y);
181
+ // parse the response, older mobilecli versions print help and exit 0 on unknown commands
182
+ this.runJsonCommand(["io", "doubletap", `${Math.round(x)},${Math.round(y)}`]);
173
183
  }
174
184
  async doubleTapByRef(ref) {
175
- // resolve the ref once: two "io tap @ref" calls would each dump the ui again and miss the double-tap window
176
- const elements = await this.getElementsOnScreen();
177
- const element = elements.find(candidate => candidate.ref === ref);
178
- if (!element) {
179
- throw new robot_1.ActionableError(`Element ${ref} was not found on the current screen. Call mobile_list_elements_on_screen again to get fresh refs`);
185
+ this.runJsonCommand(["io", "doubletap", ref]);
186
+ }
187
+ async drag(from, to, hold) {
188
+ const args = ["io", "drag", `${formatGesturePoint(from)},${formatGesturePoint(to)}`];
189
+ if (hold) {
190
+ args.push("--hold", `${Math.round(hold)}`);
180
191
  }
181
- const { x, y, width, height } = element.rect;
182
- await this.doubleTap(x + width / 2, y + height / 2);
192
+ this.runJsonCommand(args);
183
193
  }
184
194
  async longPress(x, y, duration) {
185
195
  // mobilecli rejects fractional coordinates and durations, both are parsed as integers
package/lib/server.js CHANGED
@@ -10,6 +10,7 @@ const node_fs_1 = __importDefault(require("node:fs"));
10
10
  const node_os_1 = __importDefault(require("node:os"));
11
11
  const node_path_1 = __importDefault(require("node:path"));
12
12
  const node_crypto_1 = __importDefault(require("node:crypto"));
13
+ const promises_1 = require("node:timers/promises");
13
14
  const logger_1 = require("./logger");
14
15
  const android_1 = require("./android");
15
16
  const robot_1 = require("./robot");
@@ -29,6 +30,8 @@ const DEVICE_LOG_TIMEOUT_MS = 30000;
29
30
  const DEVICE_LOG_FILTER_PATTERN = /^(pid|process|tag|level|subsystem|category|message)!?=.+$/;
30
31
  const ALLOWED_SCREENSHOT_EXTENSIONS = [".png", ".jpg", ".jpeg"];
31
32
  const DEFAULT_SCREENSHOT_MAX_SIZE = 1024;
33
+ const BATCH_DELAY_STEP = "delay";
34
+ const MAX_BATCH_DELAY_MS = 10000;
32
35
  const ALLOWED_RECORDING_EXTENSIONS = [".mp4"];
33
36
  const ALLOWED_APP_EXTENSIONS = [".apk", ".ipa", ".zip", ".app"];
34
37
  const LOGIN_PROMPT_TIMEOUT_MS = 15000;
@@ -80,6 +83,34 @@ Prefer mobile_launch_app to open an app, and mobile_open_url to open a web page,
80
83
  over navigating through the UI. mobile_open_url only accepts http:// and
81
84
  https:// URLs unless the server was started with MOBILEMCP_ALLOW_UNSAFE_URLS=1,
82
85
  so do not rely on it for app deep links.`;
86
+ // the text of a tool response, for places that can only carry text
87
+ const textOf = (response) => {
88
+ if (typeof response === "string") {
89
+ return response;
90
+ }
91
+ return response.flatMap(block => block.type === "text" ? [block.text] : []).join("\n");
92
+ };
93
+ const imagesOf = (response) => {
94
+ if (typeof response === "string") {
95
+ return [];
96
+ }
97
+ return response.filter(block => block.type === "image");
98
+ };
99
+ const toGesturePoint = (ref, x, y) => {
100
+ if (ref !== undefined) {
101
+ return { ref };
102
+ }
103
+ if (x === undefined || y === undefined) {
104
+ return undefined;
105
+ }
106
+ return { x, y };
107
+ };
108
+ const describeGesturePoint = (point) => {
109
+ if ("ref" in point) {
110
+ return `element ${point.ref}`;
111
+ }
112
+ return `coordinates ${point.x}, ${point.y}`;
113
+ };
83
114
  const createMcpServer = () => {
84
115
  const server = new server_1.McpServer({
85
116
  name: "mobile-mcp",
@@ -114,8 +145,12 @@ const createMcpServer = () => {
114
145
  const telemetry = {};
115
146
  const response = await cb(args, telemetry);
116
147
  const duration = +new Date() - start;
117
- (0, logger_1.trace)(`=> ${response}`);
118
148
  posthog("tool_invoked", { "ToolName": name, "Duration": duration, ...telemetry }).then();
149
+ if (typeof response !== "string") {
150
+ (0, logger_1.trace)(`=> ${response.length} content blocks`);
151
+ return { content: response };
152
+ }
153
+ (0, logger_1.trace)(`=> ${response}`);
119
154
  return {
120
155
  content: [{ type: "text", text: response }],
121
156
  };
@@ -556,6 +591,31 @@ const createMcpServer = () => {
556
591
  await robot.doubleTap(x, y);
557
592
  return `Double-tapped on screen at coordinates: ${x}, ${y}`;
558
593
  });
594
+ tool("mobile_drag_on_screen", "Drag On Screen", "Drag one finger from one point to another, e.g. to move a slider, reorder a list or drag and drop. Each end is either x,y coordinates or an element by its ref (e.g. \"@e5\") from the latest mobile_list_elements_on_screen result. Prefer ref when the element is listed.", {
595
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
596
+ x1: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to start the drag from, in pixels. Required unless fromRef is given"),
597
+ y1: zod_1.z.coerce.number().min(0).optional().describe("The y coordinate to start the drag from, in pixels. Required unless fromRef is given"),
598
+ x2: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to end the drag at, in pixels. Required unless toRef is given"),
599
+ y2: zod_1.z.coerce.number().min(0).optional().describe("The y coordinate to end the drag at, in pixels. Required unless toRef is given"),
600
+ fromRef: zod_1.z.string().optional().describe("Element ref to start the drag from, e.g. \"@e5\". Takes precedence over x1,y1"),
601
+ toRef: zod_1.z.string().optional().describe("Element ref to end the drag at, e.g. \"@e7\". Takes precedence over x2,y2"),
602
+ hold: zod_1.z.coerce.number().min(0).max(10000).optional().describe("How long the finger stays down before it moves, in milliseconds. Use it (e.g. 500) to pick up an item that would otherwise scroll. Defaults to 0"),
603
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, x1, y1, x2, y2, fromRef, toRef, hold }) => {
604
+ const from = toGesturePoint(fromRef, x1, y1);
605
+ if (!from) {
606
+ throw new robot_1.ActionableError("Either fromRef or both x1 and y1 must be provided");
607
+ }
608
+ const to = toGesturePoint(toRef, x2, y2);
609
+ if (!to) {
610
+ throw new robot_1.ActionableError("Either toRef or both x2 and y2 must be provided");
611
+ }
612
+ const robot = getRobotFromDevice(device);
613
+ if (!robot.drag) {
614
+ throw new robot_1.ActionableError("Dragging is not supported in legacy robot mode");
615
+ }
616
+ await robot.drag(from, to, hold);
617
+ return `Dragged from ${describeGesturePoint(from)} to ${describeGesturePoint(to)}`;
618
+ });
559
619
  tool("mobile_long_press_on_screen_at_coordinates", "Long Press Screen", "Long press on the screen, either at x,y coordinates or on an element by its ref (e.g. \"@e5\") from the latest mobile_list_elements_on_screen result. Prefer ref when the element is listed.", {
560
620
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
561
621
  x: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to long press on the screen, in pixels. Required unless ref is given"),
@@ -578,7 +638,7 @@ const createMcpServer = () => {
578
638
  await robot.longPress(x, y, pressDuration);
579
639
  return `Long pressed on screen at coordinates: ${x}, ${y} for ${pressDuration}ms`;
580
640
  });
581
- tool("mobile_list_elements_on_screen", "List Screen Elements", "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates, mobile_double_tap_on_screen or mobile_long_press_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.", {
641
+ tool("mobile_list_elements_on_screen", "List Screen Elements", "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates, mobile_double_tap_on_screen, mobile_long_press_on_screen_at_coordinates or mobile_drag_on_screen. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.", {
582
642
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
583
643
  format: zod_1.z.enum(["text", "json", "raw"]).default("text").describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array, \"raw\" is the unfiltered platform-specific hierarchy as JSON (mobilecli only). Raw nodes may be offscreen and do not have element refs"),
584
644
  }, { readOnlyHint: true, openWorldHint: true }, async ({ device, format }) => {
@@ -672,74 +732,66 @@ const createMcpServer = () => {
672
732
  (0, utils_1.writeOutputFile)(saveTo, screenshot);
673
733
  return `Screenshot saved to: ${saveTo}`;
674
734
  });
675
- server.registerTool("mobile_take_screenshot", {
676
- title: "Take Screenshot",
677
- description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so when the device reports its screen size the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.",
678
- inputSchema: zod_1.z.object({
679
- device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
680
- maxSize: zod_1.z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),
681
- scale: zod_1.z.number().gt(0).max(1).optional().describe("Scale factor (0.0-1.0). Ignored if maxSize is provided."),
682
- }),
683
- annotations: {
684
- readOnlyHint: true,
685
- openWorldHint: true,
686
- },
687
- }, async ({ device, maxSize, scale }) => {
688
- try {
689
- const robot = getRobotFromDevice(device);
690
- // mobilecli scales and converts natively; legacy robots ignore the options and return a full-size png
691
- // maxSize wins over scale (same precedence as mobilecli); the default applies only when neither is given
692
- const options = { format: "jpeg", quality: 75 };
693
- if (maxSize !== undefined) {
694
- options.maxSize = maxSize;
695
- }
696
- else if (scale !== undefined) {
697
- options.scale = scale;
698
- }
699
- else {
700
- options.maxSize = DEFAULT_SCREENSHOT_MAX_SIZE;
701
- }
702
- const screenshot = await robot.getScreenshot(options);
703
- const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;
704
- let mimeType = "image/jpeg";
705
- let screenshotSize;
706
- if (isJpeg) {
707
- screenshotSize = (0, jpeg_1.getJpegDimensions)(screenshot);
708
- }
709
- else {
710
- mimeType = "image/png";
711
- // validate we received a png, will throw exception otherwise
712
- const image = new png_1.PNG(screenshot);
713
- screenshotSize = image.getDimensions();
714
- }
715
- if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {
716
- throw new robot_1.ActionableError("Screenshot is invalid. Please try again.");
717
- }
718
- // the screenshot is downscaled (and on ios the screen is in points while pixels are not),
719
- // so tell the model how to map what it sees onto tap coordinates. see #29.
720
- const screenSize = await robot.getScreenSize();
721
- const mapping = (0, coordinate_mapping_1.describeCoordinateMapping)(screenshotSize, screenSize);
722
- const screenshot64 = screenshot.toString("base64");
723
- (0, logger_1.trace)(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);
724
- posthog("tool_invoked", {
725
- "ToolName": "mobile_take_screenshot",
726
- "ScreenshotFilesize": screenshot64.length,
727
- "ScreenshotMimeType": mimeType,
728
- }).then();
729
- const content = [];
730
- if (mapping !== null) {
731
- content.push({ type: "text", text: mapping });
732
- }
733
- content.push({ type: "image", data: screenshot64, mimeType });
734
- return { content };
735
+ tool("mobile_take_screenshot", "Take Screenshot", "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so when the device reports its screen size the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.", {
736
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
737
+ maxSize: zod_1.z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),
738
+ scale: zod_1.z.number().gt(0).max(1).optional().describe("Scale factor (0.0-1.0). Ignored if maxSize is provided."),
739
+ clip: zod_1.z.object({
740
+ x: zod_1.z.number().min(0).describe("Left edge of the region"),
741
+ y: zod_1.z.number().min(0).describe("Top edge of the region"),
742
+ width: zod_1.z.number().positive().describe("Width of the region"),
743
+ height: zod_1.z.number().positive().describe("Height of the region"),
744
+ }).optional().describe("Only capture this region of the screen, in screen coordinates (the ones used for tapping). Applied before maxSize/scale. Use it to look at one part of the screen in more detail or at a lower cost."),
745
+ }, { readOnlyHint: true, openWorldHint: true }, async ({ device, maxSize, scale, clip }, telemetry) => {
746
+ const robot = getRobotFromDevice(device);
747
+ if (clip !== undefined && !(robot instanceof mobile_device_1.MobileDevice)) {
748
+ throw new robot_1.ActionableError("Clipping a screenshot is not supported in legacy robot mode");
735
749
  }
736
- catch (err) {
737
- (0, logger_1.error)(`Error taking screenshot: ${err.message} ${err.stack}`);
738
- return {
739
- content: [{ type: "text", text: `Error: ${err.message}` }],
740
- isError: true,
741
- };
750
+ // mobilecli scales and converts natively; legacy robots ignore the options and return a full-size png
751
+ // maxSize wins over scale (same precedence as mobilecli); the default applies only when neither is given
752
+ const options = { format: "jpeg", quality: 75 };
753
+ if (maxSize !== undefined) {
754
+ options.maxSize = maxSize;
755
+ }
756
+ else if (scale !== undefined) {
757
+ options.scale = scale;
758
+ }
759
+ else {
760
+ options.maxSize = DEFAULT_SCREENSHOT_MAX_SIZE;
761
+ }
762
+ if (clip !== undefined) {
763
+ options.clip = clip;
764
+ }
765
+ const screenshot = await robot.getScreenshot(options);
766
+ const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;
767
+ let mimeType = "image/jpeg";
768
+ let screenshotSize;
769
+ if (isJpeg) {
770
+ screenshotSize = (0, jpeg_1.getJpegDimensions)(screenshot);
771
+ }
772
+ else {
773
+ mimeType = "image/png";
774
+ // validate we received a png, will throw exception otherwise
775
+ const image = new png_1.PNG(screenshot);
776
+ screenshotSize = image.getDimensions();
742
777
  }
778
+ if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {
779
+ throw new robot_1.ActionableError("Screenshot is invalid");
780
+ }
781
+ // the screenshot is downscaled (and on ios the screen is in points while pixels are not),
782
+ // so tell the model how to map what it sees onto tap coordinates. see #29.
783
+ const screenSize = await robot.getScreenSize();
784
+ const mapping = (0, coordinate_mapping_1.describeCoordinateMapping)(screenshotSize, screenSize, clip);
785
+ const screenshot64 = screenshot.toString("base64");
786
+ (0, logger_1.trace)(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);
787
+ telemetry.ScreenshotFilesize = screenshot64.length;
788
+ telemetry.ScreenshotMimeType = mimeType;
789
+ const content = [];
790
+ if (mapping !== null) {
791
+ content.push({ type: "text", text: mapping });
792
+ }
793
+ content.push({ type: "image", data: screenshot64, mimeType });
794
+ return content;
743
795
  });
744
796
  tool("mobile_set_orientation", "Set Orientation", "Change the screen orientation of the device", {
745
797
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
@@ -921,12 +973,21 @@ const createMcpServer = () => {
921
973
  const response = mobilecli.crashesGet(device, id);
922
974
  return response.data.content;
923
975
  });
924
- // mobile_take_screenshot returns an image, which the text result of a batch cannot carry
925
- const batchableToolNames = [...toolCallbacks.keys()].filter(name => name !== "mobile_take_screenshot");
926
- tool("mobile_batch_commands", "Batch Commands", `Run multiple tools in sequence in a single call, e.g. click, type, click, type. Use this to fill forms or perform multi-step flows without round-trips. The device argument is applied to every step unless a step provides its own. Tools allowed as steps: ${batchableToolNames.join(", ")}.`, {
976
+ const batchableToolNames = [...toolCallbacks.keys()];
977
+ // a step that only exists inside a batch: waiting is of no use as a tool call of its own
978
+ toolCallbacks.set(BATCH_DELAY_STEP, {
979
+ paramsSchema: {
980
+ milliseconds: zod_1.z.coerce.number().int().min(1).max(MAX_BATCH_DELAY_MS).describe("How long to wait, in milliseconds"),
981
+ },
982
+ callback: async ({ milliseconds }) => {
983
+ await (0, promises_1.setTimeout)(milliseconds);
984
+ return `Waited ${milliseconds}ms`;
985
+ },
986
+ });
987
+ tool("mobile_batch_commands", "Batch Commands", `Run multiple tools in sequence in a single call, e.g. click, type, click, type. Use this to fill forms or perform multi-step flows without round-trips. The device argument is applied to every step unless a step provides its own. Tools allowed as steps: ${batchableToolNames.join(", ")}. A step named "${BATCH_DELAY_STEP}" with arguments {"milliseconds": N} (up to ${MAX_BATCH_DELAY_MS}) waits before the next step, e.g. for an animation or a page load to finish before looking at the screen.`, {
927
988
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
928
989
  steps: zod_1.z.array(zod_1.z.object({
929
- name: zod_1.z.string().describe("Name of a tool allowed as a step, e.g. mobile_click_on_screen_at_coordinates"),
990
+ name: zod_1.z.string().describe(`Name of a tool allowed as a step, e.g. mobile_click_on_screen_at_coordinates, or "${BATCH_DELAY_STEP}"`),
930
991
  arguments: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).describe("Arguments for the tool, same as calling it directly"),
931
992
  })).min(1).describe("Tools to run, in order"),
932
993
  stopOnError: zod_1.z.boolean().optional().describe("Stop at the first failing step. Defaults to true"),
@@ -934,15 +995,13 @@ const createMcpServer = () => {
934
995
  }, { readOnlyHint: false, destructiveHint: true, openWorldHint: true }, async ({ device, steps, stopOnError, listElementsAtEnd }, telemetry) => {
935
996
  const shouldStopOnError = stopOnError ?? true;
936
997
  const results = [];
998
+ const images = [];
937
999
  telemetry.StepCount = steps.length;
938
1000
  for (let i = 0; i < steps.length; i++) {
939
1001
  const step = steps[i];
940
1002
  if (step.name === "mobile_batch_commands") {
941
1003
  throw new robot_1.ActionableError("mobile_batch_commands cannot be nested");
942
1004
  }
943
- if (step.name === "mobile_take_screenshot") {
944
- throw new robot_1.ActionableError("mobile_take_screenshot returns an image and cannot be used in a batch, use mobile_save_screenshot instead");
945
- }
946
1005
  const entry = toolCallbacks.get(step.name);
947
1006
  if (!entry) {
948
1007
  throw new robot_1.ActionableError(`Unknown tool in step ${i + 1}: ${step.name}`);
@@ -950,7 +1009,8 @@ const createMcpServer = () => {
950
1009
  try {
951
1010
  const args = zod_1.z.object(entry.paramsSchema).parse({ device, ...step.arguments });
952
1011
  const output = await entry.callback(args, {});
953
- results.push(`Step ${i + 1} (${step.name}): ${output}`);
1012
+ results.push(`Step ${i + 1} (${step.name}): ${textOf(output)}`);
1013
+ images.push(...imagesOf(output));
954
1014
  }
955
1015
  catch (error) {
956
1016
  results.push(`Step ${i + 1} (${step.name}) failed: ${error.message}`);
@@ -960,9 +1020,20 @@ const createMcpServer = () => {
960
1020
  }
961
1021
  }
962
1022
  if (listElementsAtEnd) {
963
- results.push(await toolCallbacks.get("mobile_list_elements_on_screen").callback({ device }, {}));
1023
+ // the steps already ran, so a failed listing must not throw their results away
1024
+ try {
1025
+ results.push(textOf(await toolCallbacks.get("mobile_list_elements_on_screen").callback({ device }, {})));
1026
+ }
1027
+ catch (error) {
1028
+ results.push(`Listing elements at the end failed: ${error.message}`);
1029
+ }
1030
+ }
1031
+ const text = results.join("\n");
1032
+ if (images.length === 0) {
1033
+ return text;
964
1034
  }
965
- return results.join("\n");
1035
+ // images follow the text, in the order their steps ran
1036
+ return [{ type: "text", text }, ...images];
966
1037
  });
967
1038
  return server;
968
1039
  };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mobilenext/mobile-mcp",
3
3
  "mcpName": "io.github.mobile-next/mobile-mcp",
4
- "version": "1.0.17",
4
+ "version": "1.0.18",
5
5
  "description": "Mobile MCP",
6
6
  "repository": {
7
7
  "type": "git",
@@ -34,7 +34,7 @@
34
34
  "commander": "14.0.0",
35
35
  "express": "5.1.0",
36
36
  "fast-xml-parser": "5.8.0",
37
- "mobilewright": "0.0.69",
37
+ "mobilewright": "0.0.70",
38
38
  "qs": "^6.15.0",
39
39
  "zod": "^4.2.0",
40
40
  "zod-to-json-schema": "3.25.0"