@mobilenext/mobile-mcp 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/server.js CHANGED
@@ -15,12 +15,21 @@ const android_1 = require("./android");
15
15
  const robot_1 = require("./robot");
16
16
  const ios_1 = require("./ios");
17
17
  const png_1 = require("./png");
18
- const image_utils_1 = require("./image-utils");
18
+ const jpeg_1 = require("./jpeg");
19
+ const coordinate_mapping_1 = require("./coordinate-mapping");
19
20
  const mobilecli_1 = require("./mobilecli");
20
21
  const mobile_device_1 = require("./mobile-device");
21
22
  const utils_1 = require("./utils");
23
+ const format_elements_1 = require("./format-elements");
24
+ const ALLOWED_LOG_EXTENSIONS = [".log", ".txt", ".jsonl"];
25
+ const DEFAULT_DEVICE_LOG_ENTRIES = 100;
26
+ const MAX_DEVICE_LOG_ENTRIES = 10000;
27
+ const DEVICE_LOG_TIMEOUT_MS = 30000;
28
+ const DEVICE_LOG_FILTER_PATTERN = /^(pid|process|tag|level|subsystem|category|message)!?=.+$/;
22
29
  const ALLOWED_SCREENSHOT_EXTENSIONS = [".png", ".jpg", ".jpeg"];
30
+ const DEFAULT_SCREENSHOT_MAX_SIZE = 1024;
23
31
  const ALLOWED_RECORDING_EXTENSIONS = [".mp4"];
32
+ const ALLOWED_APP_EXTENSIONS = [".apk", ".ipa", ".zip", ".app"];
24
33
  const LOGIN_PROMPT_TIMEOUT_MS = 15000;
25
34
  const getAgentVersion = () => {
26
35
  const json = require("../package.json");
@@ -42,13 +51,17 @@ const createMcpServer = () => {
42
51
  return "unknown";
43
52
  }
44
53
  };
54
+ // ponytail: batch calls tool callbacks directly, bypassing mcp transport
55
+ const toolCallbacks = new Map();
45
56
  const tool = (name, title, description, paramsSchema, annotations, cb) => {
57
+ toolCallbacks.set(name, { callback: cb, paramsSchema });
46
58
  server.registerTool(name, {
47
59
  title,
48
60
  description,
49
61
  inputSchema: paramsSchema,
50
62
  annotations,
51
63
  }, (async (args, _extra) => {
64
+ scarf();
52
65
  try {
53
66
  (0, logger_1.trace)(`Invoking ${name} with args: ${JSON.stringify(args)}`);
54
67
  const start = +new Date();
@@ -120,6 +133,15 @@ const createMcpServer = () => {
120
133
  // ignore
121
134
  }
122
135
  };
136
+ // ponytail: scarf.sh pixel, fired once per process on first tool invocation
137
+ let scarfSent = false;
138
+ const scarf = () => {
139
+ if (scarfSent || process.env.MOBILEMCP_DISABLE_TELEMETRY) {
140
+ return;
141
+ }
142
+ scarfSent = true;
143
+ fetch("https://static.scarf.sh/a.png?x-pxid=f238110d-fcbb-479a-bdda-d47716ec4f07").catch(() => { });
144
+ };
123
145
  const mobilecli = new mobilecli_1.Mobilecli();
124
146
  const activeRecordings = new Map();
125
147
  const agentVerifiedSimulators = new Set();
@@ -136,7 +158,18 @@ const createMcpServer = () => {
136
158
  throw new robot_1.ActionableError(`mobilecli is not available or not working properly. Please review the documentation at https://github.com/mobile-next/mobile-mcp/wiki for installation instructions`);
137
159
  }
138
160
  };
161
+ // cache per device id, saves ~0.2s (mobilecli --version + devices) on every tool call
162
+ const robotCache = new Map();
139
163
  const getRobotFromDevice = (deviceId) => {
164
+ const cached = robotCache.get(deviceId);
165
+ if (cached) {
166
+ return cached;
167
+ }
168
+ const robot = createRobotFromDevice(deviceId);
169
+ robotCache.set(deviceId, robot);
170
+ return robot;
171
+ };
172
+ const createRobotFromDevice = (deviceId) => {
140
173
  // from now on, we must have mobilecli working
141
174
  ensureMobilecliAvailable();
142
175
  const legacyRobot = process.env.MOBILEMCP_LEGACY_ROBOT === "1";
@@ -146,7 +179,6 @@ const createMcpServer = () => {
146
179
  const iosDevices = iosManager.listDevices();
147
180
  const iosDevice = iosDevices.find(d => d.deviceId === deviceId);
148
181
  if (iosDevice) {
149
- posthog("get_robot", { "DevicePlatform": "ios", "DeviceType": "real" }).then();
150
182
  return new ios_1.IosRobot(deviceId);
151
183
  }
152
184
  // Check if it's an Android device
@@ -154,7 +186,6 @@ const createMcpServer = () => {
154
186
  const androidDevices = androidManager.getConnectedDevices();
155
187
  const androidDevice = androidDevices.find(d => d.deviceId === deviceId);
156
188
  if (androidDevice) {
157
- posthog("get_robot", { "DevicePlatform": "android", "DeviceType": androidDevice.deviceType }).then();
158
189
  return new android_1.AndroidRobot(deviceId);
159
190
  }
160
191
  }
@@ -175,7 +206,6 @@ const createMcpServer = () => {
175
206
  }
176
207
  agentVerifiedSimulators.add(deviceId);
177
208
  }
178
- posthog("get_robot", { "DevicePlatform": device.platform, "DeviceType": device.type }).then();
179
209
  return new mobile_device_1.MobileDevice(deviceId);
180
210
  }
181
211
  }
@@ -183,7 +213,7 @@ const createMcpServer = () => {
183
213
  throw new robot_1.ActionableError(`Device "${deviceId}" not found. Use the mobile_list_available_devices tool to see available devices.`);
184
214
  };
185
215
  tool("mobile_list_available_devices", "List Devices", "List all available devices. This includes both physical mobile devices and mobile simulators and emulators. It returns both Android and iOS devices. " +
186
- "These are local devices already connected to this machine, ready to use immediately at no cost - for devices from the shared remote cloud fleet, use mobile_list_remote_devices instead.", {}, { readOnlyHint: true }, async ({}, telemetry) => {
216
+ "These are local devices already connected to this machine, ready to use immediately at no cost - for devices from the shared remote cloud fleet, use mobile_list_remote_devices instead.", {}, { readOnlyHint: true, openWorldHint: false }, async ({}, telemetry) => {
187
217
  // from today onward, we must have mobilecli working
188
218
  ensureMobilecliAvailable();
189
219
  const devices = [];
@@ -261,7 +291,7 @@ const createMcpServer = () => {
261
291
  tool("mobile_login_to_cloud_provider", "Login to Cloud Provider", "Start authenticating this machine with the remote device cloud provider. This is required once before mobile_list_remote_devices or mobile_allocate_remote_device will work; if either of those fails with an authentication error, call this tool and then retry. " +
262
292
  "This starts a browser-based device-code login and returns quickly with a URL and a one-time code - it does NOT wait for the login to complete. Show the URL and code to the user verbatim and ask them to open the URL and enter the code in their own browser. " +
263
293
  "The login keeps running in the background after this tool returns; once the user confirms they've completed it, retry the remote devices tool that originally failed. " +
264
- "Only call this after the user has explicitly asked to connect to, log into, or use remote/cloud devices - never call it speculatively, since it interrupts the user to act in their browser.", {}, {}, async ({}) => {
294
+ "Only call this after the user has explicitly asked to connect to, log into, or use remote/cloud devices - never call it speculatively, since it interrupts the user to act in their browser.", {}, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({}) => {
265
295
  ensureMobilecliAvailable();
266
296
  const child = mobilecli.spawnRemoteLogin();
267
297
  activeLoginProcesses.push(child);
@@ -321,7 +351,7 @@ const createMcpServer = () => {
321
351
  tool("mobile_list_remote_devices", "List Remote Devices", "List the catalog of device models (make, platform, OS version) available to reserve from the remote cloud device fleet. " +
322
352
  "This is different from mobile_list_available_devices, which lists real devices and simulators/emulators already connected to this local machine and ready to use immediately at no cost. " +
323
353
  "Remote devices live in a shared cloud fleet: they are not usable until reserved with mobile_allocate_remote_device, and reserving one may be a limited/billed resource. " +
324
- "Requires mobile_login_to_cloud_provider to have been called first; if this fails with an authentication error, call that tool then retry.", {}, { readOnlyHint: true }, async ({}) => {
354
+ "Requires mobile_login_to_cloud_provider to have been called first; if this fails with an authentication error, call that tool then retry.", {}, { readOnlyHint: true, openWorldHint: true }, async ({}) => {
325
355
  ensureMobilecliAvailable();
326
356
  const result = mobilecli.remoteListDevices();
327
357
  return result;
@@ -337,7 +367,7 @@ const createMcpServer = () => {
337
367
  type: zod_1.z.enum(["real"]).optional().describe("Device type filter. Currently only \"real\" (physical devices) is supported by the fleet."),
338
368
  wait: zod_1.z.boolean().optional().describe("If true, block until the device has finished allocating and is ready to use, up to timeoutSeconds. If false/omitted, this returns as soon as the reservation is made, but the device may not be immediately ready."),
339
369
  timeoutSeconds: zod_1.z.coerce.number().int().positive().optional().describe("Seconds to wait for allocation when wait is true. Defaults to 900 (15 minutes). Only relevant when wait is true."),
340
- }, { destructiveHint: true }, async ({ platform, name, version, type, wait, timeoutSeconds }) => {
370
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ platform, name, version, type, wait, timeoutSeconds }) => {
341
371
  ensureMobilecliAvailable();
342
372
  const result = mobilecli.remoteAllocate({ platform, name, version, type, wait, timeoutSeconds });
343
373
  return result;
@@ -346,23 +376,33 @@ const createMcpServer = () => {
346
376
  "Releasing is destructive to the device's state: apps installed, files pushed, and any other changes made during this session are lost, and a later mobile_allocate_remote_device call may take time and could return a different physical unit. " +
347
377
  "Only release once the whole task is finished - if there is more work to do on the same device shortly, keep holding it rather than releasing and reallocating.", {
348
378
  device: zod_1.z.string().describe("The device identifier to release back to the remote fleet"),
349
- }, { destructiveHint: true }, async ({ device }) => {
379
+ }, { readOnlyHint: false, destructiveHint: true, openWorldHint: true }, async ({ device }) => {
350
380
  ensureMobilecliAvailable();
351
381
  const result = mobilecli.remoteRelease(device);
352
382
  return result;
353
383
  });
354
384
  tool("mobile_list_apps", "List Apps", "List all the installed apps on the device", {
355
385
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
356
- }, { readOnlyHint: true }, async ({ device }) => {
386
+ }, { readOnlyHint: true, openWorldHint: false }, async ({ device }) => {
357
387
  const robot = getRobotFromDevice(device);
358
388
  const result = await robot.listApps();
359
389
  return `Found these apps on device: ${result.map(app => `${app.appName} (${app.packageName})`).join(", ")}`;
360
390
  });
391
+ tool("mobile_get_foreground_app", "Get Foreground App", "Get the app currently in the foreground on the device. Use this to verify which app or screen you are on before interacting with it.", {
392
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
393
+ }, { readOnlyHint: true, openWorldHint: false }, async ({ device }) => {
394
+ const robot = getRobotFromDevice(device);
395
+ if (!robot.getForegroundApp) {
396
+ throw new robot_1.ActionableError("Getting the foreground app is not supported in legacy robot mode");
397
+ }
398
+ const app = await robot.getForegroundApp();
399
+ return `Foreground app: ${app.appName} (${app.packageName})`;
400
+ });
361
401
  tool("mobile_launch_app", "Launch App", "Launch an app on mobile device. Use this to open a specific app. You can find the package name of the app by calling list_apps_on_device.", {
362
402
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
363
403
  packageName: zod_1.z.string().describe("The package name of the app to launch"),
364
404
  locale: zod_1.z.string().optional().describe("Comma-separated BCP 47 locale tags to launch the app with (e.g., fr-FR,en-GB)"),
365
- }, { destructiveHint: true }, async ({ device, packageName, locale }) => {
405
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, packageName, locale }) => {
366
406
  const robot = getRobotFromDevice(device);
367
407
  await robot.launchApp(packageName, locale);
368
408
  return `Launched app ${packageName}`;
@@ -370,7 +410,7 @@ const createMcpServer = () => {
370
410
  tool("mobile_terminate_app", "Terminate App", "Stop and terminate an app on mobile device", {
371
411
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
372
412
  packageName: zod_1.z.string().describe("The package name of the app to terminate"),
373
- }, { destructiveHint: true }, async ({ device, packageName }) => {
413
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, packageName }) => {
374
414
  const robot = getRobotFromDevice(device);
375
415
  await robot.terminateApp(packageName);
376
416
  return `Terminated app ${packageName}`;
@@ -378,7 +418,11 @@ const createMcpServer = () => {
378
418
  tool("mobile_install_app", "Install App", "Install an app on mobile device", {
379
419
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
380
420
  path: zod_1.z.string().describe("The path to the app file to install. For iOS simulators, provide a .zip file or a .app directory. For Android provide an .apk file. For iOS real devices provide an .ipa file"),
381
- }, { destructiveHint: true }, async ({ device, path }) => {
421
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, path }) => {
422
+ (0, utils_1.validateFileExtension)(path, ALLOWED_APP_EXTENSIONS, "install_app");
423
+ if (!node_fs_1.default.existsSync(path)) {
424
+ throw new robot_1.ActionableError(`App file not found: ${path}`);
425
+ }
382
426
  const robot = getRobotFromDevice(device);
383
427
  await robot.installApp(path);
384
428
  return `Installed app from ${path}`;
@@ -386,78 +430,70 @@ const createMcpServer = () => {
386
430
  tool("mobile_uninstall_app", "Uninstall App", "Uninstall an app from mobile device", {
387
431
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
388
432
  bundle_id: zod_1.z.string().describe("Bundle identifier (iOS) or package name (Android) of the app to be uninstalled"),
389
- }, { destructiveHint: true }, async ({ device, bundle_id }) => {
433
+ }, { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, async ({ device, bundle_id }) => {
390
434
  const robot = getRobotFromDevice(device);
391
435
  await robot.uninstallApp(bundle_id);
392
436
  return `Uninstalled app ${bundle_id}`;
393
437
  });
394
438
  tool("mobile_get_screen_size", "Get Screen Size", "Get the screen size of the mobile device in pixels", {
395
439
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
396
- }, { readOnlyHint: true }, async ({ device }) => {
440
+ }, { readOnlyHint: true, openWorldHint: false }, async ({ device }) => {
397
441
  const robot = getRobotFromDevice(device);
398
442
  const screenSize = await robot.getScreenSize();
399
443
  return `Screen size is ${screenSize.width}x${screenSize.height} pixels`;
400
444
  });
401
- tool("mobile_click_on_screen_at_coordinates", "Click Screen", "Click on the screen at given x,y coordinates. If clicking on an element, use the list_elements_on_screen tool to find the coordinates.", {
445
+ tool("mobile_click_on_screen_at_coordinates", "Click Screen", "Click on the screen, either at x,y coordinates or on an element by its ref (e.g. \"@e5\") from the latest mobile_list_elements_on_screen result. Prefer ref when the element is listed.", {
402
446
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
403
- x: zod_1.z.coerce.number().describe("The x coordinate to click on the screen, in pixels"),
404
- y: zod_1.z.coerce.number().describe("The y coordinate to click on the screen, in pixels"),
405
- }, { destructiveHint: true }, async ({ device, x, y }) => {
447
+ x: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to click on the screen, in pixels. Required unless ref is given"),
448
+ y: zod_1.z.coerce.number().min(0).optional().describe("The y coordinate to click on the screen, in pixels. Required unless ref is given"),
449
+ ref: zod_1.z.string().optional().describe("Element ref from mobile_list_elements_on_screen, e.g. \"@e5\". Takes precedence over x,y"),
450
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, x, y, ref }) => {
406
451
  const robot = getRobotFromDevice(device);
452
+ if (ref !== undefined) {
453
+ if (!robot.tapByRef) {
454
+ throw new robot_1.ActionableError("Clicking by ref is not supported in legacy robot mode");
455
+ }
456
+ await robot.tapByRef(ref);
457
+ return `Clicked on element ${ref}`;
458
+ }
459
+ if (x === undefined || y === undefined) {
460
+ throw new robot_1.ActionableError("Either ref or both x and y must be provided");
461
+ }
407
462
  await robot.tap(x, y);
408
463
  return `Clicked on screen at coordinates: ${x}, ${y}`;
409
464
  });
410
465
  tool("mobile_double_tap_on_screen", "Double Tap Screen", "Double-tap on the screen at given x,y coordinates.", {
411
466
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
412
- x: zod_1.z.coerce.number().describe("The x coordinate to double-tap, in pixels"),
413
- y: zod_1.z.coerce.number().describe("The y coordinate to double-tap, in pixels"),
414
- }, { destructiveHint: true }, async ({ device, x, y }) => {
467
+ x: zod_1.z.coerce.number().min(0).describe("The x coordinate to double-tap, in pixels"),
468
+ y: zod_1.z.coerce.number().min(0).describe("The y coordinate to double-tap, in pixels"),
469
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, x, y }) => {
415
470
  const robot = getRobotFromDevice(device);
416
471
  await robot.doubleTap(x, y);
417
472
  return `Double-tapped on screen at coordinates: ${x}, ${y}`;
418
473
  });
419
- tool("mobile_long_press_on_screen_at_coordinates", "Long Press Screen", "Long press on the screen at given x,y coordinates. If long pressing on an element, use the list_elements_on_screen tool to find the coordinates.", {
474
+ tool("mobile_long_press_on_screen_at_coordinates", "Long Press Screen", "Long press on the screen at given x,y coordinates. If long pressing on an element, use the mobile_list_elements_on_screen tool to find the coordinates.", {
420
475
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
421
- x: zod_1.z.coerce.number().describe("The x coordinate to long press on the screen, in pixels"),
422
- y: zod_1.z.coerce.number().describe("The y coordinate to long press on the screen, in pixels"),
476
+ x: zod_1.z.coerce.number().min(0).describe("The x coordinate to long press on the screen, in pixels"),
477
+ y: zod_1.z.coerce.number().min(0).describe("The y coordinate to long press on the screen, in pixels"),
423
478
  duration: zod_1.z.coerce.number().min(1).max(10000).optional().describe("Duration of the long press in milliseconds. Defaults to 500ms."),
424
- }, { destructiveHint: true }, async ({ device, x, y, duration }) => {
479
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, x, y, duration }) => {
425
480
  const robot = getRobotFromDevice(device);
426
481
  const pressDuration = duration ?? 500;
427
482
  await robot.longPress(x, y, pressDuration);
428
483
  return `Long pressed on screen at coordinates: ${x}, ${y} for ${pressDuration}ms`;
429
484
  });
430
- tool("mobile_list_elements_on_screen", "List Screen Elements", "List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.", {
431
- device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
432
- }, { readOnlyHint: true }, async ({ device }) => {
485
+ tool("mobile_list_elements_on_screen", "List Screen Elements", "List elements on screen with their ref, coordinates, and display text or accessibility label. Use the ref with mobile_click_on_screen_at_coordinates. Refs and coordinates stay valid as long as the screen does not change; re-list only after navigation or a layout change.", {
486
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
487
+ format: zod_1.z.enum(["text", "json"]).optional().describe("Output format. \"text\" (default) is one compact line per element, \"json\" is a json array"),
488
+ }, { readOnlyHint: true, openWorldHint: true }, async ({ device, format = "text" }) => {
433
489
  const robot = getRobotFromDevice(device);
434
490
  const elements = await robot.getElementsOnScreen();
435
- const result = elements.map(element => {
436
- const out = {
437
- type: element.type,
438
- text: element.text,
439
- label: element.label,
440
- name: element.name,
441
- value: element.value,
442
- identifier: element.identifier,
443
- coordinates: {
444
- x: element.rect.x,
445
- y: element.rect.y,
446
- width: element.rect.width,
447
- height: element.rect.height,
448
- },
449
- };
450
- if (element.focused) {
451
- out.focused = true;
452
- }
453
- return out;
454
- });
455
- return `Found these elements on screen: ${JSON.stringify(result)}`;
491
+ return (0, format_elements_1.formatElements)(elements, format);
456
492
  });
457
493
  tool("mobile_press_button", "Press Button", "Press a button on device", {
458
494
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
459
495
  button: zod_1.z.string().describe("The button to press. Supported buttons: BACK (android only), HOME, VOLUME_UP, VOLUME_DOWN, ENTER, DPAD_CENTER (android tv only), DPAD_UP (android tv only), DPAD_DOWN (android tv only), DPAD_LEFT (android tv only), DPAD_RIGHT (android tv only)"),
460
- }, { destructiveHint: true }, async ({ device, button }) => {
496
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, button }) => {
461
497
  const robot = getRobotFromDevice(device);
462
498
  await robot.pressButton(button);
463
499
  return `Pressed the button: ${button}`;
@@ -465,7 +501,7 @@ const createMcpServer = () => {
465
501
  tool("mobile_open_url", "Open URL", "Open a URL in browser on device", {
466
502
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
467
503
  url: zod_1.z.string().describe("The URL to open"),
468
- }, { destructiveHint: true }, async ({ device, url }) => {
504
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, url }) => {
469
505
  const allowUnsafeUrls = process.env.MOBILEMCP_ALLOW_UNSAFE_URLS === "1";
470
506
  if (!allowUnsafeUrls && !url.startsWith("http://") && !url.startsWith("https://")) {
471
507
  throw new robot_1.ActionableError("Only http:// and https:// URLs are allowed. Set MOBILEMCP_ALLOW_UNSAFE_URLS=1 to allow other URL schemes.");
@@ -477,10 +513,10 @@ const createMcpServer = () => {
477
513
  tool("mobile_swipe_on_screen", "Swipe Screen", "Swipe on the screen", {
478
514
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
479
515
  direction: zod_1.z.enum(["up", "down", "left", "right"]).describe("The direction to swipe"),
480
- x: zod_1.z.coerce.number().optional().describe("The x coordinate to start the swipe from, in pixels. If not provided, uses center of screen"),
481
- y: zod_1.z.coerce.number().optional().describe("The y coordinate to start the swipe from, in pixels. If not provided, uses center of screen"),
516
+ x: zod_1.z.coerce.number().min(0).optional().describe("The x coordinate to start the swipe from, in pixels. If not provided, uses center of screen"),
517
+ y: zod_1.z.coerce.number().min(0).optional().describe("The y coordinate to start the swipe from, in pixels. If not provided, uses center of screen"),
482
518
  distance: zod_1.z.coerce.number().optional().describe("The distance to swipe in pixels. Defaults to 400 pixels for iOS or 30% of screen dimension for Android"),
483
- }, { destructiveHint: true }, async ({ device, direction, x, y, distance }) => {
519
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, direction, x, y, distance }) => {
484
520
  const robot = getRobotFromDevice(device);
485
521
  if (x !== undefined && y !== undefined) {
486
522
  // Use coordinate-based swipe
@@ -498,7 +534,7 @@ const createMcpServer = () => {
498
534
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
499
535
  text: zod_1.z.string().describe("The text to type"),
500
536
  submit: zod_1.z.boolean().describe("Whether to submit the text. If true, the text will be submitted as if the user pressed the enter key."),
501
- }, { destructiveHint: true }, async ({ device, text, submit }) => {
537
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, async ({ device, text, submit }) => {
502
538
  const robot = getRobotFromDevice(device);
503
539
  await robot.sendKeys(text);
504
540
  if (submit) {
@@ -509,58 +545,82 @@ const createMcpServer = () => {
509
545
  tool("mobile_save_screenshot", "Save Screenshot", "Save a screenshot of the mobile device to a file", {
510
546
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
511
547
  saveTo: zod_1.z.string().describe("The path to save the screenshot to. Filename must end with .png, .jpg, or .jpeg"),
512
- }, { destructiveHint: true }, async ({ device, saveTo }) => {
548
+ maxSize: zod_1.z.number().int().positive().optional().describe("Maximum width/height in pixels, keeping aspect ratio. Omit for full size."),
549
+ scale: zod_1.z.number().gt(0).max(1).optional().describe("Scale factor (0.0-1.0). Ignored if maxSize is provided."),
550
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, saveTo, maxSize, scale }) => {
513
551
  (0, utils_1.validateFileExtension)(saveTo, ALLOWED_SCREENSHOT_EXTENSIONS, "save_screenshot");
514
552
  (0, utils_1.validateOutputPath)(saveTo);
515
553
  const robot = getRobotFromDevice(device);
516
- const screenshot = await robot.getScreenshot();
554
+ const format = saveTo.toLowerCase().endsWith(".png") ? "png" : "jpeg";
555
+ const screenshot = await robot.getScreenshot({ format, maxSize, scale });
556
+ // legacy robots ignore the requested format and always return a png
557
+ const isPng = screenshot.length > 0 && screenshot[0] === 0x89;
558
+ if (format === "jpeg" && isPng) {
559
+ throw new robot_1.ActionableError("This device only supports png screenshots, save to a .png file instead.");
560
+ }
517
561
  node_fs_1.default.writeFileSync(saveTo, screenshot);
518
562
  return `Screenshot saved to: ${saveTo}`;
519
563
  });
520
564
  server.registerTool("mobile_take_screenshot", {
521
565
  title: "Take Screenshot",
522
- description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. Do not cache this result.",
566
+ description: "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. The screenshot is usually smaller than the screen, so when the device reports its screen size the result also states how to convert positions in the screenshot into screen coordinates before tapping. Do not cache this result.",
523
567
  inputSchema: {
524
- device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
568
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
569
+ maxSize: zod_1.z.number().int().positive().optional().describe(`Maximum width/height in pixels, keeping aspect ratio. Defaults to ${DEFAULT_SCREENSHOT_MAX_SIZE}.`),
570
+ scale: zod_1.z.number().gt(0).max(1).optional().describe("Scale factor (0.0-1.0). Ignored if maxSize is provided."),
525
571
  },
526
572
  annotations: {
527
573
  readOnlyHint: true,
574
+ openWorldHint: true,
528
575
  },
529
- }, async ({ device }) => {
576
+ }, async ({ device, maxSize, scale }) => {
530
577
  try {
531
578
  const robot = getRobotFromDevice(device);
532
- const screenSize = await robot.getScreenSize();
533
- let screenshot = await robot.getScreenshot();
534
- let mimeType = "image/png";
535
- // validate we received a png, will throw exception otherwise
536
- const image = new png_1.PNG(screenshot);
537
- const pngSize = image.getDimensions();
538
- if (pngSize.width <= 0 || pngSize.height <= 0) {
539
- throw new robot_1.ActionableError("Screenshot is invalid. Please try again.");
579
+ // mobilecli scales and converts natively; legacy robots ignore the options and return a full-size png
580
+ // maxSize wins over scale (same precedence as mobilecli); the default applies only when neither is given
581
+ const options = { format: "jpeg", quality: 75 };
582
+ if (maxSize !== undefined) {
583
+ options.maxSize = maxSize;
584
+ }
585
+ else if (scale !== undefined) {
586
+ options.scale = scale;
540
587
  }
541
- if ((0, image_utils_1.isScalingAvailable)()) {
542
- (0, logger_1.trace)("Image scaling is available, resizing screenshot");
543
- const image = image_utils_1.Image.fromBuffer(screenshot);
544
- const beforeSize = screenshot.length;
545
- screenshot = image.resize(Math.floor(pngSize.width / screenSize.scale))
546
- .jpeg({ quality: 75 })
547
- .toBuffer();
548
- const afterSize = screenshot.length;
549
- (0, logger_1.trace)(`Screenshot resized from ${beforeSize} bytes to ${afterSize} bytes`);
550
- mimeType = "image/jpeg";
588
+ else {
589
+ options.maxSize = DEFAULT_SCREENSHOT_MAX_SIZE;
551
590
  }
591
+ const screenshot = await robot.getScreenshot(options);
592
+ const isJpeg = screenshot.length > 2 && screenshot[0] === 0xff && screenshot[1] === 0xd8;
593
+ let mimeType = "image/jpeg";
594
+ let screenshotSize;
595
+ if (isJpeg) {
596
+ screenshotSize = (0, jpeg_1.getJpegDimensions)(screenshot);
597
+ }
598
+ else {
599
+ mimeType = "image/png";
600
+ // validate we received a png, will throw exception otherwise
601
+ const image = new png_1.PNG(screenshot);
602
+ screenshotSize = image.getDimensions();
603
+ }
604
+ if (screenshotSize.width <= 0 || screenshotSize.height <= 0) {
605
+ throw new robot_1.ActionableError("Screenshot is invalid. Please try again.");
606
+ }
607
+ // the screenshot is downscaled (and on ios the screen is in points while pixels are not),
608
+ // so tell the model how to map what it sees onto tap coordinates. see #29.
609
+ const screenSize = await robot.getScreenSize();
610
+ const mapping = (0, coordinate_mapping_1.describeCoordinateMapping)(screenshotSize, screenSize);
552
611
  const screenshot64 = screenshot.toString("base64");
553
- (0, logger_1.trace)(`Screenshot taken: ${screenshot.length} bytes`);
612
+ (0, logger_1.trace)(`Screenshot taken: ${screenshot.length} bytes, ${screenshotSize.width}x${screenshotSize.height}`);
554
613
  posthog("tool_invoked", {
555
614
  "ToolName": "mobile_take_screenshot",
556
615
  "ScreenshotFilesize": screenshot64.length,
557
616
  "ScreenshotMimeType": mimeType,
558
- "ScreenshotWidth": pngSize.width,
559
- "ScreenshotHeight": pngSize.height,
560
617
  }).then();
561
- return {
562
- content: [{ type: "image", data: screenshot64, mimeType }]
563
- };
618
+ const content = [];
619
+ if (mapping !== null) {
620
+ content.push({ type: "text", text: mapping });
621
+ }
622
+ content.push({ type: "image", data: screenshot64, mimeType });
623
+ return { content };
564
624
  }
565
625
  catch (err) {
566
626
  (0, logger_1.error)(`Error taking screenshot: ${err.message} ${err.stack}`);
@@ -573,14 +633,69 @@ const createMcpServer = () => {
573
633
  tool("mobile_set_orientation", "Set Orientation", "Change the screen orientation of the device", {
574
634
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
575
635
  orientation: zod_1.z.enum(["portrait", "landscape"]).describe("The desired orientation"),
576
- }, { destructiveHint: true }, async ({ device, orientation }) => {
636
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, orientation }) => {
577
637
  const robot = getRobotFromDevice(device);
578
638
  await robot.setOrientation(orientation);
579
639
  return `Changed device orientation to ${orientation}`;
580
640
  });
641
+ tool("mobile_set_location", "Set Location", "Override the GPS location reported by the device, or clear the override to restore the real location. Omit latitude and longitude to clear.", {
642
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
643
+ latitude: zod_1.z.number().min(-90).max(90).optional().describe("Latitude in decimal degrees, e.g. 37.7749. Omit together with longitude to clear the override."),
644
+ longitude: zod_1.z.number().min(-180).max(180).optional().describe("Longitude in decimal degrees, e.g. -122.4194. Omit together with latitude to clear the override."),
645
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, latitude, longitude }) => {
646
+ const robot = getRobotFromDevice(device);
647
+ if (!robot.setLocation || !robot.clearLocation) {
648
+ throw new robot_1.ActionableError("Setting location is not supported in legacy robot mode");
649
+ }
650
+ if (latitude === undefined && longitude === undefined) {
651
+ await robot.clearLocation();
652
+ return "Cleared location override";
653
+ }
654
+ if (latitude === undefined || longitude === undefined) {
655
+ throw new robot_1.ActionableError("Both latitude and longitude are required to set a location");
656
+ }
657
+ await robot.setLocation(latitude, longitude);
658
+ return `Set device location to ${latitude},${longitude}`;
659
+ });
660
+ tool("mobile_clipboard", "Clipboard", "Read or replace the device clipboard. Pass text to set the clipboard, omit it to read the current clipboard.", {
661
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
662
+ text: zod_1.z.string().optional().describe("Text to place on the clipboard. Omit to read the clipboard instead."),
663
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, text }) => {
664
+ const robot = getRobotFromDevice(device);
665
+ if (!robot.getClipboard || !robot.setClipboard) {
666
+ throw new robot_1.ActionableError("Clipboard is not supported in legacy robot mode");
667
+ }
668
+ if (text === undefined) {
669
+ return await robot.getClipboard();
670
+ }
671
+ await robot.setClipboard(text);
672
+ return "Clipboard updated";
673
+ });
674
+ tool("mobile_get_device_logs", "Get Device Logs", "Collect live device logs (logcat on Android, unified log on iOS) as one JSON object per line. Only logs emitted after the call starts are captured, so trigger the behavior you want to observe right before or during the call. Stops after `limit` entries or after 30 seconds of silence. Each line of a stack trace counts as one entry, so filter by process or tag to avoid filling the limit with noise. Start with a small limit (20-50) for a quick look. For large captures, pass `saveTo` to write logs to a file and only the entry count is returned.", {
675
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
676
+ limit: zod_1.z.number().int().min(1).max(MAX_DEVICE_LOG_ENTRIES).default(DEFAULT_DEVICE_LOG_ENTRIES).describe("Stop after this many log entries"),
677
+ filter: zod_1.z.array(zod_1.z.string().regex(DEVICE_LOG_FILTER_PATTERN, "Filter must be key=value or key!=value with key in: pid, process, tag, level, subsystem, category, message")).default([]).describe("Filters, ANDed together. key=value includes, key!=value excludes. Keys: pid, process, tag, level, subsystem, category, message. Example: [\"tag=ActivityManager\", \"level=Error\", \"process!=SpringBoard\"]"),
678
+ saveTo: zod_1.z.string().optional().describe("Path to write the logs to instead of returning them. Filename must end with .log, .txt, or .jsonl"),
679
+ }, { readOnlyHint: true, openWorldHint: true }, async ({ device, limit, filter, saveTo }) => {
680
+ if (saveTo) {
681
+ (0, utils_1.validateFileExtension)(saveTo, ALLOWED_LOG_EXTENSIONS, "get_device_logs");
682
+ (0, utils_1.validateOutputPath)(saveTo);
683
+ }
684
+ const robot = getRobotFromDevice(device);
685
+ if (!robot.getLogs) {
686
+ throw new robot_1.ActionableError("Device logs are not supported in legacy robot mode");
687
+ }
688
+ const logs = await robot.getLogs(limit, filter, DEVICE_LOG_TIMEOUT_MS);
689
+ if (!saveTo) {
690
+ return logs || "No log entries received";
691
+ }
692
+ node_fs_1.default.writeFileSync(saveTo, logs + "\n");
693
+ const lineCount = logs ? logs.split("\n").length : 0;
694
+ return `Saved ${lineCount} log entries to: ${saveTo}`;
695
+ });
581
696
  tool("mobile_get_orientation", "Get Orientation", "Get the current screen orientation of the device", {
582
697
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you.")
583
- }, { readOnlyHint: true }, async ({ device }) => {
698
+ }, { readOnlyHint: true, openWorldHint: false }, async ({ device }) => {
584
699
  const robot = getRobotFromDevice(device);
585
700
  const orientation = await robot.getOrientation();
586
701
  return `Current device orientation is ${orientation}`;
@@ -589,7 +704,7 @@ const createMcpServer = () => {
589
704
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
590
705
  output: zod_1.z.string().optional().describe("The file path to save the recording to. Filename must end with .mp4. If not provided, a temporary path will be used."),
591
706
  timeLimit: zod_1.z.coerce.number().optional().describe("Maximum recording duration in seconds. The recording will stop automatically after this time."),
592
- }, { destructiveHint: true }, async ({ device, output, timeLimit }) => {
707
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device, output, timeLimit }) => {
593
708
  if (output) {
594
709
  (0, utils_1.validateFileExtension)(output, ALLOWED_RECORDING_EXTENSIONS, "start_screen_recording");
595
710
  (0, utils_1.validateOutputPath)(output);
@@ -618,7 +733,7 @@ const createMcpServer = () => {
618
733
  });
619
734
  tool("mobile_stop_screen_recording", "Stop Screen Recording", "Stop an active screen recording on a mobile device. Returns the file path, size, and approximate duration of the recording.", {
620
735
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
621
- }, { destructiveHint: true }, async ({ device }) => {
736
+ }, { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, async ({ device }) => {
622
737
  const recording = activeRecordings.get(device);
623
738
  if (!recording) {
624
739
  throw new robot_1.ActionableError(`No active recording found for device "${device}". Start a recording first with mobile_start_screen_recording.`);
@@ -646,7 +761,7 @@ const createMcpServer = () => {
646
761
  });
647
762
  tool("mobile_list_crashes", "List Crash Reports", "List crash reports available on the device", {
648
763
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
649
- }, { readOnlyHint: true }, async ({ device }) => {
764
+ }, { readOnlyHint: true, openWorldHint: true }, async ({ device }) => {
650
765
  ensureMobilecliAvailable();
651
766
  const response = mobilecli.crashesList(device);
652
767
  return JSON.stringify(response.data);
@@ -654,11 +769,52 @@ const createMcpServer = () => {
654
769
  tool("mobile_get_crash", "Get Crash Report", "Get the full content of a crash report by its ID. Use mobile_list_crashes to find available crash IDs.", {
655
770
  device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
656
771
  id: zod_1.z.string().describe("The crash report ID to retrieve"),
657
- }, { readOnlyHint: true }, async ({ device, id }) => {
772
+ }, { readOnlyHint: true, openWorldHint: true }, async ({ device, id }) => {
658
773
  ensureMobilecliAvailable();
659
774
  const response = mobilecli.crashesGet(device, id);
660
775
  return response.data.content;
661
776
  });
777
+ tool("mobile_batch_commands", "Batch Commands", "Run multiple tools in sequence in a single call, e.g. click, type, click, type. Use this to fill forms or perform multi-step flows without round-trips. The device argument is applied to every step unless a step provides its own.", {
778
+ device: zod_1.z.string().describe("The device identifier to use. Use mobile_list_available_devices to find which devices are available to you."),
779
+ steps: zod_1.z.array(zod_1.z.object({
780
+ name: zod_1.z.string().describe("Tool name, e.g. mobile_click_on_screen_at_coordinates"),
781
+ arguments: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).describe("Arguments for the tool, same as calling it directly"),
782
+ })).min(1).describe("Tools to run, in order"),
783
+ stopOnError: zod_1.z.boolean().optional().describe("Stop at the first failing step. Defaults to true"),
784
+ listElementsAtEnd: zod_1.z.boolean().optional().describe("Run mobile_list_elements_on_screen after the last step and include its result. Defaults to false"),
785
+ }, { readOnlyHint: false, destructiveHint: true, openWorldHint: true }, async ({ device, steps, stopOnError, listElementsAtEnd }, telemetry) => {
786
+ const shouldStopOnError = stopOnError ?? true;
787
+ const results = [];
788
+ telemetry.StepCount = steps.length;
789
+ for (let i = 0; i < steps.length; i++) {
790
+ const step = steps[i];
791
+ if (step.name === "mobile_batch_commands") {
792
+ throw new robot_1.ActionableError("mobile_batch_commands cannot be nested");
793
+ }
794
+ if (step.name === "mobile_take_screenshot") {
795
+ throw new robot_1.ActionableError("mobile_take_screenshot returns an image and cannot be used in a batch, use mobile_save_screenshot instead");
796
+ }
797
+ const entry = toolCallbacks.get(step.name);
798
+ if (!entry) {
799
+ throw new robot_1.ActionableError(`Unknown tool in step ${i + 1}: ${step.name}`);
800
+ }
801
+ try {
802
+ const args = zod_1.z.object(entry.paramsSchema).parse({ device, ...step.arguments });
803
+ const output = await entry.callback(args, {});
804
+ results.push(`Step ${i + 1} (${step.name}): ${output}`);
805
+ }
806
+ catch (error) {
807
+ results.push(`Step ${i + 1} (${step.name}) failed: ${error.message}`);
808
+ if (shouldStopOnError) {
809
+ break;
810
+ }
811
+ }
812
+ }
813
+ if (listElementsAtEnd) {
814
+ results.push(await toolCallbacks.get("mobile_list_elements_on_screen").callback({ device }, {}));
815
+ }
816
+ return results.join("\n");
817
+ });
662
818
  return server;
663
819
  };
664
820
  exports.createMcpServer = createMcpServer;