@mobilenext/mobile-mcp 0.0.11 → 0.0.12

package/README.md

@@ -79,6 +79,8 @@ How we help to scale mobile automation:
 claude mcp add mobile -- npx -y @mobilenext/mobile-mcp@latest ⁠
 ```
 
+[Read more in our wiki](https://github.com/mobile-next/mobile-mcp/wiki)! 🚀
+
 ## Prerequisites
 
 What you will need to connect MCP with your agent and mobile devices:
@@ -109,16 +111,10 @@ On iOS, you'll need Xcode and to run the Simulator before using Mobile MCP with
 - `xcrun simctl list`
 - `xcrun simctl boot "iPhone 16"`
 
-
 # Mobile Commands and interaction tools
 
 The commands and tools support both accessibility-based locators (preferred) and coordinate-based inputs, giving you flexibility when accessibility/automation IDs are missing for reliable and seemless automation.
 
-## mobile_install_app
-- **Description:** Installs an app onto the device/emulator
-- **Parameters:**
-  - `appPath` (string): Path or URL to the app file (e.g., .apk for Android, .ipa/.app for iOS)
-
 ## mobile_list_apps
 - **Description:** List all the installed apps on the device
 - **Parameters:**
@@ -172,8 +168,6 @@ The commands and tools support both accessibility-based locators (preferred) and
 ## mobile_type_text
 - **Description:** Types text into a focused UI element (e.g., TextField, SearchField)
 - **Parameters:**
-  - `element` (string): Human-readable element description
-  - `ref` (string): Accessibility/automation ID of the element
   - `text` (string): Text to type
   - `submit` (boolean): Whether to press Enter/Return after typing
 
@@ -200,22 +194,12 @@ The commands and tools support both accessibility-based locators (preferred) and
 
 ## mobile_take_screenshot
 - **Description:** Captures a screenshot of the current device screen
-- **Parameters:**
-  - `raw` (boolean): Return a lossless image if true; otherwise, compressed by default
+- **Parameters:** None
 
 ## mobile_get_source
 - **Description:** Fetches the current device UI structure (accessibility snapshot) (xml format)
 - **Parameters:** None
 
-## mobile_wait
-- **Description:** Waits for a specified time
-- **Parameters:**
-  - `time` (number): Time to wait in seconds (capped at 10 seconds)
-
-## mobile_close_session
-- **Description:** Closes the current device session
-- **Parameters:** None
-
 
 # Thanks to all contributors ❤️
 

package/lib/android.js

@@ -37,44 +37,61 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getConnectedDevices = exports.AndroidRobot = void 0;
+const path_1 = __importDefault(require("path"));
 const child_process_1 = require("child_process");
 const xml = __importStar(require("fast-xml-parser"));
-const path_1 = __importDefault(require("path"));
+const robot_1 = require("./robot");
+const getAdbPath = () => {
+    let executable = "adb";
+    if (process.env.ANDROID_HOME) {
+        executable = path_1.default.join(process.env.ANDROID_HOME, "platform-tools", "adb");
+    }
+    return executable;
+};
+const BUTTON_MAP = {
+    "BACK": "KEYCODE_BACK",
+    "HOME": "KEYCODE_HOME",
+    "VOLUME_UP": "KEYCODE_VOLUME_UP",
+    "VOLUME_DOWN": "KEYCODE_VOLUME_DOWN",
+    "ENTER": "KEYCODE_ENTER",
+};
+const TIMEOUT = 30000;
+const MAX_BUFFER_SIZE = 1024 * 1024 * 4;
 class AndroidRobot {
     deviceId;
     constructor(deviceId) {
         this.deviceId = deviceId;
     }
+    adb(...args) {
+        return (0, child_process_1.execFileSync)(getAdbPath(), ["-s", this.deviceId, ...args], {
+            maxBuffer: MAX_BUFFER_SIZE,
+            timeout: TIMEOUT,
+        });
+    }
     async getScreenSize() {
-        const screenSize = (0, child_process_1.execSync)(`adb -s ${this.deviceId} shell wm size`)
+        const screenSize = this.adb("shell", "wm", "size")
             .toString()
             .split(" ")
             .pop();
         if (!screenSize) {
             throw new Error("Failed to get screen size");
         }
+        const scale = 1;
         const [width, height] = screenSize.split("x").map(Number);
-        return { width, height };
-    }
-    adb(...args) {
-        let executable = "adb";
-        if (process.env.ANDROID_HOME) {
-            executable = path_1.default.join(process.env.ANDROID_HOME, "platform-tools", "adb");
-        }
-        return (0, child_process_1.execFileSync)(executable, ["-s", this.deviceId, ...args], {
-            maxBuffer: 1024 * 1024 * 4,
-            timeout: 30000,
-        });
+        return { width, height, scale };
     }
     async listApps() {
-        const result = this.adb("shell", "cmd", "package", "query-activities", "-a", "android.intent.action.MAIN", "-c", "android.intent.category.LAUNCHER")
+        return this.adb("shell", "cmd", "package", "query-activities", "-a", "android.intent.action.MAIN", "-c", "android.intent.category.LAUNCHER")
             .toString()
             .split("\n")
             .map(line => line.trim())
             .filter(line => line.startsWith("packageName="))
             .map(line => line.substring("packageName=".length))
-            .filter((value, index, self) => self.indexOf(value) === index);
-        return result;
+            .filter((value, index, self) => self.indexOf(value) === index)
+            .map(packageName => ({
+            packageName,
+            appName: packageName,
+        }));
     }
     async launchApp(packageName) {
         this.adb("shell", "monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1");
@@ -96,53 +113,45 @@ class AndroidRobot {
                 y1 = Math.floor(screenSize.height * 0.80);
                 break;
             default:
-                throw new Error(`Swipe direction "${direction}" is not supported`);
+                throw new robot_1.ActionableError(`Swipe direction "${direction}" is not supported`);
         }
         this.adb("shell", "input", "swipe", `${x0}`, `${y0}`, `${x1}`, `${y1}`, "1000");
     }
     async getScreenshot() {
         return this.adb("shell", "screencap", "-p");
     }
-    collectElements(node, screenSize) {
+    collectElements(node) {
         const elements = [];
-        const getCoordinates = (element) => {
+        const getScreenElementRect = (element) => {
             const bounds = String(element.bounds);
             const [, left, top, right, bottom] = bounds.match(/^\[(\d+),(\d+)\]\[(\d+),(\d+)\]$/)?.map(Number) || [];
-            return { left, top, right, bottom };
-        };
-        const getCenter = (coordinates) => {
-            return {
-                x: Math.floor((coordinates.left + coordinates.right) / 2),
-                y: Math.floor((coordinates.top + coordinates.bottom) / 2),
-            };
-        };
-        const normalizeCoordinates = (coordinates, screenSize) => {
             return {
-                x: Number((coordinates.x / screenSize.width).toFixed(3)),
-                y: Number((coordinates.y / screenSize.height).toFixed(3)),
+                x: left,
+                y: top,
+                width: right - left,
+                height: bottom - top,
             };
         };
         if (node.node) {
             if (Array.isArray(node.node)) {
                 for (const childNode of node.node) {
-                    elements.push(...this.collectElements(childNode, screenSize));
+                    elements.push(...this.collectElements(childNode));
                 }
             }
             else {
-                elements.push(...this.collectElements(node.node, screenSize));
+                elements.push(...this.collectElements(node.node));
             }
         }
-        if (node.text) {
-            elements.push({
-                "text": node.text,
-                "coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
-            });
-        }
-        if (node["content-desc"]) {
-            elements.push({
-                "text": node["content-desc"],
-                "coordinates": normalizeCoordinates(getCenter(getCoordinates(node)), screenSize),
-            });
+        if (node.text || node["content-desc"] || node.hint) {
+            const element = {
+                type: node.class || "text",
+                name: node.text,
+                label: node["content-desc"] || node.hint || "",
+                rect: getScreenElementRect(node),
+            };
+            if (element.rect.width > 0 && element.rect.height > 0) {
+                elements.push(element);
+            }
         }
         return elements;
     }
@@ -154,8 +163,7 @@ class AndroidRobot {
         });
         const parsedXml = parser.parse(dump);
         const hierarchy = parsedXml.hierarchy;
-        const screenSize = await this.getScreenSize();
-        const elements = this.collectElements(hierarchy.node, screenSize);
+        const elements = this.collectElements(hierarchy.node);
         return elements;
     }
     async terminateApp(packageName) {
@@ -170,17 +178,10 @@ class AndroidRobot {
         this.adb("shell", "input", "text", _text);
     }
     async pressButton(button) {
-        const _map = {
-            "BACK": "KEYCODE_BACK",
-            "HOME": "KEYCODE_HOME",
-            "VOLUME_UP": "KEYCODE_VOLUME_UP",
-            "VOLUME_DOWN": "KEYCODE_VOLUME_DOWN",
-            "ENTER": "KEYCODE_ENTER",
-        };
-        if (!_map[button]) {
-            throw new Error(`Button "${button}" is not supported`);
+        if (!BUTTON_MAP[button]) {
+            throw new robot_1.ActionableError(`Button "${button}" is not supported`);
         }
-        this.adb("shell", "input", "keyevent", _map[button]);
+        this.adb("shell", "input", "keyevent", BUTTON_MAP[button]);
     }
     async tap(x, y) {
         this.adb("shell", "input", "tap", `${x}`, `${y}`);
@@ -188,7 +189,7 @@ class AndroidRobot {
 }
 exports.AndroidRobot = AndroidRobot;
 const getConnectedDevices = () => {
-    return (0, child_process_1.execSync)(`adb devices`)
+    return (0, child_process_1.execFileSync)(getAdbPath(), ["devices"])
         .toString()
         .split("\n")
         .filter(line => !line.startsWith("List of devices attached"))

package/lib/ios.js

@@ -1,79 +1,158 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.IosManager = exports.IosRobot = void 0;
+const path_1 = __importDefault(require("path"));
+const os_1 = require("os");
+const crypto_1 = require("crypto");
 const fs_1 = require("fs");
 const child_process_1 = require("child_process");
+const net_1 = require("net");
 const webdriver_agent_1 = require("./webdriver-agent");
+const robot_1 = require("./robot");
+const WDA_PORT = 8100;
+const IOS_TUNNEL_PORT = 60105;
+const getGoIosPath = () => {
+    if (process.env.GO_IOS_PATH) {
+        return process.env.GO_IOS_PATH;
+    }
+    // fallback to go-ios in PATH via `npm install -g go-ios`
+    return "ios";
+};
 class IosRobot {
     deviceId;
-    wda;
     constructor(deviceId) {
         this.deviceId = deviceId;
-        this.wda = new webdriver_agent_1.WebDriverAgent("localhost", 8100);
+    }
+    isListeningOnPort(port) {
+        return new Promise((resolve, reject) => {
+            const client = new net_1.Socket();
+            client.connect(port, "localhost", () => {
+                client.destroy();
+                resolve(true);
+            });
+            client.on("error", (err) => {
+                resolve(false);
+            });
+        });
+    }
+    async isTunnelRunning() {
+        return await this.isListeningOnPort(IOS_TUNNEL_PORT);
+    }
+    async isWdaForwardRunning() {
+        return await this.isListeningOnPort(WDA_PORT);
+    }
+    async assertTunnelRunning() {
+        if (await this.isTunnelRequired()) {
+            if (!(await this.isTunnelRunning())) {
+                throw new robot_1.ActionableError("iOS tunnel is not running, please see https://github.com/mobile-next/mobile-mcp/wiki/");
+            }
+        }
+    }
+    async wda() {
+        await this.assertTunnelRunning();
+        if (!(await this.isWdaForwardRunning())) {
+            throw new robot_1.ActionableError("Port forwarding to WebDriverAgent is not running (tunnel okay), please see https://github.com/mobile-next/mobile-mcp/wiki/");
+        }
+        const wda = new webdriver_agent_1.WebDriverAgent("localhost", WDA_PORT);
+        if (!(await wda.isRunning())) {
+            throw new robot_1.ActionableError("WebDriverAgent is not running on device (tunnel okay, port forwarding okay), please see https://github.com/mobile-next/mobile-mcp/wiki/");
+        }
+        return wda;
     }
     async ios(...args) {
-        return (0, child_process_1.execFileSync)("ios", ["--udid", this.deviceId, ...args], {}).toString();
+        return (0, child_process_1.execFileSync)(getGoIosPath(), ["--udid", this.deviceId, ...args], {}).toString();
+    }
+    async getIosVersion() {
+        const output = await this.ios("info");
+        const json = JSON.parse(output);
+        return json.ProductVersion;
+    }
+    async isTunnelRequired() {
+        const version = await this.getIosVersion();
+        const args = version.split(".");
+        return parseInt(args[0], 10) >= 17;
     }
     async getScreenSize() {
-        return await this.wda.getScreenSize();
+        const wda = await this.wda();
+        return await wda.getScreenSize();
     }
-    swipe(direction) {
-        return Promise.resolve();
+    async swipe(direction) {
+        const wda = await this.wda();
+        await wda.swipe(direction);
     }
     async listApps() {
+        await this.assertTunnelRunning();
         const output = await this.ios("apps", "--all", "--list");
         return output
             .split("\n")
-            .map(line => line.split(" ")[0]);
+            .map(line => {
+            const [packageName, appName] = line.split(" ");
+            return {
+                packageName,
+                appName,
+            };
+        });
     }
     async launchApp(packageName) {
+        await this.assertTunnelRunning();
         await this.ios("launch", packageName);
     }
     async terminateApp(packageName) {
+        await this.assertTunnelRunning();
         await this.ios("kill", packageName);
     }
     async openUrl(url) {
-        await this.wda.withinSession(async (sessionUrl) => {
-            await fetch(`${sessionUrl}/url`, {
-                method: "POST",
-                body: JSON.stringify({ url }),
-            });
-        });
+        const wda = await this.wda();
+        await wda.openUrl(url);
     }
     async sendKeys(text) {
-        await this.wda.sendKeys(text);
+        const wda = await this.wda();
+        await wda.sendKeys(text);
     }
     async pressButton(button) {
-        await this.wda.pressButton(button);
+        const wda = await this.wda();
+        await wda.pressButton(button);
     }
     async tap(x, y) {
-        await this.wda.tap(x, y);
+        const wda = await this.wda();
+        await wda.tap(x, y);
     }
     async getElementsOnScreen() {
-        return await this.wda.getElementsOnScreen();
+        const wda = await this.wda();
+        return await wda.getElementsOnScreen();
     }
     async getScreenshot() {
-        await this.ios("screenshot", "--output", "screenshot.png");
-        const buffer = (0, fs_1.readFileSync)("screenshot.png");
-        (0, fs_1.unlinkSync)("screenshot.png");
+        await this.assertTunnelRunning();
+        const tmpFilename = path_1.default.join((0, os_1.tmpdir)(), `screenshot-${(0, crypto_1.randomBytes)(8).toString("hex")}.png`);
+        await this.ios("screenshot", "--output", tmpFilename);
+        const buffer = (0, fs_1.readFileSync)(tmpFilename);
+        (0, fs_1.unlinkSync)(tmpFilename);
         return buffer;
     }
 }
 exports.IosRobot = IosRobot;
 class IosManager {
+    async isGoIosInstalled() {
+        try {
+            const output = (0, child_process_1.execFileSync)(getGoIosPath(), ["version"], { stdio: ["pipe", "pipe", "ignore"] }).toString();
+            const json = JSON.parse(output);
+            return json.version !== undefined && (json.version.startsWith("v") || json.version === "local-build");
+        }
+        catch (error) {
+            return false;
+        }
+    }
     async listDevices() {
-        const output = (0, child_process_1.execSync)("ios list").toString();
+        if (!(await this.isGoIosInstalled())) {
+            console.error("go-ios is not installed, no physical iOS devices can be detected");
+            return [];
+        }
+        const output = (0, child_process_1.execFileSync)(getGoIosPath(), ["list"]).toString();
         const json = JSON.parse(output);
         return json.deviceList;
     }
 }
 exports.IosManager = IosManager;
-async function main() {
-    const ios = new IosRobot("4C07ED7E-AE81-412E-8AA9-1061EED59DFA");
-    const before = +new Date();
-    console.dir(await ios.getElementsOnScreen(), { depth: null });
-    const after = +new Date();
-    console.log(`Time taken: ${after - before}ms`);
-    // await ios.pressButton("VOLUME_UP");
-}
-main().then();

package/lib/iphone-simulator.js

@@ -3,21 +3,35 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.SimctlManager = exports.Simctl = void 0;
 const child_process_1 = require("child_process");
 const webdriver_agent_1 = require("./webdriver-agent");
+const robot_1 = require("./robot");
+const TIMEOUT = 30000;
+const WDA_PORT = 8100;
+const MAX_BUFFER_SIZE = 1024 * 1024 * 4;
 class Simctl {
     simulatorUuid;
-    webDriverAgent;
     constructor(simulatorUuid) {
         this.simulatorUuid = simulatorUuid;
-        this.webDriverAgent = new webdriver_agent_1.WebDriverAgent("localhost", 8100);
+    }
+    async wda() {
+        const wda = new webdriver_agent_1.WebDriverAgent("localhost", WDA_PORT);
+        if (!(await wda.isRunning())) {
+            throw new robot_1.ActionableError("WebDriverAgent is not running on device (tunnel okay, port forwarding okay), please see https://github.com/mobile-next/mobile-mcp/wiki/");
+        }
+        return wda;
     }
     simctl(...args) {
-        return (0, child_process_1.execFileSync)("xcrun", ["simctl", ...args], { maxBuffer: 1024 * 1024 * 4 });
+        return (0, child_process_1.execFileSync)("xcrun", ["simctl", ...args], {
+            timeout: TIMEOUT,
+            maxBuffer: MAX_BUFFER_SIZE,
+        });
     }
     async getScreenshot() {
         return this.simctl("io", this.simulatorUuid, "screenshot", "-");
     }
     async openUrl(url) {
-        this.simctl("openurl", this.simulatorUuid, url);
+        const wda = await this.wda();
+        await wda.openUrl(url);
+        // alternative: this.simctl("openurl", this.simulatorUuid, url);
     }
     async launchApp(packageName) {
         this.simctl("launch", this.simulatorUuid, packageName);
@@ -35,10 +49,7 @@ class Simctl {
         while ((appMatch = appRegex.exec(cleanText)) !== null) {
             // const bundleId = appMatch[1];
             const appContent = appMatch[2];
-            const appInfo = {
-                GroupContainers: {},
-                SBAppTags: []
-            };
+            const appInfo = {};
             // parse simple key-value pairs
             const keyValueRegex = /\s+(\w+)\s+=\s+([^;]+);/g;
             let keyValueMatch;
@@ -53,30 +64,6 @@ class Simctl {
                     appInfo[key] = value;
                 }
             }
-            // parse GroupContainers
-            const groupContainersMatch = appContent.match(/GroupContainers\s+=\s+\{([^}]+)\};/);
-            if (groupContainersMatch) {
-                const groupContainersContent = groupContainersMatch[1];
-                const groupRegex = /"([^"]+)"\s+=\s+"([^"]+)"/g;
-                let groupMatch;
-                while ((groupMatch = groupRegex.exec(groupContainersContent)) !== null) {
-                    const groupId = groupMatch[1];
-                    const groupPath = groupMatch[2];
-                    appInfo.GroupContainers[groupId] = groupPath;
-                }
-            }
-            // parse SBAppTags
-            const sbAppTagsMatch = appContent.match(/SBAppTags\s+=\s+\(\s*(.*?)\s*\);/);
-            if (sbAppTagsMatch) {
-                const tagsContent = sbAppTagsMatch[1].trim();
-                if (tagsContent) {
-                    const tagRegex = /"([^"]+)"/g;
-                    let tagMatch;
-                    while ((tagMatch = tagRegex.exec(tagsContent)) !== null) {
-                        appInfo.SBAppTags.push(tagMatch[1]);
-                    }
-                }
-            }
             result.push(appInfo);
         }
         return result;
@@ -84,83 +71,50 @@ class Simctl {
     async listApps() {
         const text = this.simctl("listapps", this.simulatorUuid).toString();
         const apps = this.parseIOSAppData(text);
-        return apps.map(app => app.CFBundleIdentifier);
+        return apps.map(app => ({
+            packageName: app.CFBundleIdentifier,
+            appName: app.CFBundleDisplayName,
+        }));
     }
     async getScreenSize() {
-        return this.webDriverAgent.getScreenSize();
+        const wda = await this.wda();
+        return wda.getScreenSize();
     }
     async sendKeys(keys) {
-        return this.webDriverAgent.sendKeys(keys);
+        const wda = await this.wda();
+        return wda.sendKeys(keys);
     }
     async swipe(direction) {
-        await this.webDriverAgent.withinSession(async (sessionUrl) => {
-            const x0 = 200;
-            let y0 = 600;
-            const x1 = 200;
-            let y1 = 200;
-            if (direction === "up") {
-                const tmp = y0;
-                y0 = y1;
-                y1 = tmp;
-            }
-            const url = `${sessionUrl}/actions`;
-            await fetch(url, {
-                method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify({
-                    actions: [
-                        {
-                            type: "pointer",
-                            id: "finger1",
-                            parameters: { pointerType: "touch" },
-                            actions: [
-                                { type: "pointerMove", duration: 0, x: x0, y: y0 },
-                                { type: "pointerDown", button: 0 },
-                                { type: "pointerMove", duration: 0, x: x1, y: y1 },
-                                { type: "pause", duration: 1000 },
-                                { type: "pointerUp", button: 0 }
-                            ]
-                        }
-                    ]
-                }),
-            });
-        });
+        const wda = await this.wda();
+        return wda.swipe(direction);
     }
     async tap(x, y) {
-        await this.webDriverAgent.tap(x, y);
+        const wda = await this.wda();
+        return wda.tap(x, y);
     }
     async pressButton(button) {
-        await this.webDriverAgent.pressButton(button);
+        const wda = await this.wda();
+        return wda.pressButton(button);
     }
     async getElementsOnScreen() {
-        return await this.webDriverAgent.getElementsOnScreen();
+        const wda = await this.wda();
+        return wda.getElementsOnScreen();
     }
 }
 exports.Simctl = Simctl;
 class SimctlManager {
-    parseSimulator(line) {
-        // extract device name and UUID from the line
-        const match = line.match(/(.*?)\s+\(([\w-]+)\)\s+\((\w+)\)/);
-        if (!match) {
-            return null;
-        }
-        const deviceName = match[1].trim();
-        const deviceUuid = match[2];
-        const deviceState = match[3];
-        return {
-            name: deviceName,
-            uuid: deviceUuid,
-            state: deviceState,
-        };
-    }
     listSimulators() {
-        return (0, child_process_1.execSync)(`xcrun simctl list devices`)
-            .toString()
-            .split("\n")
-            .map(line => this.parseSimulator(line))
-            .filter(simulator => simulator !== null);
+        const text = (0, child_process_1.execFileSync)("xcrun", ["simctl", "list", "devices", "-j"]).toString();
+        const json = JSON.parse(text);
+        return Object.values(json.devices).flatMap(device => {
+            return device.map(d => {
+                return {
+                    name: d.name,
+                    uuid: d.udid,
+                    state: d.state,
+                };
+            });
+        });
     }
     listBootedSimulators() {
         return this.listSimulators()

package/lib/robot.js

@@ -1,2 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActionableError = void 0;
+class ActionableError extends Error {
+    constructor(message) {
+        super(message);
+    }
+}
+exports.ActionableError = ActionableError;

package/lib/server.js

@@ -9,6 +9,7 @@ const zod_1 = require("zod");
 const sharp_1 = __importDefault(require("sharp"));
 const logger_1 = require("./logger");
 const android_1 = require("./android");
+const robot_1 = require("./robot");
 const iphone_simulator_1 = require("./iphone-simulator");
 const ios_1 = require("./ios");
 const getAgentVersion = () => {
@@ -35,18 +36,31 @@ const createMcpServer = () => {
                 };
             }
             catch (error) {
-                (0, logger_1.trace)(`Tool '${description}' failed: ${error.message} stack: ${error.stack}`);
-                return {
-                    content: [{ type: "text", text: `Error: ${error.message}` }],
-                    isError: true,
-                };
+                if (error instanceof robot_1.ActionableError) {
+                    return {
+                        content: [{ type: "text", text: `${error.message}. Please fix the issue and try again.` }],
+                    };
+                }
+                else {
+                    // a real exception
+                    (0, logger_1.trace)(`Tool '${description}' failed: ${error.message} stack: ${error.stack}`);
+                    return {
+                        content: [{ type: "text", text: `Error: ${error.message}` }],
+                        isError: true,
+                    };
+                }
             }
         };
         server.tool(name, description, paramsSchema, args => wrappedCb(args));
     };
     let robot;
     const simulatorManager = new iphone_simulator_1.SimctlManager();
-    tool("list_available_devices", "List all available devices. This includes both physical devices and simulators. If there is more than one device returned, you need to let the user select one of them.", {}, async ({}) => {
+    const requireRobot = () => {
+        if (!robot) {
+            throw new robot_1.ActionableError("No device selected. Use the mobile_use_device tool to select a device.");
+        }
+    };
+    tool("mobile_list_available_devices", "List all available devices. This includes both physical devices and simulators. If there is more than one device returned, you need to let the user select one of them.", {}, async ({}) => {
         const iosManager = new ios_1.IosManager();
         const devices = await simulatorManager.listBootedSimulators();
         const simulatorNames = devices.map(d => d.name);
@@ -54,11 +68,10 @@ const createMcpServer = () => {
         const iosDevices = await iosManager.listDevices();
         return `Found these iOS simulators: [${simulatorNames.join(".")}], iOS devices: [${iosDevices.join(",")}] and Android devices: [${androidDevices.join(",")}]`;
     });
-    tool("use_device", "Select a device to use. This can be a simulator or an Android device. Use the list_available_devices tool to get a list of available devices.", {
+    tool("mobile_use_device", "Select a device to use. This can be a simulator or an Android device. Use the list_available_devices tool to get a list of available devices.", {
         device: zod_1.z.string().describe("The name of the device to select"),
         deviceType: zod_1.z.enum(["simulator", "ios", "android"]).describe("The type of device to select"),
     }, async ({ device, deviceType }) => {
-        console.log(device, deviceType);
         switch (deviceType) {
             case "simulator":
                 robot = simulatorManager.getSimulator(device);
@@ -73,112 +86,84 @@ const createMcpServer = () => {
         return `Selected device: ${device} (${deviceType})`;
     });
     tool("mobile_list_apps", "List all the installed apps on the device", {}, async ({}) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
+        requireRobot();
         const result = await robot.listApps();
-        return `Found these packages on device: ${result.join(",")}`;
+        return `Found these apps on device: ${result.map(app => `${app.appName} (${app.packageName})`).join(", ")}`;
     });
     tool("mobile_launch_app", "Launch an app on mobile device. Use this to open a specific app. You can find the package name of the app by calling list_apps_on_device.", {
         packageName: zod_1.z.string().describe("The package name of the app to launch"),
     }, async ({ packageName }) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
+        requireRobot();
         await robot.launchApp(packageName);
         return `Launched app ${packageName}`;
     });
     tool("mobile_terminate_app", "Stop and terminate an app on mobile device", {
         packageName: zod_1.z.string().describe("The package name of the app to terminate"),
     }, async ({ packageName }) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
+        requireRobot();
         await robot.terminateApp(packageName);
         return `Terminated app ${packageName}`;
     });
     tool("mobile_get_screen_size", "Get the screen size of the mobile device in pixels", {}, async ({}) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
+        requireRobot();
         const screenSize = await robot.getScreenSize();
         return `Screen size is ${screenSize.width}x${screenSize.height} pixels`;
     });
     tool("mobile_click_on_screen_at_coordinates", "Click on the screen at given x,y coordinates", {
-        x: zod_1.z.number().describe("The x coordinate to click between 0 and 1"),
-        y: zod_1.z.number().describe("The y coordinate to click between 0 and 1"),
+        x: zod_1.z.number().describe("The x coordinate to click on the screen, in pixels"),
+        y: zod_1.z.number().describe("The y coordinate to click on the screen, in pixels"),
     }, async ({ x, y }) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
-        const screenSize = await robot.getScreenSize();
-        const x0 = Math.floor(screenSize.width * x);
-        const y0 = Math.floor(screenSize.height * y);
-        await robot.tap(x0, y0);
+        requireRobot();
+        await robot.tap(x, y);
         return `Clicked on screen at coordinates: ${x}, ${y}`;
     });
     tool("mobile_list_elements_on_screen", "List elements on screen and their coordinates, with display text or accessibility label. Do not cache this result.", {}, async ({}) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
-        const screenSize = await robot.getScreenSize();
+        requireRobot();
         const elements = await robot.getElementsOnScreen();
-        const result = [];
-        for (let i = 0; i < elements.length; i++) {
-            elements[i].rect.x0 = elements[i].rect.x0 / screenSize.width;
-            elements[i].rect.y0 = elements[i].rect.y0 / screenSize.height;
-            elements[i].rect.x1 = elements[i].rect.x1 / screenSize.width;
-            elements[i].rect.y1 = elements[i].rect.y1 / screenSize.height;
-            result.push({
-                text: elements[i].label,
-                coordinates: {
-                    x: (elements[i].rect.x0 + elements[i].rect.x1) / 2,
-                    y: (elements[i].rect.y0 + elements[i].rect.y1) / 2,
-                }
-            });
-        }
+        const result = elements.map(element => {
+            const x = Number((element.rect.x + element.rect.width / 2)).toFixed(3);
+            const y = Number((element.rect.y + element.rect.height / 2)).toFixed(3);
+            return {
+                text: element.label,
+                coordinates: { x, y }
+            };
+        });
         return `Found these elements on screen: ${JSON.stringify(result)}`;
     });
     tool("mobile_press_button", "Press a button on device", {
-        button: zod_1.z.string().describe("The button to press. Supported buttons: BACK, HOME, VOLUME_UP, VOLUME_DOWN, ENTER"),
+        button: zod_1.z.string().describe("The button to press. Supported buttons: BACK (android only), HOME, VOLUME_UP, VOLUME_DOWN, ENTER"),
     }, async ({ button }) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
-        robot.pressButton(button);
+        requireRobot();
+        await robot.pressButton(button);
         return `Pressed the button: ${button}`;
     });
     tool("mobile_open_url", "Open a URL in browser on device", {
         url: zod_1.z.string().describe("The URL to open"),
     }, async ({ url }) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
-        robot.openUrl(url);
+        requireRobot();
+        await robot.openUrl(url);
         return `Opened URL: ${url}`;
     });
     tool("swipe_on_screen", "Swipe on the screen", {
         direction: zod_1.z.enum(["up", "down"]).describe("The direction to swipe"),
     }, async ({ direction }) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
-        robot.swipe(direction);
+        requireRobot();
+        await robot.swipe(direction);
         return `Swiped ${direction} on screen`;
     });
     tool("mobile_type_keys", "Type text into the focused element", {
         text: zod_1.z.string().describe("The text to type"),
-    }, async ({ text }) => {
-        if (!robot) {
-            throw new Error("No device selected");
+        submit: zod_1.z.boolean().describe("Whether to submit the text. If true, the text will be submitted as if the user pressed the enter key."),
+    }, async ({ text, submit }) => {
+        requireRobot();
+        await robot.sendKeys(text);
+        if (submit) {
+            await robot.pressButton("ENTER");
         }
-        robot.sendKeys(text);
         return `Typed text: ${text}`;
     });
     server.tool("mobile_take_screenshot", "Take a screenshot of the mobile device. Use this to understand what's on screen, if you need to press an element that is available through view hierarchy then you must list elements on screen instead. Do not cache this result.", {}, async ({}) => {
-        if (!robot) {
-            throw new Error("No device selected");
-        }
+        requireRobot();
         try {
             const screenshot = await robot.getScreenshot();
             // Scale down the screenshot by 50%

package/lib/webdriver-agent.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WebDriverAgent = void 0;
+const robot_1 = require("./robot");
 class WebDriverAgent {
     host;
     port;
@@ -8,6 +9,17 @@ class WebDriverAgent {
         this.host = host;
         this.port = port;
     }
+    async isRunning() {
+        const url = `http://${this.host}:${this.port}/status`;
+        try {
+            const response = await fetch(url);
+            return response.status === 200;
+        }
+        catch (error) {
+            console.error(`Failed to connect to WebDriverAgent: ${error}`);
+            return false;
+        }
+    }
     async createSession() {
         const url = `http://${this.host}:${this.port}/session`;
         const response = await fetch(url, {
@@ -38,8 +50,9 @@ class WebDriverAgent {
             const response = await fetch(url);
             const json = await response.json();
             return {
-                width: json.value.screenSize.width * json.value.scale,
-                height: json.value.screenSize.height * json.value.scale,
+                width: json.value.screenSize.width,
+                height: json.value.screenSize.height,
+                scale: json.value.scale || 1,
             };
         });
     }
@@ -67,7 +80,7 @@ class WebDriverAgent {
         }
         // Type assertion to check if button is a key of _map
         if (!(button in _map)) {
-            throw new Error(`Button "${button}" is not supported`);
+            throw new robot_1.ActionableError(`Button "${button}" is not supported`);
         }
         await this.withinSession(async (sessionUrl) => {
             const url = `${sessionUrl}/wda/pressButton`;
@@ -109,20 +122,29 @@ class WebDriverAgent {
             });
         });
     }
+    isVisible(rect) {
+        return rect.x >= 0 && rect.y >= 0;
+    }
     filterSourceElements(source) {
         const output = [];
-        if (["TextField", "Button", "Switch"].includes(source.type)) {
-            output.push({
-                type: source.type,
-                label: source.label,
-                name: source.name,
-                rect: {
-                    x0: source.rect.x,
-                    y0: source.rect.y,
-                    x1: source.rect.x + source.rect.width,
-                    y1: source.rect.y + source.rect.height,
-                },
-            });
+        const acceptedTypes = ["TextField", "Button", "Switch", "Icon", "SearchField"];
+        if (acceptedTypes.includes(source.type)) {
+            if (source.isVisible === "1" && this.isVisible(source.rect)) {
+                if (source.label !== null || source.name !== null) {
+                    output.push({
+                        type: source.type,
+                        label: source.label,
+                        name: source.name,
+                        value: source.value,
+                        rect: {
+                            x: source.rect.x,
+                            y: source.rect.y,
+                            width: source.rect.width,
+                            height: source.rect.height,
+                        },
+                    });
+                }
+            }
         }
         if (source.children) {
             for (const child of source.children) {
@@ -141,5 +163,49 @@ class WebDriverAgent {
         const source = await this.getPageSource();
         return this.filterSourceElements(source.value);
     }
+    async openUrl(url) {
+        await this.withinSession(async (sessionUrl) => {
+            await fetch(`${sessionUrl}/url`, {
+                method: "POST",
+                body: JSON.stringify({ url }),
+            });
+        });
+    }
+    async swipe(direction) {
+        await this.withinSession(async (sessionUrl) => {
+            const x0 = 200;
+            let y0 = 600;
+            const x1 = 200;
+            let y1 = 200;
+            if (direction === "up") {
+                const tmp = y0;
+                y0 = y1;
+                y1 = tmp;
+            }
+            const url = `${sessionUrl}/actions`;
+            await fetch(url, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    actions: [
+                        {
+                            type: "pointer",
+                            id: "finger1",
+                            parameters: { pointerType: "touch" },
+                            actions: [
+                                { type: "pointerMove", duration: 0, x: x0, y: y0 },
+                                { type: "pointerDown", button: 0 },
+                                { type: "pointerMove", duration: 0, x: x1, y: y1 },
+                                { type: "pause", duration: 1000 },
+                                { type: "pointerUp", button: 0 }
+                            ]
+                        }
+                    ]
+                }),
+            });
+        });
+    }
 }
 exports.WebDriverAgent = WebDriverAgent;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mobilenext/mobile-mcp",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "description": "Mobile MCP",
   "repository": {
     "type": "git",
@@ -13,7 +13,7 @@
   "scripts": {
     "build": "tsc && chmod +x lib/index.js",
     "lint": "eslint .",
-    "test": "mocha --require ts-node/register test/*.ts",
+    "test": "nyc mocha --require ts-node/register test/*.ts",
     "watch": "tsc --watch",
     "clean": "rm -rf lib",
     "prepare": "husky"
@@ -24,6 +24,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.6.1",
     "fast-xml-parser": "^5.0.9",
+    "nyc": "^17.1.0",
     "sharp": "^0.33.5",
     "zod-to-json-schema": "^3.24.4"
   },