mobai-mcp 1.0.3 → 1.2.0

package/dist/index.js

@@ -198,11 +198,15 @@ const TOOLS = [
                 },
                 verbose: {
                     type: "boolean",
-                    description: "Include full element coordinates (default: false)",
+                    description: "Include detailed elements array with bounds (default: false)",
                 },
                 only_visible: {
                     type: "boolean",
-                    description: "Filter to visible elements only (default: true)",
+                    description: "Filter to only visible elements (default: true)",
+                },
+                include_keyboard: {
+                    type: "boolean",
+                    description: "Include keyboard elements in the tree (default: false). Useful for interacting with on-screen keyboards.",
                 },
             },
             required: ["device_id"],
@@ -332,6 +336,20 @@ const TOOLS = [
             required: ["device_id"],
         },
     },
+    {
+        name: "get_ocr",
+        description: "Perform OCR text recognition on the current screen (iOS only). Returns detected text with screen coordinates for tapping (already adjusted for tapping).",
+        inputSchema: {
+            type: "object",
+            properties: {
+                device_id: {
+                    type: "string",
+                    description: "Device ID",
+                },
+            },
+            required: ["device_id"],
+        },
+    },
     {
         name: "execute_dsl",
         description: `Execute a batch of automation steps using the DSL (Domain Specific Language).
@@ -580,6 +598,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     params.set("verbose", "true");
                 if (args?.only_visible === false)
                     params.set("onlyVisible", "false");
+                if (args?.include_keyboard)
+                    params.set("includeKeyboard", "true");
                 const queryString = params.toString();
                 const endpoint = `/devices/${args?.device_id}/ui-tree${queryString ? `?${queryString}` : ""}`;
                 result = await makeRequest("GET", endpoint);
@@ -619,6 +639,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             case "list_apps":
                 result = await makeRequest("GET", `/devices/${args?.device_id}/apps`);
                 break;
+            case "get_ocr":
+                result = await makeRequest("GET", `/devices/${args?.device_id}/ocr`);
+                break;
             case "execute_dsl":
                 result = await makeRequest("POST", `/devices/${args?.device_id}/dsl/execute`, args?.script, 300000 // 5 minutes
                 );
@@ -785,6 +808,7 @@ const API_REFERENCE = `# MobAI API Reference
 | /devices/{id}/screenshot | GET | Capture screenshot (saved to /tmp/mobai/screenshots/) |
 | /devices/{id}/ui-tree | GET | Get UI accessibility tree |
 | /devices/{id}/apps | GET | List installed apps |
+| /devices/{id}/ocr | GET | OCR text recognition (iOS only) |
 
 ## Bridge Control
 
@@ -889,7 +913,7 @@ The DSL (Domain Specific Language) enables batch execution of multiple automatio
 
 | Action | Description | Key Fields |
 |--------|-------------|------------|
-| observe | Get UI tree/screenshot | context, include (ui_tree, screenshot, installed_apps) |
+| observe | Get UI tree/screenshot/OCR | context, include (ui_tree, screenshot, installed_apps, ocr) |
 | tap | Tap element | predicate or coords |
 | type | Type text | text, predicate (if keyboard not open), dismiss_keyboard (default: false) |
 | press_key | Press keyboard key | key (return, tab, delete, etc.), context (optional: "web") |
@@ -956,6 +980,16 @@ Note: \`predicate\` is required if keyboard is not already open. Use \`dismiss_k
 - \`abort\`: Stop on failure (default)
 - \`skip\`: Skip failed step, continue
 - \`retry\`: Retry with delay
+
+## OCR (iOS only)
+
+Use \`include: ["ocr"]\` in observe to get text recognition when UI tree is empty:
+
+\`\`\`json
+{"action": "observe", "context": "native", "include": ["ocr"]}
+\`\`\`
+
+Returns text with coordinates for tapping (already adjusted for tapping).
 `;
 const NATIVE_RUNNER_GUIDE = `# Native App Automation Guide
 
@@ -1026,6 +1060,11 @@ The \`type\` action requires either:
 {"action": "type", "text": "username", "predicate": {"type": "input", "label": "Username"}}
 \`\`\`
 
+### Dismissing Keyboard
+- Use \`press_key: return\` to submit and close the keyboard
+- If submit is not desired, look for a "Close", "Cancel", "Done" or "Back" button in the UI tree and tap it
+- On Android, \`press_key: back\` also dismisses the keyboard
+
 ## Common Patterns
 
 ### Open App and Navigate
@@ -1097,6 +1136,7 @@ The \`type\` action requires either:
 - **Add delays after navigation** - Apps need time to render
 - **Use retry strategy** - Transient failures are common
 - **Use press_key for form navigation** - Tab between fields, Return to submit
+- **Use OCR for system dialogs (iOS)** - When UI tree is empty, use \`include: ["ocr"]\`
 `;
 const WEB_RUNNER_GUIDE = `# Web Automation Guide
 

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "mobai-mcp",
-  "version": "1.0.3",
+  "version": "1.2.0",
+  "mcpName": "io.github.mobai-app/mobai-mcp",
   "description": "MCP server for MobAI - AI-powered mobile device automation",
   "type": "module",
   "main": "dist/index.js",
@@ -9,6 +10,7 @@
   },
   "files": [
     "dist",
+    "server.json",
     "README.md",
     "LICENSE"
   ],
@@ -42,7 +44,7 @@
     "url": "https://github.com/MobAI-App/mobai-mcp/issues"
   },
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.0.0"
+    "@modelcontextprotocol/sdk": "^1.25.3"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",

package/server.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
+  "name": "io.github.mobai-app/mobai-mcp",
+  "description": "AI-powered mobile device automation for Android and iOS devices, emulators, and simulators",
+  "repository": {
+    "url": "https://github.com/MobAI-App/mobai-mcp",
+    "source": "github"
+  },
+  "version": "1.2.0",
+  "packages": [
+    {
+      "registryType": "npm",
+      "identifier": "mobai-mcp",
+      "version": "1.2.0",
+      "transport": {
+        "type": "stdio"
+      }
+    }
+  ]
+}