mobai-mcp 1.0.2 → 1.1.0

package/dist/index.js

@@ -198,11 +198,15 @@ const TOOLS = [
                 },
                 verbose: {
                     type: "boolean",
-                    description: "Include full element coordinates (default: false)",
+                    description: "Include detailed elements array with bounds (default: false)",
                 },
                 only_visible: {
                     type: "boolean",
-                    description: "Filter to visible elements only (default: true)",
+                    description: "Filter to only visible elements (default: true)",
+                },
+                include_keyboard: {
+                    type: "boolean",
+                    description: "Include keyboard elements in the tree (default: false). Useful for interacting with on-screen keyboards.",
                 },
             },
             required: ["device_id"],
@@ -332,6 +336,20 @@ const TOOLS = [
             required: ["device_id"],
         },
     },
+    {
+        name: "get_ocr",
+        description: "Perform OCR text recognition on the current screen (iOS only). Returns detected text with screen coordinates for tapping (already adjusted for tapping).",
+        inputSchema: {
+            type: "object",
+            properties: {
+                device_id: {
+                    type: "string",
+                    description: "Device ID",
+                },
+            },
+            required: ["device_id"],
+        },
+    },
     {
         name: "execute_dsl",
         description: `Execute a batch of automation steps using the DSL (Domain Specific Language).
@@ -368,6 +386,7 @@ Example DSL script:
                         steps: {
                             type: "array",
                             description: "Array of action steps",
+                            items: { type: "object" },
                         },
                         on_fail: {
                             type: "object",
@@ -579,6 +598,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
                     params.set("verbose", "true");
                 if (args?.only_visible === false)
                     params.set("onlyVisible", "false");
+                if (args?.include_keyboard)
+                    params.set("includeKeyboard", "true");
                 const queryString = params.toString();
                 const endpoint = `/devices/${args?.device_id}/ui-tree${queryString ? `?${queryString}` : ""}`;
                 result = await makeRequest("GET", endpoint);
@@ -618,6 +639,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             case "list_apps":
                 result = await makeRequest("GET", `/devices/${args?.device_id}/apps`);
                 break;
+            case "get_ocr":
+                result = await makeRequest("GET", `/devices/${args?.device_id}/ocr`);
+                break;
             case "execute_dsl":
                 result = await makeRequest("POST", `/devices/${args?.device_id}/dsl/execute`, args?.script, 300000 // 5 minutes
                 );
@@ -784,6 +808,7 @@ const API_REFERENCE = `# MobAI API Reference
 | /devices/{id}/screenshot | GET | Capture screenshot (saved to /tmp/mobai/screenshots/) |
 | /devices/{id}/ui-tree | GET | Get UI accessibility tree |
 | /devices/{id}/apps | GET | List installed apps |
+| /devices/{id}/ocr | GET | OCR text recognition (iOS only) |
 
 ## Bridge Control
 
@@ -888,7 +913,7 @@ The DSL (Domain Specific Language) enables batch execution of multiple automatio
 
 | Action | Description | Key Fields |
 |--------|-------------|------------|
-| observe | Get UI tree/screenshot | context, include (ui_tree, screenshot, installed_apps) |
+| observe | Get UI tree/screenshot/OCR | context, include (ui_tree, screenshot, installed_apps, ocr) |
 | tap | Tap element | predicate or coords |
 | type | Type text | text, predicate (if keyboard not open), dismiss_keyboard (default: false) |
 | press_key | Press keyboard key | key (return, tab, delete, etc.), context (optional: "web") |
@@ -955,6 +980,16 @@ Note: \`predicate\` is required if keyboard is not already open. Use \`dismiss_k
 - \`abort\`: Stop on failure (default)
 - \`skip\`: Skip failed step, continue
 - \`retry\`: Retry with delay
+
+## OCR (iOS only)
+
+Use \`include: ["ocr"]\` in observe to get text recognition when UI tree is empty:
+
+\`\`\`json
+{"action": "observe", "context": "native", "include": ["ocr"]}
+\`\`\`
+
+Returns text with coordinates for tapping (already adjusted for tapping).
 `;
 const NATIVE_RUNNER_GUIDE = `# Native App Automation Guide
 
@@ -1025,6 +1060,11 @@ The \`type\` action requires either:
 {"action": "type", "text": "username", "predicate": {"type": "input", "label": "Username"}}
 \`\`\`
 
+### Dismissing Keyboard
+- Use \`press_key: return\` to submit and close the keyboard
+- If submit is not desired, look for a "Close", "Cancel", "Done" or "Back" button in the UI tree and tap it
+- On Android, \`press_key: back\` also dismisses the keyboard
+
 ## Common Patterns
 
 ### Open App and Navigate
@@ -1096,6 +1136,7 @@ The \`type\` action requires either:
 - **Add delays after navigation** - Apps need time to render
 - **Use retry strategy** - Transient failures are common
 - **Use press_key for form navigation** - Tab between fields, Return to submit
+- **Use OCR for system dialogs (iOS)** - When UI tree is empty, use \`include: ["ocr"]\`
 `;
 const WEB_RUNNER_GUIDE = `# Web Automation Guide
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobai-mcp",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "MCP server for MobAI - AI-powered mobile device automation",
   "type": "module",
   "main": "dist/index.js",