mobai-mcp 2.3.1 → 2.4.0

package/dist/index.js

@@ -223,6 +223,12 @@ const TOOLS = [
             properties: {
                 device_id: { type: "string", description: "Device ID" },
                 path: { type: "string", description: "Local file path to the app (.apk or .ipa)" },
+                resign: { type: "boolean", description: "Sign the IPA with iloader before installing (iOS only)" },
+                apple_id: { type: "string", description: "Apple ID for signing (optional; used when resign=true)" },
+                password: { type: "string", description: "Apple ID password for signing (optional; uses cached credentials if empty)" },
+                profile_path: { type: "string", description: "Provisioning profile to reuse for an offline re-sign (with cert_path/key_path); falls back to online signing if invalid" },
+                cert_path: { type: "string", description: "Signing certificate PEM to reuse for an offline re-sign" },
+                key_path: { type: "string", description: "Signing private key PEM to reuse for an offline re-sign" },
             },
             required: ["device_id", "path"],
         },
@@ -242,16 +248,62 @@ const TOOLS = [
     // DSL execution
     {
         name: "execute_dsl",
-        description: `Execute a batch of DSL commands on a device. This is the primary tool for all device interaction — tap, type, swipe, observe, launch apps, assertions, web automation, and more.
+        description: `The example below is the full command surface. For richer semantics - per-action defaults, platform notes, retry/failure strategies, observe and scroll guidance, web/OCR caveats - read the MCP resource mobai://reference/device-automation. Read it the first time you hit anything the example doesn't make obvious.
 
-You MUST read the MCP resource mobai://reference/device-automation to learn how to control devices before using this tool.
+Execute a batch of DSL commands on a device. This is the primary tool for all device interaction - tap, type, swipe, observe, launch apps, assertions, web automation, and more.
 
-Input: JSON string with "version": "0.2" and "steps" array. Example:
-{"version":"0.2","steps":[
-  {"action":"open_app","bundle_id":"com.apple.Preferences"},
-  {"action":"tap","predicate":{"text_contains":"Wi-Fi"}},
-  {"action":"wait_for","predicate":{"type":"switch"},"timeout_ms":3000}
-]}`,
+Input: JSON string with "version": "0.2" and a "steps" array. Optional script-wide "params" (declared defaults, substituted as \${name}) and "on_fail" (strategy: abort|skip|retry|replan|require_user; retry also takes max_retries, retry_delay_ms, fallback_strategy). Any step may carry its own "on_fail".
+
+Every action below appears at least once - this is the full command surface (semantics, defaults, and platform notes are in mobai://reference/device-automation):
+{"version":"0.2","params":{"query":"shoes"},"on_fail":{"strategy":"retry","max_retries":2,"retry_delay_ms":1000,"fallback_strategy":{"strategy":"skip"}},"steps":[
+  {"action":"open_app","bundle_id":"com.apple.Preferences","fresh":true,"debug":false},
+  {"action":"kill_app","bundle_id":"com.apple.mobilesafari"},
+  {"action":"open_link","url":"myapp://profile/42"},
+  {"action":"tap","predicate":{"text_contains":"Wi-Fi","type":"button"}},
+  {"action":"tap","coords":{"x":200,"y":640}},
+  {"action":"double_tap","predicate":{"text":"Photo"}},
+  {"action":"two_finger_tap","predicate":{"text":"Map"}},
+  {"action":"long_press","predicate":{"text":"Item"},"duration_ms":1000},
+  {"action":"pinch","predicate":{"text_contains":"Map"},"scale":0.5,"velocity":1.0},
+  {"action":"type","text":"\${query}","predicate":{"type":"input"},"clear_first":true,"dismiss_keyboard":true},
+  {"action":"type_secret","secret_id":"test-account-password","predicate":{"type":"input","text_contains":"Password"}},
+  {"action":"clear","predicate":{"type":"input"}},
+  {"action":"toggle","predicate":{"type":"switch","text_contains":"Wi-Fi"},"state":"on"},
+  {"action":"press_key","key":"enter"},
+  {"action":"swipe","direction":"up","distance":"medium"},
+  {"action":"swipe","from_coords":{"x":200,"y":600},"to_coords":{"x":200,"y":200},"duration_ms":300},
+  {"action":"scroll","direction":"down","to_element":{"predicate":{"text":"Privacy"}},"predicate":{"type":"scrollview"},"max_scrolls":10,"amount":"page"},
+  {"action":"drag","from":{"predicate":{"text":"Item"}},"to_element":{"predicate":{"text":"Trash"}},"press_duration_ms":500,"hold_duration_ms":200,"duration_ms":500},
+  {"action":"drag","from_coords":{"x":100,"y":400},"to_coords":{"x":300,"y":400}},
+  {"action":"drag_path","points":[{"x":100,"y":400,"duration_ms":200},{"x":150,"y":300,"duration_ms":150},{"x":300,"y":500,"duration_ms":300}]},
+  {"action":"navigate","target":"home"},
+  {"action":"set_location","lat":40.7128,"lon":-74.0060},
+  {"action":"reset_location"},
+  {"action":"siri","prompt":"Search YouTube for cat videos"},
+  {"action":"if_exists","predicate":{"text":"Allow"},"then":[{"action":"tap","predicate":{"text":"Allow"}}],"else":[{"action":"tap","predicate":{"text":"Don't Allow"}}]},
+  {"action":"delay","duration_ms":1000},
+  {"action":"wait_for","predicate":{"text":"Welcome"},"timeout_ms":5000,"poll_interval_ms":500},
+  {"action":"assert_exists","predicate":{"text":"Success"},"timeout_ms":3000},
+  {"action":"assert_not_exists","predicate":{"text":"Error"}},
+  {"action":"assert_count","predicate":{"type":"cell"},"count":5},
+  {"action":"assert_screen_changed","threshold_percent":15},
+  {"action":"metrics_start","types":["system_cpu","fps"],"interval_ms":1000,"label":"login_flow","capture_logs":true,"thresholds":{"cpu_high":80,"fps_low":45}},
+  {"action":"metrics_stop","format":"summary"},
+  {"action":"record_start","file_path":"/tmp/mobai/rec"},
+  {"action":"record_stop"},
+  {"action":"select_web_context","url_contains":"google.com"},
+  {"action":"navigate","context":"web","url":"https://example.com"},
+  {"action":"tap","context":"web","predicate":{"css_selector":"button.submit"}},
+  {"action":"type","context":"web","predicate":{"css_selector":"input#email"},"text":"user@example.com","clear_first":true},
+  {"action":"execute_js","context":"web","script":"return document.title","async":false},
+  {"action":"screenshot","file_path":"/tmp/mobai","name":"final_state"},
+  {"action":"wait_for","stable":true,"timeout_ms":3000},
+  {"action":"observe","include":["ui_tree"],"only_visible":true,"filter":{"text_regex":"Settings|Wi-Fi"},"store_as":"end_state"}
+]}
+
+Predicate - object passed in a step's "predicate"/"from"/"to_element". Combine fields (AND); prefer text_contains over exact text. All native fields:
+{"text":"exact","text_contains":"substr","text_starts_with":"prefix","text_regex":"\\d+ items","value":"typed value","value_contains":"typed substr","type":"button|input|switch|text|image|cell|scrollview","accessibility_id":"login_btn","enabled":true,"visible":true,"selected":false,"index":0,"bounds_hint":"top_half|bottom_half|left_half|right_half|center","near":{"text_contains":"Email","direction":"below|above|left|right|any","max_distance":100},"parent_of":{"text":"child label"}}
+Web predicate (context:"web") uses instead: {"css_selector":"button.submit"}.`,
         inputSchema: {
             type: "object",
             properties: {
@@ -437,8 +489,23 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             }
             case "list_apps":
                 return textResult(await doGet(`/devices/${args?.device_id}/apps`));
-            case "install_app":
-                return textResult(await doPost(`/devices/${args?.device_id}/install-app`, { path: args?.path }));
+            case "install_app": {
+                const body = { path: args?.path };
+                if (args?.resign) {
+                    body.resign = true;
+                    if (args?.apple_id)
+                        body.appleId = args.apple_id;
+                    if (args?.password)
+                        body.password = args.password;
+                    if (args?.profile_path)
+                        body.profilePath = args.profile_path;
+                    if (args?.cert_path)
+                        body.certPath = args.cert_path;
+                    if (args?.key_path)
+                        body.keyPath = args.key_path;
+                }
+                return textResult(await doPost(`/devices/${args?.device_id}/install-app`, body));
+            }
             case "uninstall_app":
                 return textResult(await doDelete(`/devices/${args?.device_id}/apps/${encodeURIComponent(args?.bundle_id)}`));
             // DSL execution

package/dist/resources.js

@@ -161,7 +161,6 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
 
   <predicate context="web">
     <field name="css_selector" type="string">CSS selector</field>
-    <field name="xpath" type="string">XPath expression</field>
   </predicate>
 
   <failure-strategy>
@@ -217,6 +216,15 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
     <example>{"action": "type", "text": "Hello", "predicate": {"type": "input"}, "clear_first": true}</example>
   </action>
 
+  <action name="type_secret">
+    Type a stored credential without ever exposing its value. secret_id names a credential saved in the app; its value is resolved from the keychain at run time and typed on-device - it never appears in the script, logs, results, or the model context. Use for passwords/tokens in login steps. Same field-targeting as type.
+    <field name="secret_id" required="yes">Id of a stored credential (see list_secrets)</field>
+    <field name="predicate">Optional target field to focus first</field>
+    <field name="clear_first" type="bool"/>
+    <field name="dismiss_keyboard" type="bool" default="false"/>
+    <example>{"action": "type_secret", "secret_id": "test-account-password", "predicate": {"type": "input", "text_contains": "Password"}}</example>
+  </action>
+
   <action name="clear">
     Clear a field's text without typing. With a predicate, focuses that field first; without one, clears the currently focused field.
     <field name="predicate">Optional target element</field>
@@ -235,11 +243,13 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
 
   <action name="scroll">
     Direction = semantic (where to look), not finger movement.
-    <field name="direction" required="yes">down (look below), up (look above)</field>
+    <field name="direction" required="yes">down (look below), up (look above), left (look left), right (look right). All four directions are supported - use left/right for horizontal scrollviews (carousels, palettes, paging) instead of falling back to raw-coordinate swipe.</field>
+    <field name="predicate" type="Predicate">Targets a specific scrollable container so the gesture lands inside it (e.g. {"type": "scrollview"} or {"parent_of": {"text": "Work"}}). Without it the gesture is applied at screen center - set this when multiple scrollviews share the screen (e.g. a horizontal palette above a vertical list) to avoid scrolling the wrong one.</field>
     <field name="to_element" type="TargetElement"/>
     <field name="max_scrolls" type="int" default="10"/>
     <field name="amount">small, page, full</field>
     <example>{"action": "scroll", "direction": "down", "to_element": {"predicate": {"text": "Privacy"}}, "max_scrolls": 10}</example>
+    <example>{"action": "scroll", "direction": "right", "predicate": {"type": "scrollview"}}</example>
     <note>scroll with to_element returns "reached end of scrollable content" if the list ends before the element is found. If it returns "element not found after scrolling" instead, the list has more content — increase max_scrolls or call scroll again to continue searching.</note>
   </action>
 
@@ -337,6 +347,13 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
     <example>{"action": "reset_location"}</example>
   </action>
 
+  <action name="open_link">
+    Opens a URL or deep link on the device. The OS routes custom schemes (myapp://path) to the registered app, so this jumps straight to a screen without navigating the UI. Use for deep-link entry points instead of tapping through the app.
+    <field name="url" required="yes">Full URL or deep link, e.g. https://example.com or myapp://profile/42</field>
+    <example>{"action": "open_link", "url": "myapp://profile/42"}</example>
+    <example>{"action": "open_link", "url": "https://example.com"}</example>
+  </action>
+
   <action name="siri">
     iOS only. Sends a voice command to Siri service on iOS devices. Auto-approves consent dialogs, captures Siri's response text, then dismisses the Siri UI.
     Use for triggering SiriKit intents and App Shortcuts registered by apps (media playback, messaging, banking shortcuts, etc.).
@@ -360,7 +377,7 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
   </action>
 
   <action name="tap/type/press_key/wait_for/if_exists">
-    Same fields as native but use css_selector or xpath in predicate.
+    Same fields as native but use css_selector in predicate.
     <example>{"action": "tap", "context": "web", "predicate": {"css_selector": "button.submit"}}</example>
     <example>{"action": "type", "context": "web", "predicate": {"css_selector": "input#email"}, "text": "user@example.com", "clear_first": true}</example>
     Web press_key keys: enter, tab, delete, escape.
@@ -462,7 +479,7 @@ const TESTING_REF = `<testing-reference>
 
 <rules>
   <rule>Never ask the user for information you can get yourself — use observe, list_apps, get_ui_tree.</rule>
-  <rule>Always add wait_for before every element interaction (tap, type, toggle, long_press, double_tap, drag). Exception: the element was asserted on the immediately preceding line.</rule>
+  <rule>Add wait_for before an interaction only when the PREVIOUS action may have changed the screen and the target might not be ready yet - after opening an app, navigating, a screen transition, opening a modal/sheet, scrolling, or submitting something that reloads. If the previous action left the same screen up (typing into a field, toggling a switch, tapping a control that stays put) the element is already there - skip wait_for. Never add it before the element you just asserted.</rule>
   <rule>Always use predicates over coordinates — predicates survive layout changes.</rule>
   <rule>Always prefer UI tree and OCR over screenshots for element discovery.</rule>
   <rule>Your first action must be observe — never ask the user first.</rule>
@@ -542,6 +559,8 @@ const TESTING_REF = `<testing-reference>
     type ~"Field" → "text"             — partial match field
     type "Field" → "text" clear        — clear first
     type "Field" → "text" keep_keyboard — keep keyboard open
+    type_secret "Field" → "secret-id"  - type a stored credential (value never in script/logs)
+    type_secret "secret-id"             - type credential into focused field
     swipe up|down|left|right            — swipe direction
     swipe up distance:long              — short/medium/long
     swipe up duration:500               — custom duration (ms)
@@ -549,6 +568,7 @@ const TESTING_REF = `<testing-reference>
     scroll down                         — scroll
     scroll down to "Element"            — scroll until visible
     scroll down to "Footer" max_scrolls:10 — limit attempts
+    scroll right in type:scrollview     - target a specific container
     toggle "Wi-Fi" on|off              — toggle switch
     toggle type:switch near "Wi-Fi" on  — modifier-only
     drag "Item" to "Trash"             — drag element
@@ -568,6 +588,7 @@ const TESTING_REF = `<testing-reference>
     paste_text "Field"                  — paste clipboard into element
     set_location 40.7128,-74.0060       — simulate GPS location (lat,lon)
     reset_location                      — stop location simulation
+    open_link "myapp://profile/42"      - open a URL or deep link (routes to the app)
     siri "Search YouTube for cats"      — invoke Siri with voice command (iOS only)
     observe                             — observe screen
     screenshot "path.png"               — take screenshot

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobai-mcp",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "mcpName": "io.github.MobAI-App/mobai-mcp",
   "description": "MCP server for MobAI - AI-powered mobile device automation",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/MobAI-App/mobai-mcp",
     "source": "github"
   },
-  "version": "2.3.1",
+  "version": "2.4.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "mobai-mcp",
-      "version": "2.3.1",
+      "version": "2.4.0",
       "transport": {
         "type": "stdio"
       }