mobai-mcp 2.3.1 → 2.4.1

package/dist/index.js

@@ -223,6 +223,12 @@ const TOOLS = [
             properties: {
                 device_id: { type: "string", description: "Device ID" },
                 path: { type: "string", description: "Local file path to the app (.apk or .ipa)" },
+                resign: { type: "boolean", description: "Sign the IPA with iloader before installing (iOS only)" },
+                apple_id: { type: "string", description: "Apple ID for signing (optional; used when resign=true)" },
+                password: { type: "string", description: "Apple ID password for signing (optional; uses cached credentials if empty)" },
+                profile_path: { type: "string", description: "Provisioning profile to reuse for an offline re-sign (with cert_path/key_path); falls back to online signing if invalid" },
+                cert_path: { type: "string", description: "Signing certificate PEM to reuse for an offline re-sign" },
+                key_path: { type: "string", description: "Signing private key PEM to reuse for an offline re-sign" },
             },
             required: ["device_id", "path"],
         },
@@ -242,16 +248,65 @@ const TOOLS = [
     // DSL execution
     {
         name: "execute_dsl",
-        description: `Execute a batch of DSL commands on a device. This is the primary tool for all device interaction — tap, type, swipe, observe, launch apps, assertions, web automation, and more.
+        description: `The example below is the full command surface. For richer semantics - per-action defaults, platform notes, retry/failure strategies, observe and scroll guidance, web/OCR caveats - read the MCP resource mobai://reference/device-automation. Read it the first time you hit anything the example doesn't make obvious.
 
-You MUST read the MCP resource mobai://reference/device-automation to learn how to control devices before using this tool.
+Execute a batch of DSL commands on a device. This is the primary tool for all device interaction - tap, type, swipe, observe, launch apps, assertions, web automation, and more.
 
-Input: JSON string with "version": "0.2" and "steps" array. Example:
-{"version":"0.2","steps":[
-  {"action":"open_app","bundle_id":"com.apple.Preferences"},
-  {"action":"tap","predicate":{"text_contains":"Wi-Fi"}},
-  {"action":"wait_for","predicate":{"type":"switch"},"timeout_ms":3000}
-]}`,
+To find visual/UI bugs on the current screen, run a single {"action":"audit"} step: it returns deterministic findings (touch targets, clipping, alignment, spacing, safe area, accessibility) plus an annotated screenshot whose red boxes are labelled with the element ids in the findings. Findings are evidence, not verdicts - cross-check the annotated screenshot to decide which are real bugs.
+
+Input: JSON string with "version": "0.2" and a "steps" array. Optional script-wide "params" (declared defaults, substituted as \${name}) and "on_fail" (strategy: abort|skip|retry|replan|require_user; retry also takes max_retries, retry_delay_ms, fallback_strategy). Any step may carry its own "on_fail".
+
+Every action below appears at least once - this is the full command surface (semantics, defaults, and platform notes are in mobai://reference/device-automation):
+{"version":"0.2","params":{"query":"shoes"},"on_fail":{"strategy":"retry","max_retries":2,"retry_delay_ms":1000,"fallback_strategy":{"strategy":"skip"}},"steps":[
+  {"action":"open_app","bundle_id":"com.apple.Preferences","fresh":true,"debug":false},
+  {"action":"kill_app","bundle_id":"com.apple.mobilesafari"},
+  {"action":"open_link","url":"myapp://profile/42"},
+  {"action":"tap","predicate":{"text_contains":"Wi-Fi","type":"button"}},
+  {"action":"tap","coords":{"x":200,"y":640}},
+  {"action":"double_tap","predicate":{"text":"Photo"}},
+  {"action":"two_finger_tap","predicate":{"text":"Map"}},
+  {"action":"long_press","predicate":{"text":"Item"},"duration_ms":1000},
+  {"action":"pinch","predicate":{"text_contains":"Map"},"scale":0.5,"velocity":1.0},
+  {"action":"type","text":"\${query}","predicate":{"type":"input"},"clear_first":true,"dismiss_keyboard":true},
+  {"action":"type_secret","secret_id":"test-account-password","predicate":{"type":"input","text_contains":"Password"}},
+  {"action":"clear","predicate":{"type":"input"}},
+  {"action":"toggle","predicate":{"type":"switch","text_contains":"Wi-Fi"},"state":"on"},
+  {"action":"press_key","key":"enter"},
+  {"action":"swipe","direction":"up","distance":"medium"},
+  {"action":"swipe","from_coords":{"x":200,"y":600},"to_coords":{"x":200,"y":200},"duration_ms":300},
+  {"action":"scroll","direction":"down","to_element":{"predicate":{"text":"Privacy"}},"predicate":{"type":"scrollview"},"max_scrolls":10,"amount":"page"},
+  {"action":"drag","from":{"predicate":{"text":"Item"}},"to_element":{"predicate":{"text":"Trash"}},"press_duration_ms":500,"hold_duration_ms":200,"duration_ms":500},
+  {"action":"drag","from_coords":{"x":100,"y":400},"to_coords":{"x":300,"y":400}},
+  {"action":"drag_path","points":[{"x":100,"y":400,"duration_ms":200},{"x":150,"y":300,"duration_ms":150},{"x":300,"y":500,"duration_ms":300}]},
+  {"action":"navigate","target":"home"},
+  {"action":"set_location","lat":40.7128,"lon":-74.0060},
+  {"action":"reset_location"},
+  {"action":"siri","prompt":"Search YouTube for cat videos"},
+  {"action":"if_exists","predicate":{"text":"Allow"},"then":[{"action":"tap","predicate":{"text":"Allow"}}],"else":[{"action":"tap","predicate":{"text":"Don't Allow"}}]},
+  {"action":"delay","duration_ms":1000},
+  {"action":"wait_for","predicate":{"text":"Welcome"},"timeout_ms":5000,"poll_interval_ms":500},
+  {"action":"assert_exists","predicate":{"text":"Success"},"timeout_ms":3000},
+  {"action":"assert_not_exists","predicate":{"text":"Error"}},
+  {"action":"assert_count","predicate":{"type":"cell"},"count":5},
+  {"action":"assert_screen_changed","threshold_percent":15},
+  {"action":"metrics_start","types":["system_cpu","fps"],"interval_ms":1000,"label":"login_flow","capture_logs":true,"thresholds":{"cpu_high":80,"fps_low":45}},
+  {"action":"metrics_stop","format":"summary"},
+  {"action":"record_start","file_path":"/tmp/mobai/rec"},
+  {"action":"record_stop"},
+  {"action":"select_web_context","url_contains":"google.com"},
+  {"action":"navigate","context":"web","url":"https://example.com"},
+  {"action":"tap","context":"web","predicate":{"css_selector":"button.submit"}},
+  {"action":"type","context":"web","predicate":{"css_selector":"input#email"},"text":"user@example.com","clear_first":true},
+  {"action":"execute_js","context":"web","script":"return document.title","async":false},
+  {"action":"screenshot","file_path":"/tmp/mobai","name":"final_state"},
+  {"action":"wait_for","stable":true,"timeout_ms":3000},
+  {"action":"observe","include":["ui_tree"],"only_visible":true,"filter":{"text_regex":"Settings|Wi-Fi"},"store_as":"end_state"},
+  {"action":"audit"}
+]}
+
+Predicate - object passed in a step's "predicate"/"from"/"to_element". Combine fields (AND); prefer text_contains over exact text. All native fields:
+{"text":"exact","text_contains":"substr","text_starts_with":"prefix","text_regex":"\\d+ items","value":"typed value","value_contains":"typed substr","type":"button|input|switch|text|image|cell|scrollview","accessibility_id":"login_btn","enabled":true,"visible":true,"selected":false,"index":0,"bounds_hint":"top_half|bottom_half|left_half|right_half|center","near":{"text_contains":"Email","direction":"below|above|left|right|any","max_distance":100},"parent_of":{"text":"child label"}}
+Web predicate (context:"web") uses instead: {"css_selector":"button.submit"}.`,
         inputSchema: {
             type: "object",
             properties: {
@@ -437,8 +492,23 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
             }
             case "list_apps":
                 return textResult(await doGet(`/devices/${args?.device_id}/apps`));
-            case "install_app":
-                return textResult(await doPost(`/devices/${args?.device_id}/install-app`, { path: args?.path }));
+            case "install_app": {
+                const body = { path: args?.path };
+                if (args?.resign) {
+                    body.resign = true;
+                    if (args?.apple_id)
+                        body.appleId = args.apple_id;
+                    if (args?.password)
+                        body.password = args.password;
+                    if (args?.profile_path)
+                        body.profilePath = args.profile_path;
+                    if (args?.cert_path)
+                        body.certPath = args.cert_path;
+                    if (args?.key_path)
+                        body.keyPath = args.key_path;
+                }
+                return textResult(await doPost(`/devices/${args?.device_id}/install-app`, body));
+            }
             case "uninstall_app":
                 return textResult(await doDelete(`/devices/${args?.device_id}/apps/${encodeURIComponent(args?.bundle_id)}`));
             // DSL execution

package/dist/resources.js

@@ -92,6 +92,7 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
   <observe-guidance>
     Prefer UI tree and OCR over screenshots — screenshots consume far more context and are slower. Only use screenshots for visual verification (layout, colors, images).
     only_visible: false — use ONLY for data collection from scrollable views (returns ALL elements including off-screen). NEVER when interacting — off-screen elements cannot be tapped.
+    compact: true — token-lean ui_tree (same elements as the full tree, as flat [index] Type "label" #id [state] lines; coordinates and indentation removed, ~2x fewer tokens). Use when you select by id/index/text. Keep the default full tree when you need coordinates or spatial layout (e.g. coordinate taps, near/spatial reasoning).
   </observe-guidance>
 
   <ocr-fallback>
@@ -161,7 +162,6 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
 
   <predicate context="web">
     <field name="css_selector" type="string">CSS selector</field>
-    <field name="xpath" type="string">XPath expression</field>
   </predicate>
 
   <failure-strategy>
@@ -217,6 +217,15 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
     <example>{"action": "type", "text": "Hello", "predicate": {"type": "input"}, "clear_first": true}</example>
   </action>
 
+  <action name="type_secret">
+    Type a stored credential without ever exposing its value. secret_id names a credential saved in the app; its value is resolved from the keychain at run time and typed on-device - it never appears in the script, logs, results, or the model context. Use for passwords/tokens in login steps. Same field-targeting as type.
+    <field name="secret_id" required="yes">Id of a stored credential (see list_secrets)</field>
+    <field name="predicate">Optional target field to focus first</field>
+    <field name="clear_first" type="bool"/>
+    <field name="dismiss_keyboard" type="bool" default="false"/>
+    <example>{"action": "type_secret", "secret_id": "test-account-password", "predicate": {"type": "input", "text_contains": "Password"}}</example>
+  </action>
+
   <action name="clear">
     Clear a field's text without typing. With a predicate, focuses that field first; without one, clears the currently focused field.
     <field name="predicate">Optional target element</field>
@@ -235,11 +244,13 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
 
   <action name="scroll">
     Direction = semantic (where to look), not finger movement.
-    <field name="direction" required="yes">down (look below), up (look above)</field>
+    <field name="direction" required="yes">down (look below), up (look above), left (look left), right (look right). All four directions are supported - use left/right for horizontal scrollviews (carousels, palettes, paging) instead of falling back to raw-coordinate swipe.</field>
+    <field name="predicate" type="Predicate">Targets a specific scrollable container so the gesture lands inside it (e.g. {"type": "scrollview"} or {"parent_of": {"text": "Work"}}). Without it the gesture is applied at screen center - set this when multiple scrollviews share the screen (e.g. a horizontal palette above a vertical list) to avoid scrolling the wrong one.</field>
     <field name="to_element" type="TargetElement"/>
     <field name="max_scrolls" type="int" default="10"/>
     <field name="amount">small, page, full</field>
     <example>{"action": "scroll", "direction": "down", "to_element": {"predicate": {"text": "Privacy"}}, "max_scrolls": 10}</example>
+    <example>{"action": "scroll", "direction": "right", "predicate": {"type": "scrollview"}}</example>
     <note>scroll with to_element returns "reached end of scrollable content" if the list ends before the element is found. If it returns "element not found after scrolling" instead, the list has more content — increase max_scrolls or call scroll again to continue searching.</note>
   </action>
 
@@ -294,11 +305,17 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
     <field name="store_as" type="string">Store result in named variable</field>
     <field name="include_keyboard" type="bool" default="false"/>
     <field name="only_visible" type="bool" default="true">MUST be false when collecting data from scrollable lists/search results. MUST be true when interacting with elements — off-screen elements cannot be tapped.</field>
+    <field name="compact" type="bool" default="false">Token-lean ui_tree: same elements as the full tree, formatted as flat [index] Type "label" #id [state] lines with coordinates and indentation removed (~2x fewer tokens). Use when selecting by id/index/text; keep false when you need coordinates or spatial layout.</field>
     <field name="filter">text_regex or bounds {"x","y","width","height"} to reduce output</field>
     <note>OCR (iOS only): returns text with tap coordinates. Useful for system dialogs missing from UI tree.</note>
     <example>{"action": "observe", "include": ["ui_tree"], "filter": {"text_regex": "Settings|Wi-Fi"}}</example>
   </action>
 
+  <action name="audit">
+    <note>Runs deterministic UI checks on the current native screen and returns findings plus an annotated screenshot (path under observations.native.screenshot, with red boxes labelled by element id). Findings are EVIDENCE, not conclusions (e.g. margins are 18px/34px, not "misaligned") - cross-check the annotated screenshot to decide which are real bugs. Checks: touch_target, clipping, alignment, spacing, safe_area, accessibility. Dense grids of tiny repeated controls (paint canvas, keyboard) are skipped to avoid noise.</note>
+    <example>{"action": "audit"}</example>
+  </action>
+
   <action name="wait_for">
     <field name="predicate">Required unless stable is true</field>
     <field name="stable" type="bool" default="false">Wait for UI to stop changing</field>
@@ -337,6 +354,13 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
     <example>{"action": "reset_location"}</example>
   </action>
 
+  <action name="open_link">
+    Opens a URL or deep link on the device. The OS routes custom schemes (myapp://path) to the registered app, so this jumps straight to a screen without navigating the UI. Use for deep-link entry points instead of tapping through the app.
+    <field name="url" required="yes">Full URL or deep link, e.g. https://example.com or myapp://profile/42</field>
+    <example>{"action": "open_link", "url": "myapp://profile/42"}</example>
+    <example>{"action": "open_link", "url": "https://example.com"}</example>
+  </action>
+
   <action name="siri">
     iOS only. Sends a voice command to Siri service on iOS devices. Auto-approves consent dialogs, captures Siri's response text, then dismisses the Siri UI.
     Use for triggering SiriKit intents and App Shortcuts registered by apps (media playback, messaging, banking shortcuts, etc.).
@@ -360,7 +384,7 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
   </action>
 
   <action name="tap/type/press_key/wait_for/if_exists">
-    Same fields as native but use css_selector or xpath in predicate.
+    Same fields as native but use css_selector in predicate.
     <example>{"action": "tap", "context": "web", "predicate": {"css_selector": "button.submit"}}</example>
     <example>{"action": "type", "context": "web", "predicate": {"css_selector": "input#email"}, "text": "user@example.com", "clear_first": true}</example>
     Web press_key keys: enter, tab, delete, escape.
@@ -462,7 +486,7 @@ const TESTING_REF = `<testing-reference>
 
 <rules>
   <rule>Never ask the user for information you can get yourself — use observe, list_apps, get_ui_tree.</rule>
-  <rule>Always add wait_for before every element interaction (tap, type, toggle, long_press, double_tap, drag). Exception: the element was asserted on the immediately preceding line.</rule>
+  <rule>Add wait_for before an interaction only when the PREVIOUS action may have changed the screen and the target might not be ready yet - after opening an app, navigating, a screen transition, opening a modal/sheet, scrolling, or submitting something that reloads. If the previous action left the same screen up (typing into a field, toggling a switch, tapping a control that stays put) the element is already there - skip wait_for. Never add it before the element you just asserted.</rule>
   <rule>Always use predicates over coordinates — predicates survive layout changes.</rule>
   <rule>Always prefer UI tree and OCR over screenshots for element discovery.</rule>
   <rule>Your first action must be observe — never ask the user first.</rule>
@@ -542,6 +566,8 @@ const TESTING_REF = `<testing-reference>
     type ~"Field" → "text"             — partial match field
     type "Field" → "text" clear        — clear first
     type "Field" → "text" keep_keyboard — keep keyboard open
+    type_secret "Field" → "secret-id"  - type a stored credential (value never in script/logs)
+    type_secret "secret-id"             - type credential into focused field
     swipe up|down|left|right            — swipe direction
     swipe up distance:long              — short/medium/long
     swipe up duration:500               — custom duration (ms)
@@ -549,6 +575,7 @@ const TESTING_REF = `<testing-reference>
     scroll down                         — scroll
     scroll down to "Element"            — scroll until visible
     scroll down to "Footer" max_scrolls:10 — limit attempts
+    scroll right in type:scrollview     - target a specific container
     toggle "Wi-Fi" on|off              — toggle switch
     toggle type:switch near "Wi-Fi" on  — modifier-only
     drag "Item" to "Trash"             — drag element
@@ -568,6 +595,7 @@ const TESTING_REF = `<testing-reference>
     paste_text "Field"                  — paste clipboard into element
     set_location 40.7128,-74.0060       — simulate GPS location (lat,lon)
     reset_location                      — stop location simulation
+    open_link "myapp://profile/42"      - open a URL or deep link (routes to the app)
     siri "Search YouTube for cats"      — invoke Siri with voice command (iOS only)
     observe                             — observe screen
     screenshot "path.png"               — take screenshot

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mobai-mcp",
-  "version": "2.3.1",
+  "version": "2.4.1",
   "mcpName": "io.github.MobAI-App/mobai-mcp",
   "description": "MCP server for MobAI - AI-powered mobile device automation",
   "type": "module",

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/MobAI-App/mobai-mcp",
     "source": "github"
   },
-  "version": "2.3.1",
+  "version": "2.4.1",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "mobai-mcp",
-      "version": "2.3.1",
+      "version": "2.4.1",
       "transport": {
         "type": "stdio"
       }