mobile-automator 0.23.8

package/src/device/device-resolver.js

@@ -0,0 +1,34 @@
+'use strict';
+
+// Typed error so callers (CLI verb handlers) can map it onto the `device`
+// envelope with the carried, actionable hint.
+class DeviceResolutionError extends Error {
+  constructor(message, hint) {
+    super(message);
+    this.name = 'DeviceResolutionError';
+    this.kind = 'device';
+    this.hint = hint;
+  }
+}
+
+// mobile-mcp 0.0.55 requires a concrete device id on every action/read tool.
+// When the caller pinned nothing (no --device, no persisted selection), we
+// auto-discover: exactly one active device is used; zero or many is a clear,
+// actionable failure rather than a silent empty result.
+function resolveSingleDevice(devices) {
+  const list = Array.isArray(devices) ? devices : [];
+  if (list.length === 1) return list[0].id;
+  if (list.length === 0) {
+    throw new DeviceResolutionError(
+      'No active device or emulator found.',
+      'Start an emulator/simulator (or connect a device), or pass --device <id>.'
+    );
+  }
+  const ids = list.map((d) => (d && d.name ? `${d.id} (${d.name})` : d.id)).join(', ');
+  throw new DeviceResolutionError(
+    `Multiple active devices: ${ids}.`,
+    'Pick one with --device <id> or persist it via `mauto devices use <id>`.'
+  );
+}
+
+module.exports = { resolveSingleDevice, DeviceResolutionError };

package/src/device/element-model.js

@@ -0,0 +1,82 @@
+'use strict';
+
+// Agnostic element shape. We deliberately drop every OS-specific identifier
+// (resource_id / identifier) so downstream consumers can only target elements
+// by visible/semantic attributes (text, accessibility_label, geometry).
+
+function resolveBounds(raw) {
+  if (Array.isArray(raw.bounds) && raw.bounds.length === 4) {
+    const b = raw.bounds.map(Number);
+    if (b.every(Number.isFinite)) return b;
+  }
+  if (Array.isArray(raw.coordinates) && raw.coordinates.length === 4) {
+    const c = raw.coordinates.map(Number);
+    if (c.every(Number.isFinite)) return c;
+  }
+  if (raw.coordinates && typeof raw.coordinates === 'object' && !Array.isArray(raw.coordinates)) {
+    const { x, y, width, height } = raw.coordinates;
+    if ([x, y, width, height].every((v) => Number.isFinite(v))) {
+      return [x, y, x + width, y + height];
+    }
+  }
+  if (raw.rect && typeof raw.rect === 'object') {
+    const { x, y, width, height } = raw.rect;
+    if ([x, y, width, height].every((v) => Number.isFinite(v))) {
+      return [x, y, x + width, y + height];
+    }
+  }
+  return null;
+}
+
+function centerOf([x1, y1, x2, y2]) {
+  return [Math.round((x1 + x2) / 2), Math.round((y1 + y2) / 2)];
+}
+
+function nullable(v) {
+  return v === undefined || v === null || v === '' ? null : v;
+}
+
+function normalize(rawElements) {
+  if (!Array.isArray(rawElements)) return [];
+
+  const out = [];
+  for (const raw of rawElements) {
+    if (!raw || typeof raw !== 'object') continue;
+    const bounds = resolveBounds(raw);
+    if (!bounds) continue; // can't position it -> not useful, skip
+
+    out.push({
+      text: nullable(raw.text),
+      accessibility_label: nullable(
+        raw.accessibility_label !== undefined ? raw.accessibility_label : raw.label
+      ),
+      bounds,
+      center: centerOf(bounds),
+      type: nullable(raw.type),
+    });
+  }
+  return out;
+}
+
+// mobile-mcp 0.0.55 returns elements as the string
+// `Found these elements on screen: <JSON array>`. Earlier/other shapes may be
+// a bare array or `{elements:[...]}`. Parse all three into the raw element
+// array. Parse the WHOLE JSON array (never per-element regex) so labels
+// containing brackets/quotes/newlines survive.
+function parseElements(raw) {
+  if (Array.isArray(raw)) return raw;
+  if (raw && Array.isArray(raw.elements)) return raw.elements;
+  if (typeof raw === 'string') {
+    const start = raw.indexOf('[');
+    const end = raw.lastIndexOf(']');
+    if (start !== -1 && end > start) {
+      try {
+        const parsed = JSON.parse(raw.slice(start, end + 1));
+        if (Array.isArray(parsed)) return parsed;
+      } catch (_e) { /* fall through */ }
+    }
+  }
+  return [];
+}
+
+module.exports = { normalize, parseElements };

package/src/device/mobile-mcp-client.js

@@ -0,0 +1,163 @@
+'use strict';
+
+const path = require('path');
+const { Client } = require('@modelcontextprotocol/sdk/client/index.js');
+const { StdioClientTransport } = require('@modelcontextprotocol/sdk/client/stdio.js');
+const { injectDeviceArg } = require('./tool-args');
+const { resolveSingleDevice } = require('./device-resolver');
+const { normalizeDevices } = require('./device-model');
+
+// Resolve the mobile-mcp server entry point from the bundled, pinned
+// dependency rather than fetching `@latest` at runtime. We read the package's
+// own `bin` map (mcp-server-mobile -> lib/index.js) so the path stays correct
+// across mobile-mcp versions, then spawn it with the current Node binary.
+//
+// Resolution is done lazily inside createCall (not at module load) so that the
+// module can be required without mobile-mcp installed — the smoke test only
+// asserts the module loads and exports createCall.
+function resolveServerEntry() {
+  const pkgPath = require.resolve('@mobilenext/mobile-mcp/package.json');
+  const pkg = require('@mobilenext/mobile-mcp/package.json');
+  const pkgDir = path.dirname(pkgPath);
+  const binRel =
+    typeof pkg.bin === 'string'
+      ? pkg.bin
+      : (pkg.bin && pkg.bin['mcp-server-mobile']) || pkg.main || 'lib/index.js';
+  return path.join(pkgDir, binRel);
+}
+
+// Integration glue: spawns mobile-mcp over stdio and returns a `call`
+// function shaped exactly like the one DeviceBridge expects.
+// NOT unit-tested (would require spawning mobile-mcp); see the smoke test.
+async function createCall({ device } = {}) {
+  const serverEntry = resolveServerEntry();
+  const transport = new StdioClientTransport({
+    command: process.execPath,
+    args: [serverEntry],
+    // mobile-mcp reads the target device from the environment / its own
+    // selection; we pass it through when provided so callers can pin a device.
+    env: device ? { ...process.env, MOBILE_MCP_DEVICE: device } : process.env,
+  });
+
+  const client = new Client(
+    { name: 'mauto', version: '0.1.0' },
+    { capabilities: {} }
+  );
+
+  await client.connect(transport);
+
+  async function rawCall(toolName, args = {}) {
+    const res = await client.callTool({ name: toolName, arguments: args });
+    return parseToolResult(res);
+  }
+  const { call } = makeCall({ rawCall, device });
+
+  async function close() {
+    try {
+      await client.close();
+    } catch (_e) {
+      // best-effort teardown
+    }
+  }
+
+  return { call, close };
+}
+
+// Pure orchestration over a low-level rawCall(toolName, args) => Promise<parsed>.
+// Threads a concrete device id into every tool call except the discovery verb,
+// lazily auto-discovering a single device when none was pinned (cached after
+// the first resolution). Extracted from createCall so it is unit-testable
+// without spawning mobile-mcp.
+function makeCall({ rawCall, device = null }) {
+  let resolvedPromise = device != null ? Promise.resolve(device) : null;
+  function ensureDevice() {
+    if (!resolvedPromise) {
+      resolvedPromise = rawCall('mobile_list_available_devices', {})
+        .then((listed) => resolveSingleDevice(normalizeDevices(listed))) // throws DeviceResolutionError on 0/many
+        .catch((err) => {
+          // A failed discovery must not wedge the cache: the next call re-lists.
+          resolvedPromise = null;
+          throw err;
+        });
+    }
+    return resolvedPromise;
+  }
+  async function call(toolName, args = {}) {
+    if (toolName === 'mobile_list_available_devices') {
+      return rawCall(toolName, args);
+    }
+    // The auto-resolved device can go stale (disconnected, a new one appears);
+    // the cached id then fails every call with `Device "X" not found` until the
+    // daemon restarts. Retry ONCE by busting the cache and re-resolving — but
+    // only for an auto-resolved device (device === null): a user-pinned daemon
+    // must never silently switch devices.
+    let retried = false;
+    for (;;) {
+      const deviceId = await ensureDevice();
+      try {
+        return await rawCall(toolName, injectDeviceArg(toolName, args, deviceId));
+      } catch (err) {
+        const staleAutoResolved =
+          device == null && !retried && /Device .+ not found/i.test(err && err.message);
+        if (!staleAutoResolved) throw err;
+        retried = true;
+        resolvedPromise = null;
+      }
+    }
+  }
+  return { call };
+}
+
+// Collect the text content blocks from an MCP tool result into one string.
+function textContent(res) {
+  const content = res && res.content;
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter((c) => c && c.type === 'text' && typeof c.text === 'string')
+    .map((c) => c.text)
+    .join('\n');
+}
+
+// mobile-mcp returns *recoverable* failures (its `ActionableError`) as ordinary
+// text content WITHOUT the `isError` flag, appending this sentinel so an agent
+// reads the hint and retries. We must treat those as failures too, otherwise
+// the error text gets laundered into a successful `{ok:true}` envelope.
+const MOBILE_MCP_ACTIONABLE_SUFFIX = '. Please fix the issue and try again.';
+
+// mobile-mcp returns its payload as text content that is itself JSON.
+// Parse defensively: prefer JSON text content, fall back to raw text/struct.
+function parseToolResult(res) {
+  if (!res) return null;
+
+  // `isError` covers "real exceptions"; the sentinel covers ActionableError.
+  // Surface both so device verbs fail honestly instead of reporting ok:true.
+  if (res.isError) {
+    throw new Error(textContent(res) || 'mobile-mcp tool call failed');
+  }
+
+  if (res.structuredContent !== undefined) return res.structuredContent;
+
+  const content = res.content;
+  if (Array.isArray(content)) {
+    const texts = content
+      .filter((c) => c && c.type === 'text' && typeof c.text === 'string')
+      .map((c) => c.text);
+    for (const t of texts) {
+      try {
+        return JSON.parse(t);
+      } catch (_e) {
+        // not JSON; keep looking, fall through to raw join
+      }
+    }
+    if (texts.length) {
+      const joined = texts.join('\n');
+      if (joined.endsWith(MOBILE_MCP_ACTIONABLE_SUFFIX)) {
+        throw new Error(joined);
+      }
+      return joined;
+    }
+  }
+  return res;
+}
+
+module.exports = { createCall, makeCall, parseToolResult };

package/src/device/resolve-connection.js

@@ -0,0 +1,156 @@
+'use strict';
+
+// Decides HOW a one-shot verb gets its device connection, transparently
+// reusing a persistent daemon when one fits.
+//
+// Returns { bridge, close, source } where source is 'daemon' | 'oneshot'.
+//
+// Branches (in order):
+//   (a) daemon live AND its device pin matches the requested device
+//         -> daemon-backed bridge; close() is a NO-OP (must NOT tear the
+//            shared daemon down — other verbs depend on it).
+//   (b) device-pin MISMATCH (daemon bound to A, verb asks for B)
+//         -> one-shot connection for this call (do not silently reuse A).
+//   (c) autostart && no reachable daemon
+//         -> spawn a daemon, then connect daemon-backed (no-op close).
+//   (d) spawn failed, or autostart:false
+//         -> one-shot fallback via the real createCall (real close).
+//
+// Everything device-touching is injected so the whole matrix is unit-testable
+// with a fake daemon + fake createCall + fake spawn.
+
+const fs = require('fs');
+
+const { DeviceBridge } = require('./bridge');
+const paths = require('./session-paths');
+const sessionClient = require('./session-client');
+const sessionSpawn = require('./session-spawn');
+
+function defaultCreateCall(opts) {
+  return require('./mobile-mcp-client').createCall(opts);
+}
+
+const NOOP_CLOSE = () => Promise.resolve();
+
+// Read the live daemon's pinned device from its handle, or null. A null pin
+// means the daemon serves whatever device mobile-mcp selected (matches any
+// request that doesn't pin a specific device).
+function readHandleDevice(projectRoot) {
+  try {
+    const raw = fs.readFileSync(paths.handlePath(projectRoot), 'utf8');
+    const handle = JSON.parse(raw);
+    return handle && handle.device ? handle.device : null;
+  } catch (_) {
+    return null;
+  }
+}
+
+// Does a daemon pinned to `handleDevice` satisfy a request for `requested`?
+// - request without --device  -> reuse any live daemon.
+// - request with --device     -> only reuse when the pins are equal.
+function deviceMatches(requested, handleDevice) {
+  if (!requested) return true;
+  return requested === handleDevice;
+}
+
+// PURE decision: given the observed facts, which strategy should we run?
+//   'daemon'           — a live daemon fits; connect daemon-backed (no-op close)
+//   'oneshot'          — connect once for this call (real close)
+//   'spawn-then-daemon'— no daemon fits but autostart is on; spawn then connect
+//
+// Zero collaborators, no I/O — value-in/value-out so the branching can be tested
+// without faking a client/spawn/createCall or writing a handle to disk. The
+// effectful resolver (below) gathers the facts, calls this, and runs exactly one
+// of the three thin effect paths.
+function chooseConnectionStrategy({ alive, handleDevice, requestedDevice, autostart }) {
+  if (alive && deviceMatches(requestedDevice, handleDevice)) return 'daemon';
+  if (alive) return 'oneshot'; // live daemon, but its device pin mismatches
+  if (autostart) return 'spawn-then-daemon';
+  return 'oneshot';
+}
+
+async function resolveDeviceConnection({
+  device = null,
+  projectRoot = process.cwd(),
+  autostart = true,
+  // Injectable seams (tests override these).
+  client = sessionClient,
+  spawn = sessionSpawn,
+  createCall = defaultCreateCall,
+  idleMs = undefined,
+} = {}) {
+  const oneShot = async () => {
+    const { call, close } = await createCall({ device });
+    return { bridge: new DeviceBridge({ call, device }), close, source: 'oneshot' };
+  };
+
+  const daemonBacked = async () => {
+    const conn = await client.tryConnect(projectRoot);
+    if (!conn) return null;
+    // The verb's `device` may be null while the daemon is pinned to a device;
+    // fall back to the live handle's pin so getPlatform() resolves correctly.
+    const pinned = device || readHandleDevice(projectRoot);
+    // close() is a no-op AND we must release our socket so the daemon's idle
+    // timer can fire — wrap close to end the underlying socket but never stop
+    // the shared daemon.
+    return {
+      bridge: new DeviceBridge({ call: conn.call, device: pinned }),
+      close: async () => {
+        try {
+          await conn.close();
+        } catch (_) {
+          /* ignore */
+        }
+      },
+      source: 'daemon',
+    };
+  };
+
+  const alive = await client.isAlive(projectRoot);
+
+  if (alive) {
+    const handleDevice = readHandleDevice(projectRoot);
+    const strategy = chooseConnectionStrategy({
+      alive: true,
+      handleDevice,
+      requestedDevice: device,
+      autostart,
+    });
+    if (strategy === 'daemon') {
+      // (a) reuse the live daemon.
+      const conn = await daemonBacked();
+      if (conn) return conn;
+      // Race: daemon vanished between isAlive and connect — fall through and
+      // re-decide as if no daemon were alive.
+    } else {
+      // (b) device-pin mismatch: never silently reuse the wrong device.
+      return oneShot();
+    }
+  }
+
+  // No live daemon (or it vanished mid-flight): spawn-then-daemon vs one-shot.
+  const strategy = chooseConnectionStrategy({
+    alive: false,
+    handleDevice: null,
+    requestedDevice: device,
+    autostart,
+  });
+  if (strategy === 'spawn-then-daemon') {
+    // (c) spawn a daemon, then connect daemon-backed.
+    const started = await spawn.spawnDaemon({ projectRoot, device, idleMs });
+    if (started) {
+      const conn = await daemonBacked();
+      if (conn) return conn;
+    }
+  }
+
+  // (d) explicit one-shot, or spawn failed / unreachable.
+  return oneShot();
+}
+
+module.exports = {
+  resolveDeviceConnection,
+  chooseConnectionStrategy,
+  deviceMatches,
+  readHandleDevice,
+};

package/src/device/selection.js

@@ -0,0 +1,80 @@
+'use strict';
+
+// Persisted device selection for the `mauto` CLI.
+//
+// A chosen device id is remembered across separate one-shot verb invocations so
+// the agent does not have to re-pass --device every call. The selection lives in
+// #91's session layout (mobile-automator/.session/) as a small sidecar file,
+// SEPARATE from config.json (which is user-authored project config) and from the
+// live session handle (session.json, owned by the daemon lifecycle).
+//
+// `read` is graceful: a missing OR corrupt file returns null so a verb never
+// crashes on a half-written/garbage selection — it just behaves as "no
+// selection". The directory is created lazily on write.
+
+const fs = require('fs');
+const path = require('path');
+
+const paths = require('./session-paths');
+const { atomicWrite } = require('../util/atomic');
+
+const SELECTION_NAME = 'selection.json';
+
+function selectionPath(projectRoot) {
+  return path.join(paths.sessionDir(projectRoot), SELECTION_NAME);
+}
+
+// The persisted device id, or null when absent/corrupt.
+function read(projectRoot) {
+  try {
+    const raw = fs.readFileSync(selectionPath(projectRoot), 'utf8');
+    const parsed = JSON.parse(raw);
+    const id = parsed && parsed.device;
+    return id ? String(id) : null;
+  } catch (_) {
+    return null;
+  }
+}
+
+// Persist a device id. Creates mobile-automator/.session/ if needed. Written
+// via atomicWrite (tmp + fsync + rename) so a concurrent reader or a crash
+// sees the old-complete or new-complete selection, never a torn one.
+function write(projectRoot, id) {
+  atomicWrite(
+    selectionPath(projectRoot),
+    JSON.stringify({ device: String(id) }, null, 2) + '\n'
+  );
+  return String(id);
+}
+
+// Remove the persisted selection. Idempotent — clearing when none exists is a
+// no-op (never throws).
+function clear(projectRoot) {
+  try {
+    fs.rmSync(selectionPath(projectRoot), { force: true });
+  } catch (_) {
+    /* ignore */
+  }
+}
+
+// Pure precedence resolver: an explicit --device flag ALWAYS wins and is never
+// written to the store (it is a per-call override only); otherwise fall back to
+// the persisted selection; otherwise none. `store` is injectable so this stays
+// pure and unit-testable without touching the filesystem.
+//
+// Returns { device, source } where source is:
+//   'flag'    — explicit --device supplied
+//   'session' — resolved from the persisted selection
+//   'none'    — no device chosen (mobile-mcp will auto-select a single device)
+function resolveDevice({ explicit, projectRoot, store = module.exports } = {}) {
+  if (explicit) {
+    return { device: String(explicit), source: 'flag' };
+  }
+  const persisted = store.read(projectRoot);
+  if (persisted) {
+    return { device: persisted, source: 'session' };
+  }
+  return { device: null, source: 'none' };
+}
+
+module.exports = { selectionPath, read, write, clear, resolveDevice, SELECTION_NAME };

package/src/device/semantic-press/index.js

@@ -0,0 +1,38 @@
+'use strict';
+
+const { AndroidResolver, IOSResolver, SemanticResolutionError } = require('./resolver');
+
+const ACTION_METHOD = {
+  press_back: 'pressBack',
+  dismiss_keyboard: 'dismissKeyboard',
+  grant_permission: 'grantPermission',
+  deny_permission: 'denyPermission',
+};
+
+const SEMANTIC_ACTIONS = Object.keys(ACTION_METHOD);
+
+function isSemanticAction(token) {
+  return Object.prototype.hasOwnProperty.call(ACTION_METHOD, token);
+}
+
+const STRATEGIES = { android: AndroidResolver, ios: IOSResolver };
+
+// Pick the per-platform resolver, or hard-fail (never silently forward).
+function selectResolver(platform, bridge) {
+  const Strategy = STRATEGIES[platform];
+  if (!Strategy) {
+    throw new SemanticResolutionError(
+      `No semantic-action resolver for platform "${platform}".`,
+      'Connect a supported Android or iOS device/simulator.'
+    );
+  }
+  return new Strategy(bridge);
+}
+
+module.exports = {
+  SEMANTIC_ACTIONS,
+  ACTION_METHOD,
+  isSemanticAction,
+  selectResolver,
+  SemanticResolutionError,
+};

package/src/device/semantic-press/labels.js

@@ -0,0 +1,39 @@
+'use strict';
+
+// System permission-dialog button labels, per platform. Lowercase and
+// apostrophe-normalized so matching is locale-tolerant within English.
+const ALLOW_LABELS = {
+  android: ['allow', 'while using the app', 'allow only this time', 'ok', 'continue'],
+  ios: ['allow', 'allow while using app', 'allow once', 'ok'],
+};
+
+const DENY_LABELS = {
+  android: ['deny', "don't allow"],
+  ios: ["don't allow"],
+};
+
+function norm(s) {
+  // Map the curly apostrophe (U+2019, the ’ glyph in the regex below) onto a
+  // straight quote, so "Don’t Allow" matches the straight-quoted entries in the
+  // label sets above. The U+2019 byte must be preserved literally here; the
+  // guard assertion in labels.test.js fails loudly if it is ever flattened to a
+  // straight quote by editor/transport tooling.
+  return String(s == null ? '' : s).replace(/’/g, "'").trim().toLowerCase();
+}
+
+// First element whose visible text or accessibility label exactly equals one of
+// `labels` (after normalization). Returns the element plus the original label
+// that matched, or null.
+function findByLabels(elements, labels) {
+  const wanted = new Set(labels.map(norm));
+  for (const el of Array.isArray(elements) ? elements : []) {
+    for (const raw of [el.text, el.accessibility_label]) {
+      if (raw != null && wanted.has(norm(raw))) {
+        return { element: el, label: raw };
+      }
+    }
+  }
+  return null;
+}
+
+module.exports = { ALLOW_LABELS, DENY_LABELS, findByLabels };