dsh-mobilecode 0.1.4 → 0.2.0

package/README.md

@@ -46,14 +46,32 @@ drawer:
   failed or incomplete), the Metro state, and `complete`.
 - `device_detect` — which platforms a directory supports, the framework, and
   the first attached Android device.
+- A bundled **`device-ui-automation` playbook skill** (registered through
+  `ctx.skills.register()`, defensively — hosts without the skill service just
+  skip it): the observe-once → act-with-an-assertion → observe-again workflow,
+  which observer to reach for first, how to confirm an action landed via
+  `expect_text`/`expect_gone`, and the real-device safety rules.
 - `device_screen` — see what is on an attached Android device: a PNG screenshot
   plus the uiautomator UI hierarchy (text + pixel bounds) and **local PaddleOCR**
   text recognition (text + confidence + box), so an agent can read the screen
   and tap by coordinates. Also returns the foreground activity and screen size.
+- `device_ui_tree` — the default screen observer: the uiautomator hierarchy as a
+  typed node tree (`type`/`text`/`contentDesc`/`resourceId`/`bounds`, with
+  `enabled`/`focused`/`clickable`/`scrollable` emitted only in their interesting
+  state), case-insensitive `filter` that keeps ancestors of matches, `max_depth`,
+  and a 40 KB cap that prunes the deepest levels first. Resource-ids are the most
+  stable tap handles.
+- `device_tap_element` — tap a control by identity: `resource_id` matches the
+  node's resource-id, `text` matches its text or content-desc; exact match wins
+  over substring, nested duplicates collapse to the outermost control, ambiguity
+  lists up to 8 candidates instead of guessing, and disabled / off-screen nodes
+  are refused with the fix. `expect_text` / `expect_gone` verify the tap in the
+  same call — one round trip, no separate screenshot.
 - `device_input` — act on the device: tap / swipe / type / press a key at
   **absolute pixel coordinates** (the same space `device_screen` returns — take
   the box center `x=(x1+x2)/2, y=(y1+y2)/2`). The deterministic control loop is
-  `device_screen → device_input → device_screen`.
+  `device_ui_tree → device_tap_element`, falling back to
+  `device_screen → device_input` when a surface exposes no accessibility tree.
 - `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
   (kernel = dmesg, needs adb root — works on emulators) with an optional
   case-insensitive substring filter, capped line count.

package/lib/device-build.js

@@ -108,11 +108,19 @@ export async function capture(command, args, options = {}) {
   })
   running.stdout?.setEncoding("utf8")
   running.stdout?.on("data", (chunk) => out.push(chunk))
+  let timer
+  if (options.timeoutMs) {
+    timer = setTimeout(() => running.kill(), options.timeoutMs)
+    timer.unref?.()
+  }
   const code = await new Promise((resolve) => {
     running.once("error", () => resolve(-1))
     running.once("close", (value) => resolve(value ?? -1))
   })
-  return code === 0 ? out.join("") : ""
+  if (timer) clearTimeout(timer)
+  if (code !== 0) return ""
+  const text = out.join("")
+  return options.maxBytes && Buffer.byteLength(text) > options.maxBytes ? text.slice(0, options.maxBytes) : text
 }
 
 /** Direct child pids of `pid`. POSIX only; returns nothing when pgrep is unavailable. */

package/lib/index.js

@@ -17,8 +17,10 @@
 
 import { defineTool } from '@deepseek-ai/dsh-tools'
 import * as DeviceBuild from './device-build.js'
+import * as UiTree from './uitree.js'
 import { DevicePreviewEngine } from './device-preview.js'
 import * as Setup from './setup.js'
+import { registerMobileSkill } from './skill.js'
 
 export const name = 'mobilecode'
 export const inject = ['webServer', 'tools', 'systemPrompt']
@@ -717,6 +719,144 @@ async function captureScreenSize(serial) {
   return match ? { width: Number(match[1]), height: Number(match[2]) } : undefined
 }
 
+/** Recursive node schema is not expressible here; tree children stay open objects. */
+const UI_TREE_ITEM_SCHEMA = { type: 'object', additionalProperties: true }
+
+function deviceUiTreeTool() {
+  return defineTool({
+    name: 'device_ui_tree',
+    description: 'Dump the foreground Android app\'s uiautomator hierarchy as a compact node tree — type, text, ' +
+      'contentDesc, resourceId, pixel bounds, enabled/focused/clickable flags. The default observer for UI automation: ' +
+      'resource-ids are the most stable tap handles. Use device_tap_element to tap by identity instead of guessing ' +
+      'pixel coordinates. When the tree comes back shallow or empty on a WebView/Compose/canvas, fall back to ' +
+      'device_screen (OCR reads pixels and needs no idle).',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      max_depth: { type: 'integer', description: 'Maximum hierarchy depth to include (omit for the full tree).' },
+      filter: { type: 'string', description: 'Case-insensitive substring over text/content-desc/resource-id/type; matching nodes and their ancestors are kept.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          rotation: { type: 'integer' },
+          nodes: { type: 'integer', required: true },
+          tree: { type: 'array', required: true, items: UI_TREE_ITEM_SCHEMA },
+          truncated: { type: 'boolean' },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', nodes: 0, tree: [] }
+        const lines = [`Device ${v.serial}: ${v.nodes} nodes${v.truncated ? ' (truncated at 40 KB — narrow with max_depth or filter)' : ''}${v.rotation !== undefined ? `, rotation=${v.rotation}` : ''}`]
+        const walk = (nodes, indent) => {
+          for (const node of nodes) {
+            const label = [node.text, node.contentDesc].find(Boolean) ?? ''
+            const id = node.resourceId ? ` [${node.resourceId}]` : ''
+            const flags = []
+            if (node.enabled === false) flags.push('disabled')
+            if (node.clickable) flags.push('clickable')
+            if (node.scrollable) flags.push('scrollable')
+            const b = node.bounds
+            lines.push(`${indent}- ${node.type}${label ? ` "${label}"` : ''}${id}${flags.length > 0 ? ` (${flags.join(',')})` : ''} @${b.x},${b.y} ${b.w}x${b.h}`)
+            if (indent.length < 8) walk(node.children, indent + '  ')
+          }
+        }
+        walk(v.tree, '  ')
+        if (v.nodes === 0) lines.push('  (no nodes — the surface may expose no accessibility; try device_screen OCR)')
+        return [{ type: 'text', text: lines.join('\n') }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const parsed = await UiTree.readUiTree(serial)
+      const { tree, count } = UiTree.buildCompactTree(parsed.roots, args.max_depth, args.filter)
+      const capped = UiTree.capTreeToBytes(tree)
+      const out = {
+        serial,
+        nodes: count,
+        tree: capped.tree,
+        ...(capped.truncated ? { truncated: true } : {}),
+        ...(parsed.rotation !== undefined ? { rotation: parsed.rotation } : {}),
+      }
+      return out
+    },
+  })
+}
+
+function deviceTapElementTool() {
+  return defineTool({
+    name: 'device_tap_element',
+    description: 'Tap an Android UI element by identity — resource_id matches the node\'s resource-id; text matches its ' +
+      'text or content-desc. Exact match first, then case-insensitive substring; nested duplicates collapse to one ' +
+      'target and an ambiguous match lists up to 8 candidates instead of picking one. Disabled or off-screen elements ' +
+      'are refused with the fix. Pass expect_text / expect_gone and the tap plus its verification become one round trip.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      resource_id: { type: 'string', description: 'The resource-id to match (e.g. com.android.settings:id/search_bar).' },
+      text: { type: 'string', description: 'Text or content-desc to match. Exact wins over substring.' },
+      expect_text: { type: 'string', description: 'After the tap, re-dump and verify this text is present.' },
+      expect_gone: { type: 'string', description: 'After the tap, re-dump and verify this text is gone.' },
+      allow_offscreen: { type: 'boolean', description: 'Tap a node whose bounds lie outside the screen anyway (default false).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          tapped: { type: 'string', required: true },
+          matchedBy: { type: 'string', required: true },
+          x: { type: 'integer', required: true },
+          y: { type: 'integer', required: true },
+          expected: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              mode: { type: 'string', required: true },
+              text: { type: 'string', required: true },
+              matched: { type: 'boolean', required: true },
+            },
+          },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', tapped: '', matchedBy: '', x: 0, y: 0 }
+        const lines = [`Tapped ${v.tapped} (${v.matchedBy}) at ${v.x},${v.y} on ${v.serial}`]
+        if (v.expected) lines.push(`Expect ${v.expected.mode} "${v.expected.text}": ${v.expected.matched ? 'VERIFIED' : 'NOT matched — the action did not land as expected'}`)
+        return [{ type: 'text', text: lines.join('\n') }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const parsed = await UiTree.readUiTree(serial)
+      const selector = { identifier: args.resource_id, label: args.text }
+      const { node, matchedBy } = UiTree.resolveTapTarget(parsed.roots, selector, { tool: 'device_tap_element', allowOffscreen: args.allow_offscreen === true })
+      const center = UiTree.boundsCenter(node.bounds)
+      await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'input', 'tap', String(center.x), String(center.y)]).exit
+      const describe = () => {
+        const parts = []
+        if (node.resourceId) parts.push(`resource_id ${node.resourceId}`)
+        if (node.text) parts.push(`text ${JSON.stringify(node.text)}`)
+        if (node.contentDesc) parts.push(`content-desc ${JSON.stringify(node.contentDesc)}`)
+        return parts.join(', ') || node.type
+      }
+      const out = { serial, tapped: describe(), matchedBy, x: center.x, y: center.y }
+      const mode = args.expect_gone !== undefined && args.expect_gone !== '' ? 'gone' : (args.expect_text !== undefined && args.expect_text !== '' ? 'appear' : undefined)
+      if (mode !== undefined) {
+        const text = mode === 'gone' ? String(args.expect_gone) : String(args.expect_text)
+        await new Promise((resolve) => setTimeout(resolve, 600))
+        const fresh = await UiTree.readUiTree(serial).catch(() => undefined)
+        const labels = fresh ? UiTree.collectLabels(fresh.roots) : []
+        const matched = mode === 'gone' ? !labels.some((label) => label.toLowerCase().includes(text.toLowerCase())) : labels.some((label) => label.toLowerCase().includes(text.toLowerCase()))
+        out.expected = { mode, text, matched }
+      }
+      return out
+    },
+  })
+}
+
 function deviceLogTool(engine) {
   return defineTool({
     name: 'device_log',
@@ -898,6 +1038,10 @@ function guidance() {
     '  Pass directory when the project root is not the working directory. A run takes minutes; use a generous timeout.',
     '- device_screen: capture the attached Android screen as a PNG plus the UI hierarchy and OCR text, so you can see what',
     '  the app shows and tap by pixel coordinates. Call it after a run to confirm the app rendered, and to drive UI flows.',
+    '- device_ui_tree: the default screen observer — the uiautomator hierarchy as a typed node tree with resource-ids,',
+    '  text, and pixel bounds. Narrow with filter/max_depth; on a textless surface (WebView, Compose, canvas) use device_screen.',
+    '- device_tap_element: tap a control by resource_id or text/content-desc, with one-call verification via',
+    '  expect_text / expect_gone (no separate screenshot needed to know the tap landed).',
     '- device_input: tap/swipe/type/press on the attached Android device at ABSOLUTE pixel coordinates (take the center of a',
     '  device_screen box: x=(x1+x2)/2, y=(y1+y2)/2). The control loop is device_screen → device_input → device_screen.',
     '- device_log: read device logs (logcat main/crash/events, kernel dmesg). Call it when a run fails or an app misbehaves.',
@@ -914,6 +1058,14 @@ export function apply(ctx, config) {
   Setup.ensureHome()
   Setup.writeOcrReadme()
 
+  // The playbook skill is host-independent; register it once at apply-time.
+  let disposeSkill = () => {}
+  try {
+    disposeSkill = registerMobileSkill(ctx) ?? (() => {})
+  } catch {
+    // The skills service is optional; the plugin works without the playbook.
+  }
+
   const resolve = () => ({
     enabled: config?.enabled ?? true,
     announceToAgent: config?.announceToAgent ?? true,
@@ -959,6 +1111,8 @@ export function apply(ctx, config) {
         deviceRunTool(engine, config),
         deviceDetectTool(engine, config),
         deviceScreenTool(engine),
+        deviceUiTreeTool(),
+        deviceTapElementTool(),
         deviceLogTool(engine),
         deviceStatusTool(engine),
         deviceInputTool(),
@@ -968,6 +1122,7 @@ export function apply(ctx, config) {
   }
 
   ctx.effect(() => () => {
+    disposeSkill()
     void engine.dispose()
   }, 'dsh-mobilecode: engine')
 

package/lib/skill.js

@@ -0,0 +1,104 @@
+/**
+ * dsh-mobilecode — the plugin's bundled playbook, contributed through
+ * ctx.skills.register().
+ *
+ * A tool description answers "what does this argument mean", one tool at a
+ * time. What agents re-derive every session is the WORKFLOW between the tools
+ * — which observer to reach for, how to confirm an action landed, what a
+ * device refuses to tell you. This skill carries that workflow so the model
+ * loads it once per UI task instead of rediscovering it by trial and error.
+ *
+ * Registration is DEFENSIVE: a host without the skill service still loads the
+ * plugin, it just does not advertise the playbook (the scoped inject never
+ * runs when the service is absent).
+ *
+ * Playbook design credit: ZSeven-W/dsh-android (MIT) src/skill.ts.
+ */
+
+export const SKILL_NAME = "device-ui-automation"
+
+export const SKILL_DESCRIPTION =
+  "Drive an attached Android device through the dsh-mobilecode device_* tools: read the screen, tap by identity, " +
+  "type, and confirm that an action landed. Load this before the first device_ui_tree / device_tap_element call of a UI task."
+
+export const SKILL_WHEN_TO_USE =
+  "Any task that operates an Android app through the device_* tools — opening apps, tapping controls, filling " +
+  "fields, scrolling, reading logs, or verifying what is on screen, on an emulator or a real device."
+
+export const SKILL_CONTENT = `# Driving Android with dsh-mobilecode
+
+The loop is **observe once → act with an assertion → observe again only if the assertion could not settle it**. Everything goes through adb; an emulator and a USB phone take exactly the same tools with the same arguments.
+
+## Reading the screen
+
+| Tool | Cost | Use it for |
+| --- | --- | --- |
+| \`device_ui_tree\` | ~0.6–1.5 s | hierarchy, resource-ids, text, enabled state — the default observer |
+| \`device_screen\` | ~0.3 s + OCR | the picture for the user, plus PaddleOCR text+boxes when the tree is blind |
+| \`device_log\` | fast | what the device logs (main/crash/events buffers) |
+
+- Start with \`device_ui_tree\`: the accessibility tree carries \`resource-id\` (e.g. \`com.android.settings:id/search_bar\`), the most stable handle a control can have — far better than its text, which changes with the device language.
+- \`uiautomator\` dumps a snapshot of the CURRENT frame. If an animation is still running the tree can be a half-finished layout; when a read looks wrong, re-read once rather than reasoning about the wrong frame.
+- A shallow or empty tree is NEVER evidence that an app lacks accessibility support. Attribute an unlabeled read to ONE of three causes — (a) the depth/filter cut it: re-read wider; (b) a WebView, Compose or canvas surface that publishes little: \`device_screen\` OCR is the fallback; (c) a DEEP, unfiltered read with no labels — only then may "little accessibility information" be reported. Never jump from a shallow read to "OCR the screen".
+- The uiautomator dump fails on a continuously animating foreground (a page still loading) — the error says so. Do not retry; use \`device_screen\` (OCR reads pixels and needs no idle).
+
+## Acting
+
+- Prefer \`device_tap_element\` (resource-id / text / content-desc). Raw pixel coordinates through \`device_input action=tap\` are the last resort — they break on the next layout change.
+- When you must tap pixels, take the center of a box from \`device_ui_tree\`/\`device_screen\`: x=(x1+x2)/2, y=(y1+y2)/2. Coordinates are ABSOLUTE pixels of the current display.
+- \`device_input action=swipe\` scrolls; \`action=key\` with "back" is a first-class verb on Android — use it instead of hunting for an on-screen back arrow.
+- Typing is ASCII-only through adb. Non-ASCII text (Chinese, emoji) cannot be delivered by \`input text\` — do not retry; type via the app's own UI instead.
+
+## Never guess a package name or a control
+
+- A package name that looks plausible is routinely NOT the installed one. Check \`device_status\` / \`device_log filter=<package>\` for what is actually running before assuming.
+- Icon-only controls carry no OCR text by definition: the tree's \`content-desc\` is the only reliable handle — a bare tree means "look deeper", never "start guessing".
+
+## Confirming an action landed
+
+- Pass \`expect_text\` (or \`expect_gone\`) to \`device_tap_element\`: the tap and its verification become ONE round trip, and the result carries \`expected.matched\`.
+- Waiting for something slow (a load, an animation, a network round trip) is a \`device_ui_tree\`/\`device_screen\` re-read after a pause — one observation, not a poll loop.
+- Never compare screenshots or count pixels to decide whether something happened. Read the text back.
+
+## Logs
+
+- \`device_log\` buffers: \`main\` (default), \`crash\` (holds ONLY fatal Java/native crashes — that is where a stack trace lives), \`events\` (activity lifecycle), \`kernel\` (needs adb root; emulators usually allow it).
+- An idle emulator emits hundreds of lines a second. Narrow BEFORE widening: \`filter\` (package or tag substring) and a small \`lines\` count first.
+
+## Real devices
+
+- **Every tap on a real phone has real consequences** — posts, likes, purchases, messages. NEVER tap an unidentified control to find out what it does. If a control cannot be identified (no resource-id after a deep tree, no distinguishing text), STOP and report what you see and ask how to proceed. Do not guess coordinates on someone's live account.
+- A phone must be UNLOCKED for anything to be visible; \`device_input action=key key="wakeup"\` wakes it, but a PIN/pattern lock cannot be passed from here.
+- \`device_status\` reports each device's adb state: \`unauthorized\` means the USB-debugging prompt has not been accepted ON the device — no tool can fix that from this side.
+`
+
+/**
+ * Register the playbook when the host provides the skill service. The scoped
+ * inject is the optional-service pattern: a profile without the skill service
+ * simply never runs the callback, and the plugin loads without the playbook.
+ */
+export function registerMobileSkill(ctx) {
+  if (typeof ctx.inject === "function") {
+    const fiber = ctx.inject(["skills"], (skillCtx) => {
+      skillCtx.effect(() => skillCtx.skills.register({
+        name: SKILL_NAME,
+        description: SKILL_DESCRIPTION,
+        whenToUse: SKILL_WHEN_TO_USE,
+        content: SKILL_CONTENT,
+        source: "bundled",
+      }), "dsh-mobilecode: skill")
+    })
+    return () => { try { fiber.dispose() } catch { /* already disposed */ } }
+  }
+  // Very old hosts: direct registration when the service exists, else skip.
+  if (ctx.skills && typeof ctx.skills.register === "function") {
+    return ctx.skills.register({
+      name: SKILL_NAME,
+      description: SKILL_DESCRIPTION,
+      whenToUse: SKILL_WHEN_TO_USE,
+      content: SKILL_CONTENT,
+      source: "bundled",
+    })
+  }
+  return undefined
+}

package/lib/uitree.js

@@ -0,0 +1,638 @@
+/**
+ * dsh-mobilecode — uiautomator semantic backend.
+ *
+ * Dumps the frontmost window's view hierarchy over plain adb, parses it with a
+ * hand-written quote-aware XML reader (no runtime dependencies — uiautomator's
+ * output is a tiny attribute-only dialect), and shapes it into the compact
+ * node tree the semantic tools reason over. The selector resolver ports the
+ * dsh-android design: exact match wins over substring, nested duplicates
+ * collapse into one chain by bounds containment (the chain's outermost control
+ * is the tap target), off-screen and disabled matches are refused with
+ * actionable copy, and ambiguity lists up to 8 candidates instead of guessing.
+ *
+ * Ported design credit: ZSeven-W/dsh-android (MIT) src/uitree.ts.
+ */
+
+import { adb, capture, exec } from "./device-build.js"
+
+const DUMP_TIMEOUT_MS = 60_000
+const DUMP_MAX_BYTES = 8 * 1024 * 1024
+/** Compact tree output cap: past this the deepest levels are pruned. */
+export const UI_TREE_CAP_BYTES = 40 * 1024
+
+// ── XML ───────────────────────────────────────────────────────────────────────
+
+const NAMED_ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }
+
+/**
+ * Decode the five XML entities plus numeric character references. Unknown or
+ * malformed references stay verbatim — a literal `&` in a label must survive.
+ */
+export function decodeXmlEntities(value) {
+  if (!value.includes("&")) return value
+  return value.replace(/&(#x[0-9a-fA-F]+|#\d+|[a-zA-Z]+);/g, (match, body) => {
+    if (body.startsWith("#x") || body.startsWith("#X")) {
+      const code = Number.parseInt(body.slice(2), 16)
+      return Number.isFinite(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
+    }
+    if (body.startsWith("#")) {
+      const code = Number.parseInt(body.slice(1), 10)
+      return Number.isFinite(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match
+    }
+    return NAMED_ENTITIES[body] ?? match
+  })
+}
+
+function isSpace(ch) {
+  return ch === " " || ch === "\t" || ch === "\n" || ch === "\r"
+}
+
+/**
+ * Scan one start tag. Quote-aware: `>` inside an attribute value never ends
+ * the tag early — uiautomator labels legitimately contain that character.
+ */
+function scanStartTag(source, start) {
+  const length = source.length
+  let index = start
+  while (index < length && !isSpace(source[index]) && source[index] !== "/" && source[index] !== ">") index += 1
+  const name = source.slice(start, index)
+  const attributes = {}
+  let selfClosing = false
+  for (;;) {
+    while (index < length && isSpace(source[index])) index += 1
+    if (index >= length) return { name, attributes, selfClosing, next: -1 }
+    const ch = source[index]
+    if (ch === "/") { selfClosing = true; index += 1; continue }
+    if (ch === ">") return { name, attributes, selfClosing, next: index + 1 }
+    const nameStart = index
+    while (index < length && !isSpace(source[index]) && source[index] !== "=" && source[index] !== "/" && source[index] !== ">") index += 1
+    const attributeName = source.slice(nameStart, index)
+    while (index < length && isSpace(source[index])) index += 1
+    let raw = ""
+    if (source[index] === "=") {
+      index += 1
+      while (index < length && isSpace(source[index])) index += 1
+      const quote = source[index]
+      if (quote === '"' || quote === "'") {
+        index += 1
+        const valueStart = index
+        while (index < length && source[index] !== quote) index += 1
+        raw = source.slice(valueStart, index)
+        index += 1
+      } else {
+        const valueStart = index
+        while (index < length && !isSpace(source[index]) && source[index] !== ">") index += 1
+        raw = source.slice(valueStart, index)
+      }
+    }
+    if (attributeName !== "") attributes[attributeName] = decodeXmlEntities(raw)
+    // A degenerate attribute name (nothing consumed) would spin forever.
+    if (attributeName === "" && raw === "") index += 1
+  }
+}
+
+/**
+ * Parse an attribute-only XML document into its element forest. Prologs,
+ * comments, doctypes and CDATA are skipped; character data is ignored.
+ * Mismatched close tags unwind to the nearest matching ancestor instead of
+ * throwing — a truncated dump still yields the part that arrived.
+ */
+export function parseXmlElements(source) {
+  const roots = []
+  const stack = []
+  const length = source.length
+  let index = 0
+  while (index < length) {
+    const open = source.indexOf("<", index)
+    if (open < 0) break
+    index = open + 1
+    if (index >= length) break
+    if (source.startsWith("!--", index)) {
+      const end = source.indexOf("-->", index)
+      index = end < 0 ? length : end + 3
+      continue
+    }
+    if (source.startsWith("![CDATA[", index)) {
+      const end = source.indexOf("]]>", index)
+      index = end < 0 ? length : end + 3
+      continue
+    }
+    if (source[index] === "?" || source[index] === "!") {
+      const end = source.indexOf(">", index)
+      index = end < 0 ? length : end + 1
+      continue
+    }
+    if (source[index] === "/") {
+      const end = source.indexOf(">", index)
+      if (end < 0) break
+      const name = source.slice(index + 1, end).trim()
+      for (let depth = stack.length - 1; depth >= 0; depth -= 1) {
+        if (stack[depth].name === name) { stack.length = depth; break }
+      }
+      index = end + 1
+      continue
+    }
+    const tag = scanStartTag(source, index)
+    if (tag.next < 0) break
+    index = tag.next
+    if (tag.name === "") continue
+    const element = { name: tag.name, attributes: tag.attributes, children: [] }
+    const parent = stack[stack.length - 1]
+    if (parent === undefined) roots.push(element)
+    else parent.children.push(element)
+    if (!tag.selfClosing) stack.push(element)
+  }
+  return roots
+}
+
+// ── uiautomator hierarchy → compact nodes ─────────────────────────────────────
+
+const BOUNDS_PATTERN = /^\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]$/
+
+/** Parse `bounds="[l,t][r,b]"` into an origin+size box; unparseable → undefined. */
+export function parseBounds(raw) {
+  if (raw === undefined) return undefined
+  const match = BOUNDS_PATTERN.exec(raw.trim())
+  if (match === null) return undefined
+  const [left, top, right, bottom] = match.slice(1).map(Number)
+  if (![left, top, right, bottom].every(Number.isFinite)) return undefined
+  return { x: left, y: top, w: right - left, h: bottom - top }
+}
+
+/** `android.widget.FrameLayout` → `FrameLayout`; empty class → `Node`. */
+export function classTail(className) {
+  const trimmed = (className ?? "").trim()
+  if (trimmed === "") return "Node"
+  const tail = trimmed.slice(trimmed.lastIndexOf(".") + 1)
+  return tail === "" ? trimmed : tail
+}
+
+function attributeText(attributes, key) {
+  const value = attributes[key]
+  if (value === undefined) return undefined
+  return value.trim() === "" ? undefined : value
+}
+
+function isTrue(attributes, key) {
+  return attributes[key] === "true"
+}
+
+function toNode(element) {
+  const attributes = element.attributes
+  const node = {
+    type: classTail(attributes.class),
+    bounds: parseBounds(attributes.bounds) ?? { x: 0, y: 0, w: 0, h: 0 },
+    children: [],
+  }
+  const text = attributeText(attributes, "text")
+  if (text !== undefined) node.text = text
+  const contentDesc = attributeText(attributes, "content-desc")
+  if (contentDesc !== undefined) node.contentDesc = contentDesc
+  const resourceId = attributeText(attributes, "resource-id")
+  if (resourceId !== undefined) node.resourceId = resourceId
+  // Interesting state only: absent means enabled / not focused / not
+  // clickable / not scrollable — never "unknown".
+  if (attributes.enabled === "false") node.enabled = false
+  if (isTrue(attributes, "focused")) node.focused = true
+  if (isTrue(attributes, "clickable")) node.clickable = true
+  if (isTrue(attributes, "scrollable")) node.scrollable = true
+  for (const child of element.children) {
+    if (child.name === "node") node.children.push(toNode(child))
+  }
+  return node
+}
+
+/**
+ * Convert one uiautomator XML document into the compact node forest. The
+ * `<hierarchy>` wrapper is unwrapped; a dump without it falls back to any
+ * top-level `node` elements so a hand-trimmed fixture still parses.
+ */
+export function parseUiTree(xml) {
+  const elements = parseXmlElements(xml)
+  const hierarchy = elements.find((element) => element.name === "hierarchy")
+  const source = hierarchy?.children ?? elements
+  const roots = source.filter((element) => element.name === "node").map(toNode)
+  const rotationRaw = hierarchy?.attributes.rotation
+  const rotation = rotationRaw === undefined ? undefined : Number(rotationRaw)
+  const out = { roots }
+  if (rotation !== undefined && Number.isInteger(rotation)) out.rotation = rotation
+  return out
+}
+
+/**
+ * Strip everything around the hierarchy document. `uiautomator dump /dev/tty`
+ * writes the XML and then its own confirmation line ("UI hierchary dumped to:
+ * /dev/tty" — the typo is upstream's) onto the SAME stream, and a tty may
+ * translate `\n` into `\r\n` on the way out.
+ */
+export function extractHierarchyXml(raw) {
+  const text = raw.replace(/\r\n/g, "\n")
+  const end = text.lastIndexOf("</hierarchy>")
+  if (end >= 0) {
+    const start = text.indexOf("<")
+    return text.slice(start < 0 ? 0 : start, end + "</hierarchy>".length)
+  }
+  // A self-closed or empty hierarchy still counts as a valid (if useless) dump.
+  const empty = /<hierarchy\b[^>]*\/>/.exec(text)
+  if (empty !== null) return empty[0]
+  const snippet = text.trim().slice(0, 200)
+  throw new Error(
+    "the uiautomator dump did not contain a <hierarchy> document"
+    + (snippet === "" ? " (the device produced no output)" : `: ${snippet}`),
+  )
+}
+
+// ── dump over adb ─────────────────────────────────────────────────────────────
+
+/**
+ * Dump the frontmost window hierarchy of `serial`.
+ *
+ * Primary path: `adb exec-out uiautomator dump /dev/tty` — one round trip, no
+ * device-side file. Some vendor images refuse `/dev/tty`; the fallback writes
+ * `/sdcard/window_dump.xml`, cats it back, and removes it. "could not get
+ * idle state" earns exactly one retry after 800 ms — a transient animation
+ * settles; a continuously animating foreground (a web page with a spinner)
+ * will fail again, and the error then routes the caller to OCR instead of a
+ * retry loop.
+ */
+export async function dumpUiTreeXml(serial) {
+  const options = { timeoutMs: DUMP_TIMEOUT_MS, maxBytes: DUMP_MAX_BYTES }
+  let primaryFailure
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      const buffer = await capture(adb(), ["-s", serial, "exec-out", "uiautomator", "dump", "/dev/tty"], options)
+      if (buffer === "") throw new Error("the device produced no output")
+      return extractHierarchyXml(buffer)
+    } catch (error) {
+      primaryFailure = error instanceof Error ? error.message : String(error)
+      if (attempt === 0 && /could not get idle state/i.test(primaryFailure)) {
+        await new Promise((resolve) => setTimeout(resolve, 800))
+        continue
+      }
+      break
+    }
+  }
+  if (primaryFailure !== undefined && /could not get idle state/i.test(primaryFailure)) {
+    // The /sdcard fallback runs the SAME dump against the same never-idle
+    // foreground; paying its cost only to fail identically helps nobody.
+    throw new Error(
+      `uiautomator could not dump the window hierarchy of ${serial} (${primaryFailure}). `
+      + "The foreground app is continuously animating (web pages in a browser are the classic case), "
+      + "so uiautomator can never reach its idle state — do not retry this tool; read the screen with "
+      + "device_screen (OCR reads pixels and needs no idle).",
+    )
+  }
+  const remotePath = "/sdcard/window_dump.xml"
+  try {
+    const notice = await capture(adb(), ["-s", serial, "shell", "uiautomator", "dump", remotePath], options)
+    const buffer = await capture(adb(), ["-s", serial, "exec-out", "cat", remotePath], options)
+    const xml = extractHierarchyXml(buffer)
+    await exec(adb(), ["-s", serial, "shell", "rm", "-f", remotePath]).exit.catch(() => {})
+    if (xml.trim() === "") throw new Error(notice.trim() || "empty dump")
+    return xml
+  } catch (error) {
+    await exec(adb(), ["-s", serial, "shell", "rm", "-f", remotePath]).exit.catch(() => {})
+    const fallbackFailure = error instanceof Error ? error.message : String(error)
+    const idleStarved = /could not get idle state/i.test(`${primaryFailure} ${fallbackFailure}`)
+    throw new Error(
+      `uiautomator could not dump the window hierarchy of ${serial} `
+      + `(exec-out /dev/tty: ${primaryFailure ?? "unknown"}; ${remotePath} fallback: ${fallbackFailure}). `
+      + (idleStarved
+        ? "The foreground app is continuously animating, so uiautomator can never reach its idle state — "
+          + "do not retry this tool; read the screen with device_screen (OCR) instead."
+        : "uiautomator needs the screen ON and an idle window — wake the device (device_input action=key "
+          + 'key="wakeup"), wait for animations to settle, and retry; if it keeps failing the screen is '
+          + "likely secure (FLAG_SECURE) and only device_screen's OCR can read it."),
+    )
+  }
+}
+
+/** Dump and parse in one step. */
+export async function readUiTree(serial) {
+  return parseUiTree(await dumpUiTreeXml(serial))
+}
+
+// ── tree shaping ──────────────────────────────────────────────────────────────
+
+/** Screen bounds in display pixels, taken from the widest/tallest root. */
+export function screenBoundsOf(roots) {
+  let width = 0
+  let height = 0
+  for (const root of roots) {
+    width = Math.max(width, root.bounds.x + root.bounds.w)
+    height = Math.max(height, root.bounds.y + root.bounds.h)
+  }
+  if (width <= 0 || height <= 0) {
+    const fallback = roots.length > 0 ? roots[0].bounds : { w: 0, h: 0 }
+    width = fallback.w
+    height = fallback.h
+  }
+  return { width, height }
+}
+
+/**
+ * True when `bounds` lies ENTIRELY outside the screen. uiautomator keeps
+ * scrolled-out rows in the dump with their real (off-screen) coordinates and
+ * exposes no visibility flag, so geometry is the only signal. A zero-size box
+ * can never be tapped, so it counts as off-screen too.
+ */
+export function isOffscreenBounds(bounds, screen) {
+  if (screen.width <= 0 || screen.height <= 0) return false
+  return bounds.x + bounds.w <= 0
+    || bounds.y + bounds.h <= 0
+    || bounds.x >= screen.width
+    || bounds.y >= screen.height
+}
+
+/** Case-insensitive substring match over text, content-desc, resource-id and type. */
+export function nodeMatchesFilter(node, needle) {
+  const haystacks = [node.type, node.text, node.contentDesc, node.resourceId]
+  return haystacks.some((value) => value !== undefined && value.toLowerCase().includes(needle))
+}
+
+function copyNode(node) {
+  const copy = { type: node.type, bounds: { ...node.bounds }, children: [] }
+  if (node.text !== undefined) copy.text = node.text
+  if (node.contentDesc !== undefined) copy.contentDesc = node.contentDesc
+  if (node.resourceId !== undefined) copy.resourceId = node.resourceId
+  if (node.enabled !== undefined) copy.enabled = node.enabled
+  if (node.focused !== undefined) copy.focused = node.focused
+  if (node.clickable !== undefined) copy.clickable = node.clickable
+  if (node.scrollable !== undefined) copy.scrollable = node.scrollable
+  return copy
+}
+
+/**
+ * Build the output tree: an optional case-insensitive substring filter (a node
+ * survives when it or any descendant matches — ancestors of matches are kept
+ * so the tree stays connected) and an optional nesting depth cap.
+ */
+export function buildCompactTree(roots, maxDepth, filter) {
+  const needle = filter !== undefined && filter.trim() !== "" ? filter.trim().toLowerCase() : undefined
+  let count = 0
+  const walk = (node, depth) => {
+    const selfMatches = needle === undefined || nodeMatchesFilter(node, needle)
+    const children = []
+    if (maxDepth === undefined || depth < maxDepth) {
+      for (const child of node.children) {
+        const compact = walk(child, depth + 1)
+        if (compact !== undefined) children.push(compact)
+      }
+    }
+    if (!selfMatches && children.length === 0) return undefined
+    const copy = copyNode(node)
+    copy.children = children
+    count += 1
+    return copy
+  }
+  const tree = []
+  for (const root of roots) {
+    const compact = walk(root, 0)
+    if (compact !== undefined) tree.push(compact)
+  }
+  return { tree, count }
+}
+
+/** Depth-first flatten, roots first; depth annotates each copy. */
+export function flattenNodes(roots) {
+  const flat = []
+  const walk = (node, depth) => {
+    const copy = copyNode(node)
+    copy.depth = depth
+    flat.push(copy)
+    for (const child of node.children) walk(child, depth + 1)
+  }
+  for (const root of roots) walk(root, 0)
+  return flat
+}
+
+function treeDepth(nodes) {
+  let depth = 0
+  for (const node of nodes) {
+    if (node.children.length > 0) depth = Math.max(depth, 1 + treeDepth(node.children))
+  }
+  return depth
+}
+
+function pruneDeepestLevel(nodes) {
+  const depth = treeDepth(nodes)
+  if (depth === 0) return
+  const pruneAt = (list, level) => {
+    for (const node of list) {
+      if (level === depth - 1) node.children = []
+      else pruneAt(node.children, level + 1)
+    }
+  }
+  pruneAt(nodes, 0)
+}
+
+function treeBytes(nodes) {
+  return Buffer.byteLength(JSON.stringify(nodes), "utf8")
+}
+
+/**
+ * Fit a compact tree under `capBytes` by pruning the deepest levels first —
+ * the same strategy the `max_depth` hint offers interactively. Mutates the
+ * nodes it is handed (they are already the tool's private copies).
+ */
+export function capTreeToBytes(tree, capBytes = UI_TREE_CAP_BYTES) {
+  let truncated = treeBytes(tree) > capBytes
+  while (treeBytes(tree) > capBytes && treeDepth(tree) > 0) {
+    pruneDeepestLevel(tree)
+  }
+  if (!truncated) truncated = treeBytes(tree) > capBytes
+  return { tree, truncated }
+}
+
+// ── selector resolution ───────────────────────────────────────────────────────
+
+/** Tolerance (pixels) for containment checks — rounding, not layout, slack. */
+const BOUNDS_EPSILON = 1
+
+/** True when `outer` (approximately) contains `inner`. */
+export function containsBounds(outer, inner) {
+  return outer.x <= inner.x + BOUNDS_EPSILON
+    && outer.y <= inner.y + BOUNDS_EPSILON
+    && outer.x + outer.w >= inner.x + inner.w - BOUNDS_EPSILON
+    && outer.y + outer.h >= inner.y + inner.h - BOUNDS_EPSILON
+}
+
+/** True when two boxes are the same box (mutual containment). */
+export function sameBounds(a, b) {
+  return containsBounds(a, b) && containsBounds(b, a)
+}
+
+/**
+ * Widget classes that ARE controls even when the platform did not mark them
+ * clickable (a disabled Button reports clickable="false"). `clickable=true`
+ * remains the primary signal; this set only rescues the chain-folding step.
+ */
+const CONTROL_TYPES = new Set([
+  "Button", "ImageButton", "CompoundButton", "CheckBox", "CheckedTextView",
+  "RadioButton", "Switch", "SwitchCompat", "ToggleButton", "MaterialButton",
+  "EditText", "AutoCompleteTextView", "SearchView", "SeekBar", "RatingBar",
+  "Spinner", "TabWidget", "ActionMenuItemView", "MenuItem", "Chip",
+  "FloatingActionButton", "BottomNavigationItemView", "NavigationMenuItemView",
+])
+
+function isControl(node) {
+  return node.clickable === true || CONTROL_TYPES.has(node.type)
+}
+
+function describeCandidate(node, index) {
+  const text = node.text === undefined ? "" : ` text=${JSON.stringify(node.text)}`
+  const desc = node.contentDesc === undefined ? "" : ` content-desc=${JSON.stringify(node.contentDesc)}`
+  const id = node.resourceId === undefined ? "" : ` resource-id=${JSON.stringify(node.resourceId)}`
+  const flags = node.enabled === false ? " enabled=false" : ""
+  const bounds = `bounds={x:${node.bounds.x},y:${node.bounds.y},w:${node.bounds.w},h:${node.bounds.h}}`
+  return `${index}) type=${node.type}${text}${desc}${id}${flags} ${bounds}`
+}
+
+/** Actionable refusal when every selector match is off-screen or disabled. */
+function tapGateFailure(tool, representatives, screen, wanted, allowOffscreen) {
+  const offscreen = representatives.filter((node) => isOffscreenBounds(node.bounds, screen))
+  const disabled = representatives.filter((node) => node.enabled === false)
+  const hint = allowOffscreen ? " (allow_offscreen=true bypasses only the off-screen check — disabled stays refused)" : ""
+  if (offscreen.length > 0 && disabled.length > 0) {
+    throw new Error(
+      `${tool}: ${wanted} matched ${representatives.length} node(s) that are off-screen or disabled`
+      + " — scroll the off-screen ones into view first and enable the disabled ones" + hint,
+    )
+  }
+  if (offscreen.length > 0) {
+    const noun = representatives.length === 1 ? "matched an off-screen node" : `matched ${representatives.length} off-screen nodes`
+    throw new Error(
+      `${tool}: ${wanted} ${noun} — scroll it into view first (device_input action=swipe), `
+      + "then re-run device_ui_tree so the fresh dump re-locates it"
+      + `; pass allow_offscreen=true to tap the recorded coordinates anyway` + hint,
+    )
+  }
+  const noun = representatives.length === 1 ? "matched a disabled node" : `matched ${representatives.length} disabled nodes`
+  throw new Error(
+    `${tool}: ${wanted} ${noun} — the control is disabled, so a tap would do nothing; enable it first` + hint,
+  )
+}
+
+/**
+ * Resolve one node from a selector.
+ *
+ * `identifier` matches the resource-id; `label` matches the text OR the
+ * content-desc. Exact (case-sensitive) equality wins; otherwise
+ * case-insensitive substring. When both fields are given both must match.
+ *
+ * Nested duplicates — a list row mirrors its text onto a child TextView, and
+ * the clickable container wraps them both — collapse into ONE chain by bounds
+ * containment; the chain's outermost control (clickable, or a control widget
+ * class) is the tap target, falling back to the deepest node when the chain
+ * contains no control at all.
+ *
+ * Safety gate: matches that are off-screen or disabled are NOT tappable. When
+ * every match fails the gate the resolver throws an actionable error naming
+ * the fix; `allowOffscreen` skips only the off-screen half — a disabled node
+ * always refuses. Distinct nodes that all survive the gate raise an ambiguity
+ * error listing up to 8 candidates.
+ */
+export function resolveTapTarget(roots, selector, options = {}) {
+  const tool = options.tool ?? "device_tap_element"
+  const identifier = selector.identifier !== undefined && selector.identifier.trim() !== "" ? selector.identifier.trim() : undefined
+  const label = selector.label !== undefined && selector.label.trim() !== "" ? selector.label.trim() : undefined
+  if (identifier === undefined && label === undefined) {
+    throw new Error(
+      `${tool} requires an element selector: resource_id and/or text `
+      + "(text also matches content-desc). Run device_ui_tree to see what the screen exposes.",
+    )
+  }
+  const flat = flattenNodes(roots)
+  const matchesValue = (actual, wanted, mode) => {
+    if (actual === undefined) return false
+    return mode === "exact" ? actual === wanted : actual.toLowerCase().includes(wanted.toLowerCase())
+  }
+  const matchesNode = (node, mode) => {
+    if (identifier !== undefined && !matchesValue(node.resourceId, identifier, mode)) return false
+    if (label !== undefined && !matchesValue(node.text, label, mode) && !matchesValue(node.contentDesc, label, mode)) return false
+    return true
+  }
+  let candidates = flat.filter((node) => matchesNode(node, "exact"))
+  let matchedBy = "exact"
+  if (candidates.length === 0) {
+    candidates = flat.filter((node) => matchesNode(node, "contains"))
+    matchedBy = "contains"
+  }
+  const wantedParts = []
+  if (identifier !== undefined) wantedParts.push(`resource_id ${JSON.stringify(identifier)}`)
+  if (label !== undefined) wantedParts.push(`text ${JSON.stringify(label)}`)
+  const wanted = wantedParts.join(" and ")
+  if (candidates.length === 0) {
+    throw new Error(
+      `${tool}: no node matches ${wanted} on the current screen — run device_ui_tree to inspect what is `
+      + "actually there, or device_screen to OCR labels the view hierarchy does not carry "
+      + "(Compose/Flutter/WebView/game canvases often expose none).",
+    )
+  }
+  // Drop exact box duplicates of the same class (a wrapper listed twice).
+  const unique = candidates.filter((node, index) => !candidates
+    .slice(0, index)
+    .some((other) => other.type === node.type && sameBounds(other.bounds, node.bounds)))
+  // Group containment chains: an ancestor that mirrors its child's text is
+  // the same row, not an ambiguity.
+  const chains = []
+  for (const node of unique) {
+    const chain = chains.find((group) => group.some((other) =>
+      !sameBounds(node.bounds, other.bounds)
+      && (containsBounds(node.bounds, other.bounds) || containsBounds(other.bounds, node.bounds)),
+    ))
+    if (chain === undefined) chains.push([node])
+    else chain.push(node)
+  }
+  const representatives = chains.map((chain) => {
+    const controls = chain.filter(isControl)
+    if (controls.length > 0) {
+      // Outermost control of the chain: not contained in another control.
+      const outer = controls.find((node) => !controls.some((other) =>
+        other !== node && containsBounds(other.bounds, node.bounds) && !sameBounds(other.bounds, node.bounds),
+      ))
+      return outer ?? controls[0]
+    }
+    // No control in the chain: the deepest (most specific) node it is.
+    return chain.reduce((deepest, node) => (node.depth > deepest.depth ? node : deepest), chain[0])
+  })
+  const screen = screenBoundsOf(roots)
+  const allowOffscreen = options.allowOffscreen === true
+  const viable = representatives.filter((node) =>
+    node.enabled !== false && (allowOffscreen || !isOffscreenBounds(node.bounds, screen)),
+  )
+  if (viable.length === 0) tapGateFailure(tool, representatives, screen, wanted, allowOffscreen)
+  if (viable.length > 1) {
+    const skipped = representatives.length - viable.length
+    const skippedSentence = skipped > 0 ? ` (${skipped} skipped: off-screen or disabled)` : ""
+    const shown = representatives.slice(0, 8)
+    const more = representatives.length - shown.length
+    throw new Error(
+      `${tool}: ${representatives.length} nodes match ${wanted}${skippedSentence} — use a more specific `
+      + "selector (an exact text, a resource_id, or device_ui_tree to disambiguate). Candidates:\n"
+      + shown.map((node, index) => `  ${describeCandidate(node, index + 1)}`).join("\n")
+      + (more > 0 ? `\n  …and ${more} more` : ""),
+    )
+  }
+  return { node: viable[0], matchedBy }
+}
+
+/** Center of a box in display pixels (integers: `input tap` takes pixels). */
+export function boundsCenter(bounds) {
+  return {
+    x: Math.round(bounds.x + bounds.w / 2),
+    y: Math.round(bounds.y + bounds.h / 2),
+  }
+}
+
+/** Every label (text or content-desc) in the tree, for expect_* verification. */
+export function collectLabels(roots) {
+  const labels = []
+  const walk = (node) => {
+    if (node.text !== undefined) labels.push(node.text)
+    if (node.contentDesc !== undefined) labels.push(node.contentDesc)
+    for (const child of node.children) walk(child)
+  }
+  for (const root of roots) walk(root)
+  return labels
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-mobilecode",
   "description": "MobileCode for the dsh web GUI: detect iOS/Android projects, run serve-sim / serve-avd preview servers, and build-install-launch the app on the simulator or emulator from the session — plus agent tools (device_run, device_detect). Hot-pluggable — mounted via the profile bundle list + cordis.patch.yml, no dsh source changes.",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "type": "module",
   "packageManager": "pnpm@11.22.0",
   "engines": {