dsh-mobilecode 0.1.4 → 0.2.1

package/lib/index.js

@@ -17,8 +17,10 @@
 
 import { defineTool } from '@deepseek-ai/dsh-tools'
 import * as DeviceBuild from './device-build.js'
+import * as UiTree from './uitree.js'
 import { DevicePreviewEngine } from './device-preview.js'
 import * as Setup from './setup.js'
+import { registerMobileSkill } from './skill.js'
 
 export const name = 'mobilecode'
 export const inject = ['webServer', 'tools', 'systemPrompt']
@@ -581,9 +583,21 @@ function deviceInputTool() {
         }
         case 'text': {
           if (typeof args.text !== 'string' || args.text.length === 0) throw new Error('action=text requires a non-empty text string.')
-          const escaped = args.text.replace(/\s/g, '%s')
-          await DeviceBuild.exec(DeviceBuild.adb(), adbArgs(['input', 'text', escaped])).exit
-          return { serial, action, sent: `text "${args.text}"` }
+          if (DeviceBuild.isAsciiInput(args.text)) {
+            await DeviceBuild.exec(DeviceBuild.adb(), adbArgs(['input', 'text', DeviceBuild.escapeInputText(args.text)])).exit
+            return { serial, action, sent: `text "${args.text}"` }
+          }
+          // Non-ASCII (CJK, emoji, accented) cannot go through `input text`; the
+          // ADBKeyboard IME is the only adb path. Refuse with the fix, never mangle.
+          if (!(await DeviceBuild.adbKeyboardReady(serial))) {
+            throw new Error(
+              `device_input cannot type non-ASCII text ("${args.text}") over plain adb. Install ADBKeyboard `
+              + 'on the device (https://github.com/senzhk/ADBKeyBoard), enable it (ime enable/set '
+              + 'com.android.adbkeyboard/.AdbIME), then retry — this tool then types via its base64 broadcast.',
+            )
+          }
+          await DeviceBuild.typeViaAdbKeyboard(serial, args.text)
+          return { serial, action, sent: `text "${args.text}" (ADBKeyboard)` }
         }
         case 'key': {
           const raw = String(args.key ?? '')
@@ -599,6 +613,301 @@ function deviceInputTool() {
   })
 }
 
+/** OCR the current screen and report whether `wantedLower` appears; undefined when OCR is unavailable. */
+async function ocrHasText(serial, wantedLower) {
+  if (!DeviceBuild.ocrPython()) return undefined
+  const png = await DeviceBuild.screenCapture(serial)
+  if (!png) return undefined
+  const ocr = await DeviceBuild.ocrImage(png).catch(() => [])
+  return ocr.some((item) => String(item.text).toLowerCase().includes(wantedLower))
+}
+
+function deviceWaitForTool() {
+  return defineTool({
+    name: 'device_wait_for',
+    description: 'Wait for on-screen text to appear or disappear. Polls the uiautomator tree every ~600 ms; when the tree ' +
+      'carries no labels (WebView/Compose/canvas) it falls back to local PaddleOCR. A timeout is a normal matched:false ' +
+      'result, never an error — one call replaces an agent-side poll loop.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      text: { type: 'string', description: 'Text to wait for (case-insensitive substring).' },
+      mode: { type: 'string', enum: ['appear', 'disappear'], description: 'appear (default) waits for the text; disappear waits for it to go away.' },
+      timeout_ms: { type: 'integer', description: 'Max wait in ms (default 10000, max 60000).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          text: { type: 'string', required: true },
+          mode: { type: 'string', required: true },
+          matched: { type: 'boolean', required: true },
+          waited_ms: { type: 'integer', required: true },
+          source: { type: 'string', required: true },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', text: '', mode: 'appear', matched: false, waited_ms: 0, source: 'ui_tree' }
+        return [{ type: 'text', text: `${v.matched ? 'MATCHED' : 'TIMEOUT'}: "${v.text}" ${v.mode} (${v.source}) after ${v.waited_ms}ms on ${v.serial}` }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const text = String(args.text ?? '').trim()
+      if (text === '') throw new Error('device_wait_for requires a non-empty text.')
+      const mode = args.mode === 'disappear' ? 'disappear' : 'appear'
+      const timeout = Math.min(Math.max(args.timeout_ms ?? 10_000, 500), 60_000)
+      const wanted = text.toLowerCase()
+      const start = Date.now()
+      const deadline = start + timeout
+      let source = 'ui_tree'
+      for (;;) {
+        let labels
+        try {
+          labels = UiTree.collectLabels((await UiTree.readUiTree(serial)).roots)
+        } catch {
+          labels = undefined
+        }
+        let present
+        if (labels !== undefined && labels.length > 0) {
+          source = 'ui_tree'
+          present = labels.some((label) => label.toLowerCase().includes(wanted))
+        } else {
+          const ocr = await ocrHasText(serial, wanted)
+          if (ocr === undefined) { source = 'ui_tree'; present = false }
+          else { source = 'ocr'; present = ocr }
+        }
+        const matched = mode === 'appear' ? present : !present
+        const waited = Math.min(timeout, Date.now() - start)
+        if (matched) return { serial, text, mode, matched: true, waited_ms: waited, source }
+        if (Date.now() >= deadline) return { serial, text, mode, matched: false, waited_ms: timeout, source }
+        await new Promise((resolve) => setTimeout(resolve, source === 'ocr' ? 2000 : 600))
+      }
+    },
+  })
+}
+
+function deviceBootTool() {
+  return defineTool({
+    name: 'device_boot',
+    description: 'Boot an Android emulator AVD by name (from device_status.avds) and wait until it finishes booting. ' +
+      'If an emulator for that AVD is already running it is adopted. This is the device-centric boot — device_run is the ' +
+      'project-centric build+install+launch.',
+    parameters: {
+      avd: { type: 'string', description: 'AVD name to boot.' },
+      timeout_ms: { type: 'integer', description: 'Max wait for boot in ms (default 180000, max 600000).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          avd: { type: 'string', required: true },
+          booted: { type: 'boolean', required: true },
+          alreadyRunning: { type: 'boolean', required: true },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', avd: '', booted: false, alreadyRunning: false }
+        return [{ type: 'text', text: `${v.alreadyRunning ? 'Adopted running' : 'Booted'} emulator ${v.serial} (AVD ${v.avd}), boot completed: ${v.booted}` }]
+      },
+    },
+    async execute(args) {
+      const avd = String(args.avd ?? '').trim()
+      if (avd === '') throw new Error('device_boot requires an avd name (see device_status.avds).')
+      const timeout = Math.min(Math.max(args.timeout_ms ?? 180_000, 5_000), 600_000)
+      const rows = await DeviceBuild.devices()
+      for (const row of rows.filter((item) => item.state === 'device' && item.serial.startsWith('emulator-'))) {
+        const name = await DeviceBuild.avdName(row.serial).catch(() => undefined)
+        if (name === avd) {
+          const booted = await DeviceBuild.waitForBoot(row.serial, timeout)
+          return { serial: row.serial, avd, booted, alreadyRunning: true }
+        }
+      }
+      if (!DeviceBuild.emulatorBinary()) throw new Error('No SDK emulator binary found; cannot boot an AVD.')
+      const before = new Set(rows.map((item) => item.serial))
+      if (!DeviceBuild.bootEmulator(avd)) throw new Error(`Could not launch the emulator for AVD "${avd}".`)
+      const deadline = Date.now() + timeout
+      let serial
+      while (Date.now() < deadline) {
+        const now = await DeviceBuild.devices()
+        const fresh = now.find((item) => item.serial.startsWith('emulator-') && !before.has(item.serial))
+        if (fresh) { serial = fresh.serial; break }
+        await new Promise((resolve) => setTimeout(resolve, 1000))
+      }
+      if (!serial) throw new Error(`No emulator serial appeared for "${avd}" within ${timeout} ms.`)
+      const booted = await DeviceBuild.waitForBoot(serial, Math.max(1000, deadline - Date.now()))
+      if (!booted) throw new Error(`Emulator ${serial} ("${avd}") did not finish booting within ${timeout} ms.`)
+      return { serial, avd, booted: true, alreadyRunning: false }
+    },
+  })
+}
+
+function deviceShutdownTool() {
+  return defineTool({
+    name: 'device_shutdown',
+    description: 'Shut down an emulator (`adb emu kill`). Refuses physical devices — adb has no power-off verb for phones.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          shutdown: { type: 'boolean', required: true },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: `Shut down emulator ${value?.serial}` }],
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const isEmulator = serial.startsWith('emulator-') || (await DeviceBuild.avdName(serial).catch(() => undefined)) !== undefined
+      if (!isEmulator) {
+        throw new Error(`device_shutdown refuses ${serial}: it is a physical device and adb has no power-off verb for phones — use its own power button.`)
+      }
+      await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'emu', 'kill']).exit
+      return { serial, shutdown: true }
+    },
+  })
+}
+
+const DEVICE_ACTIONS = {
+  notifications: ['cmd', 'statusbar', 'expand-notifications'],
+  quick_settings: ['cmd', 'statusbar', 'expand-settings'],
+  collapse: ['cmd', 'statusbar', 'collapse'],
+  lock: ['input', 'keyevent', '223'],
+  wake: ['input', 'keyevent', '224'],
+  assistant: ['am', 'start', '-a', 'android.intent.action.ASSIST'],
+}
+
+function deviceActionTool() {
+  return defineTool({
+    name: 'device_action',
+    description: 'Device-level actions beyond touches: open the notification shade or quick settings, collapse the shade, ' +
+      'lock or wake the screen, launch the assistant, or rotate the display (cycles 0→90→180→270 and pins auto-rotate off).',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      action: { type: 'string', enum: [...Object.keys(DEVICE_ACTIONS), 'rotate'], description: 'Which device action to perform.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          action: { type: 'string', required: true },
+          rotation: { type: 'integer' },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: value?.rotation !== undefined ? `${value.action} on ${value.serial} → rotation ${value.rotation * 90}°` : `${value?.action} on ${value?.serial}` }],
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const action = String(args.action ?? '')
+      if (action === 'rotate') {
+        const current = Number(await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'get', 'system', 'user_rotation']))
+        const next = ((Number.isFinite(current) ? current : 0) + 1) % 4
+        await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'put', 'system', 'accelerometer_rotation', '0']).exit
+        await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'put', 'system', 'user_rotation', String(next)]).exit
+        return { serial, action, rotation: next }
+      }
+      const shell = DEVICE_ACTIONS[action]
+      if (!shell) throw new Error(`unknown action "${action}" — use ${[...Object.keys(DEVICE_ACTIONS), 'rotate'].join(', ')}.`)
+      await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', ...shell]).exit
+      return { serial, action }
+    },
+  })
+}
+
+function deviceAppsTool() {
+  return defineTool({
+    name: 'device_apps',
+    description: 'List installed Android packages (third-party by default; include_system=true adds platform apps). ' +
+      'Use it to find the real package name before device_launch_app — never guess one.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      include_system: { type: 'boolean', description: 'Include system/platform packages (default false).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          count: { type: 'integer', required: true },
+          packages: { type: 'array', required: true, items: { type: 'string' } },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', count: 0, packages: [] }
+        const shown = v.packages.slice(0, 60).join('\n  ')
+        return [{ type: 'text', text: `${v.serial}: ${v.count} packages\n  ${shown}${v.count > 60 ? `\n  … and ${v.count - 60} more` : ''}` }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'pm', 'list', 'packages', ...(args.include_system ? [] : ['-3'])])
+      const packages = output
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith('package:'))
+        .map((line) => line.slice('package:'.length).trim())
+      return { serial, count: packages.length, packages: packages.slice(0, 200) }
+    },
+  })
+}
+
+function deviceLaunchAppTool() {
+  return defineTool({
+    name: 'device_launch_app',
+    description: 'Launch an installed app by package name (or a unique substring of it). relaunch=true force-stops it first ' +
+      'for a cold start. Resolves the exact package via `pm list packages` so a wrong guess fails loudly instead of ' +
+      'opening the wrong app.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      package: { type: 'string', description: 'Package name or a unique substring of it.' },
+      relaunch: { type: 'boolean', description: 'Force-stop the app first (cold start).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          package: { type: 'string', required: true },
+          launched: { type: 'boolean', required: true },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: `Launched ${value?.package} on ${value?.serial}` }],
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      let pkg = String(args.package ?? '').trim()
+      if (pkg === '') throw new Error('device_launch_app requires a package name (or a unique substring).')
+      const listOut = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'pm', 'list', 'packages'])
+      const all = listOut
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith('package:'))
+        .map((line) => line.slice('package:'.length).trim())
+      if (!all.includes(pkg)) {
+        const matches = all.filter((item) => item.toLowerCase().includes(pkg.toLowerCase()))
+        if (matches.length === 0) throw new Error(`No installed package matches "${pkg}". Run device_apps to list them.`)
+        if (matches.length > 1) throw new Error(`"${pkg}" matches ${matches.length} packages (${matches.slice(0, 8).join(', ')}${matches.length > 8 ? ', …' : ''}) — be more specific.`)
+        pkg = matches[0]
+      }
+      if (args.relaunch) await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'am', 'force-stop', pkg]).exit
+      const code = await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'monkey', '-p', pkg, '-c', 'android.intent.category.LAUNCHER', '1']).exit
+      if (code !== 0) throw new Error(`Could not launch ${pkg} (no launcher activity, or monkey failed with exit ${code}).`)
+      return { serial, package: pkg, launched: true }
+    },
+  })
+}
+
 function deviceScreenTool(engine) {
   return defineTool({
     name: 'device_screen',
@@ -713,10 +1022,150 @@ function deviceScreenTool(engine) {
 
 async function captureScreenSize(serial) {
   const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'wm', 'size'])
-  const match = /Physical size:\s*(\d+)x(\d+)/.exec(output)
+  // An `wm size` override wins over the physical panel — the input space is the override.
+  const override = /Override size:\s*(\d+)x(\d+)/.exec(output)
+  const match = override ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
   return match ? { width: Number(match[1]), height: Number(match[2]) } : undefined
 }
 
+/** Recursive node schema is not expressible here; tree children stay open objects. */
+const UI_TREE_ITEM_SCHEMA = { type: 'object', additionalProperties: true }
+
+function deviceUiTreeTool() {
+  return defineTool({
+    name: 'device_ui_tree',
+    description: 'Dump the foreground Android app\'s uiautomator hierarchy as a compact node tree — type, text, ' +
+      'contentDesc, resourceId, pixel bounds, enabled/focused/clickable flags. The default observer for UI automation: ' +
+      'resource-ids are the most stable tap handles. Use device_tap_element to tap by identity instead of guessing ' +
+      'pixel coordinates. When the tree comes back shallow or empty on a WebView/Compose/canvas, fall back to ' +
+      'device_screen (OCR reads pixels and needs no idle).',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      max_depth: { type: 'integer', description: 'Maximum hierarchy depth to include (omit for the full tree).' },
+      filter: { type: 'string', description: 'Case-insensitive substring over text/content-desc/resource-id/type; matching nodes and their ancestors are kept.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          rotation: { type: 'integer' },
+          nodes: { type: 'integer', required: true },
+          tree: { type: 'array', required: true, items: UI_TREE_ITEM_SCHEMA },
+          truncated: { type: 'boolean' },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', nodes: 0, tree: [] }
+        const lines = [`Device ${v.serial}: ${v.nodes} nodes${v.truncated ? ' (truncated at 40 KB — narrow with max_depth or filter)' : ''}${v.rotation !== undefined ? `, rotation=${v.rotation}` : ''}`]
+        const walk = (nodes, indent) => {
+          for (const node of nodes) {
+            const label = [node.text, node.contentDesc].find(Boolean) ?? ''
+            const id = node.resourceId ? ` [${node.resourceId}]` : ''
+            const flags = []
+            if (node.enabled === false) flags.push('disabled')
+            if (node.clickable) flags.push('clickable')
+            if (node.scrollable) flags.push('scrollable')
+            const b = node.bounds
+            lines.push(`${indent}- ${node.type}${label ? ` "${label}"` : ''}${id}${flags.length > 0 ? ` (${flags.join(',')})` : ''} @${b.x},${b.y} ${b.w}x${b.h}`)
+            if (indent.length < 8) walk(node.children, indent + '  ')
+          }
+        }
+        walk(v.tree, '  ')
+        if (v.nodes === 0) lines.push('  (no nodes — the surface may expose no accessibility; try device_screen OCR)')
+        return [{ type: 'text', text: lines.join('\n') }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const parsed = await UiTree.readUiTree(serial)
+      const { tree, count } = UiTree.buildCompactTree(parsed.roots, args.max_depth, args.filter)
+      const capped = UiTree.capTreeToBytes(tree)
+      const out = {
+        serial,
+        nodes: count,
+        tree: capped.tree,
+        ...(capped.truncated ? { truncated: true } : {}),
+        ...(parsed.rotation !== undefined ? { rotation: parsed.rotation } : {}),
+      }
+      return out
+    },
+  })
+}
+
+function deviceTapElementTool() {
+  return defineTool({
+    name: 'device_tap_element',
+    description: 'Tap an Android UI element by identity — resource_id matches the node\'s resource-id; text matches its ' +
+      'text or content-desc. Exact match first, then case-insensitive substring; nested duplicates collapse to one ' +
+      'target and an ambiguous match lists up to 8 candidates instead of picking one. Disabled or off-screen elements ' +
+      'are refused with the fix. Pass expect_text / expect_gone and the tap plus its verification become one round trip.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      resource_id: { type: 'string', description: 'The resource-id to match (e.g. com.android.settings:id/search_bar).' },
+      text: { type: 'string', description: 'Text or content-desc to match. Exact wins over substring.' },
+      expect_text: { type: 'string', description: 'After the tap, re-dump and verify this text is present.' },
+      expect_gone: { type: 'string', description: 'After the tap, re-dump and verify this text is gone.' },
+      allow_offscreen: { type: 'boolean', description: 'Tap a node whose bounds lie outside the screen anyway (default false).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          tapped: { type: 'string', required: true },
+          matchedBy: { type: 'string', required: true },
+          x: { type: 'integer', required: true },
+          y: { type: 'integer', required: true },
+          expected: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              mode: { type: 'string', required: true },
+              text: { type: 'string', required: true },
+              matched: { type: 'boolean', required: true },
+            },
+          },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', tapped: '', matchedBy: '', x: 0, y: 0 }
+        const lines = [`Tapped ${v.tapped} (${v.matchedBy}) at ${v.x},${v.y} on ${v.serial}`]
+        if (v.expected) lines.push(`Expect ${v.expected.mode} "${v.expected.text}": ${v.expected.matched ? 'VERIFIED' : 'NOT matched — the action did not land as expected'}`)
+        return [{ type: 'text', text: lines.join('\n') }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const parsed = await UiTree.readUiTree(serial)
+      const selector = { identifier: args.resource_id, label: args.text }
+      const { node, matchedBy } = UiTree.resolveTapTarget(parsed.roots, selector, { tool: 'device_tap_element', allowOffscreen: args.allow_offscreen === true })
+      const center = UiTree.boundsCenter(node.bounds)
+      await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'input', 'tap', String(center.x), String(center.y)]).exit
+      const describe = () => {
+        const parts = []
+        if (node.resourceId) parts.push(`resource_id ${node.resourceId}`)
+        if (node.text) parts.push(`text ${JSON.stringify(node.text)}`)
+        if (node.contentDesc) parts.push(`content-desc ${JSON.stringify(node.contentDesc)}`)
+        return parts.join(', ') || node.type
+      }
+      const out = { serial, tapped: describe(), matchedBy, x: center.x, y: center.y }
+      const mode = args.expect_gone !== undefined && args.expect_gone !== '' ? 'gone' : (args.expect_text !== undefined && args.expect_text !== '' ? 'appear' : undefined)
+      if (mode !== undefined) {
+        const text = mode === 'gone' ? String(args.expect_gone) : String(args.expect_text)
+        await new Promise((resolve) => setTimeout(resolve, 600))
+        const fresh = await UiTree.readUiTree(serial).catch(() => undefined)
+        const labels = fresh ? UiTree.collectLabels(fresh.roots) : []
+        const matched = mode === 'gone' ? !labels.some((label) => label.toLowerCase().includes(text.toLowerCase())) : labels.some((label) => label.toLowerCase().includes(text.toLowerCase()))
+        out.expected = { mode, text, matched }
+      }
+      return out
+    },
+  })
+}
+
 function deviceLogTool(engine) {
   return defineTool({
     name: 'device_log',
@@ -898,8 +1347,19 @@ function guidance() {
     '  Pass directory when the project root is not the working directory. A run takes minutes; use a generous timeout.',
     '- device_screen: capture the attached Android screen as a PNG plus the UI hierarchy and OCR text, so you can see what',
     '  the app shows and tap by pixel coordinates. Call it after a run to confirm the app rendered, and to drive UI flows.',
+    '- device_ui_tree: the default screen observer — the uiautomator hierarchy as a typed node tree with resource-ids,',
+    '  text, and pixel bounds. Narrow with filter/max_depth; on a textless surface (WebView, Compose, canvas) use device_screen.',
+    '- device_tap_element: tap a control by resource_id or text/content-desc, with one-call verification via',
+    '  expect_text / expect_gone (no separate screenshot needed to know the tap landed).',
+    '- device_wait_for: wait for text to appear/disappear (polls the UI tree, falls back to OCR on textless surfaces);',
+    '  a timeout is a normal matched:false result, never an error. One call replaces an agent-side poll loop.',
     '- device_input: tap/swipe/type/press on the attached Android device at ABSOLUTE pixel coordinates (take the center of a',
-    '  device_screen box: x=(x1+x2)/2, y=(y1+y2)/2). The control loop is device_screen → device_input → device_screen.',
+    '  device_screen box: x=(x1+x2)/2, y=(y1+y2)/2). The control loop is device_ui_tree → device_tap_element, falling back to',
+    '  device_screen → device_input when a surface exposes no accessibility tree. Typing is ASCII over plain adb; non-ASCII',
+    '  (CJK, emoji) goes through the ADBKeyboard IME when installed, and is refused with the install hint otherwise.',
+    '- device_action: notifications / quick_settings / collapse / lock / wake / assistant / rotate.',
+    '- device_boot / device_shutdown: boot an AVD by name and wait for boot / shut an emulator down (refuses physical devices).',
+    '- device_apps / device_launch_app: list installed packages (never guess a package name) / launch one by package or unique substring.',
     '- device_log: read device logs (logcat main/crash/events, kernel dmesg). Call it when a run fails or an app misbehaves.',
     '- device_status: one normalized snapshot of attached devices, AVDs, running/parked projects, Metro and preview servers.',
     '',
@@ -914,6 +1374,14 @@ export function apply(ctx, config) {
   Setup.ensureHome()
   Setup.writeOcrReadme()
 
+  // The playbook skill is host-independent; register it once at apply-time.
+  let disposeSkill = () => {}
+  try {
+    disposeSkill = registerMobileSkill(ctx) ?? (() => {})
+  } catch {
+    // The skills service is optional; the plugin works without the playbook.
+  }
+
   const resolve = () => ({
     enabled: config?.enabled ?? true,
     announceToAgent: config?.announceToAgent ?? true,
@@ -959,6 +1427,14 @@ export function apply(ctx, config) {
         deviceRunTool(engine, config),
         deviceDetectTool(engine, config),
         deviceScreenTool(engine),
+        deviceUiTreeTool(),
+        deviceTapElementTool(),
+        deviceWaitForTool(),
+        deviceBootTool(),
+        deviceShutdownTool(),
+        deviceActionTool(),
+        deviceAppsTool(),
+        deviceLaunchAppTool(),
         deviceLogTool(engine),
         deviceStatusTool(engine),
         deviceInputTool(),
@@ -968,6 +1444,7 @@ export function apply(ctx, config) {
   }
 
   ctx.effect(() => () => {
+    disposeSkill()
     void engine.dispose()
   }, 'dsh-mobilecode: engine')
 

package/lib/skill.js

@@ -0,0 +1,104 @@
+/**
+ * dsh-mobilecode — the plugin's bundled playbook, contributed through
+ * ctx.skills.register().
+ *
+ * A tool description answers "what does this argument mean", one tool at a
+ * time. What agents re-derive every session is the WORKFLOW between the tools
+ * — which observer to reach for, how to confirm an action landed, what a
+ * device refuses to tell you. This skill carries that workflow so the model
+ * loads it once per UI task instead of rediscovering it by trial and error.
+ *
+ * Registration is DEFENSIVE: a host without the skill service still loads the
+ * plugin, it just does not advertise the playbook (the scoped inject never
+ * runs when the service is absent).
+ *
+ * Playbook design credit: ZSeven-W/dsh-android (MIT) src/skill.ts.
+ */
+
+export const SKILL_NAME = "device-ui-automation"
+
+export const SKILL_DESCRIPTION =
+  "Drive an attached Android device through the dsh-mobilecode device_* tools: read the screen, tap by identity, " +
+  "type, and confirm that an action landed. Load this before the first device_ui_tree / device_tap_element call of a UI task."
+
+export const SKILL_WHEN_TO_USE =
+  "Any task that operates an Android app through the device_* tools — opening apps, tapping controls, filling " +
+  "fields, scrolling, reading logs, or verifying what is on screen, on an emulator or a real device."
+
+export const SKILL_CONTENT = `# Driving Android with dsh-mobilecode
+
+The loop is **observe once → act with an assertion → observe again only if the assertion could not settle it**. Everything goes through adb; an emulator and a USB phone take exactly the same tools with the same arguments.
+
+## Reading the screen
+
+| Tool | Cost | Use it for |
+| --- | --- | --- |
+| \`device_ui_tree\` | ~0.6–1.5 s | hierarchy, resource-ids, text, enabled state — the default observer |
+| \`device_screen\` | ~0.3 s + OCR | the picture for the user, plus PaddleOCR text+boxes when the tree is blind |
+| \`device_log\` | fast | what the device logs (main/crash/events buffers) |
+
+- Start with \`device_ui_tree\`: the accessibility tree carries \`resource-id\` (e.g. \`com.android.settings:id/search_bar\`), the most stable handle a control can have — far better than its text, which changes with the device language.
+- \`uiautomator\` dumps a snapshot of the CURRENT frame. If an animation is still running the tree can be a half-finished layout; when a read looks wrong, re-read once rather than reasoning about the wrong frame.
+- A shallow or empty tree is NEVER evidence that an app lacks accessibility support. Attribute an unlabeled read to ONE of three causes — (a) the depth/filter cut it: re-read wider; (b) a WebView, Compose or canvas surface that publishes little: \`device_screen\` OCR is the fallback; (c) a DEEP, unfiltered read with no labels — only then may "little accessibility information" be reported. Never jump from a shallow read to "OCR the screen".
+- The uiautomator dump fails on a continuously animating foreground (a page still loading) — the error says so. Do not retry; use \`device_screen\` (OCR reads pixels and needs no idle).
+
+## Acting
+
+- Prefer \`device_tap_element\` (resource-id / text / content-desc). Raw pixel coordinates through \`device_input action=tap\` are the last resort — they break on the next layout change.
+- When you must tap pixels, take the center of a box from \`device_ui_tree\`/\`device_screen\`: x=(x1+x2)/2, y=(y1+y2)/2. Coordinates are ABSOLUTE pixels of the current display.
+- \`device_input action=swipe\` scrolls; \`action=key\` with "back" is a first-class verb on Android — use it instead of hunting for an on-screen back arrow.
+- Typing is ASCII-only through adb. Non-ASCII text (Chinese, emoji) cannot be delivered by \`input text\` — do not retry; type via the app's own UI instead.
+
+## Never guess a package name or a control
+
+- A package name that looks plausible is routinely NOT the installed one. Check \`device_status\` / \`device_log filter=<package>\` for what is actually running before assuming.
+- Icon-only controls carry no OCR text by definition: the tree's \`content-desc\` is the only reliable handle — a bare tree means "look deeper", never "start guessing".
+
+## Confirming an action landed
+
+- Pass \`expect_text\` (or \`expect_gone\`) to \`device_tap_element\`: the tap and its verification become ONE round trip, and the result carries \`expected.matched\`.
+- Waiting for something slow (a load, an animation, a network round trip) is a \`device_ui_tree\`/\`device_screen\` re-read after a pause — one observation, not a poll loop.
+- Never compare screenshots or count pixels to decide whether something happened. Read the text back.
+
+## Logs
+
+- \`device_log\` buffers: \`main\` (default), \`crash\` (holds ONLY fatal Java/native crashes — that is where a stack trace lives), \`events\` (activity lifecycle), \`kernel\` (needs adb root; emulators usually allow it).
+- An idle emulator emits hundreds of lines a second. Narrow BEFORE widening: \`filter\` (package or tag substring) and a small \`lines\` count first.
+
+## Real devices
+
+- **Every tap on a real phone has real consequences** — posts, likes, purchases, messages. NEVER tap an unidentified control to find out what it does. If a control cannot be identified (no resource-id after a deep tree, no distinguishing text), STOP and report what you see and ask how to proceed. Do not guess coordinates on someone's live account.
+- A phone must be UNLOCKED for anything to be visible; \`device_input action=key key="wakeup"\` wakes it, but a PIN/pattern lock cannot be passed from here.
+- \`device_status\` reports each device's adb state: \`unauthorized\` means the USB-debugging prompt has not been accepted ON the device — no tool can fix that from this side.
+`
+
+/**
+ * Register the playbook when the host provides the skill service. The scoped
+ * inject is the optional-service pattern: a profile without the skill service
+ * simply never runs the callback, and the plugin loads without the playbook.
+ */
+export function registerMobileSkill(ctx) {
+  if (typeof ctx.inject === "function") {
+    const fiber = ctx.inject(["skills"], (skillCtx) => {
+      skillCtx.effect(() => skillCtx.skills.register({
+        name: SKILL_NAME,
+        description: SKILL_DESCRIPTION,
+        whenToUse: SKILL_WHEN_TO_USE,
+        content: SKILL_CONTENT,
+        source: "bundled",
+      }), "dsh-mobilecode: skill")
+    })
+    return () => { try { fiber.dispose() } catch { /* already disposed */ } }
+  }
+  // Very old hosts: direct registration when the service exists, else skip.
+  if (ctx.skills && typeof ctx.skills.register === "function") {
+    return ctx.skills.register({
+      name: SKILL_NAME,
+      description: SKILL_DESCRIPTION,
+      whenToUse: SKILL_WHEN_TO_USE,
+      content: SKILL_CONTENT,
+      source: "bundled",
+    })
+  }
+  return undefined
+}