dsh-mobilecode 0.2.0 → 0.2.1

package/README.md

@@ -67,11 +67,26 @@ drawer:
   lists up to 8 candidates instead of guessing, and disabled / off-screen nodes
   are refused with the fix. `expect_text` / `expect_gone` verify the tap in the
   same call — one round trip, no separate screenshot.
+- `device_wait_for` — wait for text to appear or disappear: polls the UI tree
+  every ~600 ms and falls back to local PaddleOCR on textless surfaces (WebView /
+  Compose / canvas). A timeout is a normal `matched: false` result, never an
+  error — one call replaces an agent-side poll loop.
 - `device_input` — act on the device: tap / swipe / type / press a key at
   **absolute pixel coordinates** (the same space `device_screen` returns — take
   the box center `x=(x1+x2)/2, y=(y1+y2)/2`). The deterministic control loop is
   `device_ui_tree → device_tap_element`, falling back to
   `device_screen → device_input` when a surface exposes no accessibility tree.
+  Typing is ASCII over plain adb; non-ASCII (CJK, emoji) is routed through the
+  ADBKeyboard IME when installed and refused with the install hint otherwise.
+- `device_action` — device-level verbs beyond touches: `notifications`,
+  `quick_settings`, `collapse`, `lock`, `wake`, `assistant`, `rotate` (cycles
+  0→90→180→270 and pins auto-rotate off).
+- `device_boot` / `device_shutdown` — boot an AVD by name and wait until it
+  finishes booting (adopts a running emulator for the same AVD) / shut an
+  emulator down (`adb emu kill`; refuses physical devices).
+- `device_apps` / `device_launch_app` — list installed packages (third-party by
+  default) so a package name is never guessed / launch one by package or a
+  unique substring, with `relaunch` for a cold start.
 - `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
   (kernel = dmesg, needs adb root — works on emulators) with an optional
   case-insensitive substring filter, capped line count.

package/lib/device-build.js

@@ -740,6 +740,57 @@ export async function androidBooted(serial) {
   return output.trim() === "1"
 }
 
+/** AVD name of an emulator serial (`adb emu avd name`); undefined for physical/offline. */
+export async function avdName(serial) {
+  const output = await capture(adb(), ["-s", serial, "emu", "avd", "name"])
+  const line = output
+    .split(/\r?\n/)
+    .map((item) => item.trim())
+    .find((item) => item && !/^OK$/i.test(item) && !/^KO:/i.test(item))
+  return line && /^[A-Za-z0-9._-]+$/.test(line) ? line : undefined
+}
+
+/** Poll until sys.boot_completed == 1 or the timeout; true when booted. */
+export async function waitForBoot(serial, timeoutMs = 180_000) {
+  const deadline = Date.now() + timeoutMs
+  for (;;) {
+    if (await androidBooted(serial)) return true
+    if (Date.now() >= deadline) return false
+    await new Promise((resolve) => setTimeout(resolve, 1000))
+  }
+}
+
+/** Launch an AVD detached (survives this process); undefined when no emulator binary. */
+export function bootEmulator(avd) {
+  const emulator = emulatorBinary()
+  if (!emulator || !/^[A-Za-z0-9._-]+$/.test(avd)) return undefined
+  const child = launch(emulator, ["-avd", avd], { stdio: "ignore", detached: true })
+  child.unref?.()
+  return child
+}
+
+/** True when the ADBKeyboard IME is installed (the only way to type non-ASCII over adb). */
+export async function adbKeyboardReady(serial) {
+  const output = await capture(adb(), ["-s", serial, "shell", "ime", "list", "-s"])
+  return /com\.android\.adbkeyboard/i.test(output)
+}
+
+/** Type one string via the ADBKeyboard broadcast (base64, so any codepoint survives the shell). */
+export async function typeViaAdbKeyboard(serial, text) {
+  const msg = Buffer.from(text, "utf8").toString("base64")
+  await exec(adb(), ["-s", serial, "shell", "am", "broadcast", "-a", "ADB_INPUT_B64", "--es", "msg", msg]).exit
+}
+
+/** True when every codepoint is safe for `adb shell input text` (printable ASCII, no shell metachars). */
+export function isAsciiInput(text) {
+  return [...text].every((ch) => ch.codePointAt(0) >= 0x20 && ch.codePointAt(0) <= 0x7e)
+}
+
+/** Escape one `input text` argument for the device shell: backslash the metachars, spaces to %s. */
+export function escapeInputText(text) {
+  return text.replace(/[\\()<>|;&*~"'`$#!{}[\]]/g, "\\$&").replace(/ /g, "%s")
+}
+
 function aapt2() {
   const sdk = androidSdk()
   if (!sdk) return undefined

package/lib/index.js

@@ -583,9 +583,21 @@ function deviceInputTool() {
         }
         case 'text': {
           if (typeof args.text !== 'string' || args.text.length === 0) throw new Error('action=text requires a non-empty text string.')
-          const escaped = args.text.replace(/\s/g, '%s')
-          await DeviceBuild.exec(DeviceBuild.adb(), adbArgs(['input', 'text', escaped])).exit
-          return { serial, action, sent: `text "${args.text}"` }
+          if (DeviceBuild.isAsciiInput(args.text)) {
+            await DeviceBuild.exec(DeviceBuild.adb(), adbArgs(['input', 'text', DeviceBuild.escapeInputText(args.text)])).exit
+            return { serial, action, sent: `text "${args.text}"` }
+          }
+          // Non-ASCII (CJK, emoji, accented) cannot go through `input text`; the
+          // ADBKeyboard IME is the only adb path. Refuse with the fix, never mangle.
+          if (!(await DeviceBuild.adbKeyboardReady(serial))) {
+            throw new Error(
+              `device_input cannot type non-ASCII text ("${args.text}") over plain adb. Install ADBKeyboard `
+              + 'on the device (https://github.com/senzhk/ADBKeyBoard), enable it (ime enable/set '
+              + 'com.android.adbkeyboard/.AdbIME), then retry — this tool then types via its base64 broadcast.',
+            )
+          }
+          await DeviceBuild.typeViaAdbKeyboard(serial, args.text)
+          return { serial, action, sent: `text "${args.text}" (ADBKeyboard)` }
         }
         case 'key': {
           const raw = String(args.key ?? '')
@@ -601,6 +613,301 @@ function deviceInputTool() {
   })
 }
 
+/** OCR the current screen and report whether `wantedLower` appears; undefined when OCR is unavailable. */
+async function ocrHasText(serial, wantedLower) {
+  if (!DeviceBuild.ocrPython()) return undefined
+  const png = await DeviceBuild.screenCapture(serial)
+  if (!png) return undefined
+  const ocr = await DeviceBuild.ocrImage(png).catch(() => [])
+  return ocr.some((item) => String(item.text).toLowerCase().includes(wantedLower))
+}
+
+function deviceWaitForTool() {
+  return defineTool({
+    name: 'device_wait_for',
+    description: 'Wait for on-screen text to appear or disappear. Polls the uiautomator tree every ~600 ms; when the tree ' +
+      'carries no labels (WebView/Compose/canvas) it falls back to local PaddleOCR. A timeout is a normal matched:false ' +
+      'result, never an error — one call replaces an agent-side poll loop.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      text: { type: 'string', description: 'Text to wait for (case-insensitive substring).' },
+      mode: { type: 'string', enum: ['appear', 'disappear'], description: 'appear (default) waits for the text; disappear waits for it to go away.' },
+      timeout_ms: { type: 'integer', description: 'Max wait in ms (default 10000, max 60000).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          text: { type: 'string', required: true },
+          mode: { type: 'string', required: true },
+          matched: { type: 'boolean', required: true },
+          waited_ms: { type: 'integer', required: true },
+          source: { type: 'string', required: true },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', text: '', mode: 'appear', matched: false, waited_ms: 0, source: 'ui_tree' }
+        return [{ type: 'text', text: `${v.matched ? 'MATCHED' : 'TIMEOUT'}: "${v.text}" ${v.mode} (${v.source}) after ${v.waited_ms}ms on ${v.serial}` }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const text = String(args.text ?? '').trim()
+      if (text === '') throw new Error('device_wait_for requires a non-empty text.')
+      const mode = args.mode === 'disappear' ? 'disappear' : 'appear'
+      const timeout = Math.min(Math.max(args.timeout_ms ?? 10_000, 500), 60_000)
+      const wanted = text.toLowerCase()
+      const start = Date.now()
+      const deadline = start + timeout
+      let source = 'ui_tree'
+      for (;;) {
+        let labels
+        try {
+          labels = UiTree.collectLabels((await UiTree.readUiTree(serial)).roots)
+        } catch {
+          labels = undefined
+        }
+        let present
+        if (labels !== undefined && labels.length > 0) {
+          source = 'ui_tree'
+          present = labels.some((label) => label.toLowerCase().includes(wanted))
+        } else {
+          const ocr = await ocrHasText(serial, wanted)
+          if (ocr === undefined) { source = 'ui_tree'; present = false }
+          else { source = 'ocr'; present = ocr }
+        }
+        const matched = mode === 'appear' ? present : !present
+        const waited = Math.min(timeout, Date.now() - start)
+        if (matched) return { serial, text, mode, matched: true, waited_ms: waited, source }
+        if (Date.now() >= deadline) return { serial, text, mode, matched: false, waited_ms: timeout, source }
+        await new Promise((resolve) => setTimeout(resolve, source === 'ocr' ? 2000 : 600))
+      }
+    },
+  })
+}
+
+function deviceBootTool() {
+  return defineTool({
+    name: 'device_boot',
+    description: 'Boot an Android emulator AVD by name (from device_status.avds) and wait until it finishes booting. ' +
+      'If an emulator for that AVD is already running it is adopted. This is the device-centric boot — device_run is the ' +
+      'project-centric build+install+launch.',
+    parameters: {
+      avd: { type: 'string', description: 'AVD name to boot.' },
+      timeout_ms: { type: 'integer', description: 'Max wait for boot in ms (default 180000, max 600000).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          avd: { type: 'string', required: true },
+          booted: { type: 'boolean', required: true },
+          alreadyRunning: { type: 'boolean', required: true },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', avd: '', booted: false, alreadyRunning: false }
+        return [{ type: 'text', text: `${v.alreadyRunning ? 'Adopted running' : 'Booted'} emulator ${v.serial} (AVD ${v.avd}), boot completed: ${v.booted}` }]
+      },
+    },
+    async execute(args) {
+      const avd = String(args.avd ?? '').trim()
+      if (avd === '') throw new Error('device_boot requires an avd name (see device_status.avds).')
+      const timeout = Math.min(Math.max(args.timeout_ms ?? 180_000, 5_000), 600_000)
+      const rows = await DeviceBuild.devices()
+      for (const row of rows.filter((item) => item.state === 'device' && item.serial.startsWith('emulator-'))) {
+        const name = await DeviceBuild.avdName(row.serial).catch(() => undefined)
+        if (name === avd) {
+          const booted = await DeviceBuild.waitForBoot(row.serial, timeout)
+          return { serial: row.serial, avd, booted, alreadyRunning: true }
+        }
+      }
+      if (!DeviceBuild.emulatorBinary()) throw new Error('No SDK emulator binary found; cannot boot an AVD.')
+      const before = new Set(rows.map((item) => item.serial))
+      if (!DeviceBuild.bootEmulator(avd)) throw new Error(`Could not launch the emulator for AVD "${avd}".`)
+      const deadline = Date.now() + timeout
+      let serial
+      while (Date.now() < deadline) {
+        const now = await DeviceBuild.devices()
+        const fresh = now.find((item) => item.serial.startsWith('emulator-') && !before.has(item.serial))
+        if (fresh) { serial = fresh.serial; break }
+        await new Promise((resolve) => setTimeout(resolve, 1000))
+      }
+      if (!serial) throw new Error(`No emulator serial appeared for "${avd}" within ${timeout} ms.`)
+      const booted = await DeviceBuild.waitForBoot(serial, Math.max(1000, deadline - Date.now()))
+      if (!booted) throw new Error(`Emulator ${serial} ("${avd}") did not finish booting within ${timeout} ms.`)
+      return { serial, avd, booted: true, alreadyRunning: false }
+    },
+  })
+}
+
+function deviceShutdownTool() {
+  return defineTool({
+    name: 'device_shutdown',
+    description: 'Shut down an emulator (`adb emu kill`). Refuses physical devices — adb has no power-off verb for phones.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          shutdown: { type: 'boolean', required: true },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: `Shut down emulator ${value?.serial}` }],
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const isEmulator = serial.startsWith('emulator-') || (await DeviceBuild.avdName(serial).catch(() => undefined)) !== undefined
+      if (!isEmulator) {
+        throw new Error(`device_shutdown refuses ${serial}: it is a physical device and adb has no power-off verb for phones — use its own power button.`)
+      }
+      await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'emu', 'kill']).exit
+      return { serial, shutdown: true }
+    },
+  })
+}
+
+const DEVICE_ACTIONS = {
+  notifications: ['cmd', 'statusbar', 'expand-notifications'],
+  quick_settings: ['cmd', 'statusbar', 'expand-settings'],
+  collapse: ['cmd', 'statusbar', 'collapse'],
+  lock: ['input', 'keyevent', '223'],
+  wake: ['input', 'keyevent', '224'],
+  assistant: ['am', 'start', '-a', 'android.intent.action.ASSIST'],
+}
+
+function deviceActionTool() {
+  return defineTool({
+    name: 'device_action',
+    description: 'Device-level actions beyond touches: open the notification shade or quick settings, collapse the shade, ' +
+      'lock or wake the screen, launch the assistant, or rotate the display (cycles 0→90→180→270 and pins auto-rotate off).',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      action: { type: 'string', enum: [...Object.keys(DEVICE_ACTIONS), 'rotate'], description: 'Which device action to perform.' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          action: { type: 'string', required: true },
+          rotation: { type: 'integer' },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: value?.rotation !== undefined ? `${value.action} on ${value.serial} → rotation ${value.rotation * 90}°` : `${value?.action} on ${value?.serial}` }],
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const action = String(args.action ?? '')
+      if (action === 'rotate') {
+        const current = Number(await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'get', 'system', 'user_rotation']))
+        const next = ((Number.isFinite(current) ? current : 0) + 1) % 4
+        await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'put', 'system', 'accelerometer_rotation', '0']).exit
+        await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'put', 'system', 'user_rotation', String(next)]).exit
+        return { serial, action, rotation: next }
+      }
+      const shell = DEVICE_ACTIONS[action]
+      if (!shell) throw new Error(`unknown action "${action}" — use ${[...Object.keys(DEVICE_ACTIONS), 'rotate'].join(', ')}.`)
+      await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', ...shell]).exit
+      return { serial, action }
+    },
+  })
+}
+
+function deviceAppsTool() {
+  return defineTool({
+    name: 'device_apps',
+    description: 'List installed Android packages (third-party by default; include_system=true adds platform apps). ' +
+      'Use it to find the real package name before device_launch_app — never guess one.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      include_system: { type: 'boolean', description: 'Include system/platform packages (default false).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          count: { type: 'integer', required: true },
+          packages: { type: 'array', required: true, items: { type: 'string' } },
+        },
+      },
+      render: (_args, value) => {
+        const v = value ?? { serial: '', count: 0, packages: [] }
+        const shown = v.packages.slice(0, 60).join('\n  ')
+        return [{ type: 'text', text: `${v.serial}: ${v.count} packages\n  ${shown}${v.count > 60 ? `\n  … and ${v.count - 60} more` : ''}` }]
+      },
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'pm', 'list', 'packages', ...(args.include_system ? [] : ['-3'])])
+      const packages = output
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith('package:'))
+        .map((line) => line.slice('package:'.length).trim())
+      return { serial, count: packages.length, packages: packages.slice(0, 200) }
+    },
+  })
+}
+
+function deviceLaunchAppTool() {
+  return defineTool({
+    name: 'device_launch_app',
+    description: 'Launch an installed app by package name (or a unique substring of it). relaunch=true force-stops it first ' +
+      'for a cold start. Resolves the exact package via `pm list packages` so a wrong guess fails loudly instead of ' +
+      'opening the wrong app.',
+    parameters: {
+      serial: { type: 'string', description: 'Android device serial. Omit to use the first attached device.' },
+      package: { type: 'string', description: 'Package name or a unique substring of it.' },
+      relaunch: { type: 'boolean', description: 'Force-stop the app first (cold start).' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          serial: { type: 'string', required: true },
+          package: { type: 'string', required: true },
+          launched: { type: 'boolean', required: true },
+        },
+      },
+      render: (_args, value) => [{ type: 'text', text: `Launched ${value?.package} on ${value?.serial}` }],
+    },
+    async execute(args) {
+      const serial = await requireAndroidDevice(args.serial)
+      let pkg = String(args.package ?? '').trim()
+      if (pkg === '') throw new Error('device_launch_app requires a package name (or a unique substring).')
+      const listOut = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'pm', 'list', 'packages'])
+      const all = listOut
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith('package:'))
+        .map((line) => line.slice('package:'.length).trim())
+      if (!all.includes(pkg)) {
+        const matches = all.filter((item) => item.toLowerCase().includes(pkg.toLowerCase()))
+        if (matches.length === 0) throw new Error(`No installed package matches "${pkg}". Run device_apps to list them.`)
+        if (matches.length > 1) throw new Error(`"${pkg}" matches ${matches.length} packages (${matches.slice(0, 8).join(', ')}${matches.length > 8 ? ', …' : ''}) — be more specific.`)
+        pkg = matches[0]
+      }
+      if (args.relaunch) await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'am', 'force-stop', pkg]).exit
+      const code = await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', 'monkey', '-p', pkg, '-c', 'android.intent.category.LAUNCHER', '1']).exit
+      if (code !== 0) throw new Error(`Could not launch ${pkg} (no launcher activity, or monkey failed with exit ${code}).`)
+      return { serial, package: pkg, launched: true }
+    },
+  })
+}
+
 function deviceScreenTool(engine) {
   return defineTool({
     name: 'device_screen',
@@ -715,7 +1022,9 @@ function deviceScreenTool(engine) {
 
 async function captureScreenSize(serial) {
   const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'wm', 'size'])
-  const match = /Physical size:\s*(\d+)x(\d+)/.exec(output)
+  // An `wm size` override wins over the physical panel — the input space is the override.
+  const override = /Override size:\s*(\d+)x(\d+)/.exec(output)
+  const match = override ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
   return match ? { width: Number(match[1]), height: Number(match[2]) } : undefined
 }
 
@@ -1042,8 +1351,15 @@ function guidance() {
     '  text, and pixel bounds. Narrow with filter/max_depth; on a textless surface (WebView, Compose, canvas) use device_screen.',
     '- device_tap_element: tap a control by resource_id or text/content-desc, with one-call verification via',
     '  expect_text / expect_gone (no separate screenshot needed to know the tap landed).',
+    '- device_wait_for: wait for text to appear/disappear (polls the UI tree, falls back to OCR on textless surfaces);',
+    '  a timeout is a normal matched:false result, never an error. One call replaces an agent-side poll loop.',
     '- device_input: tap/swipe/type/press on the attached Android device at ABSOLUTE pixel coordinates (take the center of a',
-    '  device_screen box: x=(x1+x2)/2, y=(y1+y2)/2). The control loop is device_screen → device_input → device_screen.',
+    '  device_screen box: x=(x1+x2)/2, y=(y1+y2)/2). The control loop is device_ui_tree → device_tap_element, falling back to',
+    '  device_screen → device_input when a surface exposes no accessibility tree. Typing is ASCII over plain adb; non-ASCII',
+    '  (CJK, emoji) goes through the ADBKeyboard IME when installed, and is refused with the install hint otherwise.',
+    '- device_action: notifications / quick_settings / collapse / lock / wake / assistant / rotate.',
+    '- device_boot / device_shutdown: boot an AVD by name and wait for boot / shut an emulator down (refuses physical devices).',
+    '- device_apps / device_launch_app: list installed packages (never guess a package name) / launch one by package or unique substring.',
     '- device_log: read device logs (logcat main/crash/events, kernel dmesg). Call it when a run fails or an app misbehaves.',
     '- device_status: one normalized snapshot of attached devices, AVDs, running/parked projects, Metro and preview servers.',
     '',
@@ -1113,6 +1429,12 @@ export function apply(ctx, config) {
         deviceScreenTool(engine),
         deviceUiTreeTool(),
         deviceTapElementTool(),
+        deviceWaitForTool(),
+        deviceBootTool(),
+        deviceShutdownTool(),
+        deviceActionTool(),
+        deviceAppsTool(),
+        deviceLaunchAppTool(),
         deviceLogTool(engine),
         deviceStatusTool(engine),
         deviceInputTool(),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-mobilecode",
   "description": "MobileCode for the dsh web GUI: detect iOS/Android projects, run serve-sim / serve-avd preview servers, and build-install-launch the app on the simulator or emulator from the session — plus agent tools (device_run, device_detect). Hot-pluggable — mounted via the profile bundle list + cordis.patch.yml, no dsh source changes.",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "type": "module",
   "packageManager": "pnpm@11.22.0",
   "engines": {