dsh-mobilecode 0.1.4 → 0.2.1

package/README.md

@@ -46,14 +46,47 @@ drawer:
   failed or incomplete), the Metro state, and `complete`.
 - `device_detect` — which platforms a directory supports, the framework, and
   the first attached Android device.
+- A bundled **`device-ui-automation` playbook skill** (registered through
+  `ctx.skills.register()`, defensively — hosts without the skill service just
+  skip it): the observe-once → act-with-an-assertion → observe-again workflow,
+  which observer to reach for first, how to confirm an action landed via
+  `expect_text`/`expect_gone`, and the real-device safety rules.
 - `device_screen` — see what is on an attached Android device: a PNG screenshot
   plus the uiautomator UI hierarchy (text + pixel bounds) and **local PaddleOCR**
   text recognition (text + confidence + box), so an agent can read the screen
   and tap by coordinates. Also returns the foreground activity and screen size.
+- `device_ui_tree` — the default screen observer: the uiautomator hierarchy as a
+  typed node tree (`type`/`text`/`contentDesc`/`resourceId`/`bounds`, with
+  `enabled`/`focused`/`clickable`/`scrollable` emitted only in their interesting
+  state), case-insensitive `filter` that keeps ancestors of matches, `max_depth`,
+  and a 40 KB cap that prunes the deepest levels first. Resource-ids are the most
+  stable tap handles.
+- `device_tap_element` — tap a control by identity: `resource_id` matches the
+  node's resource-id, `text` matches its text or content-desc; exact match wins
+  over substring, nested duplicates collapse to the outermost control, ambiguity
+  lists up to 8 candidates instead of guessing, and disabled / off-screen nodes
+  are refused with the fix. `expect_text` / `expect_gone` verify the tap in the
+  same call — one round trip, no separate screenshot.
+- `device_wait_for` — wait for text to appear or disappear: polls the UI tree
+  every ~600 ms and falls back to local PaddleOCR on textless surfaces (WebView /
+  Compose / canvas). A timeout is a normal `matched: false` result, never an
+  error — one call replaces an agent-side poll loop.
 - `device_input` — act on the device: tap / swipe / type / press a key at
   **absolute pixel coordinates** (the same space `device_screen` returns — take
   the box center `x=(x1+x2)/2, y=(y1+y2)/2`). The deterministic control loop is
-  `device_screen → device_input → device_screen`.
+  `device_ui_tree → device_tap_element`, falling back to
+  `device_screen → device_input` when a surface exposes no accessibility tree.
+  Typing is ASCII over plain adb; non-ASCII (CJK, emoji) is routed through the
+  ADBKeyboard IME when installed and refused with the install hint otherwise.
+- `device_action` — device-level verbs beyond touches: `notifications`,
+  `quick_settings`, `collapse`, `lock`, `wake`, `assistant`, `rotate` (cycles
+  0→90→180→270 and pins auto-rotate off).
+- `device_boot` / `device_shutdown` — boot an AVD by name and wait until it
+  finishes booting (adopts a running emulator for the same AVD) / shut an
+  emulator down (`adb emu kill`; refuses physical devices).
+- `device_apps` / `device_launch_app` — list installed packages (third-party by
+  default) so a package name is never guessed / launch one by package or a
+  unique substring, with `relaunch` for a cold start.
 - `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
   (kernel = dmesg, needs adb root — works on emulators) with an optional
   case-insensitive substring filter, capped line count.

package/lib/device-build.js

@@ -108,11 +108,19 @@ export async function capture(command, args, options = {}) {
   })
   running.stdout?.setEncoding("utf8")
   running.stdout?.on("data", (chunk) => out.push(chunk))
+  let timer
+  if (options.timeoutMs) {
+    timer = setTimeout(() => running.kill(), options.timeoutMs)
+    timer.unref?.()
+  }
   const code = await new Promise((resolve) => {
     running.once("error", () => resolve(-1))
     running.once("close", (value) => resolve(value ?? -1))
   })
-  return code === 0 ? out.join("") : ""
+  if (timer) clearTimeout(timer)
+  if (code !== 0) return ""
+  const text = out.join("")
+  return options.maxBytes && Buffer.byteLength(text) > options.maxBytes ? text.slice(0, options.maxBytes) : text
 }
 
 /** Direct child pids of `pid`. POSIX only; returns nothing when pgrep is unavailable. */
@@ -732,6 +740,57 @@ export async function androidBooted(serial) {
   return output.trim() === "1"
 }
 
+/** AVD name of an emulator serial (`adb emu avd name`); undefined for physical/offline. */
+export async function avdName(serial) {
+  const output = await capture(adb(), ["-s", serial, "emu", "avd", "name"])
+  const line = output
+    .split(/\r?\n/)
+    .map((item) => item.trim())
+    .find((item) => item && !/^OK$/i.test(item) && !/^KO:/i.test(item))
+  return line && /^[A-Za-z0-9._-]+$/.test(line) ? line : undefined
+}
+
+/** Poll until sys.boot_completed == 1 or the timeout; true when booted. */
+export async function waitForBoot(serial, timeoutMs = 180_000) {
+  const deadline = Date.now() + timeoutMs
+  for (;;) {
+    if (await androidBooted(serial)) return true
+    if (Date.now() >= deadline) return false
+    await new Promise((resolve) => setTimeout(resolve, 1000))
+  }
+}
+
+/** Launch an AVD detached (survives this process); undefined when no emulator binary. */
+export function bootEmulator(avd) {
+  const emulator = emulatorBinary()
+  if (!emulator || !/^[A-Za-z0-9._-]+$/.test(avd)) return undefined
+  const child = launch(emulator, ["-avd", avd], { stdio: "ignore", detached: true })
+  child.unref?.()
+  return child
+}
+
+/** True when the ADBKeyboard IME is installed (the only way to type non-ASCII over adb). */
+export async function adbKeyboardReady(serial) {
+  const output = await capture(adb(), ["-s", serial, "shell", "ime", "list", "-s"])
+  return /com\.android\.adbkeyboard/i.test(output)
+}
+
+/** Type one string via the ADBKeyboard broadcast (base64, so any codepoint survives the shell). */
+export async function typeViaAdbKeyboard(serial, text) {
+  const msg = Buffer.from(text, "utf8").toString("base64")
+  await exec(adb(), ["-s", serial, "shell", "am", "broadcast", "-a", "ADB_INPUT_B64", "--es", "msg", msg]).exit
+}
+
+/** True when every codepoint is safe for `adb shell input text` (printable ASCII, no shell metachars). */
+export function isAsciiInput(text) {
+  return [...text].every((ch) => ch.codePointAt(0) >= 0x20 && ch.codePointAt(0) <= 0x7e)
+}
+
+/** Escape one `input text` argument for the device shell: backslash the metachars, spaces to %s. */
+export function escapeInputText(text) {
+  return text.replace(/[\\()<>|;&*~"'`$#!{}[\]]/g, "\\$&").replace(/ /g, "%s")
+}
+
 function aapt2() {
   const sdk = androidSdk()
   if (!sdk) return undefined