dsh-mobilecode 0.2.0 → 0.3.0

package/README.md

@@ -67,11 +67,54 @@ drawer:
   lists up to 8 candidates instead of guessing, and disabled / off-screen nodes
   are refused with the fix. `expect_text` / `expect_gone` verify the tap in the
   same call — one round trip, no separate screenshot.
+- `device_wait_for` — wait for text to appear or disappear: polls the UI tree
+  every ~600 ms and falls back to local PaddleOCR on textless surfaces (WebView /
+  Compose / canvas). A timeout is a normal `matched: false` result, never an
+  error — one call replaces an agent-side poll loop.
 - `device_input` — act on the device: tap / swipe / type / press a key at
   **absolute pixel coordinates** (the same space `device_screen` returns — take
   the box center `x=(x1+x2)/2, y=(y1+y2)/2`). The deterministic control loop is
   `device_ui_tree → device_tap_element`, falling back to
   `device_screen → device_input` when a surface exposes no accessibility tree.
+  Typing is ASCII over plain adb; non-ASCII (CJK, emoji) is routed through the
+  ADBKeyboard IME when installed and refused with the install hint otherwise.
+- `device_action` — device-level verbs beyond touches: `notifications`,
+  `quick_settings`, `collapse`, `lock`, `wake`, `assistant`, `rotate` (cycles
+  0→90→180→270 and pins auto-rotate off).
+- `device_boot` / `device_shutdown` — boot an AVD by name and wait until it
+  finishes booting (adopts a running emulator for the same AVD) / shut an
+  emulator down (`adb emu kill`; refuses physical devices).
+- `device_apps` / `device_launch_app` — list installed packages (third-party by
+  default) so a package name is never guessed / launch one by package or a
+  unique substring, with `relaunch` for a cold start.
+- `device_stream` — drive the live screen stream the panel shows: `start` an
+  online device (returns a signed `streamUrl`), `status`, or `stop`. Agents that
+  just need to see the screen should prefer `device_screen` / `device_ui_tree`.
+
+**Live device stream (the panel)**
+
+The Devices pane shows a real-time mirror of the attached device. It is produced
+**in-process** — no inner loopback port, no external helper:
+
+- ONE persistent `adb exec-out "while :; do screencap -p; done"` child streams
+  ~8 fps with zero per-frame process cost (spawning adb per frame caps at ~5 fps
+  and 100% churn). A quote-safe PNG splitter cuts the concatenated output into
+  frames by walking chunk headers (no marker scanning, no false positives).
+- The browser `<img>` reads a `multipart/x-mixed-replace` body served straight
+  from the latest-frame buffer. Backpressure is **latest-wins**: a slow tab skips
+  frames instead of building an unbounded queue or watching a growing delay.
+- A consumer refcount + idle timeout stops the loop when nobody is watching; a
+  keep-alive restarts a crashed loop; switching devices retires the old child.
+- **Tap or drag directly on the screen** to drive the device; a Back / Home /
+  Recents / Rotate / Power bar sits below it, and a device picker switches which
+  online device streams (it never boots one — use `device_boot` for that).
+- **Security**: every stream route sits behind a loopback + trusted-browser
+  transport fence (peer address, loopback `Host`, `Sec-Fetch-Site` / `Origin` —
+  so a LAN client cannot spoof localhost and a DNS-rebinding `Host` is rejected),
+  and the stream URL is an **HMAC-SHA256 capability** signed with a per-install
+  key (`~/.dsh/mobilecode/stream-access.key`, `0600`), expiring within 10 minutes
+  and re-minted automatically. Coordinates are normalized 0..1 of the streamed
+  frame, so one mapping serves every rotation.
 - `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
   (kernel = dmesg, needs adb root — works on emulators) with an optional
   case-insensitive substring filter, capped line count.

package/lib/android-stream.js

@@ -0,0 +1,378 @@
+/**
+ * dsh-mobilecode — host-side lifecycle manager for the one live Android stream.
+ *
+ * Ported (lean) from ZSeven-W/dsh-android (android-host.ts, MIT). The stream is
+ * in-process: one AdbFrameLoop per streamed serial, a consumer refcount with an
+ * idle timeout, and a keep-alive that restarts a crashed loop. Emulators and
+ * physical devices share this path — the serial is the only identity.
+ *
+ * The control surface takes NORMALIZED 0..1 coordinates of the streamed frame
+ * (the panel's <img> is scaled, so the browser knows fractions, not pixels) and
+ * maps them onto `adb shell input` pixels using the latest frame's own size.
+ */
+
+import * as DeviceBuild from './device-build.js'
+import { AdbFrameLoop } from './frame-source.js'
+
+const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60 * 1000
+const DEFAULT_RESTART_DELAY_MS = 5000
+const KEEP_ALIVE_TICK_MS = 1000
+const FIRST_FRAME_TIMEOUT_MS = 15000
+const CONTROL_TIMEOUT_MS = 30000
+const MAX_SWIPE_MS = 5000
+
+/** Navigation/hardware buttons the panel may press. */
+export const ANDROID_BUTTONS = {
+  home: 'KEYCODE_HOME',
+  back: 'KEYCODE_BACK',
+  recents: 'KEYCODE_APP_SWITCH',
+  power: 'KEYCODE_POWER',
+  volume_up: 'KEYCODE_VOLUME_UP',
+  volume_down: 'KEYCODE_VOLUME_DOWN',
+  menu: 'KEYCODE_MENU',
+  enter: 'KEYCODE_ENTER',
+  delete: 'KEYCODE_DEL',
+}
+
+/** Clockwise user_rotation cycle (Surface.ROTATION_0..270). */
+export const ROTATION_CYCLE = [0, 1, 2, 3]
+
+function errorMessage(error) {
+  return error instanceof Error ? error.message : String(error)
+}
+
+function requireNormalized(x, y) {
+  if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || x > 1 || y < 0 || y > 1) {
+    throw new RangeError('dsh-mobilecode: tap/drag coordinates must be normalized 0..1 of the streamed frame')
+  }
+}
+
+/** Lifecycle manager for the (single) in-process Android device stream. */
+export class AndroidStreamHost {
+  #options
+  #loop
+  #starting
+  #launchQueue = Promise.resolve()
+  #consumers = 0
+  #keepAliveTimer
+  #idleTimer
+  #restarts = 0
+  #startedAt
+  #exitAt
+  #lastError
+  #lastSerial
+  #intentionalStop = false
+  #disposed = false
+  #frameSubscribers = new Set()
+
+  constructor(options = {}) {
+    this.#options = {
+      restartDelayMs: options.restartDelayMs ?? DEFAULT_RESTART_DELAY_MS,
+      idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
+      firstFrameTimeoutMs: options.firstFrameTimeoutMs ?? FIRST_FRAME_TIMEOUT_MS,
+    }
+  }
+
+  get running() {
+    return this.#loop?.running === true
+  }
+
+  get streamedSerial() {
+    return this.running ? this.#loop?.serial : undefined
+  }
+
+  get latestFrame() {
+    return this.#loop?.latestFrame
+  }
+
+  /** Observe every decoded frame; the returned function unsubscribes. */
+  subscribeFrames(subscriber) {
+    this.#frameSubscribers.add(subscriber)
+    return () => {
+      this.#frameSubscribers.delete(subscriber)
+    }
+  }
+
+  /**
+   * Make sure the frame loop is live for `serial`. Concurrent callers share one
+   * launch; a call for a different serial retires the current loop first (one
+   * streamed device at a time).
+   */
+  async ensureStreaming({ serial }) {
+    if (this.#disposed) throw new Error('dsh-mobilecode: the stream host is disposed')
+    if (typeof serial !== 'string' || serial === '') throw new TypeError('dsh-mobilecode: ensureStreaming requires a non-empty serial')
+    const startFor = async () => {
+      const current = this.#loop
+      if (current !== undefined && current.running && current.serial === serial) return this.#infoOf(current)
+      if (current !== undefined && current.serial !== serial) {
+        current.stop()
+        this.#loop = undefined
+      }
+      return this.#startFor(serial)
+    }
+    let starting = this.#starting
+    if (starting !== undefined) {
+      try {
+        await starting
+      } catch {
+        // A failed shared launch is settled; retry below.
+      }
+      if (this.running && this.#loop?.serial === serial) {
+        this.#armIdle()
+        return this.#infoOf(this.#loop)
+      }
+      starting = undefined
+    }
+    starting = this.#serializeLaunch(startFor)
+    this.#starting = starting
+    try {
+      const info = await starting
+      this.#lastError = undefined
+      this.#armIdle()
+      return info
+    } catch (error) {
+      this.#lastError = errorMessage(error)
+      throw error
+    } finally {
+      if (this.#starting === starting) this.#starting = undefined
+    }
+  }
+
+  /** Start the crash keep-alive loop (restarts an unintentionally dead loop). */
+  startKeepAlive() {
+    if (this.#keepAliveTimer !== undefined || this.#disposed) return
+    this.#keepAliveTimer = setInterval(() => {
+      void this.#keepAliveTick().catch(() => {})
+    }, KEEP_ALIVE_TICK_MS)
+    this.#keepAliveTimer.unref?.()
+  }
+
+  stopKeepAlive() {
+    if (this.#keepAliveTimer !== undefined) clearInterval(this.#keepAliveTimer)
+    this.#keepAliveTimer = undefined
+  }
+
+  /** Stop the stream intentionally (keep-alive will not fight it). */
+  async stop() {
+    this.#clearIdle()
+    this.#intentionalStop = true
+    this.#exitAt = undefined
+    const loop = this.#loop
+    this.#loop = undefined
+    this.#startedAt = undefined
+    loop?.stop()
+    await this.#starting?.catch(() => {})
+    const landed = this.#loop
+    if (landed !== undefined) {
+      this.#loop = undefined
+      landed.stop()
+    }
+  }
+
+  /** Hold the stream alive for one consumer; release exactly once. */
+  acquire() {
+    this.#consumers += 1
+    this.#armIdle()
+    let released = false
+    return () => {
+      if (released) return
+      released = true
+      this.#consumers = Math.max(0, this.#consumers - 1)
+      this.#armIdle()
+    }
+  }
+
+  status() {
+    const loop = this.#loop
+    const frame = loop?.latestFrame
+    return {
+      running: this.running,
+      ...(loop === undefined ? {} : { serial: loop.serial }),
+      restarts: this.#restarts,
+      ...(this.#lastError === undefined ? {} : { lastError: this.#lastError }),
+      consumers: this.#consumers,
+      ...(frame === undefined ? {} : { frameSequence: frame.sequence, lastFrameAt: frame.at, width: frame.width, height: frame.height }),
+      stderr: loop?.stderrLines ?? [],
+    }
+  }
+
+  dispose() {
+    this.#disposed = true
+    this.stopKeepAlive()
+    this.#frameSubscribers.clear()
+    return this.stop()
+  }
+
+  // ── control surface (normalized 0..1 of the streamed frame) ────────────────
+
+  async tap(serial, x, y) {
+    requireNormalized(x, y)
+    const point = await this.#pixels(serial, x, y)
+    await this.#shell(serial, ['input', 'tap', String(point.x), String(point.y)])
+  }
+
+  async drag(serial, drag) {
+    requireNormalized(drag.fromX, drag.fromY)
+    requireNormalized(drag.toX, drag.toY)
+    const from = await this.#pixels(serial, drag.fromX, drag.fromY)
+    const to = await this.#pixels(serial, drag.toX, drag.toY)
+    const durationMs = Math.min(MAX_SWIPE_MS, Math.max(20, Math.round((drag.duration ?? 0.3) * 1000)))
+    await this.#shell(serial, ['input', 'swipe', String(from.x), String(from.y), String(to.x), String(to.y), String(durationMs)])
+  }
+
+  async button(serial, name = 'home') {
+    const keycode = ANDROID_BUTTONS[name] ?? (/^KEYCODE_[A-Z0-9_]+$/.test(name) ? name : undefined)
+    if (keycode === undefined) throw new Error(`unknown button "${name}"; expected one of ${Object.keys(ANDROID_BUTTONS).join(', ')} or a KEYCODE_* name`)
+    await this.#shell(serial, ['input', 'keyevent', keycode])
+  }
+
+  /** ASCII via `input text`; non-ASCII via the ADBKeyboard IME, else refused. */
+  async type(serial, text) {
+    if (typeof text !== 'string' || text === '') throw new TypeError('dsh-mobilecode: type requires a non-empty text')
+    if (DeviceBuild.isAsciiInput(text)) {
+      await this.#shell(serial, ['input', 'text', DeviceBuild.escapeInputText(text)])
+      return
+    }
+    if (!(await DeviceBuild.adbKeyboardReady(serial))) {
+      throw new Error('the text contains non-ASCII characters that `input text` cannot deliver; install the ADBKeyboard IME (github.com/senzhk/ADBKeyBoard) and select it')
+    }
+    await DeviceBuild.typeViaAdbKeyboard(serial, text)
+  }
+
+  async rotate(serial, rotation) {
+    if (!ROTATION_CYCLE.includes(rotation)) throw new RangeError('dsh-mobilecode: rotation must be 0, 1, 2 or 3')
+    await this.#shell(serial, ['settings', 'put', 'system', 'accelerometer_rotation', '0'])
+    await this.#shell(serial, ['settings', 'put', 'system', 'user_rotation', String(rotation)])
+  }
+
+  async getRotation(serial) {
+    try {
+      const value = Number((await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'get', 'system', 'user_rotation'])).trim())
+      return ROTATION_CYCLE.includes(value) ? value : 0
+    } catch {
+      return 0
+    }
+  }
+
+  /** The online device list for the panel picker (serial + kind). */
+  async listDevices() {
+    const rows = await DeviceBuild.devices()
+    return rows
+      .filter((row) => row.state === 'device')
+      .map((row) => ({ serial: row.serial, kind: row.serial.startsWith('emulator-') ? 'emulator' : 'physical' }))
+  }
+
+  async #shell(serial, shell) {
+    const code = await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', ...shell], { timeoutMs: CONTROL_TIMEOUT_MS }).exit
+    if (code !== 0) throw new Error(`adb shell ${shell.join(' ')} failed (exit ${code})`)
+  }
+
+  async #keepAliveTick() {
+    if (this.#disposed || this.#intentionalStop) return
+    const exitAt = this.#exitAt
+    const serial = this.#lastSerial
+    if (exitAt === undefined || serial === undefined) return
+    if (Date.now() - exitAt < this.#options.restartDelayMs) return
+    this.#exitAt = undefined
+    this.#restarts += 1
+    try {
+      await this.ensureStreaming({ serial })
+    } catch (error) {
+      this.#lastError = errorMessage(error)
+      if (this.#exitAt === undefined) this.#exitAt = Date.now()
+    }
+  }
+
+  async #startFor(serial) {
+    if (this.#disposed) throw new Error('dsh-mobilecode: the stream host is disposed')
+    const online = await DeviceBuild.devices()
+    if (!online.some((row) => row.serial === serial && row.state === 'device')) {
+      const known = online.find((row) => row.serial === serial)
+      throw new Error(
+        known === undefined
+          ? `no connected device has the serial ${serial}`
+          : `device ${serial} is ${known.state}, not ready to stream`,
+      )
+    }
+    const loop = new AdbFrameLoop(serial, {
+      onFrame: (frame) => {
+        for (const subscriber of this.#frameSubscribers) {
+          try {
+            subscriber(frame)
+          } catch {
+            // One broken consumer must not stall the fan-out.
+          }
+        }
+      },
+      onExit: (detail) => {
+        if (this.#loop !== loop) return
+        this.#loop = undefined
+        this.#startedAt = undefined
+        this.#lastError = `the screencap loop for ${serial} died (${detail})`
+        this.#exitAt = Date.now()
+      },
+    })
+    this.#loop = loop
+    this.#lastSerial = serial
+    this.#intentionalStop = false
+    loop.reset()
+    loop.start()
+    const frame = await loop.waitForFrame(this.#options.firstFrameTimeoutMs)
+    if (frame === undefined) {
+      const stderr = loop.stderrLines.join('\n')
+      loop.stop()
+      if (this.#loop === loop) this.#loop = undefined
+      throw new Error(`no frame arrived from ${serial} within ${this.#options.firstFrameTimeoutMs} ms${stderr === '' ? '' : `: ${stderr}`}`)
+    }
+    this.#startedAt = Date.now()
+    this.#exitAt = undefined
+    return this.#infoOf(loop)
+  }
+
+  #infoOf(loop) {
+    const frame = loop.latestFrame
+    return { serial: loop.serial, ...(frame === undefined ? {} : { width: frame.width, height: frame.height }) }
+  }
+
+  /** Normalized frame coordinates → `input` pixels via the live frame size. */
+  async #pixels(serial, x, y) {
+    const frame = this.streamedSerial === serial ? this.latestFrame : undefined
+    if (frame !== undefined) return { x: Math.round(x * frame.width), y: Math.round(y * frame.height) }
+    const size = await screenSize(serial)
+    return { x: Math.round(x * size.width), y: Math.round(y * size.height) }
+  }
+
+  #armIdle() {
+    this.#clearIdle()
+    const idleMs = this.#options.idleTimeoutMs
+    if (idleMs <= 0) return
+    this.#idleTimer = setTimeout(() => {
+      this.#idleTimer = undefined
+      if (this.#consumers > 0) {
+        this.#armIdle()
+        return
+      }
+      void this.stop()
+    }, idleMs)
+    this.#idleTimer.unref?.()
+  }
+
+  #clearIdle() {
+    if (this.#idleTimer !== undefined) clearTimeout(this.#idleTimer)
+    this.#idleTimer = undefined
+  }
+
+  #serializeLaunch(task) {
+    const run = this.#launchQueue.then(task, task)
+    this.#launchQueue = run.then(() => undefined, () => undefined)
+    return run
+  }
+}
+
+/** Override-aware physical screen size for the non-streaming control fallback. */
+async function screenSize(serial) {
+  const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'wm', 'size'])
+  const match = /Override size:\s*(\d+)x(\d+)/.exec(output) ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
+  if (!match) throw new Error(`cannot read the screen size of ${serial}`)
+  return { width: Number(match[1]), height: Number(match[2]) }
+}

package/lib/client.js

@@ -110,6 +110,17 @@ window.__ModuleLoader__.load({
 .mc-section-head { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; padding: 2px 0 10px; }
 .mc-section-head .mc-hint { flex: 1; min-width: 200px; }
 .mc-section .mc-tabs { padding: 0; margin-bottom: 4px; }
+/* ── live device stream ── */
+.mc-live { display: flex; flex-direction: column; gap: 8px; }
+.mc-live-bar { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
+.mc-live-bar select { background: light-dark(#fff, #20242b); color: inherit; border: 1px solid light-dark(rgba(0,0,0,.18), rgba(255,255,255,.18)); border-radius: 6px; padding: 3px 6px; font: inherit; font-size: 12px; max-width: 200px; }
+.mc-live-stage { position: relative; align-self: center; background: #000; border-radius: 10px; overflow: hidden; line-height: 0; box-shadow: inset 0 0 0 1px light-dark(rgba(0,0,0,.12), rgba(255,255,255,.14)); }
+.mc-live-stage img { display: block; max-width: 100%; max-height: 60vh; width: auto; height: auto; cursor: crosshair; touch-action: none; user-select: none; }
+.mc-live-stage .mc-live-off { display: flex; align-items: center; justify-content: center; width: 260px; height: 460px; color: #9aa0a8; font-size: 13px; line-height: 1.5; text-align: center; }
+.mc-live-nav { display: flex; align-items: center; justify-content: center; gap: 8px; }
+.mc-live-nav button { border: 1px solid light-dark(rgba(0,0,0,.16), rgba(255,255,255,.18)); background: light-dark(#fff, #20242b); color: inherit; border-radius: 999px; width: 40px; height: 32px; cursor: pointer; font-size: 14px; }
+.mc-live-nav button:hover { background: light-dark(rgba(0,0,0,.06), rgba(255,255,255,.1)); }
+.mc-live-cap { font-size: 11px; color: light-dark(#6b7078, #9aa0a8); text-align: center; }
 `;
 		//#endregion
 
@@ -148,6 +159,20 @@ window.__ModuleLoader__.load({
 			if (!response.ok) throw new Error("HTTP " + response.status);
 			return response.json();
 		};
+		// POST that surfaces the server's JSON error copy (stream routes return
+		// human-readable 4xx bodies the generic post() would flatten to "HTTP 409").
+		const streamPost = async (path, body) => {
+			const response = await fetch(API_BASE + path, {
+				method: "POST",
+				headers: { "content-type": "application/json" },
+				body: JSON.stringify(body ?? {}),
+			});
+			const text = await response.text();
+			let json;
+			try { json = JSON.parse(text); } catch { json = {}; }
+			if (!response.ok) throw new Error(json.error || ("HTTP " + response.status));
+			return json;
+		};
 		const copyText = async (text) => {
 			try { await navigator.clipboard.writeText(text); return true; } catch { return false; }
 		};
@@ -259,6 +284,7 @@ window.__ModuleLoader__.load({
 							);
 						}),
 					),
+					h(LiveDeviceCard, null),
 					platforms.map((platform) => {
 						const srv = server(platform);
 						const bld = build(platform);
@@ -333,6 +359,113 @@ window.__ModuleLoader__.load({
 			);
 		}
 
+		/** Live device stream: an <img> fed by the in-process multipart route, with
+		 *  tap/drag control and the Back/Home/Recents/rotate/power nav bar. */
+		function LiveDeviceCard() {
+			const [devices, setDevices] = useState([]);
+			const [serial, setSerial] = useState("");
+			const [streamUrl, setStreamUrl] = useState("");
+			const [error, setError] = useState("");
+			const imgRef = useRef(null);
+			const dragRef = useRef(null);
+			const grantTimer = useRef(null);
+
+			const grant = async (device) => {
+				if (!device) return;
+				try {
+					const r = await streamPost("/stream/grant", { device });
+					setStreamUrl(r.streamUrl);
+					setSerial(r.device);
+					setError("");
+					clearTimeout(grantTimer.current);
+					// Re-mint a minute before the 10-minute capability expires.
+					const ms = Math.max(30000, (r.expiresAt - Date.now()) - 60000);
+					grantTimer.current = setTimeout(() => { grant(r.device).catch(() => {}); }, ms);
+				} catch (err) {
+					setError(err instanceof Error ? err.message : String(err));
+					setStreamUrl("");
+				}
+			};
+
+			const refreshDevices = async () => {
+				try {
+					const r = await streamPost("/stream/devices", {});
+					setDevices(r.devices);
+					setSerial((cur) => cur || (r.devices.find((d) => d.streaming)?.serial ?? r.devices[0]?.serial ?? ""));
+				} catch (err) {
+					setError(err instanceof Error ? err.message : String(err));
+				}
+			};
+
+			useEffect(() => {
+				refreshDevices();
+				const timer = setInterval(refreshDevices, 5000);
+				return () => { clearInterval(timer); clearTimeout(grantTimer.current); };
+			}, []);
+
+			// Auto-grant once a serial is known but no stream is up yet.
+			useEffect(() => { if (serial !== "" && streamUrl === "") grant(serial).catch(() => {}); }, [serial, streamUrl]);
+
+			const control = async (action) => {
+				if (serial === "") return;
+				try { await streamPost("/stream/control", { device: serial, action }); }
+				catch (err) { setError(err instanceof Error ? err.message : String(err)); }
+			};
+
+			const norm = (clientX, clientY) => {
+				const el = imgRef.current;
+				if (!el) return null;
+				const rect = el.getBoundingClientRect();
+				if (rect.width === 0 || rect.height === 0) return null;
+				return {
+					x: Math.min(1, Math.max(0, (clientX - rect.left) / rect.width)),
+					y: Math.min(1, Math.max(0, (clientY - rect.top) / rect.height)),
+				};
+			};
+			const onDown = (e) => { const p = norm(e.clientX, e.clientY); if (p) dragRef.current = p; };
+			const onUp = (e) => {
+				const start = dragRef.current;
+				dragRef.current = null;
+				const end = norm(e.clientX, e.clientY);
+				if (!start || !end) return;
+				if (Math.hypot(end.x - start.x, end.y - start.y) > 0.02) {
+					control({ kind: "drag", fromX: start.x, fromY: start.y, toX: end.x, toY: end.y });
+				} else {
+					control({ kind: "tap", x: end.x, y: end.y });
+				}
+			};
+
+			return h("div", { className: "mc-card mc-live" },
+				h("div", { className: "mc-card-head" },
+					h("span", null, "Live device"),
+					h("span", { className: "sp" }),
+					devices.length > 0
+						? h("select", {
+							value: serial,
+							onChange: (e) => { setSerial(e.target.value); setStreamUrl(""); setError(""); },
+						}, devices.map((d) => h("option", { key: d.serial, value: d.serial }, d.serial + (d.kind === "emulator" ? " · emu" : " · phone"))))
+						: h("span", { className: "mc-hint" }, "no device"),
+					h("button", { className: "mc-btn", disabled: serial === "", onClick: () => grant(serial) }, "Reconnect"),
+				),
+				error !== "" && h("div", { className: "mc-error" }, error),
+				h("div", { className: "mc-live-stage" },
+					streamUrl !== ""
+						? h("img", { ref: imgRef, src: streamUrl, alt: "device screen", draggable: false, onPointerDown: onDown, onPointerUp: onUp, onError: () => grant(serial) })
+						: h("div", { className: "mc-live-off" }, devices.length === 0
+							? "No device attached. Boot one with device_boot, or press Run app."
+							: "Connecting to " + serial + "…"),
+				),
+				h("div", { className: "mc-live-nav" },
+					h("button", { title: "Back", disabled: serial === "", onClick: () => control({ kind: "button", name: "back" }) }, "◁"),
+					h("button", { title: "Home", disabled: serial === "", onClick: () => control({ kind: "button", name: "home" }) }, "○"),
+					h("button", { title: "Recents", disabled: serial === "", onClick: () => control({ kind: "button", name: "recents" }) }, "▢"),
+					h("button", { title: "Rotate", disabled: serial === "", onClick: () => control({ kind: "rotate" }) }, "⟳"),
+					h("button", { title: "Power", disabled: serial === "", onClick: () => control({ kind: "button", name: "power" }) }, "⏻"),
+				),
+				h("div", { className: "mc-live-cap" }, "tap or drag on the screen to drive the device"),
+			);
+		}
+
 		//#region setup UI (welcome + settings)
 
 		/** PaddleOCR status card: fetch /ocr, Install button, log tail. */

package/lib/device-build.js

@@ -740,6 +740,57 @@ export async function androidBooted(serial) {
   return output.trim() === "1"
 }
 
+/** AVD name of an emulator serial (`adb emu avd name`); undefined for physical/offline. */
+export async function avdName(serial) {
+  const output = await capture(adb(), ["-s", serial, "emu", "avd", "name"])
+  const line = output
+    .split(/\r?\n/)
+    .map((item) => item.trim())
+    .find((item) => item && !/^OK$/i.test(item) && !/^KO:/i.test(item))
+  return line && /^[A-Za-z0-9._-]+$/.test(line) ? line : undefined
+}
+
+/** Poll until sys.boot_completed == 1 or the timeout; true when booted. */
+export async function waitForBoot(serial, timeoutMs = 180_000) {
+  const deadline = Date.now() + timeoutMs
+  for (;;) {
+    if (await androidBooted(serial)) return true
+    if (Date.now() >= deadline) return false
+    await new Promise((resolve) => setTimeout(resolve, 1000))
+  }
+}
+
+/** Launch an AVD detached (survives this process); undefined when no emulator binary. */
+export function bootEmulator(avd) {
+  const emulator = emulatorBinary()
+  if (!emulator || !/^[A-Za-z0-9._-]+$/.test(avd)) return undefined
+  const child = launch(emulator, ["-avd", avd], { stdio: "ignore", detached: true })
+  child.unref?.()
+  return child
+}
+
+/** True when the ADBKeyboard IME is installed (the only way to type non-ASCII over adb). */
+export async function adbKeyboardReady(serial) {
+  const output = await capture(adb(), ["-s", serial, "shell", "ime", "list", "-s"])
+  return /com\.android\.adbkeyboard/i.test(output)
+}
+
+/** Type one string via the ADBKeyboard broadcast (base64, so any codepoint survives the shell). */
+export async function typeViaAdbKeyboard(serial, text) {
+  const msg = Buffer.from(text, "utf8").toString("base64")
+  await exec(adb(), ["-s", serial, "shell", "am", "broadcast", "-a", "ADB_INPUT_B64", "--es", "msg", msg]).exit
+}
+
+/** True when every codepoint is safe for `adb shell input text` (printable ASCII, no shell metachars). */
+export function isAsciiInput(text) {
+  return [...text].every((ch) => ch.codePointAt(0) >= 0x20 && ch.codePointAt(0) <= 0x7e)
+}
+
+/** Escape one `input text` argument for the device shell: backslash the metachars, spaces to %s. */
+export function escapeInputText(text) {
+  return text.replace(/[\\()<>|;&*~"'`$#!{}[\]]/g, "\\$&").replace(/ /g, "%s")
+}
+
 function aapt2() {
   const sdk = androidSdk()
   if (!sdk) return undefined