dsh-mobilecode 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -87,6 +87,45 @@ drawer:
87
87
  - `device_apps` / `device_launch_app` — list installed packages (third-party by
88
88
  default) so a package name is never guessed / launch one by package or a
89
89
  unique substring, with `relaunch` for a cold start.
90
+ - `device_stream` — drive the live screen stream the panel shows: `start` an
91
+ online device (returns a signed `streamUrl`), `status`, or `stop`. Agents that
92
+ just need to see the screen should prefer `device_screen` / `device_ui_tree`.
93
+
94
+ **Live device stream (the panel)**
95
+
96
+ The Devices pane shows a real-time mirror of the attached device. It is produced
97
+ **in-process** — no inner loopback port, no external helper:
98
+
99
+ - ONE persistent `adb exec-out "while :; do screencap -p; done"` child streams
100
+ ~8 fps with zero per-frame process cost (spawning adb per frame caps at ~5 fps
101
+ and 100% churn). A quote-safe PNG splitter cuts the concatenated output into
102
+ frames by walking chunk headers (no marker scanning, no false positives).
103
+ - The browser `<img>` reads a `multipart/x-mixed-replace` body served straight
104
+ from the latest-frame buffer. Backpressure is **latest-wins**: a slow tab skips
105
+ frames instead of building an unbounded queue or watching a growing delay.
106
+ - A consumer refcount + idle timeout stops the loop when nobody is watching; a
107
+ keep-alive restarts a crashed loop; switching devices retires the old child.
108
+ - **Tap or drag directly on the screen** to drive the device; a Back / Home /
109
+ Recents / Rotate / Power bar sits below it, and a device picker switches which
110
+ online device streams (it never boots one — use `device_boot` for that).
111
+ - **Security**: every stream route sits behind a loopback + trusted-browser
112
+ transport fence (peer address, loopback `Host`, `Sec-Fetch-Site` / `Origin` —
113
+ so a LAN client cannot spoof localhost and a DNS-rebinding `Host` is rejected),
114
+ and the stream URL is an **HMAC-SHA256 capability** signed with a per-install
115
+ key (`~/.dsh/mobilecode/stream-access.key`, `0600`), expiring within 10 minutes
116
+ and re-minted automatically. Coordinates are normalized 0..1 of the streamed
117
+ frame, so one mapping serves every rotation.
118
+
119
+ **Multimodal screenshots**
120
+
121
+ When the routed model declares image input, `device_screen` delivers the
122
+ screenshot **as an image block** — the model literally sees the screen instead of
123
+ reading a file path. This mirrors the in-tree `read_image` tool: the PNG is
124
+ committed to DSH's durable attachment store (`ctx.get('attachments').saveImage`)
125
+ and returned as a `{type:'image', attachment}` content block, gated on
126
+ `llm.resolveModelInfo(...).inputModalities`. It **degrades, never refuses**: a
127
+ text-only route, a headless profile, or a host without the attachment store keeps
128
+ the plain JSON summary (path + UI tree + OCR) with no new error.
90
129
  - `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
91
130
  (kernel = dmesg, needs adb root — works on emulators) with an optional
92
131
  case-insensitive substring filter, capped line count.
@@ -0,0 +1,378 @@
1
+ /**
2
+ * dsh-mobilecode — host-side lifecycle manager for the one live Android stream.
3
+ *
4
+ * Ported (lean) from ZSeven-W/dsh-android (android-host.ts, MIT). The stream is
5
+ * in-process: one AdbFrameLoop per streamed serial, a consumer refcount with an
6
+ * idle timeout, and a keep-alive that restarts a crashed loop. Emulators and
7
+ * physical devices share this path — the serial is the only identity.
8
+ *
9
+ * The control surface takes NORMALIZED 0..1 coordinates of the streamed frame
10
+ * (the panel's <img> is scaled, so the browser knows fractions, not pixels) and
11
+ * maps them onto `adb shell input` pixels using the latest frame's own size.
12
+ */
13
+
14
+ import * as DeviceBuild from './device-build.js'
15
+ import { AdbFrameLoop } from './frame-source.js'
16
+
17
+ const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60 * 1000
18
+ const DEFAULT_RESTART_DELAY_MS = 5000
19
+ const KEEP_ALIVE_TICK_MS = 1000
20
+ const FIRST_FRAME_TIMEOUT_MS = 15000
21
+ const CONTROL_TIMEOUT_MS = 30000
22
+ const MAX_SWIPE_MS = 5000
23
+
24
+ /** Navigation/hardware buttons the panel may press. */
25
+ export const ANDROID_BUTTONS = {
26
+ home: 'KEYCODE_HOME',
27
+ back: 'KEYCODE_BACK',
28
+ recents: 'KEYCODE_APP_SWITCH',
29
+ power: 'KEYCODE_POWER',
30
+ volume_up: 'KEYCODE_VOLUME_UP',
31
+ volume_down: 'KEYCODE_VOLUME_DOWN',
32
+ menu: 'KEYCODE_MENU',
33
+ enter: 'KEYCODE_ENTER',
34
+ delete: 'KEYCODE_DEL',
35
+ }
36
+
37
+ /** Clockwise user_rotation cycle (Surface.ROTATION_0..270). */
38
+ export const ROTATION_CYCLE = [0, 1, 2, 3]
39
+
40
+ function errorMessage(error) {
41
+ return error instanceof Error ? error.message : String(error)
42
+ }
43
+
44
+ function requireNormalized(x, y) {
45
+ if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || x > 1 || y < 0 || y > 1) {
46
+ throw new RangeError('dsh-mobilecode: tap/drag coordinates must be normalized 0..1 of the streamed frame')
47
+ }
48
+ }
49
+
50
+ /** Lifecycle manager for the (single) in-process Android device stream. */
51
+ export class AndroidStreamHost {
52
+ #options
53
+ #loop
54
+ #starting
55
+ #launchQueue = Promise.resolve()
56
+ #consumers = 0
57
+ #keepAliveTimer
58
+ #idleTimer
59
+ #restarts = 0
60
+ #startedAt
61
+ #exitAt
62
+ #lastError
63
+ #lastSerial
64
+ #intentionalStop = false
65
+ #disposed = false
66
+ #frameSubscribers = new Set()
67
+
68
+ constructor(options = {}) {
69
+ this.#options = {
70
+ restartDelayMs: options.restartDelayMs ?? DEFAULT_RESTART_DELAY_MS,
71
+ idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
72
+ firstFrameTimeoutMs: options.firstFrameTimeoutMs ?? FIRST_FRAME_TIMEOUT_MS,
73
+ }
74
+ }
75
+
76
+ get running() {
77
+ return this.#loop?.running === true
78
+ }
79
+
80
+ get streamedSerial() {
81
+ return this.running ? this.#loop?.serial : undefined
82
+ }
83
+
84
+ get latestFrame() {
85
+ return this.#loop?.latestFrame
86
+ }
87
+
88
+ /** Observe every decoded frame; the returned function unsubscribes. */
89
+ subscribeFrames(subscriber) {
90
+ this.#frameSubscribers.add(subscriber)
91
+ return () => {
92
+ this.#frameSubscribers.delete(subscriber)
93
+ }
94
+ }
95
+
96
+ /**
97
+ * Make sure the frame loop is live for `serial`. Concurrent callers share one
98
+ * launch; a call for a different serial retires the current loop first (one
99
+ * streamed device at a time).
100
+ */
101
+ async ensureStreaming({ serial }) {
102
+ if (this.#disposed) throw new Error('dsh-mobilecode: the stream host is disposed')
103
+ if (typeof serial !== 'string' || serial === '') throw new TypeError('dsh-mobilecode: ensureStreaming requires a non-empty serial')
104
+ const startFor = async () => {
105
+ const current = this.#loop
106
+ if (current !== undefined && current.running && current.serial === serial) return this.#infoOf(current)
107
+ if (current !== undefined && current.serial !== serial) {
108
+ current.stop()
109
+ this.#loop = undefined
110
+ }
111
+ return this.#startFor(serial)
112
+ }
113
+ let starting = this.#starting
114
+ if (starting !== undefined) {
115
+ try {
116
+ await starting
117
+ } catch {
118
+ // A failed shared launch is settled; retry below.
119
+ }
120
+ if (this.running && this.#loop?.serial === serial) {
121
+ this.#armIdle()
122
+ return this.#infoOf(this.#loop)
123
+ }
124
+ starting = undefined
125
+ }
126
+ starting = this.#serializeLaunch(startFor)
127
+ this.#starting = starting
128
+ try {
129
+ const info = await starting
130
+ this.#lastError = undefined
131
+ this.#armIdle()
132
+ return info
133
+ } catch (error) {
134
+ this.#lastError = errorMessage(error)
135
+ throw error
136
+ } finally {
137
+ if (this.#starting === starting) this.#starting = undefined
138
+ }
139
+ }
140
+
141
+ /** Start the crash keep-alive loop (restarts an unintentionally dead loop). */
142
+ startKeepAlive() {
143
+ if (this.#keepAliveTimer !== undefined || this.#disposed) return
144
+ this.#keepAliveTimer = setInterval(() => {
145
+ void this.#keepAliveTick().catch(() => {})
146
+ }, KEEP_ALIVE_TICK_MS)
147
+ this.#keepAliveTimer.unref?.()
148
+ }
149
+
150
+ stopKeepAlive() {
151
+ if (this.#keepAliveTimer !== undefined) clearInterval(this.#keepAliveTimer)
152
+ this.#keepAliveTimer = undefined
153
+ }
154
+
155
+ /** Stop the stream intentionally (keep-alive will not fight it). */
156
+ async stop() {
157
+ this.#clearIdle()
158
+ this.#intentionalStop = true
159
+ this.#exitAt = undefined
160
+ const loop = this.#loop
161
+ this.#loop = undefined
162
+ this.#startedAt = undefined
163
+ loop?.stop()
164
+ await this.#starting?.catch(() => {})
165
+ const landed = this.#loop
166
+ if (landed !== undefined) {
167
+ this.#loop = undefined
168
+ landed.stop()
169
+ }
170
+ }
171
+
172
+ /** Hold the stream alive for one consumer; release exactly once. */
173
+ acquire() {
174
+ this.#consumers += 1
175
+ this.#armIdle()
176
+ let released = false
177
+ return () => {
178
+ if (released) return
179
+ released = true
180
+ this.#consumers = Math.max(0, this.#consumers - 1)
181
+ this.#armIdle()
182
+ }
183
+ }
184
+
185
+ status() {
186
+ const loop = this.#loop
187
+ const frame = loop?.latestFrame
188
+ return {
189
+ running: this.running,
190
+ ...(loop === undefined ? {} : { serial: loop.serial }),
191
+ restarts: this.#restarts,
192
+ ...(this.#lastError === undefined ? {} : { lastError: this.#lastError }),
193
+ consumers: this.#consumers,
194
+ ...(frame === undefined ? {} : { frameSequence: frame.sequence, lastFrameAt: frame.at, width: frame.width, height: frame.height }),
195
+ stderr: loop?.stderrLines ?? [],
196
+ }
197
+ }
198
+
199
+ dispose() {
200
+ this.#disposed = true
201
+ this.stopKeepAlive()
202
+ this.#frameSubscribers.clear()
203
+ return this.stop()
204
+ }
205
+
206
+ // ── control surface (normalized 0..1 of the streamed frame) ────────────────
207
+
208
+ async tap(serial, x, y) {
209
+ requireNormalized(x, y)
210
+ const point = await this.#pixels(serial, x, y)
211
+ await this.#shell(serial, ['input', 'tap', String(point.x), String(point.y)])
212
+ }
213
+
214
+ async drag(serial, drag) {
215
+ requireNormalized(drag.fromX, drag.fromY)
216
+ requireNormalized(drag.toX, drag.toY)
217
+ const from = await this.#pixels(serial, drag.fromX, drag.fromY)
218
+ const to = await this.#pixels(serial, drag.toX, drag.toY)
219
+ const durationMs = Math.min(MAX_SWIPE_MS, Math.max(20, Math.round((drag.duration ?? 0.3) * 1000)))
220
+ await this.#shell(serial, ['input', 'swipe', String(from.x), String(from.y), String(to.x), String(to.y), String(durationMs)])
221
+ }
222
+
223
+ async button(serial, name = 'home') {
224
+ const keycode = ANDROID_BUTTONS[name] ?? (/^KEYCODE_[A-Z0-9_]+$/.test(name) ? name : undefined)
225
+ if (keycode === undefined) throw new Error(`unknown button "${name}"; expected one of ${Object.keys(ANDROID_BUTTONS).join(', ')} or a KEYCODE_* name`)
226
+ await this.#shell(serial, ['input', 'keyevent', keycode])
227
+ }
228
+
229
+ /** ASCII via `input text`; non-ASCII via the ADBKeyboard IME, else refused. */
230
+ async type(serial, text) {
231
+ if (typeof text !== 'string' || text === '') throw new TypeError('dsh-mobilecode: type requires a non-empty text')
232
+ if (DeviceBuild.isAsciiInput(text)) {
233
+ await this.#shell(serial, ['input', 'text', DeviceBuild.escapeInputText(text)])
234
+ return
235
+ }
236
+ if (!(await DeviceBuild.adbKeyboardReady(serial))) {
237
+ throw new Error('the text contains non-ASCII characters that `input text` cannot deliver; install the ADBKeyboard IME (github.com/senzhk/ADBKeyBoard) and select it')
238
+ }
239
+ await DeviceBuild.typeViaAdbKeyboard(serial, text)
240
+ }
241
+
242
+ async rotate(serial, rotation) {
243
+ if (!ROTATION_CYCLE.includes(rotation)) throw new RangeError('dsh-mobilecode: rotation must be 0, 1, 2 or 3')
244
+ await this.#shell(serial, ['settings', 'put', 'system', 'accelerometer_rotation', '0'])
245
+ await this.#shell(serial, ['settings', 'put', 'system', 'user_rotation', String(rotation)])
246
+ }
247
+
248
+ async getRotation(serial) {
249
+ try {
250
+ const value = Number((await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'get', 'system', 'user_rotation'])).trim())
251
+ return ROTATION_CYCLE.includes(value) ? value : 0
252
+ } catch {
253
+ return 0
254
+ }
255
+ }
256
+
257
+ /** The online device list for the panel picker (serial + kind). */
258
+ async listDevices() {
259
+ const rows = await DeviceBuild.devices()
260
+ return rows
261
+ .filter((row) => row.state === 'device')
262
+ .map((row) => ({ serial: row.serial, kind: row.serial.startsWith('emulator-') ? 'emulator' : 'physical' }))
263
+ }
264
+
265
+ async #shell(serial, shell) {
266
+ const code = await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', ...shell], { timeoutMs: CONTROL_TIMEOUT_MS }).exit
267
+ if (code !== 0) throw new Error(`adb shell ${shell.join(' ')} failed (exit ${code})`)
268
+ }
269
+
270
+ async #keepAliveTick() {
271
+ if (this.#disposed || this.#intentionalStop) return
272
+ const exitAt = this.#exitAt
273
+ const serial = this.#lastSerial
274
+ if (exitAt === undefined || serial === undefined) return
275
+ if (Date.now() - exitAt < this.#options.restartDelayMs) return
276
+ this.#exitAt = undefined
277
+ this.#restarts += 1
278
+ try {
279
+ await this.ensureStreaming({ serial })
280
+ } catch (error) {
281
+ this.#lastError = errorMessage(error)
282
+ if (this.#exitAt === undefined) this.#exitAt = Date.now()
283
+ }
284
+ }
285
+
286
+ async #startFor(serial) {
287
+ if (this.#disposed) throw new Error('dsh-mobilecode: the stream host is disposed')
288
+ const online = await DeviceBuild.devices()
289
+ if (!online.some((row) => row.serial === serial && row.state === 'device')) {
290
+ const known = online.find((row) => row.serial === serial)
291
+ throw new Error(
292
+ known === undefined
293
+ ? `no connected device has the serial ${serial}`
294
+ : `device ${serial} is ${known.state}, not ready to stream`,
295
+ )
296
+ }
297
+ const loop = new AdbFrameLoop(serial, {
298
+ onFrame: (frame) => {
299
+ for (const subscriber of this.#frameSubscribers) {
300
+ try {
301
+ subscriber(frame)
302
+ } catch {
303
+ // One broken consumer must not stall the fan-out.
304
+ }
305
+ }
306
+ },
307
+ onExit: (detail) => {
308
+ if (this.#loop !== loop) return
309
+ this.#loop = undefined
310
+ this.#startedAt = undefined
311
+ this.#lastError = `the screencap loop for ${serial} died (${detail})`
312
+ this.#exitAt = Date.now()
313
+ },
314
+ })
315
+ this.#loop = loop
316
+ this.#lastSerial = serial
317
+ this.#intentionalStop = false
318
+ loop.reset()
319
+ loop.start()
320
+ const frame = await loop.waitForFrame(this.#options.firstFrameTimeoutMs)
321
+ if (frame === undefined) {
322
+ const stderr = loop.stderrLines.join('\n')
323
+ loop.stop()
324
+ if (this.#loop === loop) this.#loop = undefined
325
+ throw new Error(`no frame arrived from ${serial} within ${this.#options.firstFrameTimeoutMs} ms${stderr === '' ? '' : `: ${stderr}`}`)
326
+ }
327
+ this.#startedAt = Date.now()
328
+ this.#exitAt = undefined
329
+ return this.#infoOf(loop)
330
+ }
331
+
332
+ #infoOf(loop) {
333
+ const frame = loop.latestFrame
334
+ return { serial: loop.serial, ...(frame === undefined ? {} : { width: frame.width, height: frame.height }) }
335
+ }
336
+
337
+ /** Normalized frame coordinates → `input` pixels via the live frame size. */
338
+ async #pixels(serial, x, y) {
339
+ const frame = this.streamedSerial === serial ? this.latestFrame : undefined
340
+ if (frame !== undefined) return { x: Math.round(x * frame.width), y: Math.round(y * frame.height) }
341
+ const size = await screenSize(serial)
342
+ return { x: Math.round(x * size.width), y: Math.round(y * size.height) }
343
+ }
344
+
345
+ #armIdle() {
346
+ this.#clearIdle()
347
+ const idleMs = this.#options.idleTimeoutMs
348
+ if (idleMs <= 0) return
349
+ this.#idleTimer = setTimeout(() => {
350
+ this.#idleTimer = undefined
351
+ if (this.#consumers > 0) {
352
+ this.#armIdle()
353
+ return
354
+ }
355
+ void this.stop()
356
+ }, idleMs)
357
+ this.#idleTimer.unref?.()
358
+ }
359
+
360
+ #clearIdle() {
361
+ if (this.#idleTimer !== undefined) clearTimeout(this.#idleTimer)
362
+ this.#idleTimer = undefined
363
+ }
364
+
365
+ #serializeLaunch(task) {
366
+ const run = this.#launchQueue.then(task, task)
367
+ this.#launchQueue = run.then(() => undefined, () => undefined)
368
+ return run
369
+ }
370
+ }
371
+
372
+ /** Override-aware physical screen size for the non-streaming control fallback. */
373
+ async function screenSize(serial) {
374
+ const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'wm', 'size'])
375
+ const match = /Override size:\s*(\d+)x(\d+)/.exec(output) ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
376
+ if (!match) throw new Error(`cannot read the screen size of ${serial}`)
377
+ return { width: Number(match[1]), height: Number(match[2]) }
378
+ }
package/lib/client.js CHANGED
@@ -110,6 +110,17 @@ window.__ModuleLoader__.load({
110
110
  .mc-section-head { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; padding: 2px 0 10px; }
111
111
  .mc-section-head .mc-hint { flex: 1; min-width: 200px; }
112
112
  .mc-section .mc-tabs { padding: 0; margin-bottom: 4px; }
113
+ /* ── live device stream ── */
114
+ .mc-live { display: flex; flex-direction: column; gap: 8px; }
115
+ .mc-live-bar { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
116
+ .mc-live-bar select { background: light-dark(#fff, #20242b); color: inherit; border: 1px solid light-dark(rgba(0,0,0,.18), rgba(255,255,255,.18)); border-radius: 6px; padding: 3px 6px; font: inherit; font-size: 12px; max-width: 200px; }
117
+ .mc-live-stage { position: relative; align-self: center; background: #000; border-radius: 10px; overflow: hidden; line-height: 0; box-shadow: inset 0 0 0 1px light-dark(rgba(0,0,0,.12), rgba(255,255,255,.14)); }
118
+ .mc-live-stage img { display: block; max-width: 100%; max-height: 60vh; width: auto; height: auto; cursor: crosshair; touch-action: none; user-select: none; }
119
+ .mc-live-stage .mc-live-off { display: flex; align-items: center; justify-content: center; width: 260px; height: 460px; color: #9aa0a8; font-size: 13px; line-height: 1.5; text-align: center; }
120
+ .mc-live-nav { display: flex; align-items: center; justify-content: center; gap: 8px; }
121
+ .mc-live-nav button { border: 1px solid light-dark(rgba(0,0,0,.16), rgba(255,255,255,.18)); background: light-dark(#fff, #20242b); color: inherit; border-radius: 999px; width: 40px; height: 32px; cursor: pointer; font-size: 14px; }
122
+ .mc-live-nav button:hover { background: light-dark(rgba(0,0,0,.06), rgba(255,255,255,.1)); }
123
+ .mc-live-cap { font-size: 11px; color: light-dark(#6b7078, #9aa0a8); text-align: center; }
113
124
  `;
114
125
  //#endregion
115
126
 
@@ -148,6 +159,20 @@ window.__ModuleLoader__.load({
148
159
  if (!response.ok) throw new Error("HTTP " + response.status);
149
160
  return response.json();
150
161
  };
162
+ // POST that surfaces the server's JSON error copy (stream routes return
163
+ // human-readable 4xx bodies the generic post() would flatten to "HTTP 409").
164
+ const streamPost = async (path, body) => {
165
+ const response = await fetch(API_BASE + path, {
166
+ method: "POST",
167
+ headers: { "content-type": "application/json" },
168
+ body: JSON.stringify(body ?? {}),
169
+ });
170
+ const text = await response.text();
171
+ let json;
172
+ try { json = JSON.parse(text); } catch { json = {}; }
173
+ if (!response.ok) throw new Error(json.error || ("HTTP " + response.status));
174
+ return json;
175
+ };
151
176
  const copyText = async (text) => {
152
177
  try { await navigator.clipboard.writeText(text); return true; } catch { return false; }
153
178
  };
@@ -259,6 +284,7 @@ window.__ModuleLoader__.load({
259
284
  );
260
285
  }),
261
286
  ),
287
+ h(LiveDeviceCard, null),
262
288
  platforms.map((platform) => {
263
289
  const srv = server(platform);
264
290
  const bld = build(platform);
@@ -333,6 +359,113 @@ window.__ModuleLoader__.load({
333
359
  );
334
360
  }
335
361
 
362
+ /** Live device stream: an <img> fed by the in-process multipart route, with
363
+ * tap/drag control and the Back/Home/Recents/rotate/power nav bar. */
364
+ function LiveDeviceCard() {
365
+ const [devices, setDevices] = useState([]);
366
+ const [serial, setSerial] = useState("");
367
+ const [streamUrl, setStreamUrl] = useState("");
368
+ const [error, setError] = useState("");
369
+ const imgRef = useRef(null);
370
+ const dragRef = useRef(null);
371
+ const grantTimer = useRef(null);
372
+
373
+ const grant = async (device) => {
374
+ if (!device) return;
375
+ try {
376
+ const r = await streamPost("/stream/grant", { device });
377
+ setStreamUrl(r.streamUrl);
378
+ setSerial(r.device);
379
+ setError("");
380
+ clearTimeout(grantTimer.current);
381
+ // Re-mint a minute before the 10-minute capability expires.
382
+ const ms = Math.max(30000, (r.expiresAt - Date.now()) - 60000);
383
+ grantTimer.current = setTimeout(() => { grant(r.device).catch(() => {}); }, ms);
384
+ } catch (err) {
385
+ setError(err instanceof Error ? err.message : String(err));
386
+ setStreamUrl("");
387
+ }
388
+ };
389
+
390
+ const refreshDevices = async () => {
391
+ try {
392
+ const r = await streamPost("/stream/devices", {});
393
+ setDevices(r.devices);
394
+ setSerial((cur) => cur || (r.devices.find((d) => d.streaming)?.serial ?? r.devices[0]?.serial ?? ""));
395
+ } catch (err) {
396
+ setError(err instanceof Error ? err.message : String(err));
397
+ }
398
+ };
399
+
400
+ useEffect(() => {
401
+ refreshDevices();
402
+ const timer = setInterval(refreshDevices, 5000);
403
+ return () => { clearInterval(timer); clearTimeout(grantTimer.current); };
404
+ }, []);
405
+
406
+ // Auto-grant once a serial is known but no stream is up yet.
407
+ useEffect(() => { if (serial !== "" && streamUrl === "") grant(serial).catch(() => {}); }, [serial, streamUrl]);
408
+
409
+ const control = async (action) => {
410
+ if (serial === "") return;
411
+ try { await streamPost("/stream/control", { device: serial, action }); }
412
+ catch (err) { setError(err instanceof Error ? err.message : String(err)); }
413
+ };
414
+
415
+ const norm = (clientX, clientY) => {
416
+ const el = imgRef.current;
417
+ if (!el) return null;
418
+ const rect = el.getBoundingClientRect();
419
+ if (rect.width === 0 || rect.height === 0) return null;
420
+ return {
421
+ x: Math.min(1, Math.max(0, (clientX - rect.left) / rect.width)),
422
+ y: Math.min(1, Math.max(0, (clientY - rect.top) / rect.height)),
423
+ };
424
+ };
425
+ const onDown = (e) => { const p = norm(e.clientX, e.clientY); if (p) dragRef.current = p; };
426
+ const onUp = (e) => {
427
+ const start = dragRef.current;
428
+ dragRef.current = null;
429
+ const end = norm(e.clientX, e.clientY);
430
+ if (!start || !end) return;
431
+ if (Math.hypot(end.x - start.x, end.y - start.y) > 0.02) {
432
+ control({ kind: "drag", fromX: start.x, fromY: start.y, toX: end.x, toY: end.y });
433
+ } else {
434
+ control({ kind: "tap", x: end.x, y: end.y });
435
+ }
436
+ };
437
+
438
+ return h("div", { className: "mc-card mc-live" },
439
+ h("div", { className: "mc-card-head" },
440
+ h("span", null, "Live device"),
441
+ h("span", { className: "sp" }),
442
+ devices.length > 0
443
+ ? h("select", {
444
+ value: serial,
445
+ onChange: (e) => { setSerial(e.target.value); setStreamUrl(""); setError(""); },
446
+ }, devices.map((d) => h("option", { key: d.serial, value: d.serial }, d.serial + (d.kind === "emulator" ? " · emu" : " · phone"))))
447
+ : h("span", { className: "mc-hint" }, "no device"),
448
+ h("button", { className: "mc-btn", disabled: serial === "", onClick: () => grant(serial) }, "Reconnect"),
449
+ ),
450
+ error !== "" && h("div", { className: "mc-error" }, error),
451
+ h("div", { className: "mc-live-stage" },
452
+ streamUrl !== ""
453
+ ? h("img", { ref: imgRef, src: streamUrl, alt: "device screen", draggable: false, onPointerDown: onDown, onPointerUp: onUp, onError: () => grant(serial) })
454
+ : h("div", { className: "mc-live-off" }, devices.length === 0
455
+ ? "No device attached. Boot one with device_boot, or press Run app."
456
+ : "Connecting to " + serial + "…"),
457
+ ),
458
+ h("div", { className: "mc-live-nav" },
459
+ h("button", { title: "Back", disabled: serial === "", onClick: () => control({ kind: "button", name: "back" }) }, "◁"),
460
+ h("button", { title: "Home", disabled: serial === "", onClick: () => control({ kind: "button", name: "home" }) }, "○"),
461
+ h("button", { title: "Recents", disabled: serial === "", onClick: () => control({ kind: "button", name: "recents" }) }, "▢"),
462
+ h("button", { title: "Rotate", disabled: serial === "", onClick: () => control({ kind: "rotate" }) }, "⟳"),
463
+ h("button", { title: "Power", disabled: serial === "", onClick: () => control({ kind: "button", name: "power" }) }, "⏻"),
464
+ ),
465
+ h("div", { className: "mc-live-cap" }, "tap or drag on the screen to drive the device"),
466
+ );
467
+ }
468
+
336
469
  //#region setup UI (welcome + settings)
337
470
 
338
471
  /** PaddleOCR status card: fetch /ocr, Install button, log tail. */