dsh-mobilecode 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -0
- package/lib/android-stream.js +378 -0
- package/lib/client.js +133 -0
- package/lib/frame-source.js +274 -0
- package/lib/index.js +234 -6
- package/lib/stream-access.js +220 -0
- package/lib/vision.js +130 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -87,6 +87,45 @@ drawer:
|
|
|
87
87
|
- `device_apps` / `device_launch_app` — list installed packages (third-party by
|
|
88
88
|
default) so a package name is never guessed / launch one by package or a
|
|
89
89
|
unique substring, with `relaunch` for a cold start.
|
|
90
|
+
- `device_stream` — drive the live screen stream the panel shows: `start` an
|
|
91
|
+
online device (returns a signed `streamUrl`), `status`, or `stop`. Agents that
|
|
92
|
+
just need to see the screen should prefer `device_screen` / `device_ui_tree`.
|
|
93
|
+
|
|
94
|
+
**Live device stream (the panel)**
|
|
95
|
+
|
|
96
|
+
The Devices pane shows a real-time mirror of the attached device. It is produced
|
|
97
|
+
**in-process** — no inner loopback port, no external helper:
|
|
98
|
+
|
|
99
|
+
- ONE persistent `adb exec-out "while :; do screencap -p; done"` child streams
|
|
100
|
+
~8 fps with zero per-frame process cost (spawning adb per frame caps at ~5 fps
|
|
101
|
+
and 100% churn). A quote-safe PNG splitter cuts the concatenated output into
|
|
102
|
+
frames by walking chunk headers (no marker scanning, no false positives).
|
|
103
|
+
- The browser `<img>` reads a `multipart/x-mixed-replace` body served straight
|
|
104
|
+
from the latest-frame buffer. Backpressure is **latest-wins**: a slow tab skips
|
|
105
|
+
frames instead of building an unbounded queue or watching a growing delay.
|
|
106
|
+
- A consumer refcount + idle timeout stops the loop when nobody is watching; a
|
|
107
|
+
keep-alive restarts a crashed loop; switching devices retires the old child.
|
|
108
|
+
- **Tap or drag directly on the screen** to drive the device; a Back / Home /
|
|
109
|
+
Recents / Rotate / Power bar sits below it, and a device picker switches which
|
|
110
|
+
online device streams (it never boots one — use `device_boot` for that).
|
|
111
|
+
- **Security**: every stream route sits behind a loopback + trusted-browser
|
|
112
|
+
transport fence (peer address, loopback `Host`, `Sec-Fetch-Site` / `Origin` —
|
|
113
|
+
so a LAN client cannot spoof localhost and a DNS-rebinding `Host` is rejected),
|
|
114
|
+
and the stream URL is an **HMAC-SHA256 capability** signed with a per-install
|
|
115
|
+
key (`~/.dsh/mobilecode/stream-access.key`, `0600`), expiring within 10 minutes
|
|
116
|
+
and re-minted automatically. Coordinates are normalized 0..1 of the streamed
|
|
117
|
+
frame, so one mapping serves every rotation.
|
|
118
|
+
|
|
119
|
+
**Multimodal screenshots**
|
|
120
|
+
|
|
121
|
+
When the routed model declares image input, `device_screen` delivers the
|
|
122
|
+
screenshot **as an image block** — the model literally sees the screen instead of
|
|
123
|
+
reading a file path. This mirrors the in-tree `read_image` tool: the PNG is
|
|
124
|
+
committed to DSH's durable attachment store (`ctx.get('attachments').saveImage`)
|
|
125
|
+
and returned as a `{type:'image', attachment}` content block, gated on
|
|
126
|
+
`llm.resolveModelInfo(...).inputModalities`. It **degrades, never refuses**: a
|
|
127
|
+
text-only route, a headless profile, or a host without the attachment store keeps
|
|
128
|
+
the plain JSON summary (path + UI tree + OCR) with no new error.
|
|
90
129
|
- `device_log` — device logs: logcat `main`/`crash`/`events`/`kernel` buffers
|
|
91
130
|
(kernel = dmesg, needs adb root — works on emulators) with an optional
|
|
92
131
|
case-insensitive substring filter, capped line count.
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-mobilecode — host-side lifecycle manager for the one live Android stream.
|
|
3
|
+
*
|
|
4
|
+
* Ported (lean) from ZSeven-W/dsh-android (android-host.ts, MIT). The stream is
|
|
5
|
+
* in-process: one AdbFrameLoop per streamed serial, a consumer refcount with an
|
|
6
|
+
* idle timeout, and a keep-alive that restarts a crashed loop. Emulators and
|
|
7
|
+
* physical devices share this path — the serial is the only identity.
|
|
8
|
+
*
|
|
9
|
+
* The control surface takes NORMALIZED 0..1 coordinates of the streamed frame
|
|
10
|
+
* (the panel's <img> is scaled, so the browser knows fractions, not pixels) and
|
|
11
|
+
* maps them onto `adb shell input` pixels using the latest frame's own size.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import * as DeviceBuild from './device-build.js'
|
|
15
|
+
import { AdbFrameLoop } from './frame-source.js'
|
|
16
|
+
|
|
17
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60 * 1000
|
|
18
|
+
const DEFAULT_RESTART_DELAY_MS = 5000
|
|
19
|
+
const KEEP_ALIVE_TICK_MS = 1000
|
|
20
|
+
const FIRST_FRAME_TIMEOUT_MS = 15000
|
|
21
|
+
const CONTROL_TIMEOUT_MS = 30000
|
|
22
|
+
const MAX_SWIPE_MS = 5000
|
|
23
|
+
|
|
24
|
+
/** Navigation/hardware buttons the panel may press. */
|
|
25
|
+
export const ANDROID_BUTTONS = {
|
|
26
|
+
home: 'KEYCODE_HOME',
|
|
27
|
+
back: 'KEYCODE_BACK',
|
|
28
|
+
recents: 'KEYCODE_APP_SWITCH',
|
|
29
|
+
power: 'KEYCODE_POWER',
|
|
30
|
+
volume_up: 'KEYCODE_VOLUME_UP',
|
|
31
|
+
volume_down: 'KEYCODE_VOLUME_DOWN',
|
|
32
|
+
menu: 'KEYCODE_MENU',
|
|
33
|
+
enter: 'KEYCODE_ENTER',
|
|
34
|
+
delete: 'KEYCODE_DEL',
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Clockwise user_rotation cycle (Surface.ROTATION_0..270). */
|
|
38
|
+
export const ROTATION_CYCLE = [0, 1, 2, 3]
|
|
39
|
+
|
|
40
|
+
function errorMessage(error) {
|
|
41
|
+
return error instanceof Error ? error.message : String(error)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function requireNormalized(x, y) {
|
|
45
|
+
if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || x > 1 || y < 0 || y > 1) {
|
|
46
|
+
throw new RangeError('dsh-mobilecode: tap/drag coordinates must be normalized 0..1 of the streamed frame')
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Lifecycle manager for the (single) in-process Android device stream. */
|
|
51
|
+
export class AndroidStreamHost {
|
|
52
|
+
#options
|
|
53
|
+
#loop
|
|
54
|
+
#starting
|
|
55
|
+
#launchQueue = Promise.resolve()
|
|
56
|
+
#consumers = 0
|
|
57
|
+
#keepAliveTimer
|
|
58
|
+
#idleTimer
|
|
59
|
+
#restarts = 0
|
|
60
|
+
#startedAt
|
|
61
|
+
#exitAt
|
|
62
|
+
#lastError
|
|
63
|
+
#lastSerial
|
|
64
|
+
#intentionalStop = false
|
|
65
|
+
#disposed = false
|
|
66
|
+
#frameSubscribers = new Set()
|
|
67
|
+
|
|
68
|
+
constructor(options = {}) {
|
|
69
|
+
this.#options = {
|
|
70
|
+
restartDelayMs: options.restartDelayMs ?? DEFAULT_RESTART_DELAY_MS,
|
|
71
|
+
idleTimeoutMs: options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS,
|
|
72
|
+
firstFrameTimeoutMs: options.firstFrameTimeoutMs ?? FIRST_FRAME_TIMEOUT_MS,
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
get running() {
|
|
77
|
+
return this.#loop?.running === true
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
get streamedSerial() {
|
|
81
|
+
return this.running ? this.#loop?.serial : undefined
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
get latestFrame() {
|
|
85
|
+
return this.#loop?.latestFrame
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Observe every decoded frame; the returned function unsubscribes. */
|
|
89
|
+
subscribeFrames(subscriber) {
|
|
90
|
+
this.#frameSubscribers.add(subscriber)
|
|
91
|
+
return () => {
|
|
92
|
+
this.#frameSubscribers.delete(subscriber)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Make sure the frame loop is live for `serial`. Concurrent callers share one
|
|
98
|
+
* launch; a call for a different serial retires the current loop first (one
|
|
99
|
+
* streamed device at a time).
|
|
100
|
+
*/
|
|
101
|
+
async ensureStreaming({ serial }) {
|
|
102
|
+
if (this.#disposed) throw new Error('dsh-mobilecode: the stream host is disposed')
|
|
103
|
+
if (typeof serial !== 'string' || serial === '') throw new TypeError('dsh-mobilecode: ensureStreaming requires a non-empty serial')
|
|
104
|
+
const startFor = async () => {
|
|
105
|
+
const current = this.#loop
|
|
106
|
+
if (current !== undefined && current.running && current.serial === serial) return this.#infoOf(current)
|
|
107
|
+
if (current !== undefined && current.serial !== serial) {
|
|
108
|
+
current.stop()
|
|
109
|
+
this.#loop = undefined
|
|
110
|
+
}
|
|
111
|
+
return this.#startFor(serial)
|
|
112
|
+
}
|
|
113
|
+
let starting = this.#starting
|
|
114
|
+
if (starting !== undefined) {
|
|
115
|
+
try {
|
|
116
|
+
await starting
|
|
117
|
+
} catch {
|
|
118
|
+
// A failed shared launch is settled; retry below.
|
|
119
|
+
}
|
|
120
|
+
if (this.running && this.#loop?.serial === serial) {
|
|
121
|
+
this.#armIdle()
|
|
122
|
+
return this.#infoOf(this.#loop)
|
|
123
|
+
}
|
|
124
|
+
starting = undefined
|
|
125
|
+
}
|
|
126
|
+
starting = this.#serializeLaunch(startFor)
|
|
127
|
+
this.#starting = starting
|
|
128
|
+
try {
|
|
129
|
+
const info = await starting
|
|
130
|
+
this.#lastError = undefined
|
|
131
|
+
this.#armIdle()
|
|
132
|
+
return info
|
|
133
|
+
} catch (error) {
|
|
134
|
+
this.#lastError = errorMessage(error)
|
|
135
|
+
throw error
|
|
136
|
+
} finally {
|
|
137
|
+
if (this.#starting === starting) this.#starting = undefined
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Start the crash keep-alive loop (restarts an unintentionally dead loop). */
|
|
142
|
+
startKeepAlive() {
|
|
143
|
+
if (this.#keepAliveTimer !== undefined || this.#disposed) return
|
|
144
|
+
this.#keepAliveTimer = setInterval(() => {
|
|
145
|
+
void this.#keepAliveTick().catch(() => {})
|
|
146
|
+
}, KEEP_ALIVE_TICK_MS)
|
|
147
|
+
this.#keepAliveTimer.unref?.()
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
stopKeepAlive() {
|
|
151
|
+
if (this.#keepAliveTimer !== undefined) clearInterval(this.#keepAliveTimer)
|
|
152
|
+
this.#keepAliveTimer = undefined
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Stop the stream intentionally (keep-alive will not fight it). */
|
|
156
|
+
async stop() {
|
|
157
|
+
this.#clearIdle()
|
|
158
|
+
this.#intentionalStop = true
|
|
159
|
+
this.#exitAt = undefined
|
|
160
|
+
const loop = this.#loop
|
|
161
|
+
this.#loop = undefined
|
|
162
|
+
this.#startedAt = undefined
|
|
163
|
+
loop?.stop()
|
|
164
|
+
await this.#starting?.catch(() => {})
|
|
165
|
+
const landed = this.#loop
|
|
166
|
+
if (landed !== undefined) {
|
|
167
|
+
this.#loop = undefined
|
|
168
|
+
landed.stop()
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Hold the stream alive for one consumer; release exactly once. */
|
|
173
|
+
acquire() {
|
|
174
|
+
this.#consumers += 1
|
|
175
|
+
this.#armIdle()
|
|
176
|
+
let released = false
|
|
177
|
+
return () => {
|
|
178
|
+
if (released) return
|
|
179
|
+
released = true
|
|
180
|
+
this.#consumers = Math.max(0, this.#consumers - 1)
|
|
181
|
+
this.#armIdle()
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
status() {
|
|
186
|
+
const loop = this.#loop
|
|
187
|
+
const frame = loop?.latestFrame
|
|
188
|
+
return {
|
|
189
|
+
running: this.running,
|
|
190
|
+
...(loop === undefined ? {} : { serial: loop.serial }),
|
|
191
|
+
restarts: this.#restarts,
|
|
192
|
+
...(this.#lastError === undefined ? {} : { lastError: this.#lastError }),
|
|
193
|
+
consumers: this.#consumers,
|
|
194
|
+
...(frame === undefined ? {} : { frameSequence: frame.sequence, lastFrameAt: frame.at, width: frame.width, height: frame.height }),
|
|
195
|
+
stderr: loop?.stderrLines ?? [],
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
dispose() {
|
|
200
|
+
this.#disposed = true
|
|
201
|
+
this.stopKeepAlive()
|
|
202
|
+
this.#frameSubscribers.clear()
|
|
203
|
+
return this.stop()
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// ── control surface (normalized 0..1 of the streamed frame) ────────────────
|
|
207
|
+
|
|
208
|
+
async tap(serial, x, y) {
|
|
209
|
+
requireNormalized(x, y)
|
|
210
|
+
const point = await this.#pixels(serial, x, y)
|
|
211
|
+
await this.#shell(serial, ['input', 'tap', String(point.x), String(point.y)])
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async drag(serial, drag) {
|
|
215
|
+
requireNormalized(drag.fromX, drag.fromY)
|
|
216
|
+
requireNormalized(drag.toX, drag.toY)
|
|
217
|
+
const from = await this.#pixels(serial, drag.fromX, drag.fromY)
|
|
218
|
+
const to = await this.#pixels(serial, drag.toX, drag.toY)
|
|
219
|
+
const durationMs = Math.min(MAX_SWIPE_MS, Math.max(20, Math.round((drag.duration ?? 0.3) * 1000)))
|
|
220
|
+
await this.#shell(serial, ['input', 'swipe', String(from.x), String(from.y), String(to.x), String(to.y), String(durationMs)])
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async button(serial, name = 'home') {
|
|
224
|
+
const keycode = ANDROID_BUTTONS[name] ?? (/^KEYCODE_[A-Z0-9_]+$/.test(name) ? name : undefined)
|
|
225
|
+
if (keycode === undefined) throw new Error(`unknown button "${name}"; expected one of ${Object.keys(ANDROID_BUTTONS).join(', ')} or a KEYCODE_* name`)
|
|
226
|
+
await this.#shell(serial, ['input', 'keyevent', keycode])
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** ASCII via `input text`; non-ASCII via the ADBKeyboard IME, else refused. */
|
|
230
|
+
async type(serial, text) {
|
|
231
|
+
if (typeof text !== 'string' || text === '') throw new TypeError('dsh-mobilecode: type requires a non-empty text')
|
|
232
|
+
if (DeviceBuild.isAsciiInput(text)) {
|
|
233
|
+
await this.#shell(serial, ['input', 'text', DeviceBuild.escapeInputText(text)])
|
|
234
|
+
return
|
|
235
|
+
}
|
|
236
|
+
if (!(await DeviceBuild.adbKeyboardReady(serial))) {
|
|
237
|
+
throw new Error('the text contains non-ASCII characters that `input text` cannot deliver; install the ADBKeyboard IME (github.com/senzhk/ADBKeyBoard) and select it')
|
|
238
|
+
}
|
|
239
|
+
await DeviceBuild.typeViaAdbKeyboard(serial, text)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async rotate(serial, rotation) {
|
|
243
|
+
if (!ROTATION_CYCLE.includes(rotation)) throw new RangeError('dsh-mobilecode: rotation must be 0, 1, 2 or 3')
|
|
244
|
+
await this.#shell(serial, ['settings', 'put', 'system', 'accelerometer_rotation', '0'])
|
|
245
|
+
await this.#shell(serial, ['settings', 'put', 'system', 'user_rotation', String(rotation)])
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async getRotation(serial) {
|
|
249
|
+
try {
|
|
250
|
+
const value = Number((await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'settings', 'get', 'system', 'user_rotation'])).trim())
|
|
251
|
+
return ROTATION_CYCLE.includes(value) ? value : 0
|
|
252
|
+
} catch {
|
|
253
|
+
return 0
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** The online device list for the panel picker (serial + kind). */
|
|
258
|
+
async listDevices() {
|
|
259
|
+
const rows = await DeviceBuild.devices()
|
|
260
|
+
return rows
|
|
261
|
+
.filter((row) => row.state === 'device')
|
|
262
|
+
.map((row) => ({ serial: row.serial, kind: row.serial.startsWith('emulator-') ? 'emulator' : 'physical' }))
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
async #shell(serial, shell) {
|
|
266
|
+
const code = await DeviceBuild.exec(DeviceBuild.adb(), ['-s', serial, 'shell', ...shell], { timeoutMs: CONTROL_TIMEOUT_MS }).exit
|
|
267
|
+
if (code !== 0) throw new Error(`adb shell ${shell.join(' ')} failed (exit ${code})`)
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async #keepAliveTick() {
|
|
271
|
+
if (this.#disposed || this.#intentionalStop) return
|
|
272
|
+
const exitAt = this.#exitAt
|
|
273
|
+
const serial = this.#lastSerial
|
|
274
|
+
if (exitAt === undefined || serial === undefined) return
|
|
275
|
+
if (Date.now() - exitAt < this.#options.restartDelayMs) return
|
|
276
|
+
this.#exitAt = undefined
|
|
277
|
+
this.#restarts += 1
|
|
278
|
+
try {
|
|
279
|
+
await this.ensureStreaming({ serial })
|
|
280
|
+
} catch (error) {
|
|
281
|
+
this.#lastError = errorMessage(error)
|
|
282
|
+
if (this.#exitAt === undefined) this.#exitAt = Date.now()
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
async #startFor(serial) {
|
|
287
|
+
if (this.#disposed) throw new Error('dsh-mobilecode: the stream host is disposed')
|
|
288
|
+
const online = await DeviceBuild.devices()
|
|
289
|
+
if (!online.some((row) => row.serial === serial && row.state === 'device')) {
|
|
290
|
+
const known = online.find((row) => row.serial === serial)
|
|
291
|
+
throw new Error(
|
|
292
|
+
known === undefined
|
|
293
|
+
? `no connected device has the serial ${serial}`
|
|
294
|
+
: `device ${serial} is ${known.state}, not ready to stream`,
|
|
295
|
+
)
|
|
296
|
+
}
|
|
297
|
+
const loop = new AdbFrameLoop(serial, {
|
|
298
|
+
onFrame: (frame) => {
|
|
299
|
+
for (const subscriber of this.#frameSubscribers) {
|
|
300
|
+
try {
|
|
301
|
+
subscriber(frame)
|
|
302
|
+
} catch {
|
|
303
|
+
// One broken consumer must not stall the fan-out.
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
},
|
|
307
|
+
onExit: (detail) => {
|
|
308
|
+
if (this.#loop !== loop) return
|
|
309
|
+
this.#loop = undefined
|
|
310
|
+
this.#startedAt = undefined
|
|
311
|
+
this.#lastError = `the screencap loop for ${serial} died (${detail})`
|
|
312
|
+
this.#exitAt = Date.now()
|
|
313
|
+
},
|
|
314
|
+
})
|
|
315
|
+
this.#loop = loop
|
|
316
|
+
this.#lastSerial = serial
|
|
317
|
+
this.#intentionalStop = false
|
|
318
|
+
loop.reset()
|
|
319
|
+
loop.start()
|
|
320
|
+
const frame = await loop.waitForFrame(this.#options.firstFrameTimeoutMs)
|
|
321
|
+
if (frame === undefined) {
|
|
322
|
+
const stderr = loop.stderrLines.join('\n')
|
|
323
|
+
loop.stop()
|
|
324
|
+
if (this.#loop === loop) this.#loop = undefined
|
|
325
|
+
throw new Error(`no frame arrived from ${serial} within ${this.#options.firstFrameTimeoutMs} ms${stderr === '' ? '' : `: ${stderr}`}`)
|
|
326
|
+
}
|
|
327
|
+
this.#startedAt = Date.now()
|
|
328
|
+
this.#exitAt = undefined
|
|
329
|
+
return this.#infoOf(loop)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
#infoOf(loop) {
|
|
333
|
+
const frame = loop.latestFrame
|
|
334
|
+
return { serial: loop.serial, ...(frame === undefined ? {} : { width: frame.width, height: frame.height }) }
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** Normalized frame coordinates → `input` pixels via the live frame size. */
|
|
338
|
+
async #pixels(serial, x, y) {
|
|
339
|
+
const frame = this.streamedSerial === serial ? this.latestFrame : undefined
|
|
340
|
+
if (frame !== undefined) return { x: Math.round(x * frame.width), y: Math.round(y * frame.height) }
|
|
341
|
+
const size = await screenSize(serial)
|
|
342
|
+
return { x: Math.round(x * size.width), y: Math.round(y * size.height) }
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
#armIdle() {
|
|
346
|
+
this.#clearIdle()
|
|
347
|
+
const idleMs = this.#options.idleTimeoutMs
|
|
348
|
+
if (idleMs <= 0) return
|
|
349
|
+
this.#idleTimer = setTimeout(() => {
|
|
350
|
+
this.#idleTimer = undefined
|
|
351
|
+
if (this.#consumers > 0) {
|
|
352
|
+
this.#armIdle()
|
|
353
|
+
return
|
|
354
|
+
}
|
|
355
|
+
void this.stop()
|
|
356
|
+
}, idleMs)
|
|
357
|
+
this.#idleTimer.unref?.()
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
#clearIdle() {
|
|
361
|
+
if (this.#idleTimer !== undefined) clearTimeout(this.#idleTimer)
|
|
362
|
+
this.#idleTimer = undefined
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
#serializeLaunch(task) {
|
|
366
|
+
const run = this.#launchQueue.then(task, task)
|
|
367
|
+
this.#launchQueue = run.then(() => undefined, () => undefined)
|
|
368
|
+
return run
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Override-aware physical screen size for the non-streaming control fallback. */
|
|
373
|
+
async function screenSize(serial) {
|
|
374
|
+
const output = await DeviceBuild.capture(DeviceBuild.adb(), ['-s', serial, 'shell', 'wm', 'size'])
|
|
375
|
+
const match = /Override size:\s*(\d+)x(\d+)/.exec(output) ?? /Physical size:\s*(\d+)x(\d+)/.exec(output)
|
|
376
|
+
if (!match) throw new Error(`cannot read the screen size of ${serial}`)
|
|
377
|
+
return { width: Number(match[1]), height: Number(match[2]) }
|
|
378
|
+
}
|
package/lib/client.js
CHANGED
|
@@ -110,6 +110,17 @@ window.__ModuleLoader__.load({
|
|
|
110
110
|
.mc-section-head { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; padding: 2px 0 10px; }
|
|
111
111
|
.mc-section-head .mc-hint { flex: 1; min-width: 200px; }
|
|
112
112
|
.mc-section .mc-tabs { padding: 0; margin-bottom: 4px; }
|
|
113
|
+
/* ── live device stream ── */
|
|
114
|
+
.mc-live { display: flex; flex-direction: column; gap: 8px; }
|
|
115
|
+
.mc-live-bar { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
|
|
116
|
+
.mc-live-bar select { background: light-dark(#fff, #20242b); color: inherit; border: 1px solid light-dark(rgba(0,0,0,.18), rgba(255,255,255,.18)); border-radius: 6px; padding: 3px 6px; font: inherit; font-size: 12px; max-width: 200px; }
|
|
117
|
+
.mc-live-stage { position: relative; align-self: center; background: #000; border-radius: 10px; overflow: hidden; line-height: 0; box-shadow: inset 0 0 0 1px light-dark(rgba(0,0,0,.12), rgba(255,255,255,.14)); }
|
|
118
|
+
.mc-live-stage img { display: block; max-width: 100%; max-height: 60vh; width: auto; height: auto; cursor: crosshair; touch-action: none; user-select: none; }
|
|
119
|
+
.mc-live-stage .mc-live-off { display: flex; align-items: center; justify-content: center; width: 260px; height: 460px; color: #9aa0a8; font-size: 13px; line-height: 1.5; text-align: center; }
|
|
120
|
+
.mc-live-nav { display: flex; align-items: center; justify-content: center; gap: 8px; }
|
|
121
|
+
.mc-live-nav button { border: 1px solid light-dark(rgba(0,0,0,.16), rgba(255,255,255,.18)); background: light-dark(#fff, #20242b); color: inherit; border-radius: 999px; width: 40px; height: 32px; cursor: pointer; font-size: 14px; }
|
|
122
|
+
.mc-live-nav button:hover { background: light-dark(rgba(0,0,0,.06), rgba(255,255,255,.1)); }
|
|
123
|
+
.mc-live-cap { font-size: 11px; color: light-dark(#6b7078, #9aa0a8); text-align: center; }
|
|
113
124
|
`;
|
|
114
125
|
//#endregion
|
|
115
126
|
|
|
@@ -148,6 +159,20 @@ window.__ModuleLoader__.load({
|
|
|
148
159
|
if (!response.ok) throw new Error("HTTP " + response.status);
|
|
149
160
|
return response.json();
|
|
150
161
|
};
|
|
162
|
+
// POST that surfaces the server's JSON error copy (stream routes return
|
|
163
|
+
// human-readable 4xx bodies the generic post() would flatten to "HTTP 409").
|
|
164
|
+
const streamPost = async (path, body) => {
|
|
165
|
+
const response = await fetch(API_BASE + path, {
|
|
166
|
+
method: "POST",
|
|
167
|
+
headers: { "content-type": "application/json" },
|
|
168
|
+
body: JSON.stringify(body ?? {}),
|
|
169
|
+
});
|
|
170
|
+
const text = await response.text();
|
|
171
|
+
let json;
|
|
172
|
+
try { json = JSON.parse(text); } catch { json = {}; }
|
|
173
|
+
if (!response.ok) throw new Error(json.error || ("HTTP " + response.status));
|
|
174
|
+
return json;
|
|
175
|
+
};
|
|
151
176
|
const copyText = async (text) => {
|
|
152
177
|
try { await navigator.clipboard.writeText(text); return true; } catch { return false; }
|
|
153
178
|
};
|
|
@@ -259,6 +284,7 @@ window.__ModuleLoader__.load({
|
|
|
259
284
|
);
|
|
260
285
|
}),
|
|
261
286
|
),
|
|
287
|
+
h(LiveDeviceCard, null),
|
|
262
288
|
platforms.map((platform) => {
|
|
263
289
|
const srv = server(platform);
|
|
264
290
|
const bld = build(platform);
|
|
@@ -333,6 +359,113 @@ window.__ModuleLoader__.load({
|
|
|
333
359
|
);
|
|
334
360
|
}
|
|
335
361
|
|
|
362
|
+
/** Live device stream: an <img> fed by the in-process multipart route, with
|
|
363
|
+
* tap/drag control and the Back/Home/Recents/rotate/power nav bar. */
|
|
364
|
+
function LiveDeviceCard() {
|
|
365
|
+
const [devices, setDevices] = useState([]);
|
|
366
|
+
const [serial, setSerial] = useState("");
|
|
367
|
+
const [streamUrl, setStreamUrl] = useState("");
|
|
368
|
+
const [error, setError] = useState("");
|
|
369
|
+
const imgRef = useRef(null);
|
|
370
|
+
const dragRef = useRef(null);
|
|
371
|
+
const grantTimer = useRef(null);
|
|
372
|
+
|
|
373
|
+
const grant = async (device) => {
|
|
374
|
+
if (!device) return;
|
|
375
|
+
try {
|
|
376
|
+
const r = await streamPost("/stream/grant", { device });
|
|
377
|
+
setStreamUrl(r.streamUrl);
|
|
378
|
+
setSerial(r.device);
|
|
379
|
+
setError("");
|
|
380
|
+
clearTimeout(grantTimer.current);
|
|
381
|
+
// Re-mint a minute before the 10-minute capability expires.
|
|
382
|
+
const ms = Math.max(30000, (r.expiresAt - Date.now()) - 60000);
|
|
383
|
+
grantTimer.current = setTimeout(() => { grant(r.device).catch(() => {}); }, ms);
|
|
384
|
+
} catch (err) {
|
|
385
|
+
setError(err instanceof Error ? err.message : String(err));
|
|
386
|
+
setStreamUrl("");
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
const refreshDevices = async () => {
|
|
391
|
+
try {
|
|
392
|
+
const r = await streamPost("/stream/devices", {});
|
|
393
|
+
setDevices(r.devices);
|
|
394
|
+
setSerial((cur) => cur || (r.devices.find((d) => d.streaming)?.serial ?? r.devices[0]?.serial ?? ""));
|
|
395
|
+
} catch (err) {
|
|
396
|
+
setError(err instanceof Error ? err.message : String(err));
|
|
397
|
+
}
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
useEffect(() => {
|
|
401
|
+
refreshDevices();
|
|
402
|
+
const timer = setInterval(refreshDevices, 5000);
|
|
403
|
+
return () => { clearInterval(timer); clearTimeout(grantTimer.current); };
|
|
404
|
+
}, []);
|
|
405
|
+
|
|
406
|
+
// Auto-grant once a serial is known but no stream is up yet.
|
|
407
|
+
useEffect(() => { if (serial !== "" && streamUrl === "") grant(serial).catch(() => {}); }, [serial, streamUrl]);
|
|
408
|
+
|
|
409
|
+
const control = async (action) => {
|
|
410
|
+
if (serial === "") return;
|
|
411
|
+
try { await streamPost("/stream/control", { device: serial, action }); }
|
|
412
|
+
catch (err) { setError(err instanceof Error ? err.message : String(err)); }
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
const norm = (clientX, clientY) => {
|
|
416
|
+
const el = imgRef.current;
|
|
417
|
+
if (!el) return null;
|
|
418
|
+
const rect = el.getBoundingClientRect();
|
|
419
|
+
if (rect.width === 0 || rect.height === 0) return null;
|
|
420
|
+
return {
|
|
421
|
+
x: Math.min(1, Math.max(0, (clientX - rect.left) / rect.width)),
|
|
422
|
+
y: Math.min(1, Math.max(0, (clientY - rect.top) / rect.height)),
|
|
423
|
+
};
|
|
424
|
+
};
|
|
425
|
+
const onDown = (e) => { const p = norm(e.clientX, e.clientY); if (p) dragRef.current = p; };
|
|
426
|
+
const onUp = (e) => {
|
|
427
|
+
const start = dragRef.current;
|
|
428
|
+
dragRef.current = null;
|
|
429
|
+
const end = norm(e.clientX, e.clientY);
|
|
430
|
+
if (!start || !end) return;
|
|
431
|
+
if (Math.hypot(end.x - start.x, end.y - start.y) > 0.02) {
|
|
432
|
+
control({ kind: "drag", fromX: start.x, fromY: start.y, toX: end.x, toY: end.y });
|
|
433
|
+
} else {
|
|
434
|
+
control({ kind: "tap", x: end.x, y: end.y });
|
|
435
|
+
}
|
|
436
|
+
};
|
|
437
|
+
|
|
438
|
+
return h("div", { className: "mc-card mc-live" },
|
|
439
|
+
h("div", { className: "mc-card-head" },
|
|
440
|
+
h("span", null, "Live device"),
|
|
441
|
+
h("span", { className: "sp" }),
|
|
442
|
+
devices.length > 0
|
|
443
|
+
? h("select", {
|
|
444
|
+
value: serial,
|
|
445
|
+
onChange: (e) => { setSerial(e.target.value); setStreamUrl(""); setError(""); },
|
|
446
|
+
}, devices.map((d) => h("option", { key: d.serial, value: d.serial }, d.serial + (d.kind === "emulator" ? " · emu" : " · phone"))))
|
|
447
|
+
: h("span", { className: "mc-hint" }, "no device"),
|
|
448
|
+
h("button", { className: "mc-btn", disabled: serial === "", onClick: () => grant(serial) }, "Reconnect"),
|
|
449
|
+
),
|
|
450
|
+
error !== "" && h("div", { className: "mc-error" }, error),
|
|
451
|
+
h("div", { className: "mc-live-stage" },
|
|
452
|
+
streamUrl !== ""
|
|
453
|
+
? h("img", { ref: imgRef, src: streamUrl, alt: "device screen", draggable: false, onPointerDown: onDown, onPointerUp: onUp, onError: () => grant(serial) })
|
|
454
|
+
: h("div", { className: "mc-live-off" }, devices.length === 0
|
|
455
|
+
? "No device attached. Boot one with device_boot, or press Run app."
|
|
456
|
+
: "Connecting to " + serial + "…"),
|
|
457
|
+
),
|
|
458
|
+
h("div", { className: "mc-live-nav" },
|
|
459
|
+
h("button", { title: "Back", disabled: serial === "", onClick: () => control({ kind: "button", name: "back" }) }, "◁"),
|
|
460
|
+
h("button", { title: "Home", disabled: serial === "", onClick: () => control({ kind: "button", name: "home" }) }, "○"),
|
|
461
|
+
h("button", { title: "Recents", disabled: serial === "", onClick: () => control({ kind: "button", name: "recents" }) }, "▢"),
|
|
462
|
+
h("button", { title: "Rotate", disabled: serial === "", onClick: () => control({ kind: "rotate" }) }, "⟳"),
|
|
463
|
+
h("button", { title: "Power", disabled: serial === "", onClick: () => control({ kind: "button", name: "power" }) }, "⏻"),
|
|
464
|
+
),
|
|
465
|
+
h("div", { className: "mc-live-cap" }, "tap or drag on the screen to drive the device"),
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
|
|
336
469
|
//#region setup UI (welcome + settings)
|
|
337
470
|
|
|
338
471
|
/** PaddleOCR status card: fetch /ocr, Install button, log tail. */
|