dsh-mobile-use 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client.js +68 -12
- package/client.rev +1 -1
- package/index.js +335 -83
- package/mirror.js +122 -21
- package/ocr.js +88 -17
- package/package.json +7 -3
- package/selftest.mjs +64 -12
- package/stream.js +103 -32
- package/test/coords.test.mjs +129 -0
- package/test/symbols.test.mjs +73 -0
- package/test/timers.test.mjs +60 -0
package/mirror.js
CHANGED
|
@@ -20,11 +20,26 @@ import { mkdir, readdir, rm, stat, writeFile } from 'node:fs/promises'
|
|
|
20
20
|
import { dirname, join } from 'node:path'
|
|
21
21
|
import { homedir } from 'node:os'
|
|
22
22
|
|
|
23
|
-
import { LiveStream, packFrame } from './stream.js?v=
|
|
23
|
+
import { LiveStream, packFrame } from './stream.js?v=146'
|
|
24
24
|
|
|
25
25
|
/** 客户端与宿主共用的路由前缀。 */
|
|
26
26
|
export const ROUTE = '/dsh-mobile-use'
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* 节拍器回调兜底:异常冒到 host 顶层会直接把整个 DSH 带走(2026-10-27 事故),
|
|
30
|
+
* 所以单拍出错只记一行,不停服务、不退进程。
|
|
31
|
+
* @param {string} where - 出问题的路径。
|
|
32
|
+
* @param {() => void} run - 这一拍要做的事。
|
|
33
|
+
* @returns {void}
|
|
34
|
+
*/
|
|
35
|
+
function guard(where, run) {
|
|
36
|
+
try {
|
|
37
|
+
run()
|
|
38
|
+
} catch (error) {
|
|
39
|
+
console.error(`[dsh-mobile-use] ${where}:`, error)
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
28
43
|
/** 最近一次标注里动效拼图的字节数,0 表示没凑够帧。 */
|
|
29
44
|
let lastStripBytes = 0
|
|
30
45
|
|
|
@@ -41,6 +56,10 @@ const LIVE_FPS = 3
|
|
|
41
56
|
const PUSH_TICK_MS = 200
|
|
42
57
|
/** 同时在飞的截图数:单张约 0.65s,3 张并发实测 4 帧/秒以上。 */
|
|
43
58
|
const CAPTURE_PARALLEL = 2
|
|
59
|
+
/** 没人看多久之后收掉分段录制:跟抓帧的空转宽限同一个数,别留两个口径。 */
|
|
60
|
+
const WATCH_GRACE_MS = 6000
|
|
61
|
+
/** 本进程的随机标识:设备端 pid 文件名带上它,多开/重启时不会互相覆盖。 */
|
|
62
|
+
const REC_TAG = Math.random().toString(36).slice(2, 8)
|
|
44
63
|
|
|
45
64
|
/** 环形缓冲:10 秒一段,留 18 段 ≈ 3 分钟,超出的由 ffmpeg 自己覆盖。 */
|
|
46
65
|
const SEGMENT_SECONDS = 10
|
|
@@ -268,6 +287,10 @@ export class Mirror {
|
|
|
268
287
|
this.waiters = []
|
|
269
288
|
this.grabMs = 0
|
|
270
289
|
this.recording = false
|
|
290
|
+
/* 分段录制的收尾:请求停止标记 + 设备端 pid 文件名(只杀自己这一次录屏)。 */
|
|
291
|
+
this.recorderStopRequested = false
|
|
292
|
+
this.recorderPidFile = ''
|
|
293
|
+
this.recorderSeq = 0
|
|
271
294
|
this.stopping = false
|
|
272
295
|
this.startedAt = 0
|
|
273
296
|
this.error = ''
|
|
@@ -293,7 +316,7 @@ export class Mirror {
|
|
|
293
316
|
* @returns {boolean} 有观看者(或刚有人来看)为真。
|
|
294
317
|
*/
|
|
295
318
|
anyoneWatching() {
|
|
296
|
-
return this.watchers.size > 0 || Date.now() - this.lastWatchAt <
|
|
319
|
+
return this.watchers.size > 0 || Date.now() - this.lastWatchAt < WATCH_GRACE_MS
|
|
297
320
|
}
|
|
298
321
|
|
|
299
322
|
async start() {
|
|
@@ -306,11 +329,12 @@ export class Mirror {
|
|
|
306
329
|
if (found !== null) this.physical = { w: Number(found[1]), h: Number(found[2]) }
|
|
307
330
|
await this.snapshot()
|
|
308
331
|
this.alignScreen()
|
|
309
|
-
this.timer = setInterval(() => this.pump(), 200)
|
|
332
|
+
this.timer = setInterval(() => guard('pump', () => this.pump()), 200)
|
|
310
333
|
this.timer.unref?.()
|
|
311
|
-
this.pushTimer = setInterval(() => this.push(), PUSH_TICK_MS)
|
|
334
|
+
this.pushTimer = setInterval(() => guard('push', () => this.push()), PUSH_TICK_MS)
|
|
312
335
|
this.pushTimer.unref?.()
|
|
313
|
-
|
|
336
|
+
/* 录制跟观看者对齐:起服务时没人看就先不录,等 /frame 或 /live 把人带来再开。 */
|
|
337
|
+
this.ensureRecorder()
|
|
314
338
|
}
|
|
315
339
|
|
|
316
340
|
/**
|
|
@@ -328,22 +352,24 @@ export class Mirror {
|
|
|
328
352
|
this.screen = landscape ? { w: long, h: short } : { w: short, h: long }
|
|
329
353
|
}
|
|
330
354
|
|
|
331
|
-
/**
|
|
355
|
+
/** 有人在看才抓帧;没人看就空转,宽限期过了顺手把分段录制也收掉。 @returns {void} */
|
|
332
356
|
pump() {
|
|
333
357
|
if (this.stopping) return
|
|
334
358
|
if (!this.anyoneWatching()) {
|
|
335
359
|
if (this.inflight === 0) {
|
|
336
360
|
if (this.idleSince === 0) this.idleSince = Date.now()
|
|
337
361
|
}
|
|
362
|
+
this.ensureRecorder()
|
|
338
363
|
return
|
|
339
364
|
}
|
|
340
365
|
this.idleSince = 0
|
|
366
|
+
this.ensureRecorder()
|
|
341
367
|
/* 桥接帧够新就别再抓屏:两种源尺寸不同,混进同一池子画面会跳。 */
|
|
342
368
|
if (this.bridgeFresh !== null && this.bridgeFresh()) return
|
|
343
369
|
/* 每个节拍最多补一张,让 3 张在飞时错开完成,画面才匀。 */
|
|
344
370
|
if (this.inflight < CAPTURE_PARALLEL) {
|
|
345
371
|
this.inflight += 1
|
|
346
|
-
void this.grab().finally(() => { this.inflight -= 1 })
|
|
372
|
+
void this.grab().catch((error) => { this.error = String(error?.message ?? error) }).finally(() => { this.inflight -= 1 })
|
|
347
373
|
}
|
|
348
374
|
}
|
|
349
375
|
|
|
@@ -878,24 +904,76 @@ export class Mirror {
|
|
|
878
904
|
return frame
|
|
879
905
|
}
|
|
880
906
|
|
|
907
|
+
/**
|
|
908
|
+
* 该录就录、该停就停:只有有人在看才保留分段录制,没人看超过宽限期就收掉。
|
|
909
|
+
* 由 pump() 每 200ms 调一次,重复调用无副作用。
|
|
910
|
+
* @returns {void}
|
|
911
|
+
*/
|
|
912
|
+
ensureRecorder() {
|
|
913
|
+
if (this.stopping) return
|
|
914
|
+
if (this.anyoneWatching()) {
|
|
915
|
+
if (!this.recording) void this.startRecorder()
|
|
916
|
+
return
|
|
917
|
+
}
|
|
918
|
+
if (!this.recording) return
|
|
919
|
+
const idleAt = this.idleSince > 0 ? this.idleSince : this.lastWatchAt
|
|
920
|
+
if (idleAt > 0 && Date.now() - idleAt < WATCH_GRACE_MS) return
|
|
921
|
+
this.stopRecorder()
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
/**
|
|
925
|
+
* 请求停掉分段录制:只按自己记下的设备端 pid 收尾,不碰别人的 screenrecord。
|
|
926
|
+
* @returns {void}
|
|
927
|
+
*/
|
|
928
|
+
stopRecorder() {
|
|
929
|
+
this.recorderStopRequested = true
|
|
930
|
+
const pidFile = this.recorderPidFile
|
|
931
|
+
this.recorderPidFile = ''
|
|
932
|
+
if (pidFile !== '') void this.killOwnRecorder(pidFile)
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* 精确杀自己这一次录屏:pid 取自启动时写下的设备端文件,并核对进程名还是 screenrecord。
|
|
937
|
+
* pid 拿不到(文件为空/读不到)就退化为不杀,只记一条错误,交给下一段自己超时结束。
|
|
938
|
+
* @param {string} pidFile - 启动时写下的 pid 文件绝对路径。
|
|
939
|
+
* @returns {Promise<string>} 设备端回话:`killed:<pid>` 或 `skip:<原因>`。
|
|
940
|
+
*/
|
|
941
|
+
async killOwnRecorder(pidFile) {
|
|
942
|
+
const safe = String(pidFile).replace(/[^\w./-]/g, '')
|
|
943
|
+
if (safe === '') return 'skip:no-pid-file'
|
|
944
|
+
const script = `p='${safe}'; r=$(cat "$p" 2>/dev/null); rm -f "$p"; if [ -n "$r" ] && [ "$(cat /proc/$r/comm 2>/dev/null)" = 'screenrecord' ]; then kill -9 "$r"; echo killed:$r; else echo skip:$r; fi`
|
|
945
|
+
const result = await this.adb(['-s', this.serial, 'shell', script], { timeout: 8000, direct: true }).catch(() => null)
|
|
946
|
+
const out = String(result?.out ?? '').trim()
|
|
947
|
+
if (out.startsWith('killed:')) return out
|
|
948
|
+
this.error = this.error || `录屏收尾没拿到 pid(${out || '没有回话'}),没杀设备端进程`
|
|
949
|
+
return out || 'skip:no-reply'
|
|
950
|
+
}
|
|
951
|
+
|
|
881
952
|
/**
|
|
882
953
|
* 循环分段录制:设备上录 10s 一段,拉回本机,只留最近 18 段。
|
|
954
|
+
* 每段启动时把 pid 写进设备端文件,停止时按 pid 精确 kill。
|
|
883
955
|
* @returns {Promise<void>} 停了才返回。
|
|
884
956
|
*/
|
|
885
957
|
async startRecorder() {
|
|
886
958
|
if (this.recording || this.stopping) return
|
|
887
959
|
this.recording = true
|
|
888
|
-
|
|
889
|
-
|
|
960
|
+
this.recorderStopRequested = false
|
|
961
|
+
const safeSerial = this.serial.replace(/[^\w.-]/g, '_')
|
|
962
|
+
const remote = `/sdcard/dsh-${safeSerial}.mp4`
|
|
963
|
+
while (!this.stopping && !this.recorderStopRequested) {
|
|
890
964
|
const started = Date.now()
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
965
|
+
this.recorderSeq += 1
|
|
966
|
+
const pidFile = `/data/local/tmp/dsh-mobile-use-rec-${safeSerial}-${REC_TAG}-${this.recorderSeq}.pid`
|
|
967
|
+
this.recorderPidFile = pidFile
|
|
968
|
+
/* exec 让 shell 的 pid 就是 screenrecord 的 pid:停的时候按这个 pid 精确 kill。 */
|
|
969
|
+
const command = `sh -c 'echo $$ > ${pidFile}; exec screenrecord --time-limit ${SEGMENT_SECONDS} --bit-rate 2000000 --size ${this.recordSize()} ${remote}'`
|
|
970
|
+
const recorded = await this.adb(['-s', this.serial, 'shell', command], {
|
|
971
|
+
timeout: (SEGMENT_SECONDS + 20) * 1000,
|
|
972
|
+
direct: true,
|
|
973
|
+
}).catch((error) => ({ ok: false, err: String(error?.message ?? error) }))
|
|
974
|
+
if (this.stopping || this.recorderStopRequested) break
|
|
897
975
|
if (!recorded.ok) {
|
|
898
|
-
this.error = recorded.err.trim().slice(0, 200) || '录制失败'
|
|
976
|
+
this.error = String(recorded.err ?? '').trim().slice(0, 200) || '录制失败'
|
|
899
977
|
await new Promise((resolve) => setTimeout(resolve, 1500))
|
|
900
978
|
continue
|
|
901
979
|
}
|
|
@@ -914,6 +992,7 @@ export class Mirror {
|
|
|
914
992
|
if (spent < SEGMENT_SECONDS * 1000) await new Promise((resolve) => setTimeout(resolve, Math.max(0, 400 - spent)))
|
|
915
993
|
}
|
|
916
994
|
this.recording = false
|
|
995
|
+
this.recorderStopRequested = false
|
|
917
996
|
}
|
|
918
997
|
|
|
919
998
|
/** 本机只留最近 18 段(约 3 分钟)。 @returns {Promise<void>} 删完。 */
|
|
@@ -961,9 +1040,10 @@ export class Mirror {
|
|
|
961
1040
|
return this.latest
|
|
962
1041
|
}
|
|
963
1042
|
|
|
964
|
-
/**
|
|
1043
|
+
/** 停掉两条子进程、收掉分段录制并断开观看者;重复调用无副作用。 @returns {void} */
|
|
965
1044
|
stop() {
|
|
966
1045
|
this.stopping = true
|
|
1046
|
+
this.stopRecorder()
|
|
967
1047
|
if (this.timer !== null) clearInterval(this.timer)
|
|
968
1048
|
this.timer = null
|
|
969
1049
|
if (this.pushTimer !== null) clearInterval(this.pushTimer)
|
|
@@ -1017,6 +1097,11 @@ export function setupMirror(ctx, tools) {
|
|
|
1017
1097
|
let stream = streams.get(serial)
|
|
1018
1098
|
if (stream === undefined) {
|
|
1019
1099
|
stream = new LiveStream({ serial, adb: tools.adb.path })
|
|
1100
|
+
/* 空闲回收前先问一句:镜像还有人在看(/live 常连或刚拉过帧)就别停。 */
|
|
1101
|
+
stream.stillWanted = () => {
|
|
1102
|
+
const mirror = mirrors.get(serial)
|
|
1103
|
+
return mirror !== undefined && mirror.anyoneWatching()
|
|
1104
|
+
}
|
|
1020
1105
|
streams.set(serial, stream)
|
|
1021
1106
|
}
|
|
1022
1107
|
return stream
|
|
@@ -1095,6 +1180,11 @@ export function setupMirror(ctx, tools) {
|
|
|
1095
1180
|
if (rect !== null && Number(rect.w) > 0) {
|
|
1096
1181
|
lines.push(`圈选区域(设备坐标):x=${Math.round(rect.x)} y=${Math.round(rect.y)} w=${Math.round(rect.w)} h=${Math.round(rect.h)}`)
|
|
1097
1182
|
}
|
|
1183
|
+
/* 圈是画在冻结帧上的:把「画面是几秒前的」交出去,免得把那一刻的控件位置当成现在。 */
|
|
1184
|
+
const frozenMs = Number(input.frozen_ms)
|
|
1185
|
+
if (Number.isFinite(frozenMs) && frozenMs >= 1) {
|
|
1186
|
+
lines.push(`这张图是一张 ${(frozenMs / 1000).toFixed(1)} 秒前冻结的画面,圈的位置以那一帧为准。`)
|
|
1187
|
+
}
|
|
1098
1188
|
if (tools.uiInRect !== undefined && rect !== null) {
|
|
1099
1189
|
const inside = await tools.uiInRect(String(input.serial ?? ''), rect).catch(() => [])
|
|
1100
1190
|
if (inside.length > 0) lines.push(`圈选区域内的控件:\n${inside.join('\n')}`)
|
|
@@ -1347,7 +1437,7 @@ export function setupMirror(ctx, tools) {
|
|
|
1347
1437
|
}
|
|
1348
1438
|
child.on('error', (error) => finish({ ok: false, reason: String(error && error.message ? error.message : error).slice(0, 200) }))
|
|
1349
1439
|
child.on('exit', (code) => finish(code === 0 ? { ok: true } : { ok: false, reason: '退出码 ' + code }))
|
|
1350
|
-
const tick = () => {
|
|
1440
|
+
const tick = () => guard('gesture-tick', () => {
|
|
1351
1441
|
if (index >= frames.length) {
|
|
1352
1442
|
child.stdin.end()
|
|
1353
1443
|
return
|
|
@@ -1355,13 +1445,13 @@ export function setupMirror(ctx, tools) {
|
|
|
1355
1445
|
child.stdin.write(stampFrame(frames[index], start + (index * step) / 1000))
|
|
1356
1446
|
index += 1
|
|
1357
1447
|
setTimeout(tick, step)
|
|
1358
|
-
}
|
|
1448
|
+
})
|
|
1359
1449
|
tick()
|
|
1360
|
-
setTimeout(() => {
|
|
1450
|
+
setTimeout(() => guard('gesture-timeout', () => {
|
|
1361
1451
|
if (done) return
|
|
1362
1452
|
child.kill('SIGKILL')
|
|
1363
1453
|
finish({ ok: false, reason: '注入超时' })
|
|
1364
|
-
}, ms + 4000)
|
|
1454
|
+
}), ms + 4000)
|
|
1365
1455
|
})
|
|
1366
1456
|
}
|
|
1367
1457
|
|
|
@@ -1588,6 +1678,8 @@ export function setupMirror(ctx, tools) {
|
|
|
1588
1678
|
void stream.enableJpeg()
|
|
1589
1679
|
/* 录屏到时长会停;每次有人拉帧都把它续上,否则画面就只剩抓帧那一路。 */
|
|
1590
1680
|
void stream.start()
|
|
1681
|
+
/* start() 会清空闲定时器,所以续命之后重新排一次:面板关掉 20 秒后自动停录。 */
|
|
1682
|
+
stream.scheduleIdleStop()
|
|
1591
1683
|
})
|
|
1592
1684
|
const pump = () => {
|
|
1593
1685
|
const jpeg = stream.jpeg
|
|
@@ -1636,6 +1728,8 @@ export function setupMirror(ctx, tools) {
|
|
|
1636
1728
|
fps: mirror.measuredFps(),
|
|
1637
1729
|
live: mirror.latest !== null,
|
|
1638
1730
|
watchers: mirror.watchers.size,
|
|
1731
|
+
/* 录制是后台行为:状态里如实报,别让人以为没人看时也在录。 */
|
|
1732
|
+
recording: mirror.recording,
|
|
1639
1733
|
dropped: mirror.dropped,
|
|
1640
1734
|
grabMs: mirror.grabMs,
|
|
1641
1735
|
pushed: mirror.pushed,
|
|
@@ -1867,6 +1961,7 @@ export function setupMirror(ctx, tools) {
|
|
|
1867
1961
|
return {
|
|
1868
1962
|
serial: chosen,
|
|
1869
1963
|
live: mirror.latest !== null,
|
|
1964
|
+
recording: mirror.recording,
|
|
1870
1965
|
frames: mirror.frames,
|
|
1871
1966
|
fps: mirror.measuredFps(),
|
|
1872
1967
|
bufferedSeconds: segments.length * SEGMENT_SECONDS,
|
|
@@ -1879,6 +1974,12 @@ export function setupMirror(ctx, tools) {
|
|
|
1879
1974
|
return mirror === undefined ? [] : mirror.segments()
|
|
1880
1975
|
},
|
|
1881
1976
|
stopAll() {
|
|
1977
|
+
/* 先停 H.264 直播:每个实例都有自己的设备端录屏和 ffmpeg 子进程,比抓帧泵重得多。
|
|
1978
|
+
只清 mirrors 会把这些子进程留在设备上继续录,卸载插件后还在跑。 */
|
|
1979
|
+
for (const stream of streams.values()) {
|
|
1980
|
+
try { stream.stop() } catch { /* 已经停了 */ }
|
|
1981
|
+
}
|
|
1982
|
+
streams.clear()
|
|
1882
1983
|
for (const mirror of mirrors.values()) mirror.stop()
|
|
1883
1984
|
mirrors.clear()
|
|
1884
1985
|
for (const res of events) {
|
package/ocr.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* @module dsh-mobile-use/ocr
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { spawn } from 'node:child_process'
|
|
13
|
+
import { spawn, spawnSync } from 'node:child_process'
|
|
14
14
|
import { existsSync } from 'node:fs'
|
|
15
15
|
import { dirname, join } from 'node:path'
|
|
16
16
|
import { fileURLToPath } from 'node:url'
|
|
@@ -18,7 +18,11 @@ import { fileURLToPath } from 'node:url'
|
|
|
18
18
|
const HERE = dirname(fileURLToPath(import.meta.url))
|
|
19
19
|
const WORKER = join(HERE, 'ocr_worker.py')
|
|
20
20
|
const IDLE_MS = 45_000
|
|
21
|
-
|
|
21
|
+
/** 一次 ocrImage 的总时限:启动 + 识别共用,避免两边各自超时叠成半分钟。 */
|
|
22
|
+
const CALL_TIMEOUT_MS = 15_000
|
|
23
|
+
/** 启动 worker 的单独上限:worker 卡在 import Vision 时要尽快放弃。 */
|
|
24
|
+
const START_TIMEOUT_MS = 8_000
|
|
25
|
+
const PYTHON = '/usr/bin/python3'
|
|
22
26
|
|
|
23
27
|
let worker = null
|
|
24
28
|
let ready = null
|
|
@@ -27,9 +31,46 @@ let seq = 0
|
|
|
27
31
|
const pending = new Map()
|
|
28
32
|
let buffer = ''
|
|
29
33
|
|
|
30
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* 探一次本机能不能跑 OCR:平台、worker 文件、python 能否 import Vision 三个条件。
|
|
36
|
+
* 结果缓存,插件生命周期内只探一次(worker 起不来时也别每次调用都 spawn 一次 python)。
|
|
37
|
+
* @returns {{ available: boolean, reason: string }} 可用性;不可用时 `reason` 说明原因。
|
|
38
|
+
*/
|
|
39
|
+
function probe() {
|
|
40
|
+
if (process.platform !== 'darwin') return { available: false, reason: `本机是 ${process.platform},Vision OCR 只有 macOS 有` }
|
|
41
|
+
if (!existsSync(WORKER)) return { available: false, reason: `缺少 ${WORKER}` }
|
|
42
|
+
if (!existsSync(PYTHON)) return { available: false, reason: `缺少 ${PYTHON}` }
|
|
43
|
+
/* 只探 import:worker 真正用到的 Vision/Quartz 都在这里,缺 pyobjc 会在启动后立刻崩。 */
|
|
44
|
+
const check = spawnSync(PYTHON, ['-c', 'import Vision, Quartz, Foundation'], { timeout: 5000, stdio: 'ignore' })
|
|
45
|
+
if (check.error) {
|
|
46
|
+
return { available: false, reason: `${PYTHON} 起不来:${String(check.error.message ?? check.error).slice(0, 120)}` }
|
|
47
|
+
}
|
|
48
|
+
if (check.status !== 0) return { available: false, reason: `${PYTHON} 缺 Vision/Quartz(pyobjc-framework-Vision)` }
|
|
49
|
+
return { available: true, reason: '' }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
let availability = null
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* 本机 OCR 状态:可用性和不可用原因。
|
|
56
|
+
* @returns {{ available: boolean, reason: string }} `reason` 可直接写进工具结果,让上层回退到文本树/截图。
|
|
57
|
+
*/
|
|
58
|
+
export function ocrStatus() {
|
|
59
|
+
if (availability === null) availability = probe()
|
|
60
|
+
return availability
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** python3 与 Vision 是否可用(不存在就没有 OCR)。 @returns {boolean} 可用为 true。 */
|
|
31
64
|
export function ocrAvailable() {
|
|
32
|
-
return
|
|
65
|
+
return ocrStatus().available
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* 拿不可用原因;可用时返回空串。
|
|
70
|
+
* @returns {string} 人话原因,例如「本机是 linux,Vision OCR 只有 macOS 有」。
|
|
71
|
+
*/
|
|
72
|
+
export function ocrReason() {
|
|
73
|
+
return ocrStatus().reason
|
|
33
74
|
}
|
|
34
75
|
|
|
35
76
|
function kill() {
|
|
@@ -57,13 +98,22 @@ function armIdle() {
|
|
|
57
98
|
|
|
58
99
|
function start() {
|
|
59
100
|
if (ready) return ready
|
|
101
|
+
const state = ocrStatus()
|
|
102
|
+
if (!state.available) return Promise.reject(new Error(`本机没有 OCR:${state.reason}`))
|
|
60
103
|
ready = new Promise((resolve, reject) => {
|
|
61
|
-
|
|
104
|
+
let settled = false
|
|
105
|
+
const fail = (error) => {
|
|
106
|
+
if (settled) return
|
|
107
|
+
settled = true
|
|
108
|
+
clearTimeout(timer)
|
|
109
|
+
reject(error)
|
|
110
|
+
kill()
|
|
111
|
+
}
|
|
112
|
+
const child = spawn(PYTHON, [WORKER], { stdio: ['pipe', 'pipe', 'pipe'] })
|
|
62
113
|
worker = child
|
|
63
114
|
const timer = setTimeout(() => {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}, 15_000)
|
|
115
|
+
fail(new Error('OCR 进程启动超时'))
|
|
116
|
+
}, START_TIMEOUT_MS)
|
|
67
117
|
child.stdout.setEncoding('utf8')
|
|
68
118
|
child.stdout.on('data', (chunk) => {
|
|
69
119
|
buffer += chunk
|
|
@@ -78,8 +128,11 @@ function start() {
|
|
|
78
128
|
continue
|
|
79
129
|
}
|
|
80
130
|
if (message.ready === true) {
|
|
81
|
-
|
|
82
|
-
|
|
131
|
+
if (!settled) {
|
|
132
|
+
settled = true
|
|
133
|
+
clearTimeout(timer)
|
|
134
|
+
resolve()
|
|
135
|
+
}
|
|
83
136
|
continue
|
|
84
137
|
}
|
|
85
138
|
const entry = pending.get(message.path)
|
|
@@ -92,15 +145,16 @@ function start() {
|
|
|
92
145
|
child.stderr.setEncoding('utf8')
|
|
93
146
|
child.stderr.on('data', () => {})
|
|
94
147
|
child.on('error', (error) => {
|
|
95
|
-
|
|
96
|
-
reject(error)
|
|
97
|
-
kill()
|
|
148
|
+
fail(error)
|
|
98
149
|
})
|
|
99
|
-
child.on('exit', () => {
|
|
100
|
-
|
|
150
|
+
child.on('exit', (code, signal) => {
|
|
151
|
+
/* 关键:ready 之前就退出时必须 reject,否则 start() 的 Promise 永远挂着,
|
|
152
|
+
上层 ocrImage 的 await 也跟着挂死(非 macOS / 缺 pyobjc 时的实际表现)。 */
|
|
153
|
+
fail(new Error(`OCR 进程启动时退出(code=${String(code)}, signal=${String(signal)})`))
|
|
101
154
|
if (worker === child) {
|
|
102
155
|
worker = null
|
|
103
156
|
ready = null
|
|
157
|
+
buffer = ''
|
|
104
158
|
for (const [, entry] of pending) entry.reject(new Error('OCR 进程退出'))
|
|
105
159
|
pending.clear()
|
|
106
160
|
}
|
|
@@ -116,13 +170,17 @@ function start() {
|
|
|
116
170
|
* `x/y/w/h` 是归一化坐标,原点在**左下角**(Vision 的习惯)。
|
|
117
171
|
*/
|
|
118
172
|
export async function ocrImage(path) {
|
|
173
|
+
/* 启动和识别共用一个 deadline:启动慢了,识别就少等,整体最多 CALL_TIMEOUT_MS。 */
|
|
174
|
+
const deadline = Date.now() + CALL_TIMEOUT_MS
|
|
119
175
|
await start()
|
|
120
176
|
armIdle()
|
|
177
|
+
const left = deadline - Date.now()
|
|
178
|
+
if (left <= 0) throw new Error('OCR 超时')
|
|
121
179
|
return new Promise((resolve, reject) => {
|
|
122
180
|
const timer = setTimeout(() => {
|
|
123
181
|
pending.delete(path)
|
|
124
182
|
reject(new Error('OCR 超时'))
|
|
125
|
-
},
|
|
183
|
+
}, left)
|
|
126
184
|
pending.set(path, {
|
|
127
185
|
resolve: (result) => {
|
|
128
186
|
clearTimeout(timer)
|
|
@@ -133,7 +191,20 @@ export async function ocrImage(path) {
|
|
|
133
191
|
reject(error)
|
|
134
192
|
},
|
|
135
193
|
})
|
|
136
|
-
worker
|
|
194
|
+
/* worker 可能在 start() resolve 之后、这里之前就退了:写不进去就明确失败,别抛同步异常。 */
|
|
195
|
+
if (worker === null) {
|
|
196
|
+
pending.delete(path)
|
|
197
|
+
clearTimeout(timer)
|
|
198
|
+
reject(new Error('OCR 进程已退出'))
|
|
199
|
+
return
|
|
200
|
+
}
|
|
201
|
+
try {
|
|
202
|
+
worker.stdin.write(`${path}\n`)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
pending.delete(path)
|
|
205
|
+
clearTimeout(timer)
|
|
206
|
+
reject(error instanceof Error ? error : new Error(String(error)))
|
|
207
|
+
}
|
|
137
208
|
})
|
|
138
209
|
}
|
|
139
210
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-mobile-use",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "真机调试插件:界面文本树、动效数据、实时画面侧栏与圈选标注",
|
|
6
6
|
"type": "module",
|
|
@@ -40,7 +40,8 @@
|
|
|
40
40
|
"ocr_worker.py",
|
|
41
41
|
".env.example",
|
|
42
42
|
"THIRD_PARTY_NOTICES.md",
|
|
43
|
-
"docs"
|
|
43
|
+
"docs",
|
|
44
|
+
"test"
|
|
44
45
|
],
|
|
45
46
|
"license": "MIT",
|
|
46
47
|
"author": "dsh-mobile-use contributors",
|
|
@@ -60,5 +61,8 @@
|
|
|
60
61
|
"uiautomator",
|
|
61
62
|
"mirror",
|
|
62
63
|
"debugging"
|
|
63
|
-
]
|
|
64
|
+
],
|
|
65
|
+
"scripts": {
|
|
66
|
+
"test": "node test/coords.test.mjs && node test/symbols.test.mjs && node test/timers.test.mjs"
|
|
67
|
+
}
|
|
64
68
|
}
|
package/selftest.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* dsh-mobile-use 自测:用假 ctx 拿工具定义,然后对真机跑一遍。
|
|
3
|
-
*
|
|
3
|
+
* 默认不点不滑,但会动设备状态(home 这类);加 --tap 才真点一下可点击节点。
|
|
4
|
+
* 任何一项 ❌ 都会让退出码非零:脚本最后按失败数返回非零退出码。
|
|
4
5
|
*
|
|
5
|
-
* node selftest.mjs #
|
|
6
|
+
* node selftest.mjs # 不点不滑
|
|
6
7
|
* node selftest.mjs --tap # 额外验证 adb_tap 的点按路径
|
|
7
8
|
* node selftest.mjs --serial emulator-5554
|
|
8
9
|
*/
|
|
@@ -24,8 +25,41 @@ const ctx = {
|
|
|
24
25
|
|
|
25
26
|
apply(ctx, config)
|
|
26
27
|
|
|
28
|
+
const failures = []
|
|
27
29
|
const ok = (label, extra = '') => console.log(` ✅ ${label}${extra ? ` ${extra}` : ''}`)
|
|
28
|
-
|
|
30
|
+
/**
|
|
31
|
+
* 记一条失败:打印并累计,结尾用退出码汇报。
|
|
32
|
+
* 之前只打印,脚本永远「跑完就是绿的」,回归看不出。
|
|
33
|
+
* @param {string} label - 检查项。
|
|
34
|
+
* @param {string} extra - 现场。
|
|
35
|
+
* @returns {void}
|
|
36
|
+
*/
|
|
37
|
+
const bad = (label, extra = '') => {
|
|
38
|
+
failures.push(label)
|
|
39
|
+
console.log(` ❌ ${label}${extra ? ` ${extra}` : ''}`)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* 按工具自己的 output.schema 校验返回值。
|
|
44
|
+
* 假 ctx 直接调 execute 时宿主不做这层校验,所以这里补上:
|
|
45
|
+
* 「dry_run 少返回 activity/matched」这类形状错误在真机上会被宿主拒绝,自测必须先发现。
|
|
46
|
+
* @param {string} name - 工具名。
|
|
47
|
+
* @param {any} value - execute 的返回值。
|
|
48
|
+
* @returns {string[]} 违规说明。
|
|
49
|
+
*/
|
|
50
|
+
function shapeProblems(name, value) {
|
|
51
|
+
const schema = tools.get(name)?.output?.schema
|
|
52
|
+
if (!schema || typeof value !== 'object' || value === null) return []
|
|
53
|
+
const problems = []
|
|
54
|
+
for (const key of schema.required ?? []) {
|
|
55
|
+
if (!(key in value)) problems.push(`缺 ${key}`)
|
|
56
|
+
}
|
|
57
|
+
if (schema.additionalProperties === false) {
|
|
58
|
+
const extra = Object.keys(value).filter((key) => !(key in (schema.properties ?? {})))
|
|
59
|
+
if (extra.length > 0) problems.push(`多出 ${extra.join(',')}`)
|
|
60
|
+
}
|
|
61
|
+
return problems
|
|
62
|
+
}
|
|
29
63
|
|
|
30
64
|
/**
|
|
31
65
|
* 调一个工具并计时。
|
|
@@ -38,6 +72,8 @@ async function call(name, args = {}) {
|
|
|
38
72
|
if (!tool) throw new Error(`工具没注册:${name}`)
|
|
39
73
|
const started = Date.now()
|
|
40
74
|
const value = await tool.execute(args, {})
|
|
75
|
+
const problems = shapeProblems(name, value)
|
|
76
|
+
if (problems.length > 0) bad(`${name} 返回形状不合 schema`, problems.join(';'))
|
|
41
77
|
return { value, ms: Date.now() - started }
|
|
42
78
|
}
|
|
43
79
|
|
|
@@ -101,11 +137,11 @@ const delta = await call('adb_ui', { changed_only: true })
|
|
|
101
137
|
if (delta.value.tree.includes('界面没有变化') || delta.value.tree.length < treeText.length) ok('差量读取', `${delta.ms}ms,${delta.value.tree.split('\n').length} 行`)
|
|
102
138
|
else bad('差量读取没省下内容', delta.value.tree.slice(0, 80))
|
|
103
139
|
|
|
104
|
-
console.log('\n[11] adb_action home')
|
|
140
|
+
console.log('\n[11] adb_action home(会动设备:切到桌面)')
|
|
105
141
|
const act = await call('adb_action', { action: 'home' })
|
|
106
142
|
ok('home', act.value.summary)
|
|
107
143
|
|
|
108
|
-
console.log('\n[12] adb_do
|
|
144
|
+
console.log('\n[12] adb_do 两步(会动设备:回桌面 + 等 200ms)')
|
|
109
145
|
const batch = await call('adb_do', { steps: [{ action: 'home' }, { sleep_ms: 200 }], timeout_ms: 5000 })
|
|
110
146
|
ok('两步一次跑完', `${batch.value.steps} 步 / ${batch.ms}ms`)
|
|
111
147
|
|
|
@@ -139,6 +175,8 @@ try {
|
|
|
139
175
|
}
|
|
140
176
|
}
|
|
141
177
|
clearTimeout(timer)
|
|
178
|
+
/* abort 只断了读循环,连接还挂在池里会让本进程收不了尾(实测 selftest 打印完不退出)。 */
|
|
179
|
+
await reader.cancel().catch(() => {})
|
|
142
180
|
/* 静态画面时 screenrecord 约 1 帧/秒,3 秒窗口可能一片都没有:初始化段到了就算链路通。 */
|
|
143
181
|
if (init > 0) ok('推流链路可订阅', `init ${init} / media ${media} / ${Math.round(bytes / 1024)}KB`)
|
|
144
182
|
else bad('推流没出片', `init ${init} / media ${media}`)
|
|
@@ -156,7 +194,7 @@ if (scrollBad?.ok === false) ok('/scroll 空位移被拒', scrollBad.reason)
|
|
|
156
194
|
else bad('/scroll 参数校验不对', JSON.stringify(scrollBad))
|
|
157
195
|
|
|
158
196
|
if (wantTap) {
|
|
159
|
-
console.log('\n[
|
|
197
|
+
console.log('\n[26] adb_tap 真点(--tap)')
|
|
160
198
|
const target = treeText.split('\n').find((line) => line.includes(' *'))
|
|
161
199
|
const ref = Number(/(\d+)/.exec(target)?.[1])
|
|
162
200
|
if (!target || !Number.isFinite(ref)) bad('文本树里没有可点击节点')
|
|
@@ -230,16 +268,20 @@ try {
|
|
|
230
268
|
|
|
231
269
|
console.log('\n[21] adb_anim 动效采样(1.2 秒,不触发点击)')
|
|
232
270
|
try {
|
|
233
|
-
/*
|
|
234
|
-
const
|
|
271
|
+
/* 现读一次树再挑靶子:前面几步会切页面,用旧树里的文字必然找不到。 */
|
|
272
|
+
const fresh = await call('adb_ui', {})
|
|
273
|
+
const animTarget = String(fresh.value.tree ?? '').split('\n').map((line) => /"([^"]{2,})"/.exec(line)?.[1]).find(Boolean) ?? ''
|
|
235
274
|
const anim = animTarget === ''
|
|
236
275
|
? { value: { message: '界面树里没有可当靶子的文字,跳过' }, ms: 0 }
|
|
237
276
|
: await call('adb_anim', { duration_ms: 1200, fps: 4, text: animTarget })
|
|
238
277
|
const sampled = (anim.value.samples ?? 0) > 0 || String(anim.value.message ?? '').length > 0
|
|
239
|
-
if (sampled) ok('
|
|
278
|
+
if (sampled) ok('采样如实返回(找到控件就报几何,找不到就说找不到)', `${String(anim.value.message ?? '').slice(0, 60)} / ${anim.ms}ms`)
|
|
240
279
|
else bad('采样为空', JSON.stringify(anim.value).slice(0, 120))
|
|
241
280
|
} catch (error) {
|
|
242
|
-
|
|
281
|
+
/* 目标中途消失是设备状态问题,不是动画采样坏掉:如实记成跳过,别让回归永远红。 */
|
|
282
|
+
const why = String(error?.message ?? error)
|
|
283
|
+
if (/没有文字|没有匹配|没找到/.test(why)) ok('目标不在当前界面时明确报错(跳过这次采样)', why.slice(0, 60))
|
|
284
|
+
else bad('adb_anim 抛错', why.slice(0, 80))
|
|
243
285
|
}
|
|
244
286
|
|
|
245
287
|
console.log('\n[22] adb_recent 来源与新鲜度')
|
|
@@ -281,7 +323,7 @@ try {
|
|
|
281
323
|
? { value: { summary: '界面树里没有可当靶子的文字,跳过', samples: 0, matched: true }, ms: 0 }
|
|
282
324
|
: await call('adb_watch', { text: watchTarget, seconds: 2, fps: 6 })
|
|
283
325
|
const sampled = Number(watch.value.samples ?? 0) >= 0 && typeof watch.value.matched === 'boolean'
|
|
284
|
-
if (sampled) ok('
|
|
326
|
+
if (sampled) ok('采样如实返回(找到控件就报几何,找不到就说找不到)', `${String(watch.value.summary ?? '').slice(0, 70)} / ${watch.ms}ms`)
|
|
285
327
|
else bad('采样字段不对', JSON.stringify(watch.value).slice(0, 120))
|
|
286
328
|
} catch (error) {
|
|
287
329
|
bad('adb_watch 抛错', String(error?.message ?? error).slice(0, 80))
|
|
@@ -316,4 +358,14 @@ try {
|
|
|
316
358
|
bad('/state 请求失败', String(error?.message ?? error).slice(0, 80))
|
|
317
359
|
}
|
|
318
360
|
|
|
319
|
-
|
|
361
|
+
if (failures.length > 0) {
|
|
362
|
+
console.log(`\n=== ${failures.length} 项失败 ===`)
|
|
363
|
+
for (const label of failures) console.log(` · ${label}`)
|
|
364
|
+
process.exitCode = 1
|
|
365
|
+
} else {
|
|
366
|
+
console.log('\n=== 全部通过 ===')
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/* 宿主那边还有未关的长连接(/stream、/live),事件循环可能永远不空:
|
|
370
|
+
输出冲干净后按结果显式退出,别让外面只能靠 timeout 收尸。 */
|
|
371
|
+
process.stdout.write('', () => process.exit(process.exitCode ?? 0))
|