dsh-pf-stream 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.en.md +53 -28
  2. package/README.md +55 -30
  3. package/client.js +115 -10
  4. package/lib/asr/catalog.js +3 -4
  5. package/lib/asr/catalog.js.map +1 -1
  6. package/lib/asr/install.js +79 -54
  7. package/lib/asr/install.js.map +1 -1
  8. package/lib/asr/verify.js +12 -3
  9. package/lib/asr/verify.js.map +1 -1
  10. package/lib/audio/asrProviders.js +11 -2
  11. package/lib/audio/asrProviders.js.map +1 -1
  12. package/lib/audio/asrRouter.d.ts +0 -2
  13. package/lib/audio/asrRouter.js +17 -27
  14. package/lib/audio/asrRouter.js.map +1 -1
  15. package/lib/audio/subtitleExtractor.d.ts +5 -0
  16. package/lib/audio/subtitleExtractor.js +35 -5
  17. package/lib/audio/subtitleExtractor.js.map +1 -1
  18. package/lib/audio/transcriptCache.d.ts +8 -0
  19. package/lib/audio/transcriptCache.js +19 -1
  20. package/lib/audio/transcriptCache.js.map +1 -1
  21. package/lib/entry/adapter.js +7 -0
  22. package/lib/entry/adapter.js.map +1 -1
  23. package/lib/entry/legacyNormalizer.js +12 -5
  24. package/lib/entry/legacyNormalizer.js.map +1 -1
  25. package/lib/host/api.js +2 -1
  26. package/lib/host/api.js.map +1 -1
  27. package/lib/host/runtime.js +7 -2
  28. package/lib/host/runtime.js.map +1 -1
  29. package/lib/session/state.js +0 -3
  30. package/lib/session/state.js.map +1 -1
  31. package/lib/stage0/decoder.d.ts +4 -2
  32. package/lib/stage0/decoder.js +10 -5
  33. package/lib/stage0/decoder.js.map +1 -1
  34. package/lib/stage0/ffmpeg.js +19 -5
  35. package/lib/stage0/ffmpeg.js.map +1 -1
  36. package/lib/stage0/probe.d.ts +5 -0
  37. package/lib/stage0/probe.js +49 -34
  38. package/lib/stage0/probe.js.map +1 -1
  39. package/lib/stage2/switchDetector.d.ts +1 -2
  40. package/lib/stage2/switchDetector.js +1 -2
  41. package/lib/stage2/switchDetector.js.map +1 -1
  42. package/lib/stage3/deterministicRouter.js +1 -0
  43. package/lib/stage3/deterministicRouter.js.map +1 -1
  44. package/lib/stage3/streamSampler.js +28 -20
  45. package/lib/stage3/streamSampler.js.map +1 -1
  46. package/lib/stage4/viewport.js +18 -7
  47. package/lib/stage4/viewport.js.map +1 -1
  48. package/lib/stage5/controller.js +8 -0
  49. package/lib/stage5/controller.js.map +1 -1
  50. package/lib/tool/handler.js +16 -1
  51. package/lib/tool/handler.js.map +1 -1
  52. package/lib/tool/schema.js +3 -1
  53. package/lib/tool/schema.js.map +1 -1
  54. package/lib/types.d.ts +4 -1
  55. package/lib/utils/pts.d.ts +5 -1
  56. package/lib/utils/pts.js +13 -1
  57. package/lib/utils/pts.js.map +1 -1
  58. package/package.json +8 -2
package/README.en.md CHANGED
@@ -1,8 +1,31 @@
1
- # dsh-pf-stream
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/icon.svg" width="72" height="72" alt="Video understanding">
3
+ </p>
2
4
 
3
- [中文](https://github.com/Weilv-D/dsh-pf-stream/blob/main/README.md) | **English**
5
+ <h1 align="center">Video understanding</h1>
4
6
 
5
- Let the agent watch videos: adaptive sampling, plus the spoken lines.
7
+ <p align="center">Frames, motion and speech from a video, aligned on a timeline for the agent.</p>
8
+
9
+ <p align="center">
10
+ <a href="https://www.npmjs.com/package/dsh-pf-stream"><img src="https://img.shields.io/npm/v/dsh-pf-stream?color=2f6feb" alt="npm version"></a>
11
+ <img src="https://img.shields.io/badge/license-Apache--2.0-2ea44f" alt="license">
12
+ <img src="https://img.shields.io/badge/DSH-plugin-6e56cf" alt="DSH plugin">
13
+ </p>
14
+
15
+ <p align="center"><a href="README.md">中文</a> · <b>English</b></p>
16
+
17
+ ---
18
+
19
+ ## Quick start
20
+
21
+ | Step | Do |
22
+ | :--- | :--- |
23
+ | 1 | Install ffmpeg ([commands below](#1-install-ffmpeg)); `ffmpeg -version` must print output |
24
+ | 2 | Left sidebar **Plugins** → **Video understanding** → switch it on |
25
+ | 3 | Click **Download local model** (~190 MB), then **Test transcription** |
26
+ | 4 | Give the agent a video path and ask |
27
+
28
+ Video works without transcription. Only videos that have no subtitles and need speech text require step 3.
6
29
 
7
30
  ## What it does
8
31
 
@@ -24,12 +47,12 @@ ffprobe -version
24
47
  Both must print output. If not:
25
48
 
26
49
  | OS | Command |
27
- | --- | --- |
50
+ | :--- | :--- |
28
51
  | Windows | `winget install Gyan.FFmpeg` |
29
52
  | macOS | `brew install ffmpeg` |
30
53
  | Ubuntu / Debian | `sudo apt install ffmpeg` |
31
54
 
32
- Open a new terminal and check again. To keep it out of PATH, set `ffmpegPath` and `ffprobePath` in the plugin config.
55
+ > Open a new terminal and check again. To keep it out of PATH, set `ffmpegPath` and `ffprobePath` in the plugin config.
33
56
 
34
57
  ### 2. Install the plugin in DSH
35
58
 
@@ -56,30 +79,32 @@ The transcription engine decides whether videos without subtitles get speech tex
56
79
 
57
80
  To open the settings: **Plugins in the left sidebar → Video understanding → the panel below the description**. Use the same path later to change settings.
58
81
 
82
+ <p><img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/docs/assets/panel.png" width="720" alt="Transcription settings panel on the plugin detail page: decoder, local transcription and cloud transcription cards"></p>
83
+
59
84
  Three cards at the top report video decode (ffmpeg), local transcription, and cloud transcription. Pick one path:
60
85
 
61
86
  | Path | Best for | What to do |
62
- | --- | --- | --- |
87
+ | :--- | :--- | :--- |
63
88
  | Local (recommended) | Free, offline, no API key | Click "Download local model" (~190 MB), then "Test transcription" |
64
89
  | Cloud | You have a key, or the machine is slow | Pick OpenAI / Groq / a custom compatible endpoint, paste the key, save, then test |
65
90
  | Skip for now | Videos have subtitles | Use it as is; speech-only videos get no transcript |
66
91
 
67
- Local model steps:
92
+ ### Local model
68
93
 
69
94
  1. Click "Download local model". The progress bar shows bytes received; cancel at any time.
70
95
  2. Clicking again after an interruption resumes the download.
71
96
  3. When done, the card reads "Ready · whisper.cpp".
72
97
  4. Click "Test transcription". "Local engine available" means it works.
73
98
 
74
- Cloud key steps:
99
+ One-click install currently supports Windows x64. On other systems, install whisper.cpp and set `audio.asr.local.binPath` in the config file, or use cloud transcription.
100
+
101
+ ### Cloud key
75
102
 
76
103
  1. Choose a provider under "Cloud transcription key".
77
104
  2. Paste the API key and click "Save". After saving, only "Configured" is shown; the key is never echoed back.
78
105
  3. Click "Test transcription" to verify the key and network.
79
106
 
80
- The default policy is `auto`: use a cloud key when configured, otherwise the local engine. Under "Advanced" you can switch to local only, cloud only, or off.
81
-
82
- One-click install currently supports Windows x64. On other systems, install whisper.cpp and set `audio.asr.local.binPath` in the config file, or use cloud transcription.
107
+ > **Auto policy**: cloud is used when key, endpoint and model are all set (faster, billed per use); otherwise the local engine. The badge next to the panel title shows the engine in use, such as "Transcription ready · Cloud". Under "Advanced" you can switch to local only, cloud only, or off.
83
108
 
84
109
  ## Use it
85
110
 
@@ -102,7 +127,7 @@ The text block is structured metadata: each canvas packet reports time, form, an
102
127
  Four ways to look, by question:
103
128
 
104
129
  | Goal | Ask |
105
- | --- | --- |
130
+ | :--- | :--- |
106
131
  | Overview first | "Look at this video" — streaming sampling; motion as process, settle as detail |
107
132
  | One exact moment | "What is written at 12.9s" — deterministic high-resolution frame |
108
133
  | Even coverage | "One frame every 5 seconds" — uniform sampling |
@@ -112,34 +137,32 @@ Four ways to look, by question:
112
137
 
113
138
  ### Dual-scale streaming perception
114
139
 
115
- Video is not sampled at a fixed rate. While the picture moves, low-resolution process units (P) record change; when it settles, high-resolution freeze units (F) keep detail. A single canvas packet carries both process and result, so the model does not need to re-watch a clip to recover one detail.
140
+ Sampling is event-driven rather than fixed-rate. During motion, low-resolution process units (P) record state transitions; at settle, high-resolution freeze units (F) preserve detail. Both are delivered within a single canvas packet, so one observation carries the temporal process and the steady-state result together, and the model need not re-watch a clip to recover a detail.
116
141
 
117
- The sampling cadence follows the picture: fast change lowers density to avoid duplicates; slow change waits for a settle before freezing. Long static stretches keep a heartbeat instead of repeating images.
142
+ Density adapts to picture activity: rapid change lowers the sampling rate to suppress redundant frames, while slow change extends the wait until a settle is detected; long static intervals maintain a temporal reference through heartbeats without retransmitting images.
118
143
 
119
144
  ### Canvas packing and metering
120
145
 
121
- A frame is divided into 42 px superpixel-aligned units. P or F units are taken as needed and packed into one canvas. One call can carry several packets; each packet holds only the units that are new, and older packets collapse into reference hashes instead of resending images. Metering follows the canvas grid (`r × (c + 1) + 2`).
146
+ Each frame is partitioned on a 42 px superpixel grid; P or F units are selected as needed and assembled into a canvas. A call may carry multiple packets, and each packet contains only the units new relative to the previous one; historical packets collapse into content-hash references, keeping context bounded in long sessions. Metering follows canvas grid cells (`r × (c + 1) + 2`).
122
147
 
123
148
  ### Deterministic inspection
124
149
 
125
- Besides streaming, three paths are deterministic: `probe` reads metadata only; `timestamps` returns exact frames in milliseconds; `uniform` samples at a fixed interval; `roi_hint` crops a region. These do not depend on motion detection, so results are reproducible and suited to reading text, numbers, and small objects.
150
+ Beyond streaming, four deterministic paths are provided: `probe` parses container metadata only; `timestamps` extracts exact frames at millisecond timestamps; `uniform` samples at a fixed interval; `roi_hint` crops and magnifies a normalized region. None depends on motion detection; outputs are reproducible and suited to verifying text, numerals, and small objects.
126
151
 
127
- ### Three sources for speech
152
+ ### Cascaded speech sources
128
153
 
129
- 1. **Subtitles**: a sidecar `.srt` / `.vtt` with the same name first, then embedded soft subtitles.
130
- 2. **Acoustic check**: without subtitles, silence detection and a speech-ratio estimate run first; an all-silent video skips transcription.
131
- 3. **Transcription**: local whisper.cpp, or OpenAI / Groq / a custom compatible endpoint. Segments are mounted on packets by PTS and stitched across block boundaries with a ±300 ms tolerance.
154
+ Speech text is obtained through a three-stage cascade: a sidecar `.srt` / `.vtt` is read first, then embedded soft subtitles; without subtitles, silence detection and a speech-ratio estimate run, and an all-silent video skips transcription; finally ASR runs, either local whisper.cpp or OpenAI / Groq / a custom compatible endpoint. Segments are aligned by PTS and mounted on the corresponding packets, stitched across block boundaries with a ±300 ms tolerance.
132
155
 
133
- The default is "picture first, speech later": when the soft deadline (`graceMs`, 8 s by default) passes, the visual result returns and transcription continues in the background; the next call on the same video hits the disk cache.
156
+ The default policy is vision-first with background transcription: once a call exceeds the soft deadline (`graceMs`, 8 s by default), the visual result returns and transcription continues in the background; later calls on the same video hit the disk cache.
134
157
 
135
- ### Old regions
158
+ ### Cold regions in long videos
136
159
 
137
- Canvas packets outside the window collapse into references (anchor hash, time, and transcript text) instead of resending images. A timestamp call brings the detail back when needed.
160
+ Historical packets outside the current window collapse into references (anchor hash, time, and transcript text) rather than retransmitting images; a timestamp call restores the detail when needed.
138
161
 
139
162
  ## Troubleshooting
140
163
 
141
164
  | Symptom | Read |
142
- | --- | --- |
165
+ | :--- | :--- |
143
166
  | ffmpeg not found | [ffmpeg](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#找不到-ffmpeg) |
144
167
  | Speech but no transcript | [No transcript](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#有声音但没有台词) |
145
168
  | Model download fails or is slow | [Download](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#下载失败) |
@@ -148,10 +171,12 @@ Canvas packets outside the window collapse into references (anchor hash, time, a
148
171
 
149
172
  ## Docs
150
173
 
151
- - [Usage Guide](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/usage-guide.md) (English)
152
- - [使用指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/使用指南.md) (Chinese)
153
- - [排障](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md) (Chinese)
154
- - [开发者指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/开发者指南.md) (Chinese): architecture, host contracts, tests, release
174
+ | Document | Contents |
175
+ | :--- | :--- |
176
+ | [Usage Guide](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/usage-guide.md) | Usage, parameters, output, metering, every config key |
177
+ | [使用指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/使用指南.md) | Chinese usage guide |
178
+ | [排障](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md) | Troubleshooting by symptom (Chinese) |
179
+ | [开发者指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/开发者指南.md) | Architecture, host contracts, tests, release (Chinese) |
155
180
 
156
181
  ## Development
157
182
 
package/README.md CHANGED
@@ -1,18 +1,41 @@
1
- # dsh-pf-stream
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/icon.svg" width="72" height="72" alt="视频理解">
3
+ </p>
2
4
 
3
- **中文** | [English](https://github.com/Weilv-D/dsh-pf-stream/blob/main/README.en.md)
5
+ <h1 align="center">视频理解</h1>
4
6
 
5
- 让智能体看懂视频:动态抽帧,加上台词。
7
+ <p align="center">让智能体看懂视频:画面、动作与台词,按时间对齐。</p>
8
+
9
+ <p align="center">
10
+ <a href="https://www.npmjs.com/package/dsh-pf-stream"><img src="https://img.shields.io/npm/v/dsh-pf-stream?color=2f6feb" alt="npm 版本"></a>
11
+ <img src="https://img.shields.io/badge/license-Apache--2.0-2ea44f" alt="许可证">
12
+ <img src="https://img.shields.io/badge/DSH-plugin-6e56cf" alt="DSH 插件">
13
+ </p>
14
+
15
+ <p align="center"><b>中文</b> · <a href="README.en.md">English</a></p>
16
+
17
+ ---
18
+
19
+ ## 30 秒上手
20
+
21
+ | 步骤 | 做什么 |
22
+ | :--- | :--- |
23
+ | 1 | 安装 ffmpeg([命令见下](#1-安装-ffmpeg)),确认 `ffmpeg -version` 有输出 |
24
+ | 2 | 左侧栏「插件」→ 点「视频理解」→ 打开右上角开关 |
25
+ | 3 | 详情页点「下载本地模型」,约 190 MB;完成后点「测试转写」 |
26
+ | 4 | 在对话里给视频路径,直接提问 |
27
+
28
+ 不配置转写也能看画面。只有「没有字幕、又需要台词」的视频,才需要第 3 步。
6
29
 
7
30
  ## 能做什么
8
31
 
9
32
  - **看画面**:画面运动时记录过程,停稳时定格细节;指定时间点可以高清复核。
10
33
  - **听声音**:有字幕就用字幕;没有字幕时,配置转写引擎后生成台词,并按时间挂到对应画面。
11
- - **按需配置**:不配置转写也能处理画面;配置后可以处理带音轨的视频。
34
+ - **按需配置**:默认不转写也能处理画面;配置后可以处理带音轨的视频。
12
35
 
13
36
  ## 安装
14
37
 
15
- ### 第一步:安装 ffmpeg
38
+ ### 1. 安装 ffmpeg
16
39
 
17
40
  插件用 ffmpeg 解码视频。先确认它能运行:
18
41
 
@@ -24,14 +47,14 @@ ffprobe -version
24
47
  两条都要有输出。没有的话:
25
48
 
26
49
  | 系统 | 安装命令 |
27
- | --- | --- |
50
+ | :--- | :--- |
28
51
  | Windows | `winget install Gyan.FFmpeg` |
29
52
  | macOS | `brew install ffmpeg` |
30
53
  | Ubuntu / Debian | `sudo apt install ffmpeg` |
31
54
 
32
- 装完新开一个终端再验证。不想放进 PATH,可以在插件配置里写 `ffmpegPath` 与 `ffprobePath` 的绝对路径。
55
+ > 装完新开一个终端再验证。不想放进 PATH,可以在插件配置里写 `ffmpegPath` 与 `ffprobePath` 的绝对路径。
33
56
 
34
- ### 第二步:在 DSH 里安装插件
57
+ ### 2. 在 DSH 里安装插件
35
58
 
36
59
  1. 打开 DSH,点左侧栏的「插件」。
37
60
  2. 在列表里找「视频理解」(包名 `dsh-pf-stream`)。
@@ -56,30 +79,32 @@ ffprobe -version
56
79
 
57
80
  进入配置页:**左侧栏「插件」→ 点「视频理解」→ 详情页描述下方的面板**。以后要改设置,走同一条路。
58
81
 
82
+ <p><img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/docs/assets/panel.png" width="720" alt="插件详情页的转写设置面板:视频解码、本地转写与云端转写三张状态卡"></p>
83
+
59
84
  面板顶部有三张状态卡:视频解码(ffmpeg)、本地转写、云端转写。按需要选一条路线:
60
85
 
61
86
  | 路线 | 适合 | 操作 |
62
- | --- | --- | --- |
87
+ | :--- | :--- | :--- |
63
88
  | 本地转写(推荐) | 免费、离线、不填密钥 | 点「下载本地模型」,约 190 MB;下载完点「测试转写」 |
64
89
  | 云端转写 | 已有 API Key,或机器性能较弱 | 选 OpenAI / Groq / 自定义兼容端点,填 Key,保存后点「测试转写」 |
65
90
  | 暂不配置 | 视频都有字幕 | 直接使用;无字幕的语音视频不会生成转写文本 |
66
91
 
67
- 本地模型的下载步骤:
92
+ ### 本地模型
68
93
 
69
94
  1. 点「下载本地模型」。进度条显示已下载大小,可随时取消。
70
95
  2. 下载中断后再次点击会续传,不用从头开始。
71
96
  3. 完成后状态卡显示「已就绪 · whisper.cpp」。
72
97
  4. 点「测试转写」。返回「本地引擎可用」即配置成功。
73
98
 
74
- 云端密钥的保存步骤:
99
+ 一键安装当前支持 Windows x64。其他系统请先安装 whisper.cpp,在配置文件 `audio.asr.local.binPath` 填写可执行文件路径,或改用云端转写。
100
+
101
+ ### 云端密钥
75
102
 
76
103
  1. 在「云端转写密钥」里选服务商。
77
104
  2. 粘贴 API Key,点「保存」。保存后只显示「已配置」,不再回显密钥。
78
105
  3. 点「测试转写」验证密钥与网络。
79
106
 
80
- 默认策略是 `auto`:云端密钥、端点、模型都齐时用云端(更快,按量计费);否则用本地引擎。面板标题旁会显示当前实际使用的引擎,如「转写可用 · 云端」。可以在「高级」里改成仅本地、仅云端或不转写。
81
-
82
- 一键安装当前支持 Windows x64。其他系统请先安装 whisper.cpp,在配置文件 `audio.asr.local.binPath` 填写可执行文件路径,或改用云端转写。
107
+ > **自动策略**:云端密钥、端点、模型都齐时用云端(更快,按量计费);否则用本地引擎。面板标题旁会显示当前实际使用的引擎,如「转写可用 · 云端」。可以在「高级」里改成仅本地、仅云端或不转写。
83
108
 
84
109
  ## 开始使用
85
110
 
@@ -102,7 +127,7 @@ Audio @Packet #42: [12.1s-12.9s] "这件事我们准备了三年" | [13.0s-13.4s
102
127
  常用问法对应四种取帧方式:
103
128
 
104
129
  | 想要 | 说法 |
105
- | --- | --- |
130
+ | :--- | :--- |
106
131
  | 先看整体 | 「看看这个视频」——流式采样,运动记过程、停稳给细节 |
107
132
  | 只看某一秒 | 「第 12.9 秒写了什么」——定点高清帧 |
108
133
  | 均匀看一遍 | 「每 5 秒抽一帧」——等间隔抽帧 |
@@ -112,34 +137,32 @@ Audio @Packet #42: [12.1s-12.9s] "这件事我们准备了三年" | [13.0s-13.4s
112
137
 
113
138
  ### 双尺度流式感知
114
139
 
115
- 视频不再平均抽帧。画面运动时,插件用低分辨率的过程单元(P)记录变化;画面停稳后,用高分辨率的定格单元(F)保留细节。同一个画布包里既有过程也有结果,模型不用为了一个细节反复回看整段视频。
140
+ 采样过程不再采用固定帧率,而由画面变化事件驱动:运动阶段以低分辨率过程单元(P)记录状态转移,停稳后以高分辨率定格单元(F)保留细节;两类单元在同一画布包内交付,使单次观测同时包含时序过程与稳态结果,模型无需为恢复某一细节而回看整段视频。
116
141
 
117
- 采样节奏由画面变化驱动:变化快时降低采样密度、避免重复帧;变化慢时等待停稳再定格。长静止画面用心跳保持时间感,不重复发图。
142
+ 采样密度随画面活动度自适应调整:快速变化时降低采样率以抑制冗余帧,缓慢变化时延长等待直至检测到停稳;长静止区间以心跳信号维持时间基准,不重复传输图像。
118
143
 
119
144
  ### 画布打包与计费
120
145
 
121
- 一帧画面被切成 42px 超像素对齐的单元,按需取 P 或 F,再拼成一张画布。一次调用可以带多个画布包;每个包只包含当前新增的单元,历史包以引用哈希折叠,不重复传图。计费按画布网格计算(`r × (c + 1) + 2`)。
146
+ 每帧按 42 px 超像素网格切分为单元,按需选取 P 或 F 单元并拼装为画布。单次调用可携带多个画布包;每个包仅包含相对上一包新增的单元,历史包以内容哈希引用折叠,使长会话中的上下文规模保持有界。计费按画布网格单元数计算(`r × (c + 1) + 2`)。
122
147
 
123
148
  ### 确定性检视
124
149
 
125
- 流式采样之外,另外三条路径是确定性的:`probe` 只读元数据;`timestamps` 按毫秒取指定帧;`uniform` 按固定间隔取帧;`roi_hint` 裁剪指定区域。这几种不依赖运动检测,结果可复现,适合核对文字、数字与小目标。
150
+ 流式采样之外另设四条确定性路径:`probe` 仅解析容器元数据;`timestamps` 按毫秒级时间戳抽取精确帧;`uniform` 以固定间隔均匀采样;`roi_hint` 对指定归一化区域裁剪放大。上述路径不依赖运动检测,输出可复现,适用于文字、数字与小目标的核对。
126
151
 
127
- ### 台词的三级来源
152
+ ### 台词的级联来源
128
153
 
129
- 1. **字幕**:优先用外挂同名 `.srt` / `.vtt`,其次内嵌软字幕。
130
- 2. **声学判断**:没有字幕时先做静音检测与语音占比估计;全程静音的视频直接跳过转写。
131
- 3. **语音转写**:本地 whisper.cpp,或 OpenAI / Groq / 自定义兼容端点。转写结果按 PTS 挂到对应画布包,跨块边界按 ±300ms 缝合。
154
+ 转写文本按三级级联获取:优先读取同名外挂 `.srt` / `.vtt`,其次解析内嵌软字幕;无字幕时执行静音检测与语音占比估计,全静音视频直接跳过转写;最后调用语音识别,可选本地 whisper.cpp 或 OpenAI / Groq / 自定义兼容端点。识别结果按 PTS 对齐并挂载至相应画布包,跨块边界以 ±300 ms 容差缝合。
132
155
 
133
- 默认「先回画面、后台补台词」:首次调用超过软截止(`graceMs`,默认 8 秒)就先返回视觉结果,转写在后台继续;同一视频再调用时直接命中磁盘缓存。
156
+ 默认策略为「视觉先行、转写后台补齐」:首次调用超过软截止(`graceMs`,默认 8 s)时先行返回视觉结果,转写任务在后台继续;同一视频的后续调用直接命中磁盘缓存。
134
157
 
135
158
  ### 长视频的冷区
136
159
 
137
- 超出窗口的历史画布包会折叠成引用(锚点哈希 + 时间 + 台词文本),不重复传图。需要回看时再按时间点取帧。
160
+ 超出当前窗口的历史画布包折叠为引用(锚点哈希、时间与台词文本),不再重复传输图像;需要复核时按时间戳重新取帧。
138
161
 
139
162
  ## 遇到问题
140
163
 
141
164
  | 现象 | 先看这里 |
142
- | --- | --- |
165
+ | :--- | :--- |
143
166
  | 提示找不到 ffmpeg | [排障:ffmpeg](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#找不到-ffmpeg) |
144
167
  | 有声音,但没有台词 | [排障:没有台词](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#有声音但没有台词) |
145
168
  | 下载模型失败或很慢 | [排障:下载失败](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#下载失败) |
@@ -148,10 +171,12 @@ Audio @Packet #42: [12.1s-12.9s] "这件事我们准备了三年" | [13.0s-13.4s
148
171
 
149
172
  ## 文档
150
173
 
151
- - [使用指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/使用指南.md):用法、参数、返回内容、计费、全部配置项
152
- - [排障](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md):按现象查原因
153
- - [开发者指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/开发者指南.md):架构、宿主契约、测试与发布
154
- - [Usage Guide](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/usage-guide.md)(English)
174
+ | 文档 | 内容 |
175
+ | :--- | :--- |
176
+ | [使用指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/使用指南.md) | 用法、参数、返回内容、计费、全部配置项 |
177
+ | [排障](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md) | 按现象查原因 |
178
+ | [开发者指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/开发者指南.md) | 架构、宿主契约、测试与发布 |
179
+ | [Usage Guide](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/usage-guide.md) | 英文使用指南 |
155
180
 
156
181
  ## 本地开发
157
182
 
package/client.js CHANGED
@@ -23,6 +23,8 @@ window.__ModuleLoader__.load({
23
23
  panelSubtitle:
24
24
  '以下设置用于启用语音识别与转写。未配置时不影响视频播放,但不会生成转写文本;完成配置后,可对含有效音轨的视频进行转写。',
25
25
  refresh: '刷新',
26
+ github: '在 GitHub 上点个 Star',
27
+ githubAria: '在 GitHub 上打开项目仓库',
26
28
  checking: '正在读取状态…',
27
29
  ffmpeg: '视频解码',
28
30
  ffmpegOk: '已就绪',
@@ -89,6 +91,17 @@ window.__ModuleLoader__.load({
89
91
  loadFailed: '读取状态失败',
90
92
  installStarted: '下载已开始',
91
93
  waitDownload: '下载在后台进行,画面处理照常。',
94
+ engineLegacy: '旧版 provider 直连(0.2.x 配置)',
95
+ presetCustom: '自定义 OpenAI 兼容端点',
96
+ errInvalidField: '字段值不合法,请检查输入',
97
+ errJsonRequired: '请求必须是 JSON 格式',
98
+ errCrossOrigin: '跨站请求被拒绝',
99
+ errCredentialRejected: '凭据服务拒绝',
100
+ errKeyTooLong: '密钥过长',
101
+ errUnknownPreset: '未知服务商',
102
+ errNotFound: '接口不存在',
103
+ errMethodNotAllowed: '请求方法不被允许',
104
+ errInternal: '服务内部错误',
92
105
  };
93
106
 
94
107
  const EN = {
@@ -96,6 +109,8 @@ window.__ModuleLoader__.load({
96
109
  panelSubtitle:
97
110
  'These settings enable speech recognition and transcription. Without them, video playback is unaffected, but no transcript is generated. Once configured, videos with a usable audio track can be transcribed.',
98
111
  refresh: 'Refresh',
112
+ github: 'Star this project on GitHub',
113
+ githubAria: 'Open the project repository on GitHub',
99
114
  checking: 'Reading status…',
100
115
  ffmpeg: 'Video decoding',
101
116
  ffmpegOk: 'Ready',
@@ -162,6 +177,17 @@ window.__ModuleLoader__.load({
162
177
  loadFailed: 'Failed to read status',
163
178
  installStarted: 'Download started',
164
179
  waitDownload: 'The download runs in the background; video processing continues.',
180
+ engineLegacy: 'Legacy provider passthrough (0.2.x config)',
181
+ presetCustom: 'Custom OpenAI-compatible endpoint',
182
+ errInvalidField: 'Invalid field value; check the input',
183
+ errJsonRequired: 'The request must be JSON',
184
+ errCrossOrigin: 'Cross-origin request rejected',
185
+ errCredentialRejected: 'Credential store rejected the request',
186
+ errKeyTooLong: 'The key is too long',
187
+ errUnknownPreset: 'Unknown provider',
188
+ errNotFound: 'Endpoint not found',
189
+ errMethodNotAllowed: 'HTTP method not allowed',
190
+ errInternal: 'Internal server error',
165
191
  };
166
192
 
167
193
  function detectLang() {
@@ -172,6 +198,29 @@ window.__ModuleLoader__.load({
172
198
  }
173
199
  }
174
200
 
201
+ /** 服务端错误码 → 词典键(call() 抛出的 error.code;未知码回退原始消息)。 */
202
+ const ERROR_KEYS = {
203
+ invalid_field: 'errInvalidField',
204
+ json_required: 'errJsonRequired',
205
+ cross_origin: 'errCrossOrigin',
206
+ credential_rejected: 'errCredentialRejected',
207
+ key_too_long: 'errKeyTooLong',
208
+ unknown_preset: 'errUnknownPreset',
209
+ not_found: 'errNotFound',
210
+ method_not_allowed: 'errMethodNotAllowed',
211
+ internal_error: 'errInternal',
212
+ };
213
+
214
+ function translateError(t, cause) {
215
+ const code = cause && cause.code;
216
+ const key = ERROR_KEYS[code];
217
+ if (!key) return cause instanceof Error ? cause.message : String(cause);
218
+ const prefix = t(key);
219
+ // credential_rejected 服务端附带原因:拼在翻译后
220
+ const detail = cause instanceof Error ? cause.message : String(cause);
221
+ return code === 'credential_rejected' && detail ? prefix + ':' + detail : prefix;
222
+ }
223
+
175
224
  function makeTranslator(locale) {
176
225
  const lang = detectLang();
177
226
  const own = lang === 'zh' ? ZH : EN;
@@ -298,6 +347,21 @@ window.__ModuleLoader__.load({
298
347
  lineHeight: 1.2,
299
348
  whiteSpace: 'nowrap',
300
349
  },
350
+ iconButton: {
351
+ appearance: 'none',
352
+ border: '.5px solid ' + PALETTE.hairline,
353
+ background: PALETTE.glassStrong,
354
+ color: PALETTE.sub,
355
+ borderRadius: 9,
356
+ width: 30,
357
+ height: 30,
358
+ display: 'inline-flex',
359
+ alignItems: 'center',
360
+ justifyContent: 'center',
361
+ cursor: 'pointer',
362
+ textDecoration: 'none',
363
+ flex: 'none',
364
+ },
301
365
  primary: {
302
366
  appearance: 'none',
303
367
  border: '.5px solid color-mix(in srgb, var(--dsw-alias-brand-primary) 45%, transparent)',
@@ -610,9 +674,13 @@ window.__ModuleLoader__.load({
610
674
  function useStatus() {
611
675
  const [status, setStatus] = React.useState(null);
612
676
  const [error, setError] = React.useState('');
677
+ // 安装相位经 ref 传递:tick 闭包不能读 state(闭包捕获的是挂载帧的 null,
678
+ // 曾使安装期间的 1s 快刷永不生效)
679
+ const installPhaseRef = React.useRef(null);
613
680
  const load = React.useCallback(async function () {
614
681
  try {
615
682
  const data = await call('/status');
683
+ installPhaseRef.current = data && data.install ? data.install.phase : null;
616
684
  setStatus(data);
617
685
  setError('');
618
686
  } catch (cause) {
@@ -626,8 +694,8 @@ window.__ModuleLoader__.load({
626
694
  if (!alive) return;
627
695
  await load();
628
696
  if (!alive) return;
629
- const busy = status && status.install &&
630
- ['preparing', 'downloading', 'verifying', 'extracting'].indexOf(status.install.phase) >= 0;
697
+ const phase = installPhaseRef.current;
698
+ const busy = phase && ['preparing', 'downloading', 'verifying', 'extracting'].indexOf(phase) >= 0;
631
699
  timer = setTimeout(tick, busy ? 1000 : 6000);
632
700
  }
633
701
  void tick();
@@ -654,8 +722,11 @@ window.__ModuleLoader__.load({
654
722
 
655
723
  React.useEffect(function () {
656
724
  if (!status) return;
657
- setCustomBase(status.cloud.baseUrl || '');
658
- setCustomModel(status.cloud.model || '');
725
+ // 回填仅覆盖未被聚焦编辑的输入(请求在途时用户已开始输入的场景)
726
+ const active = typeof document !== 'undefined' && document.activeElement;
727
+ const activeField = active && active.getAttribute ? active.getAttribute('data-field') : null;
728
+ if (activeField !== 'baseUrl') setCustomBase(status.cloud.baseUrl || '');
729
+ if (activeField !== 'model') setCustomModel(status.cloud.model || '');
659
730
  }, [status && status.cloud && status.cloud.preset]);
660
731
 
661
732
  function run(name, action) {
@@ -669,7 +740,7 @@ window.__ModuleLoader__.load({
669
740
  return state.reload();
670
741
  })
671
742
  .catch(function (cause) {
672
- setFailure(cause instanceof Error ? cause.message : String(cause));
743
+ setFailure(translateError(t, cause));
673
744
  })
674
745
  .then(function () {
675
746
  setBusy('');
@@ -687,12 +758,16 @@ window.__ModuleLoader__.load({
687
758
  const ffmpeg = status.ffmpeg || {};
688
759
  const localReady = status.engine && status.engine.ready;
689
760
  const cloudReady = status.cloud && status.cloud.keyConfigured && status.cloud.baseUrl && status.cloud.model;
690
- const ready = ffmpeg.available && (localReady || cloudReady);
761
+ // 徽标口径取服务端权威结论(engine=off / audio.mode=off 时 ready=false),
762
+ // 本地自算会出现「转写可用 · 未启用」的自相矛盾徽标
763
+ const ready = status.ready === true;
691
764
  const job = status.install;
692
765
  const installing =
693
766
  job && ['preparing', 'downloading', 'verifying', 'extracting'].indexOf(job.phase) >= 0;
694
767
  const preset = status.cloud.preset;
695
768
  const custom = status.cloud.editable;
769
+ const presetInfo = (status.presets || []).find(function (item) { return item.id === preset; });
770
+ const keyUrl = presetInfo ? presetInfo.keyUrl : '';
696
771
  const activeKind = status.active
697
772
  ? status.active.kind
698
773
  : status.mode === 'local'
@@ -712,6 +787,7 @@ window.__ModuleLoader__.load({
712
787
  ['local', t('engineLocal')],
713
788
  ['cloud', t('engineCloud')],
714
789
  ['off', t('engineOff')],
790
+ ['legacy', t('engineLegacy')],
715
791
  ];
716
792
 
717
793
  return h(
@@ -723,6 +799,24 @@ window.__ModuleLoader__.load({
723
799
  h('h3', { style: styles.title }, t('panelTitle')),
724
800
  h('span', { style: styles.badge }, ready ? t('badgeReady') + ' · ' + activeLabel : t('badgePending')),
725
801
  h('span', { style: { flex: '1 1 auto' } }),
802
+ h(
803
+ 'a',
804
+ {
805
+ href: 'https://github.com/Weilv-D/dsh-pf-stream',
806
+ target: '_blank',
807
+ rel: 'noreferrer',
808
+ title: t('github'),
809
+ 'aria-label': t('githubAria'),
810
+ style: styles.iconButton,
811
+ },
812
+ h(
813
+ 'svg',
814
+ { viewBox: '0 0 16 16', width: 16, height: 16, fill: 'currentColor', 'aria-hidden': 'true' },
815
+ h('path', {
816
+ d: 'M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z',
817
+ }),
818
+ ),
819
+ ),
726
820
  h(
727
821
  'button',
728
822
  { type: 'button', style: styles.button, onClick: function () { void state.reload(); } },
@@ -825,9 +919,16 @@ window.__ModuleLoader__.load({
825
919
  h(SelectField, {
826
920
  value: preset,
827
921
  options: (status.presets || []).map(function (item) {
828
- return { value: item.id, label: item.label };
922
+ // custom 的服务端 label 是中文兜底,英文界面走本地词典
923
+ return {
924
+ value: item.id,
925
+ label: item.id === 'custom' ? t('presetCustom') : item.label,
926
+ };
829
927
  }),
830
928
  onChange: function (value) {
929
+ // 切换服务商时清空未保存的密钥草稿:
930
+ // 密钥按服务商独立存储(keyRef 各自独立),跨服务商误存会双向错乱
931
+ setKeyDraft('');
831
932
  void run('preset', function () {
832
933
  return call('/config', { field: 'preset', value: value });
833
934
  });
@@ -841,6 +942,7 @@ window.__ModuleLoader__.load({
841
942
  style: styles.input,
842
943
  value: customBase,
843
944
  placeholder: 'https://api.example.com',
945
+ 'data-field': 'baseUrl',
844
946
  onChange: function (event) { setCustomBase(event.target.value); },
845
947
  onBlur: function () {
846
948
  void run('baseUrl', function () {
@@ -857,6 +959,7 @@ window.__ModuleLoader__.load({
857
959
  style: styles.input,
858
960
  value: customModel,
859
961
  placeholder: 'whisper-1',
962
+ 'data-field': 'model',
860
963
  onChange: function (event) { setCustomModel(event.target.value); },
861
964
  onBlur: function () {
862
965
  void run('model', function () {
@@ -918,9 +1021,9 @@ window.__ModuleLoader__.load({
918
1021
  },
919
1022
  t('clear'),
920
1023
  ),
921
- status.cloud.keyUrl
1024
+ keyUrl
922
1025
  ? h('a', {
923
- href: status.cloud.keyUrl,
1026
+ href: keyUrl,
924
1027
  target: '_blank',
925
1028
  rel: 'noreferrer',
926
1029
  style: { fontSize: 12, color: PALETTE.brand, textDecoration: 'none', paddingBottom: 8 },
@@ -961,6 +1064,7 @@ window.__ModuleLoader__.load({
961
1064
  h('input', {
962
1065
  style: styles.input,
963
1066
  defaultValue: (status.config && status.config.language) || '',
1067
+ key: 'language-' + ((status.config && status.config.language) || ''),
964
1068
  placeholder: t('languageAuto'),
965
1069
  onBlur: function (event) {
966
1070
  const value = event.target.value;
@@ -993,7 +1097,8 @@ window.__ModuleLoader__.load({
993
1097
  style: styles.input,
994
1098
  type: 'number',
995
1099
  min: 0,
996
- max: 32,
1100
+ max: 16,
1101
+ key: 'threads-' + ((status.config && status.config.threads) || 0),
997
1102
  defaultValue: (status.config && status.config.threads) || 0,
998
1103
  onBlur: function (event) {
999
1104
  const value = Number(event.target.value || 0);
@@ -139,8 +139,8 @@ export function buildLocalArgs(engine, ctx) {
139
139
  return [
140
140
  '--model',
141
141
  ctx.modelName,
142
- '--language',
143
- lang === 'auto' ? 'auto' : lang,
142
+ // Python 系 CLI 的 --language 不接受 'auto' 字面量:省略即自动检测
143
+ ...(lang === 'auto' ? [] : ['--language', lang]),
144
144
  '--task',
145
145
  'transcribe',
146
146
  '--output_format',
@@ -156,8 +156,7 @@ export function buildLocalArgs(engine, ctx) {
156
156
  return [
157
157
  '--model',
158
158
  ctx.modelName,
159
- '--language',
160
- lang,
159
+ ...(lang === 'auto' ? [] : ['--language', lang]),
161
160
  '--output_format',
162
161
  'srt',
163
162
  '--output_dir',