dsh-pf-stream 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +53 -28
- package/README.md +55 -30
- package/client.js +115 -10
- package/lib/asr/catalog.js +3 -4
- package/lib/asr/catalog.js.map +1 -1
- package/lib/asr/install.js +79 -54
- package/lib/asr/install.js.map +1 -1
- package/lib/asr/verify.js +12 -3
- package/lib/asr/verify.js.map +1 -1
- package/lib/audio/asrProviders.js +11 -2
- package/lib/audio/asrProviders.js.map +1 -1
- package/lib/audio/asrRouter.d.ts +0 -2
- package/lib/audio/asrRouter.js +17 -27
- package/lib/audio/asrRouter.js.map +1 -1
- package/lib/audio/subtitleExtractor.d.ts +5 -0
- package/lib/audio/subtitleExtractor.js +35 -5
- package/lib/audio/subtitleExtractor.js.map +1 -1
- package/lib/audio/transcriptCache.d.ts +8 -0
- package/lib/audio/transcriptCache.js +19 -1
- package/lib/audio/transcriptCache.js.map +1 -1
- package/lib/entry/adapter.js +7 -0
- package/lib/entry/adapter.js.map +1 -1
- package/lib/entry/legacyNormalizer.js +12 -5
- package/lib/entry/legacyNormalizer.js.map +1 -1
- package/lib/host/api.js +2 -1
- package/lib/host/api.js.map +1 -1
- package/lib/host/runtime.js +7 -2
- package/lib/host/runtime.js.map +1 -1
- package/lib/session/state.js +0 -3
- package/lib/session/state.js.map +1 -1
- package/lib/stage0/decoder.d.ts +4 -2
- package/lib/stage0/decoder.js +10 -5
- package/lib/stage0/decoder.js.map +1 -1
- package/lib/stage0/ffmpeg.js +19 -5
- package/lib/stage0/ffmpeg.js.map +1 -1
- package/lib/stage0/probe.d.ts +5 -0
- package/lib/stage0/probe.js +49 -34
- package/lib/stage0/probe.js.map +1 -1
- package/lib/stage2/switchDetector.d.ts +1 -2
- package/lib/stage2/switchDetector.js +1 -2
- package/lib/stage2/switchDetector.js.map +1 -1
- package/lib/stage3/deterministicRouter.js +1 -0
- package/lib/stage3/deterministicRouter.js.map +1 -1
- package/lib/stage3/streamSampler.js +28 -20
- package/lib/stage3/streamSampler.js.map +1 -1
- package/lib/stage4/viewport.js +18 -7
- package/lib/stage4/viewport.js.map +1 -1
- package/lib/stage5/controller.js +8 -0
- package/lib/stage5/controller.js.map +1 -1
- package/lib/tool/handler.js +16 -1
- package/lib/tool/handler.js.map +1 -1
- package/lib/tool/schema.js +3 -1
- package/lib/tool/schema.js.map +1 -1
- package/lib/types.d.ts +4 -1
- package/lib/utils/pts.d.ts +5 -1
- package/lib/utils/pts.js +13 -1
- package/lib/utils/pts.js.map +1 -1
- package/package.json +8 -2
package/README.en.md
CHANGED
|
@@ -1,8 +1,31 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/icon.svg" width="72" height="72" alt="Video understanding">
|
|
3
|
+
</p>
|
|
2
4
|
|
|
3
|
-
|
|
5
|
+
<h1 align="center">Video understanding</h1>
|
|
4
6
|
|
|
5
|
-
|
|
7
|
+
<p align="center">Frames, motion and speech from a video, aligned on a timeline for the agent.</p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://www.npmjs.com/package/dsh-pf-stream"><img src="https://img.shields.io/npm/v/dsh-pf-stream?color=2f6feb" alt="npm version"></a>
|
|
11
|
+
<img src="https://img.shields.io/badge/license-Apache--2.0-2ea44f" alt="license">
|
|
12
|
+
<img src="https://img.shields.io/badge/DSH-plugin-6e56cf" alt="DSH plugin">
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
<p align="center"><a href="README.md">中文</a> · <b>English</b></p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Quick start
|
|
20
|
+
|
|
21
|
+
| Step | Do |
|
|
22
|
+
| :--- | :--- |
|
|
23
|
+
| 1 | Install ffmpeg ([commands below](#1-install-ffmpeg)); `ffmpeg -version` must print output |
|
|
24
|
+
| 2 | Left sidebar **Plugins** → **Video understanding** → switch it on |
|
|
25
|
+
| 3 | Click **Download local model** (~190 MB), then **Test transcription** |
|
|
26
|
+
| 4 | Give the agent a video path and ask |
|
|
27
|
+
|
|
28
|
+
Video works without transcription. Only videos that have no subtitles and need speech text require step 3.
|
|
6
29
|
|
|
7
30
|
## What it does
|
|
8
31
|
|
|
@@ -24,12 +47,12 @@ ffprobe -version
|
|
|
24
47
|
Both must print output. If not:
|
|
25
48
|
|
|
26
49
|
| OS | Command |
|
|
27
|
-
|
|
|
50
|
+
| :--- | :--- |
|
|
28
51
|
| Windows | `winget install Gyan.FFmpeg` |
|
|
29
52
|
| macOS | `brew install ffmpeg` |
|
|
30
53
|
| Ubuntu / Debian | `sudo apt install ffmpeg` |
|
|
31
54
|
|
|
32
|
-
Open a new terminal and check again. To keep it out of PATH, set `ffmpegPath` and `ffprobePath` in the plugin config.
|
|
55
|
+
> Open a new terminal and check again. To keep it out of PATH, set `ffmpegPath` and `ffprobePath` in the plugin config.
|
|
33
56
|
|
|
34
57
|
### 2. Install the plugin in DSH
|
|
35
58
|
|
|
@@ -56,30 +79,32 @@ The transcription engine decides whether videos without subtitles get speech tex
|
|
|
56
79
|
|
|
57
80
|
To open the settings: **Plugins in the left sidebar → Video understanding → the panel below the description**. Use the same path later to change settings.
|
|
58
81
|
|
|
82
|
+
<p><img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/docs/assets/panel.png" width="720" alt="Transcription settings panel on the plugin detail page: decoder, local transcription and cloud transcription cards"></p>
|
|
83
|
+
|
|
59
84
|
Three cards at the top report video decode (ffmpeg), local transcription, and cloud transcription. Pick one path:
|
|
60
85
|
|
|
61
86
|
| Path | Best for | What to do |
|
|
62
|
-
|
|
|
87
|
+
| :--- | :--- | :--- |
|
|
63
88
|
| Local (recommended) | Free, offline, no API key | Click "Download local model" (~190 MB), then "Test transcription" |
|
|
64
89
|
| Cloud | You have a key, or the machine is slow | Pick OpenAI / Groq / a custom compatible endpoint, paste the key, save, then test |
|
|
65
90
|
| Skip for now | Videos have subtitles | Use it as is; speech-only videos get no transcript |
|
|
66
91
|
|
|
67
|
-
Local model
|
|
92
|
+
### Local model
|
|
68
93
|
|
|
69
94
|
1. Click "Download local model". The progress bar shows bytes received; cancel at any time.
|
|
70
95
|
2. Clicking again after an interruption resumes the download.
|
|
71
96
|
3. When done, the card reads "Ready · whisper.cpp".
|
|
72
97
|
4. Click "Test transcription". "Local engine available" means it works.
|
|
73
98
|
|
|
74
|
-
|
|
99
|
+
One-click install currently supports Windows x64. On other systems, install whisper.cpp and set `audio.asr.local.binPath` in the config file, or use cloud transcription.
|
|
100
|
+
|
|
101
|
+
### Cloud key
|
|
75
102
|
|
|
76
103
|
1. Choose a provider under "Cloud transcription key".
|
|
77
104
|
2. Paste the API key and click "Save". After saving, only "Configured" is shown; the key is never echoed back.
|
|
78
105
|
3. Click "Test transcription" to verify the key and network.
|
|
79
106
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
One-click install currently supports Windows x64. On other systems, install whisper.cpp and set `audio.asr.local.binPath` in the config file, or use cloud transcription.
|
|
107
|
+
> **Auto policy**: cloud is used when key, endpoint and model are all set (faster, billed per use); otherwise the local engine. The badge next to the panel title shows the engine in use, such as "Transcription ready · Cloud". Under "Advanced" you can switch to local only, cloud only, or off.
|
|
83
108
|
|
|
84
109
|
## Use it
|
|
85
110
|
|
|
@@ -102,7 +127,7 @@ The text block is structured metadata: each canvas packet reports time, form, an
|
|
|
102
127
|
Four ways to look, by question:
|
|
103
128
|
|
|
104
129
|
| Goal | Ask |
|
|
105
|
-
|
|
|
130
|
+
| :--- | :--- |
|
|
106
131
|
| Overview first | "Look at this video" — streaming sampling; motion as process, settle as detail |
|
|
107
132
|
| One exact moment | "What is written at 12.9s" — deterministic high-resolution frame |
|
|
108
133
|
| Even coverage | "One frame every 5 seconds" — uniform sampling |
|
|
@@ -112,34 +137,32 @@ Four ways to look, by question:
|
|
|
112
137
|
|
|
113
138
|
### Dual-scale streaming perception
|
|
114
139
|
|
|
115
|
-
|
|
140
|
+
Sampling is event-driven rather than fixed-rate. During motion, low-resolution process units (P) record state transitions; at settle, high-resolution freeze units (F) preserve detail. Both are delivered within a single canvas packet, so one observation carries the temporal process and the steady-state result together, and the model need not re-watch a clip to recover a detail.
|
|
116
141
|
|
|
117
|
-
|
|
142
|
+
Density adapts to picture activity: rapid change lowers the sampling rate to suppress redundant frames, while slow change extends the wait until a settle is detected; long static intervals maintain a temporal reference through heartbeats without retransmitting images.
|
|
118
143
|
|
|
119
144
|
### Canvas packing and metering
|
|
120
145
|
|
|
121
|
-
|
|
146
|
+
Each frame is partitioned on a 42 px superpixel grid; P or F units are selected as needed and assembled into a canvas. A call may carry multiple packets, and each packet contains only the units new relative to the previous one; historical packets collapse into content-hash references, keeping context bounded in long sessions. Metering follows canvas grid cells (`r × (c + 1) + 2`).
|
|
122
147
|
|
|
123
148
|
### Deterministic inspection
|
|
124
149
|
|
|
125
|
-
|
|
150
|
+
Beyond streaming, four deterministic paths are provided: `probe` parses container metadata only; `timestamps` extracts exact frames at millisecond timestamps; `uniform` samples at a fixed interval; `roi_hint` crops and magnifies a normalized region. None depends on motion detection; outputs are reproducible and suited to verifying text, numerals, and small objects.
|
|
126
151
|
|
|
127
|
-
###
|
|
152
|
+
### Cascaded speech sources
|
|
128
153
|
|
|
129
|
-
|
|
130
|
-
2. **Acoustic check**: without subtitles, silence detection and a speech-ratio estimate run first; an all-silent video skips transcription.
|
|
131
|
-
3. **Transcription**: local whisper.cpp, or OpenAI / Groq / a custom compatible endpoint. Segments are mounted on packets by PTS and stitched across block boundaries with a ±300 ms tolerance.
|
|
154
|
+
Speech text is obtained through a three-stage cascade: a sidecar `.srt` / `.vtt` is read first, then embedded soft subtitles; without subtitles, silence detection and a speech-ratio estimate run, and an all-silent video skips transcription; finally ASR runs, either local whisper.cpp or OpenAI / Groq / a custom compatible endpoint. Segments are aligned by PTS and mounted on the corresponding packets, stitched across block boundaries with a ±300 ms tolerance.
|
|
132
155
|
|
|
133
|
-
The default is
|
|
156
|
+
The default policy is vision-first with background transcription: once a call exceeds the soft deadline (`graceMs`, 8 s by default), the visual result returns and transcription continues in the background; later calls on the same video hit the disk cache.
|
|
134
157
|
|
|
135
|
-
###
|
|
158
|
+
### Cold regions in long videos
|
|
136
159
|
|
|
137
|
-
|
|
160
|
+
Historical packets outside the current window collapse into references (anchor hash, time, and transcript text) rather than retransmitting images; a timestamp call restores the detail when needed.
|
|
138
161
|
|
|
139
162
|
## Troubleshooting
|
|
140
163
|
|
|
141
164
|
| Symptom | Read |
|
|
142
|
-
|
|
|
165
|
+
| :--- | :--- |
|
|
143
166
|
| ffmpeg not found | [ffmpeg](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#找不到-ffmpeg) |
|
|
144
167
|
| Speech but no transcript | [No transcript](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#有声音但没有台词) |
|
|
145
168
|
| Model download fails or is slow | [Download](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#下载失败) |
|
|
@@ -148,10 +171,12 @@ Canvas packets outside the window collapse into references (anchor hash, time, a
|
|
|
148
171
|
|
|
149
172
|
## Docs
|
|
150
173
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
174
|
+
| Document | Contents |
|
|
175
|
+
| :--- | :--- |
|
|
176
|
+
| [Usage Guide](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/usage-guide.md) | Usage, parameters, output, metering, every config key |
|
|
177
|
+
| [使用指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/使用指南.md) | Chinese usage guide |
|
|
178
|
+
| [排障](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md) | Troubleshooting by symptom (Chinese) |
|
|
179
|
+
| [开发者指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/开发者指南.md) | Architecture, host contracts, tests, release (Chinese) |
|
|
155
180
|
|
|
156
181
|
## Development
|
|
157
182
|
|
package/README.md
CHANGED
|
@@ -1,18 +1,41 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/icon.svg" width="72" height="72" alt="视频理解">
|
|
3
|
+
</p>
|
|
2
4
|
|
|
3
|
-
|
|
5
|
+
<h1 align="center">视频理解</h1>
|
|
4
6
|
|
|
5
|
-
|
|
7
|
+
<p align="center">让智能体看懂视频:画面、动作与台词,按时间对齐。</p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://www.npmjs.com/package/dsh-pf-stream"><img src="https://img.shields.io/npm/v/dsh-pf-stream?color=2f6feb" alt="npm 版本"></a>
|
|
11
|
+
<img src="https://img.shields.io/badge/license-Apache--2.0-2ea44f" alt="许可证">
|
|
12
|
+
<img src="https://img.shields.io/badge/DSH-plugin-6e56cf" alt="DSH 插件">
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
<p align="center"><b>中文</b> · <a href="README.en.md">English</a></p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## 30 秒上手
|
|
20
|
+
|
|
21
|
+
| 步骤 | 做什么 |
|
|
22
|
+
| :--- | :--- |
|
|
23
|
+
| 1 | 安装 ffmpeg([命令见下](#1-安装-ffmpeg)),确认 `ffmpeg -version` 有输出 |
|
|
24
|
+
| 2 | 左侧栏「插件」→ 点「视频理解」→ 打开右上角开关 |
|
|
25
|
+
| 3 | 详情页点「下载本地模型」,约 190 MB;完成后点「测试转写」 |
|
|
26
|
+
| 4 | 在对话里给视频路径,直接提问 |
|
|
27
|
+
|
|
28
|
+
不配置转写也能看画面。只有「没有字幕、又需要台词」的视频,才需要第 3 步。
|
|
6
29
|
|
|
7
30
|
## 能做什么
|
|
8
31
|
|
|
9
32
|
- **看画面**:画面运动时记录过程,停稳时定格细节;指定时间点可以高清复核。
|
|
10
33
|
- **听声音**:有字幕就用字幕;没有字幕时,配置转写引擎后生成台词,并按时间挂到对应画面。
|
|
11
|
-
-
|
|
34
|
+
- **按需配置**:默认不转写也能处理画面;配置后可以处理带音轨的视频。
|
|
12
35
|
|
|
13
36
|
## 安装
|
|
14
37
|
|
|
15
|
-
###
|
|
38
|
+
### 1. 安装 ffmpeg
|
|
16
39
|
|
|
17
40
|
插件用 ffmpeg 解码视频。先确认它能运行:
|
|
18
41
|
|
|
@@ -24,14 +47,14 @@ ffprobe -version
|
|
|
24
47
|
两条都要有输出。没有的话:
|
|
25
48
|
|
|
26
49
|
| 系统 | 安装命令 |
|
|
27
|
-
|
|
|
50
|
+
| :--- | :--- |
|
|
28
51
|
| Windows | `winget install Gyan.FFmpeg` |
|
|
29
52
|
| macOS | `brew install ffmpeg` |
|
|
30
53
|
| Ubuntu / Debian | `sudo apt install ffmpeg` |
|
|
31
54
|
|
|
32
|
-
装完新开一个终端再验证。不想放进 PATH,可以在插件配置里写 `ffmpegPath` 与 `ffprobePath` 的绝对路径。
|
|
55
|
+
> 装完新开一个终端再验证。不想放进 PATH,可以在插件配置里写 `ffmpegPath` 与 `ffprobePath` 的绝对路径。
|
|
33
56
|
|
|
34
|
-
###
|
|
57
|
+
### 2. 在 DSH 里安装插件
|
|
35
58
|
|
|
36
59
|
1. 打开 DSH,点左侧栏的「插件」。
|
|
37
60
|
2. 在列表里找「视频理解」(包名 `dsh-pf-stream`)。
|
|
@@ -56,30 +79,32 @@ ffprobe -version
|
|
|
56
79
|
|
|
57
80
|
进入配置页:**左侧栏「插件」→ 点「视频理解」→ 详情页描述下方的面板**。以后要改设置,走同一条路。
|
|
58
81
|
|
|
82
|
+
<p><img src="https://raw.githubusercontent.com/Weilv-D/dsh-pf-stream/main/docs/assets/panel.png" width="720" alt="插件详情页的转写设置面板:视频解码、本地转写与云端转写三张状态卡"></p>
|
|
83
|
+
|
|
59
84
|
面板顶部有三张状态卡:视频解码(ffmpeg)、本地转写、云端转写。按需要选一条路线:
|
|
60
85
|
|
|
61
86
|
| 路线 | 适合 | 操作 |
|
|
62
|
-
|
|
|
87
|
+
| :--- | :--- | :--- |
|
|
63
88
|
| 本地转写(推荐) | 免费、离线、不填密钥 | 点「下载本地模型」,约 190 MB;下载完点「测试转写」 |
|
|
64
89
|
| 云端转写 | 已有 API Key,或机器性能较弱 | 选 OpenAI / Groq / 自定义兼容端点,填 Key,保存后点「测试转写」 |
|
|
65
90
|
| 暂不配置 | 视频都有字幕 | 直接使用;无字幕的语音视频不会生成转写文本 |
|
|
66
91
|
|
|
67
|
-
|
|
92
|
+
### 本地模型
|
|
68
93
|
|
|
69
94
|
1. 点「下载本地模型」。进度条显示已下载大小,可随时取消。
|
|
70
95
|
2. 下载中断后再次点击会续传,不用从头开始。
|
|
71
96
|
3. 完成后状态卡显示「已就绪 · whisper.cpp」。
|
|
72
97
|
4. 点「测试转写」。返回「本地引擎可用」即配置成功。
|
|
73
98
|
|
|
74
|
-
|
|
99
|
+
一键安装当前支持 Windows x64。其他系统请先安装 whisper.cpp,在配置文件 `audio.asr.local.binPath` 填写可执行文件路径,或改用云端转写。
|
|
100
|
+
|
|
101
|
+
### 云端密钥
|
|
75
102
|
|
|
76
103
|
1. 在「云端转写密钥」里选服务商。
|
|
77
104
|
2. 粘贴 API Key,点「保存」。保存后只显示「已配置」,不再回显密钥。
|
|
78
105
|
3. 点「测试转写」验证密钥与网络。
|
|
79
106
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
一键安装当前支持 Windows x64。其他系统请先安装 whisper.cpp,在配置文件 `audio.asr.local.binPath` 填写可执行文件路径,或改用云端转写。
|
|
107
|
+
> **自动策略**:云端密钥、端点、模型都齐时用云端(更快,按量计费);否则用本地引擎。面板标题旁会显示当前实际使用的引擎,如「转写可用 · 云端」。可以在「高级」里改成仅本地、仅云端或不转写。
|
|
83
108
|
|
|
84
109
|
## 开始使用
|
|
85
110
|
|
|
@@ -102,7 +127,7 @@ Audio @Packet #42: [12.1s-12.9s] "这件事我们准备了三年" | [13.0s-13.4s
|
|
|
102
127
|
常用问法对应四种取帧方式:
|
|
103
128
|
|
|
104
129
|
| 想要 | 说法 |
|
|
105
|
-
|
|
|
130
|
+
| :--- | :--- |
|
|
106
131
|
| 先看整体 | 「看看这个视频」——流式采样,运动记过程、停稳给细节 |
|
|
107
132
|
| 只看某一秒 | 「第 12.9 秒写了什么」——定点高清帧 |
|
|
108
133
|
| 均匀看一遍 | 「每 5 秒抽一帧」——等间隔抽帧 |
|
|
@@ -112,34 +137,32 @@ Audio @Packet #42: [12.1s-12.9s] "这件事我们准备了三年" | [13.0s-13.4s
|
|
|
112
137
|
|
|
113
138
|
### 双尺度流式感知
|
|
114
139
|
|
|
115
|
-
|
|
140
|
+
采样过程不再采用固定帧率,而由画面变化事件驱动:运动阶段以低分辨率过程单元(P)记录状态转移,停稳后以高分辨率定格单元(F)保留细节;两类单元在同一画布包内交付,使单次观测同时包含时序过程与稳态结果,模型无需为恢复某一细节而回看整段视频。
|
|
116
141
|
|
|
117
|
-
|
|
142
|
+
采样密度随画面活动度自适应调整:快速变化时降低采样率以抑制冗余帧,缓慢变化时延长等待直至检测到停稳;长静止区间以心跳信号维持时间基准,不重复传输图像。
|
|
118
143
|
|
|
119
144
|
### 画布打包与计费
|
|
120
145
|
|
|
121
|
-
|
|
146
|
+
每帧按 42 px 超像素网格切分为单元,按需选取 P 或 F 单元并拼装为画布。单次调用可携带多个画布包;每个包仅包含相对上一包新增的单元,历史包以内容哈希引用折叠,使长会话中的上下文规模保持有界。计费按画布网格单元数计算(`r × (c + 1) + 2`)。
|
|
122
147
|
|
|
123
148
|
### 确定性检视
|
|
124
149
|
|
|
125
|
-
|
|
150
|
+
流式采样之外另设四条确定性路径:`probe` 仅解析容器元数据;`timestamps` 按毫秒级时间戳抽取精确帧;`uniform` 以固定间隔均匀采样;`roi_hint` 对指定归一化区域裁剪放大。上述路径不依赖运动检测,输出可复现,适用于文字、数字与小目标的核对。
|
|
126
151
|
|
|
127
|
-
###
|
|
152
|
+
### 台词的级联来源
|
|
128
153
|
|
|
129
|
-
|
|
130
|
-
2. **声学判断**:没有字幕时先做静音检测与语音占比估计;全程静音的视频直接跳过转写。
|
|
131
|
-
3. **语音转写**:本地 whisper.cpp,或 OpenAI / Groq / 自定义兼容端点。转写结果按 PTS 挂到对应画布包,跨块边界按 ±300ms 缝合。
|
|
154
|
+
转写文本按三级级联获取:优先读取同名外挂 `.srt` / `.vtt`,其次解析内嵌软字幕;无字幕时执行静音检测与语音占比估计,全静音视频直接跳过转写;最后调用语音识别,可选本地 whisper.cpp 或 OpenAI / Groq / 自定义兼容端点。识别结果按 PTS 对齐并挂载至相应画布包,跨块边界以 ±300 ms 容差缝合。
|
|
132
155
|
|
|
133
|
-
|
|
156
|
+
默认策略为「视觉先行、转写后台补齐」:首次调用超过软截止(`graceMs`,默认 8 s)时先行返回视觉结果,转写任务在后台继续;同一视频的后续调用直接命中磁盘缓存。
|
|
134
157
|
|
|
135
158
|
### 长视频的冷区
|
|
136
159
|
|
|
137
|
-
|
|
160
|
+
超出当前窗口的历史画布包折叠为引用(锚点哈希、时间与台词文本),不再重复传输图像;需要复核时按时间戳重新取帧。
|
|
138
161
|
|
|
139
162
|
## 遇到问题
|
|
140
163
|
|
|
141
164
|
| 现象 | 先看这里 |
|
|
142
|
-
|
|
|
165
|
+
| :--- | :--- |
|
|
143
166
|
| 提示找不到 ffmpeg | [排障:ffmpeg](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#找不到-ffmpeg) |
|
|
144
167
|
| 有声音,但没有台词 | [排障:没有台词](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#有声音但没有台词) |
|
|
145
168
|
| 下载模型失败或很慢 | [排障:下载失败](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md#下载失败) |
|
|
@@ -148,10 +171,12 @@ Audio @Packet #42: [12.1s-12.9s] "这件事我们准备了三年" | [13.0s-13.4s
|
|
|
148
171
|
|
|
149
172
|
## 文档
|
|
150
173
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
174
|
+
| 文档 | 内容 |
|
|
175
|
+
| :--- | :--- |
|
|
176
|
+
| [使用指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/使用指南.md) | 用法、参数、返回内容、计费、全部配置项 |
|
|
177
|
+
| [排障](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/排障.md) | 按现象查原因 |
|
|
178
|
+
| [开发者指南](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/开发者指南.md) | 架构、宿主契约、测试与发布 |
|
|
179
|
+
| [Usage Guide](https://github.com/Weilv-D/dsh-pf-stream/blob/main/docs/usage-guide.md) | 英文使用指南 |
|
|
155
180
|
|
|
156
181
|
## 本地开发
|
|
157
182
|
|
package/client.js
CHANGED
|
@@ -23,6 +23,8 @@ window.__ModuleLoader__.load({
|
|
|
23
23
|
panelSubtitle:
|
|
24
24
|
'以下设置用于启用语音识别与转写。未配置时不影响视频播放,但不会生成转写文本;完成配置后,可对含有效音轨的视频进行转写。',
|
|
25
25
|
refresh: '刷新',
|
|
26
|
+
github: '在 GitHub 上点个 Star',
|
|
27
|
+
githubAria: '在 GitHub 上打开项目仓库',
|
|
26
28
|
checking: '正在读取状态…',
|
|
27
29
|
ffmpeg: '视频解码',
|
|
28
30
|
ffmpegOk: '已就绪',
|
|
@@ -89,6 +91,17 @@ window.__ModuleLoader__.load({
|
|
|
89
91
|
loadFailed: '读取状态失败',
|
|
90
92
|
installStarted: '下载已开始',
|
|
91
93
|
waitDownload: '下载在后台进行,画面处理照常。',
|
|
94
|
+
engineLegacy: '旧版 provider 直连(0.2.x 配置)',
|
|
95
|
+
presetCustom: '自定义 OpenAI 兼容端点',
|
|
96
|
+
errInvalidField: '字段值不合法,请检查输入',
|
|
97
|
+
errJsonRequired: '请求必须是 JSON 格式',
|
|
98
|
+
errCrossOrigin: '跨站请求被拒绝',
|
|
99
|
+
errCredentialRejected: '凭据服务拒绝',
|
|
100
|
+
errKeyTooLong: '密钥过长',
|
|
101
|
+
errUnknownPreset: '未知服务商',
|
|
102
|
+
errNotFound: '接口不存在',
|
|
103
|
+
errMethodNotAllowed: '请求方法不被允许',
|
|
104
|
+
errInternal: '服务内部错误',
|
|
92
105
|
};
|
|
93
106
|
|
|
94
107
|
const EN = {
|
|
@@ -96,6 +109,8 @@ window.__ModuleLoader__.load({
|
|
|
96
109
|
panelSubtitle:
|
|
97
110
|
'These settings enable speech recognition and transcription. Without them, video playback is unaffected, but no transcript is generated. Once configured, videos with a usable audio track can be transcribed.',
|
|
98
111
|
refresh: 'Refresh',
|
|
112
|
+
github: 'Star this project on GitHub',
|
|
113
|
+
githubAria: 'Open the project repository on GitHub',
|
|
99
114
|
checking: 'Reading status…',
|
|
100
115
|
ffmpeg: 'Video decoding',
|
|
101
116
|
ffmpegOk: 'Ready',
|
|
@@ -162,6 +177,17 @@ window.__ModuleLoader__.load({
|
|
|
162
177
|
loadFailed: 'Failed to read status',
|
|
163
178
|
installStarted: 'Download started',
|
|
164
179
|
waitDownload: 'The download runs in the background; video processing continues.',
|
|
180
|
+
engineLegacy: 'Legacy provider passthrough (0.2.x config)',
|
|
181
|
+
presetCustom: 'Custom OpenAI-compatible endpoint',
|
|
182
|
+
errInvalidField: 'Invalid field value; check the input',
|
|
183
|
+
errJsonRequired: 'The request must be JSON',
|
|
184
|
+
errCrossOrigin: 'Cross-origin request rejected',
|
|
185
|
+
errCredentialRejected: 'Credential store rejected the request',
|
|
186
|
+
errKeyTooLong: 'The key is too long',
|
|
187
|
+
errUnknownPreset: 'Unknown provider',
|
|
188
|
+
errNotFound: 'Endpoint not found',
|
|
189
|
+
errMethodNotAllowed: 'HTTP method not allowed',
|
|
190
|
+
errInternal: 'Internal server error',
|
|
165
191
|
};
|
|
166
192
|
|
|
167
193
|
function detectLang() {
|
|
@@ -172,6 +198,29 @@ window.__ModuleLoader__.load({
|
|
|
172
198
|
}
|
|
173
199
|
}
|
|
174
200
|
|
|
201
|
+
/** 服务端错误码 → 词典键(call() 抛出的 error.code;未知码回退原始消息)。 */
|
|
202
|
+
const ERROR_KEYS = {
|
|
203
|
+
invalid_field: 'errInvalidField',
|
|
204
|
+
json_required: 'errJsonRequired',
|
|
205
|
+
cross_origin: 'errCrossOrigin',
|
|
206
|
+
credential_rejected: 'errCredentialRejected',
|
|
207
|
+
key_too_long: 'errKeyTooLong',
|
|
208
|
+
unknown_preset: 'errUnknownPreset',
|
|
209
|
+
not_found: 'errNotFound',
|
|
210
|
+
method_not_allowed: 'errMethodNotAllowed',
|
|
211
|
+
internal_error: 'errInternal',
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
function translateError(t, cause) {
|
|
215
|
+
const code = cause && cause.code;
|
|
216
|
+
const key = ERROR_KEYS[code];
|
|
217
|
+
if (!key) return cause instanceof Error ? cause.message : String(cause);
|
|
218
|
+
const prefix = t(key);
|
|
219
|
+
// credential_rejected 服务端附带原因:拼在翻译后
|
|
220
|
+
const detail = cause instanceof Error ? cause.message : String(cause);
|
|
221
|
+
return code === 'credential_rejected' && detail ? prefix + ':' + detail : prefix;
|
|
222
|
+
}
|
|
223
|
+
|
|
175
224
|
function makeTranslator(locale) {
|
|
176
225
|
const lang = detectLang();
|
|
177
226
|
const own = lang === 'zh' ? ZH : EN;
|
|
@@ -298,6 +347,21 @@ window.__ModuleLoader__.load({
|
|
|
298
347
|
lineHeight: 1.2,
|
|
299
348
|
whiteSpace: 'nowrap',
|
|
300
349
|
},
|
|
350
|
+
iconButton: {
|
|
351
|
+
appearance: 'none',
|
|
352
|
+
border: '.5px solid ' + PALETTE.hairline,
|
|
353
|
+
background: PALETTE.glassStrong,
|
|
354
|
+
color: PALETTE.sub,
|
|
355
|
+
borderRadius: 9,
|
|
356
|
+
width: 30,
|
|
357
|
+
height: 30,
|
|
358
|
+
display: 'inline-flex',
|
|
359
|
+
alignItems: 'center',
|
|
360
|
+
justifyContent: 'center',
|
|
361
|
+
cursor: 'pointer',
|
|
362
|
+
textDecoration: 'none',
|
|
363
|
+
flex: 'none',
|
|
364
|
+
},
|
|
301
365
|
primary: {
|
|
302
366
|
appearance: 'none',
|
|
303
367
|
border: '.5px solid color-mix(in srgb, var(--dsw-alias-brand-primary) 45%, transparent)',
|
|
@@ -610,9 +674,13 @@ window.__ModuleLoader__.load({
|
|
|
610
674
|
function useStatus() {
|
|
611
675
|
const [status, setStatus] = React.useState(null);
|
|
612
676
|
const [error, setError] = React.useState('');
|
|
677
|
+
// 安装相位经 ref 传递:tick 闭包不能读 state(闭包捕获的是挂载帧的 null,
|
|
678
|
+
// 曾使安装期间的 1s 快刷永不生效)
|
|
679
|
+
const installPhaseRef = React.useRef(null);
|
|
613
680
|
const load = React.useCallback(async function () {
|
|
614
681
|
try {
|
|
615
682
|
const data = await call('/status');
|
|
683
|
+
installPhaseRef.current = data && data.install ? data.install.phase : null;
|
|
616
684
|
setStatus(data);
|
|
617
685
|
setError('');
|
|
618
686
|
} catch (cause) {
|
|
@@ -626,8 +694,8 @@ window.__ModuleLoader__.load({
|
|
|
626
694
|
if (!alive) return;
|
|
627
695
|
await load();
|
|
628
696
|
if (!alive) return;
|
|
629
|
-
const
|
|
630
|
-
|
|
697
|
+
const phase = installPhaseRef.current;
|
|
698
|
+
const busy = phase && ['preparing', 'downloading', 'verifying', 'extracting'].indexOf(phase) >= 0;
|
|
631
699
|
timer = setTimeout(tick, busy ? 1000 : 6000);
|
|
632
700
|
}
|
|
633
701
|
void tick();
|
|
@@ -654,8 +722,11 @@ window.__ModuleLoader__.load({
|
|
|
654
722
|
|
|
655
723
|
React.useEffect(function () {
|
|
656
724
|
if (!status) return;
|
|
657
|
-
|
|
658
|
-
|
|
725
|
+
// 回填仅覆盖未被聚焦编辑的输入(请求在途时用户已开始输入的场景)
|
|
726
|
+
const active = typeof document !== 'undefined' && document.activeElement;
|
|
727
|
+
const activeField = active && active.getAttribute ? active.getAttribute('data-field') : null;
|
|
728
|
+
if (activeField !== 'baseUrl') setCustomBase(status.cloud.baseUrl || '');
|
|
729
|
+
if (activeField !== 'model') setCustomModel(status.cloud.model || '');
|
|
659
730
|
}, [status && status.cloud && status.cloud.preset]);
|
|
660
731
|
|
|
661
732
|
function run(name, action) {
|
|
@@ -669,7 +740,7 @@ window.__ModuleLoader__.load({
|
|
|
669
740
|
return state.reload();
|
|
670
741
|
})
|
|
671
742
|
.catch(function (cause) {
|
|
672
|
-
setFailure(
|
|
743
|
+
setFailure(translateError(t, cause));
|
|
673
744
|
})
|
|
674
745
|
.then(function () {
|
|
675
746
|
setBusy('');
|
|
@@ -687,12 +758,16 @@ window.__ModuleLoader__.load({
|
|
|
687
758
|
const ffmpeg = status.ffmpeg || {};
|
|
688
759
|
const localReady = status.engine && status.engine.ready;
|
|
689
760
|
const cloudReady = status.cloud && status.cloud.keyConfigured && status.cloud.baseUrl && status.cloud.model;
|
|
690
|
-
|
|
761
|
+
// 徽标口径取服务端权威结论(engine=off / audio.mode=off 时 ready=false),
|
|
762
|
+
// 本地自算会出现「转写可用 · 未启用」的自相矛盾徽标
|
|
763
|
+
const ready = status.ready === true;
|
|
691
764
|
const job = status.install;
|
|
692
765
|
const installing =
|
|
693
766
|
job && ['preparing', 'downloading', 'verifying', 'extracting'].indexOf(job.phase) >= 0;
|
|
694
767
|
const preset = status.cloud.preset;
|
|
695
768
|
const custom = status.cloud.editable;
|
|
769
|
+
const presetInfo = (status.presets || []).find(function (item) { return item.id === preset; });
|
|
770
|
+
const keyUrl = presetInfo ? presetInfo.keyUrl : '';
|
|
696
771
|
const activeKind = status.active
|
|
697
772
|
? status.active.kind
|
|
698
773
|
: status.mode === 'local'
|
|
@@ -712,6 +787,7 @@ window.__ModuleLoader__.load({
|
|
|
712
787
|
['local', t('engineLocal')],
|
|
713
788
|
['cloud', t('engineCloud')],
|
|
714
789
|
['off', t('engineOff')],
|
|
790
|
+
['legacy', t('engineLegacy')],
|
|
715
791
|
];
|
|
716
792
|
|
|
717
793
|
return h(
|
|
@@ -723,6 +799,24 @@ window.__ModuleLoader__.load({
|
|
|
723
799
|
h('h3', { style: styles.title }, t('panelTitle')),
|
|
724
800
|
h('span', { style: styles.badge }, ready ? t('badgeReady') + ' · ' + activeLabel : t('badgePending')),
|
|
725
801
|
h('span', { style: { flex: '1 1 auto' } }),
|
|
802
|
+
h(
|
|
803
|
+
'a',
|
|
804
|
+
{
|
|
805
|
+
href: 'https://github.com/Weilv-D/dsh-pf-stream',
|
|
806
|
+
target: '_blank',
|
|
807
|
+
rel: 'noreferrer',
|
|
808
|
+
title: t('github'),
|
|
809
|
+
'aria-label': t('githubAria'),
|
|
810
|
+
style: styles.iconButton,
|
|
811
|
+
},
|
|
812
|
+
h(
|
|
813
|
+
'svg',
|
|
814
|
+
{ viewBox: '0 0 16 16', width: 16, height: 16, fill: 'currentColor', 'aria-hidden': 'true' },
|
|
815
|
+
h('path', {
|
|
816
|
+
d: 'M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z',
|
|
817
|
+
}),
|
|
818
|
+
),
|
|
819
|
+
),
|
|
726
820
|
h(
|
|
727
821
|
'button',
|
|
728
822
|
{ type: 'button', style: styles.button, onClick: function () { void state.reload(); } },
|
|
@@ -825,9 +919,16 @@ window.__ModuleLoader__.load({
|
|
|
825
919
|
h(SelectField, {
|
|
826
920
|
value: preset,
|
|
827
921
|
options: (status.presets || []).map(function (item) {
|
|
828
|
-
|
|
922
|
+
// custom 的服务端 label 是中文兜底,英文界面走本地词典
|
|
923
|
+
return {
|
|
924
|
+
value: item.id,
|
|
925
|
+
label: item.id === 'custom' ? t('presetCustom') : item.label,
|
|
926
|
+
};
|
|
829
927
|
}),
|
|
830
928
|
onChange: function (value) {
|
|
929
|
+
// 切换服务商时清空未保存的密钥草稿:
|
|
930
|
+
// 密钥按服务商独立存储(keyRef 各自独立),跨服务商误存会双向错乱
|
|
931
|
+
setKeyDraft('');
|
|
831
932
|
void run('preset', function () {
|
|
832
933
|
return call('/config', { field: 'preset', value: value });
|
|
833
934
|
});
|
|
@@ -841,6 +942,7 @@ window.__ModuleLoader__.load({
|
|
|
841
942
|
style: styles.input,
|
|
842
943
|
value: customBase,
|
|
843
944
|
placeholder: 'https://api.example.com',
|
|
945
|
+
'data-field': 'baseUrl',
|
|
844
946
|
onChange: function (event) { setCustomBase(event.target.value); },
|
|
845
947
|
onBlur: function () {
|
|
846
948
|
void run('baseUrl', function () {
|
|
@@ -857,6 +959,7 @@ window.__ModuleLoader__.load({
|
|
|
857
959
|
style: styles.input,
|
|
858
960
|
value: customModel,
|
|
859
961
|
placeholder: 'whisper-1',
|
|
962
|
+
'data-field': 'model',
|
|
860
963
|
onChange: function (event) { setCustomModel(event.target.value); },
|
|
861
964
|
onBlur: function () {
|
|
862
965
|
void run('model', function () {
|
|
@@ -918,9 +1021,9 @@ window.__ModuleLoader__.load({
|
|
|
918
1021
|
},
|
|
919
1022
|
t('clear'),
|
|
920
1023
|
),
|
|
921
|
-
|
|
1024
|
+
keyUrl
|
|
922
1025
|
? h('a', {
|
|
923
|
-
href:
|
|
1026
|
+
href: keyUrl,
|
|
924
1027
|
target: '_blank',
|
|
925
1028
|
rel: 'noreferrer',
|
|
926
1029
|
style: { fontSize: 12, color: PALETTE.brand, textDecoration: 'none', paddingBottom: 8 },
|
|
@@ -961,6 +1064,7 @@ window.__ModuleLoader__.load({
|
|
|
961
1064
|
h('input', {
|
|
962
1065
|
style: styles.input,
|
|
963
1066
|
defaultValue: (status.config && status.config.language) || '',
|
|
1067
|
+
key: 'language-' + ((status.config && status.config.language) || ''),
|
|
964
1068
|
placeholder: t('languageAuto'),
|
|
965
1069
|
onBlur: function (event) {
|
|
966
1070
|
const value = event.target.value;
|
|
@@ -993,7 +1097,8 @@ window.__ModuleLoader__.load({
|
|
|
993
1097
|
style: styles.input,
|
|
994
1098
|
type: 'number',
|
|
995
1099
|
min: 0,
|
|
996
|
-
max:
|
|
1100
|
+
max: 16,
|
|
1101
|
+
key: 'threads-' + ((status.config && status.config.threads) || 0),
|
|
997
1102
|
defaultValue: (status.config && status.config.threads) || 0,
|
|
998
1103
|
onBlur: function (event) {
|
|
999
1104
|
const value = Number(event.target.value || 0);
|
package/lib/asr/catalog.js
CHANGED
|
@@ -139,8 +139,8 @@ export function buildLocalArgs(engine, ctx) {
|
|
|
139
139
|
return [
|
|
140
140
|
'--model',
|
|
141
141
|
ctx.modelName,
|
|
142
|
-
|
|
143
|
-
lang === 'auto' ? '
|
|
142
|
+
// Python 系 CLI 的 --language 不接受 'auto' 字面量:省略即自动检测
|
|
143
|
+
...(lang === 'auto' ? [] : ['--language', lang]),
|
|
144
144
|
'--task',
|
|
145
145
|
'transcribe',
|
|
146
146
|
'--output_format',
|
|
@@ -156,8 +156,7 @@ export function buildLocalArgs(engine, ctx) {
|
|
|
156
156
|
return [
|
|
157
157
|
'--model',
|
|
158
158
|
ctx.modelName,
|
|
159
|
-
'--language',
|
|
160
|
-
lang,
|
|
159
|
+
...(lang === 'auto' ? [] : ['--language', lang]),
|
|
161
160
|
'--output_format',
|
|
162
161
|
'srt',
|
|
163
162
|
'--output_dir',
|