dsh-voice-mode 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +53 -47
- package/README.md +80 -64
- package/assets/architecture.svg +1 -159
- package/assets/demo.gif +0 -0
- package/lib/client.js +720 -90
- package/lib/index.js +1151 -229
- package/lib/tts-vits-worker.cjs +147 -0
- package/package.json +13 -6
- package/scripts/prefetch.mjs +174 -50
package/assets/architecture.svg
CHANGED
|
@@ -1,159 +1 @@
|
|
|
1
|
-
<svg xmlns="http://www.w3.org/2000/svg" width="
|
|
2
|
-
<title id="title">dsh-voice-mode 原理架构图</title>
|
|
3
|
-
<desc id="desc">展示语音输入、模型输出朗读与开口打断三条数据流，以及全局单活指针的会话隔离。</desc>
|
|
4
|
-
<defs>
|
|
5
|
-
<marker id="arrow" markerWidth="8" markerHeight="8" refX="7" refY="4" orient="auto" markerUnits="strokeWidth">
|
|
6
|
-
<path d="M0,0 L8,4 L0,8 Z" fill="#66737f"/>
|
|
7
|
-
</marker>
|
|
8
|
-
<marker id="arrowGreen" markerWidth="8" markerHeight="8" refX="7" refY="4" orient="auto" markerUnits="strokeWidth">
|
|
9
|
-
<path d="M0,0 L8,4 L0,8 Z" fill="#2ea043"/>
|
|
10
|
-
</marker>
|
|
11
|
-
<style>
|
|
12
|
-
text { fill:#333; font-family:"Noto Sans SC","Microsoft YaHei",Arial,sans-serif; }
|
|
13
|
-
.title { font-size:25px; font-weight:700; }
|
|
14
|
-
.subtitle { font-size:12px; fill:#66737f; }
|
|
15
|
-
.lane { fill:#faf9f5; stroke:#dedbd2; stroke-width:1.2; }
|
|
16
|
-
.lane-blue { fill:#f5f8fa; stroke:#d7e0e6; stroke-width:1.2; }
|
|
17
|
-
.lane-green { fill:#f6faf6; stroke:#cfe4d3; stroke-width:1.2; }
|
|
18
|
-
.lane-title { font-size:17px; font-weight:700; }
|
|
19
|
-
.lane-no { font:700 12px ui-monospace,SFMono-Regular,Consolas,monospace; fill:#fff; }
|
|
20
|
-
.node { fill:#fffdf7; stroke:#b8b4a8; stroke-width:1.4; rx:10; }
|
|
21
|
-
.node-blue { fill:#eef4f7; stroke:#9fb3bf; stroke-width:1.4; rx:10; }
|
|
22
|
-
.node-green { fill:#eff8f0; stroke:#2ea043; stroke-width:1.7; rx:10; }
|
|
23
|
-
.node-title { font-size:13px; font-weight:700; text-anchor:middle; }
|
|
24
|
-
.node-detail { font-size:10.5px; fill:#59636c; text-anchor:middle; }
|
|
25
|
-
.mono { font-family:ui-monospace,SFMono-Regular,Consolas,"Liberation Mono",monospace; }
|
|
26
|
-
.flow { fill:none; stroke:#66737f; stroke-width:1.8; marker-end:url(#arrow); }
|
|
27
|
-
.flow-green { fill:none; stroke:#2ea043; stroke-width:2; marker-end:url(#arrowGreen); }
|
|
28
|
-
.label-bg { fill:#fff; stroke:#d8dde1; stroke-width:.8; rx:7; }
|
|
29
|
-
.label { font-size:9.5px; fill:#59636c; text-anchor:middle; }
|
|
30
|
-
.note { fill:#eff8f0; stroke:#2ea043; stroke-width:1.3; rx:10; }
|
|
31
|
-
.note-title { font-size:12px; font-weight:700; fill:#237a34; }
|
|
32
|
-
.note-text { font-size:10.5px; fill:#46534a; }
|
|
33
|
-
.dash { fill:none; stroke:#2ea043; stroke-width:1.2; stroke-dasharray:4 4; }
|
|
34
|
-
</style>
|
|
35
|
-
</defs>
|
|
36
|
-
|
|
37
|
-
<rect width="1200" height="640" fill="#fff"/>
|
|
38
|
-
<text class="title" x="32" y="38">dsh-voice-mode 原理架构</text>
|
|
39
|
-
<text class="subtitle" x="32" y="58">浏览器采集与播放 · 宿主端 ASR / TTS · 无损模型流旁路</text>
|
|
40
|
-
<text class="note-text" x="32" y="75">全局单活指针 activeVoiceSession：同一时刻仅一个会话处于语音模式；普通会话 llm/stream 直达、零开销</text>
|
|
41
|
-
|
|
42
|
-
<rect class="note" x="768" y="16" width="400" height="51"/>
|
|
43
|
-
<text class="note-title" x="785" y="36">全局单活指针 <tspan class="mono">activeVoiceSession</tspan></text>
|
|
44
|
-
<text class="note-text" x="785" y="54">仅指定会话进入语音旁路；普通会话 <tspan class="mono">llm/stream</tspan> 零开销直达</text>
|
|
45
|
-
|
|
46
|
-
<!-- 输入链路 -->
|
|
47
|
-
<rect class="lane" x="20" y="82" width="1160" height="178" rx="14"/>
|
|
48
|
-
<circle cx="47" cy="108" r="14" fill="#2ea043"/><text class="lane-no" x="47" y="112" text-anchor="middle">01</text>
|
|
49
|
-
<text class="lane-title" x="70" y="114">输入链路 · 采集、断句与识别</text>
|
|
50
|
-
|
|
51
|
-
<rect class="node" x="40" y="137" width="155" height="82"/>
|
|
52
|
-
<text class="node-title" x="117.5" y="162">浏览器麦克风</text>
|
|
53
|
-
<text class="node-detail mono" x="117.5" y="181">getUserMedia · 16k mono</text>
|
|
54
|
-
<text class="node-detail mono" x="117.5" y="198">echoCancellation</text>
|
|
55
|
-
|
|
56
|
-
<rect class="node-blue" x="225" y="137" width="175" height="82"/>
|
|
57
|
-
<text class="node-title" x="312.5" y="160">RMS VAD</text>
|
|
58
|
-
<text class="node-detail" x="312.5" y="179">持续聆听 · VAD 自动断句 ·</text>
|
|
59
|
-
<text class="node-detail" x="312.5" y="197"><tspan class="mono">hold</tspan> 模式按住说话绕过 VAD</text>
|
|
60
|
-
|
|
61
|
-
<rect class="node-blue" x="430" y="127" width="205" height="102"/>
|
|
62
|
-
<text class="node-title mono" x="532.5" y="151">POST /voice-mode/asr</text>
|
|
63
|
-
<text class="node-detail mono" x="532.5" y="170">f32 LE PCM</text>
|
|
64
|
-
<text class="node-detail" x="532.5" y="188">宿主 sherpa-onnx Node WASM</text>
|
|
65
|
-
<text class="node-detail mono" x="532.5" y="206">zipformer2 · 流式识别</text>
|
|
66
|
-
<text class="node-detail" x="532.5" y="220">模型懒下载 160MB · <tspan class="mono">.part</tspan> 续传</text>
|
|
67
|
-
|
|
68
|
-
<rect class="node" x="665" y="137" width="145" height="82"/>
|
|
69
|
-
<text class="node-title" x="737.5" y="162">识别结果</text>
|
|
70
|
-
<text class="node-detail" x="737.5" y="181"><tspan class="mono">partial</tspan> 预览字幕</text>
|
|
71
|
-
<text class="node-detail" x="737.5" y="198">定稿</text>
|
|
72
|
-
|
|
73
|
-
<rect class="node" x="840" y="137" width="135" height="82"/>
|
|
74
|
-
<text class="node-title" x="907.5" y="167">composer 草稿</text>
|
|
75
|
-
<text class="node-detail" x="907.5" y="189">写入输入框</text>
|
|
76
|
-
|
|
77
|
-
<rect class="node-green" x="1005" y="137" width="145" height="82"/>
|
|
78
|
-
<text class="node-title" x="1077.5" y="160">自动发送</text>
|
|
79
|
-
<text class="node-detail mono" x="1077.5" y="179">autoSend</text>
|
|
80
|
-
<text class="node-detail" x="1077.5" y="197"><tspan class="mono">Ctrl</tspan> / <tspan class="mono">hold</tspan> 松手强制</text>
|
|
81
|
-
|
|
82
|
-
<path class="flow" d="M195 178 H225"/><path class="flow" d="M400 178 H430"/><path class="flow" d="M635 178 H665"/><path class="flow" d="M810 178 H840"/><path class="flow-green" d="M975 178 H1005"/>
|
|
83
|
-
<rect class="label-bg" x="198" y="151" width="24" height="16"/><text class="label" x="210" y="162">PCM</text>
|
|
84
|
-
<rect class="label-bg" x="402" y="151" width="26" height="16"/><text class="label" x="415" y="162">POST</text>
|
|
85
|
-
<rect class="label-bg" x="638" y="151" width="24" height="16"/><text class="label" x="650" y="162">增量</text>
|
|
86
|
-
<rect class="label-bg" x="812" y="151" width="26" height="16"/><text class="label" x="825" y="162">定稿</text>
|
|
87
|
-
<rect class="label-bg" x="977" y="151" width="26" height="16"/><text class="label" x="990" y="162">发送</text>
|
|
88
|
-
|
|
89
|
-
<!-- 输出链路 -->
|
|
90
|
-
<rect class="lane-blue" x="20" y="276" width="1160" height="178" rx="14"/>
|
|
91
|
-
<circle cx="47" cy="302" r="14" fill="#2ea043"/><text class="lane-no" x="47" y="306" text-anchor="middle">02</text>
|
|
92
|
-
<text class="lane-title" x="70" y="308">输出链路 · 无损旁路、逐句合成与播放</text>
|
|
93
|
-
|
|
94
|
-
<rect class="node" x="40" y="337" width="135" height="76"/>
|
|
95
|
-
<text class="node-title" x="107.5" y="365">model stream</text>
|
|
96
|
-
<text class="node-detail" x="107.5" y="386">模型原始输出</text>
|
|
97
|
-
|
|
98
|
-
<rect class="node-green" x="205" y="327" width="155" height="96"/>
|
|
99
|
-
<text class="node-title mono" x="282.5" y="351">llm/stream tap</text>
|
|
100
|
-
<text class="node-detail" x="282.5" y="371">chunk 原样透传</text>
|
|
101
|
-
<text class="node-detail" x="282.5" y="389">仅单活语音会话旁观</text>
|
|
102
|
-
<text class="node-detail" x="282.5" y="406">不阻塞模型流</text>
|
|
103
|
-
|
|
104
|
-
<rect class="node-blue" x="390" y="337" width="135" height="76"/>
|
|
105
|
-
<text class="node-title mono" x="457.5" y="365">text-delta</text>
|
|
106
|
-
<text class="node-detail" x="457.5" y="386">过滤文本增量</text>
|
|
107
|
-
|
|
108
|
-
<rect class="node-blue" x="555" y="337" width="155" height="76"/>
|
|
109
|
-
<text class="node-title mono" x="632.5" y="361">SentenceSegmenter</text>
|
|
110
|
-
<text class="node-detail" x="632.5" y="384">中文标点切句</text>
|
|
111
|
-
|
|
112
|
-
<rect class="node-blue" x="740" y="327" width="155" height="96"/>
|
|
113
|
-
<text class="node-title mono" x="817.5" y="351">TtsQueue</text>
|
|
114
|
-
<text class="node-detail" x="817.5" y="371"><tspan class="mono">msedge-tts</tspan> 逐句合成</text>
|
|
115
|
-
<text class="node-detail" x="817.5" y="390"><tspan class="mono">epoch</tspan> 版本化</text>
|
|
116
|
-
<text class="node-detail" x="817.5" y="407">按会话隔离</text>
|
|
117
|
-
|
|
118
|
-
<rect class="node-blue" x="925" y="327" width="205" height="96"/>
|
|
119
|
-
<text class="node-title mono" x="1027.5" y="351">SSE /voice-mode/stream</text>
|
|
120
|
-
<text class="node-detail mono" x="1027.5" y="372">event: audio</text>
|
|
121
|
-
<text class="node-detail mono" x="1027.5" y="390">{ text, base64 MP3 }</text>
|
|
122
|
-
<text class="node-detail" x="1027.5" y="407">浏览器播放 + 右下角字幕浮层</text>
|
|
123
|
-
|
|
124
|
-
<path class="flow" d="M175 375 H205"/><path class="flow" d="M360 375 H390"/><path class="flow" d="M525 375 H555"/><path class="flow" d="M710 375 H740"/><path class="flow-green" d="M895 375 H925"/>
|
|
125
|
-
<rect class="label-bg" x="177" y="348" width="26" height="16"/><text class="label" x="190" y="359">无损</text>
|
|
126
|
-
<rect class="label-bg" x="362" y="348" width="26" height="16"/><text class="label" x="375" y="359">旁观</text>
|
|
127
|
-
<rect class="label-bg" x="527" y="348" width="26" height="16"/><text class="label" x="540" y="359">增量</text>
|
|
128
|
-
<rect class="label-bg" x="712" y="348" width="26" height="16"/><text class="label" x="725" y="359">逐句</text>
|
|
129
|
-
<rect class="label-bg" x="897" y="348" width="26" height="16"/><text class="label" x="910" y="359">音频</text>
|
|
130
|
-
<path class="dash" d="M900 67 V92 H282 V327"/>
|
|
131
|
-
|
|
132
|
-
<!-- 打断链路 -->
|
|
133
|
-
<rect class="lane-green" x="20" y="470" width="1160" height="150" rx="14"/>
|
|
134
|
-
<circle cx="47" cy="496" r="14" fill="#2ea043"/><text class="lane-no" x="47" y="500" text-anchor="middle">03</text>
|
|
135
|
-
<text class="lane-title" x="70" y="502">打断 <tspan class="mono">barge-in</tspan> · 即刻静音并取消当前回合</text>
|
|
136
|
-
|
|
137
|
-
<rect class="node-green" x="60" y="529" width="235" height="67"/>
|
|
138
|
-
<text class="node-title" x="177.5" y="553">开口说话</text>
|
|
139
|
-
<text class="node-detail" x="177.5" y="574">高门槛能量 + 持续时长 · 三档灵敏度</text>
|
|
140
|
-
|
|
141
|
-
<rect class="node" x="355" y="529" width="180" height="67"/>
|
|
142
|
-
<text class="node-title" x="445" y="553">本地播放静音</text>
|
|
143
|
-
<text class="node-detail" x="445" y="574">浏览器立即停止旧音频</text>
|
|
144
|
-
|
|
145
|
-
<rect class="node-green" x="595" y="519" width="245" height="87"/>
|
|
146
|
-
<text class="node-title mono" x="717.5" y="543">POST /voice-mode/cancel</text>
|
|
147
|
-
<text class="node-detail mono" x="717.5" y="565">TTS epoch++</text>
|
|
148
|
-
<text class="node-detail" x="717.5" y="584">作废队列与在途句</text>
|
|
149
|
-
|
|
150
|
-
<rect class="node-blue" x="900" y="519" width="240" height="87"/>
|
|
151
|
-
<text class="node-title mono" x="1020" y="546">session.cancel(keepInbox)</text>
|
|
152
|
-
<text class="node-detail" x="1020" y="568">取消运行中回合</text>
|
|
153
|
-
<text class="node-detail" x="1020" y="586">保留新消息</text>
|
|
154
|
-
|
|
155
|
-
<path class="flow-green" d="M295 562 H355"/><path class="flow-green" d="M535 562 H595"/><path class="flow-green" d="M840 562 H900"/>
|
|
156
|
-
<rect class="label-bg" x="307" y="535" width="36" height="16"/><text class="label" x="325" y="546">触发</text>
|
|
157
|
-
<rect class="label-bg" x="547" y="535" width="36" height="16"/><text class="label" x="565" y="546">同时</text>
|
|
158
|
-
<rect class="label-bg" x="852" y="535" width="36" height="16"/><text class="label" x="870" y="546">取消</text>
|
|
159
|
-
</svg>
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" font-family="Segoe UI, system-ui, sans-serif" font-size="14" viewBox="0 0 960 640"><rect width="960" height="640" fill="#0d1117"/><text x="480" y="30" fill="#e6edf3" font-size="20" font-weight="700" text-anchor="middle">dsh-voice-mode — 语音双工数据流</text><rect width="200" height="64" x="30" y="60" fill="#161b22" stroke="#58a6ff" stroke-width="1.5" rx="10"/><text x="130" y="86" fill="#58a6ff" font-weight="700" text-anchor="middle">麦克风 Mic</text><text x="130" y="108" fill="#8b949e" font-size="12" text-anchor="middle">16kHz 单声道 · AEC 回声消除</text><rect width="200" height="76" x="30" y="150" fill="#161b22" stroke="#58a6ff" stroke-width="1.5" rx="10"/><text x="130" y="178" fill="#58a6ff" font-weight="700" text-anchor="middle">浏览器 VAD 分段</text><text x="130" y="198" fill="#8b949e" font-size="12" text-anchor="middle">静音 700ms 断句 / 按住说门控</text><text x="130" y="216" fill="#8b949e" font-size="12" text-anchor="middle">增量传输（每 0.9s 只传新增）</text><rect width="200" height="64" x="30" y="280" fill="#161b22" stroke="#3fb950" stroke-width="1.5" rx="10"/><text x="130" y="306" fill="#3fb950" font-weight="700" text-anchor="middle">草稿 → 自动发送</text><text x="130" y="328" fill="#8b949e" font-size="12" text-anchor="middle">autoSend / 按住说松手发送</text><rect width="200" height="76" x="30" y="410" fill="#161b22" stroke="#3fb950" stroke-width="1.5" rx="10"/><text x="130" y="438" fill="#3fb950" font-weight="700" text-anchor="middle">播放 + 字幕浮层</text><text x="130" y="458" fill="#8b949e" font-size="12" text-anchor="middle">SSE 音频帧（WAV/MP3）</text><text x="130" y="476" fill="#8b949e" font-size="12" text-anchor="middle">逐段顺序 · 世代链防重播</text><rect width="290" height="64" x="330" y="60" fill="#161b22" stroke="#58a6ff" stroke-width="1.5" rx="10"/><text x="475" y="86" fill="#58a6ff" font-weight="700" text-anchor="middle">zipformer2 流式识别</text><text x="475" y="108" fill="#8b949e" font-size="12" text-anchor="middle">host 本地 WASM · 启动预热 · SHA256 校验</text><rect width="290" height="56" x="330" y="150" fill="#161b22" stroke="#d29922" stroke-width="1.5" rx="10"/><text x="475" y="173" fill="#d29922" font-weight="700" text-anchor="middle">SenseVoice 定稿重译</text><text x="475" y="193" fill="#8b949e" font-size="12" text-anchor="middle">带标点 · 数字归一化（中英日韩粤）</text><rect width="290" height="64" x="330" y="232" fill="#161b22" stroke="#a371f7" stroke-width="1.5" rx="10"/><text x="475" y="258" fill="#a371f7" font-weight="700" text-anchor="middle">llm/stream 无损 tap → 分句</text><text x="475" y="280" fill="#8b949e" font-size="12" text-anchor="middle">text-delta 过滤 · markdown 剥离消毒</text><rect width="290" height="64" x="330" y="322" fill="#161b22" stroke="#a371f7" stroke-width="1.5" rx="10"/><text x="475" y="348" fill="#a371f7" font-weight="700" text-anchor="middle">TTS 队列（epoch 打断）</text><text x="475" y="370" fill="#8b949e" font-size="12" text-anchor="middle">打断即杀在途合成 · 积压作废</text><rect width="290" height="96" x="330" y="412" fill="#161b22" stroke="#3fb950" stroke-width="1.5" rx="10"/><text x="475" y="438" fill="#3fb950" font-weight="700" text-anchor="middle">本地 TTS 引擎（子进程 fork）</text><text x="475" y="460" fill="#8b949e" font-size="12" text-anchor="middle">VITS 纯中文（WASM，可选）</text><text x="475" y="478" fill="#8b949e" font-size="12" text-anchor="middle">Kokoro int8 中英（原生，109MB）</text><text x="475" y="496" fill="#8b949e" font-size="12" text-anchor="middle">Edge 云端（默认，322 音色）</text><rect width="240" height="200" x="690" y="60" fill="#161b22" stroke="#30363d" stroke-width="1" rx="10"/><text x="810" y="88" fill="#e6edf3" font-weight="700" text-anchor="middle">安全面（四层）</text><text x="710" y="116" fill="#8b949e" font-size="12">① 会话存在性校验</text><text x="710" y="136" fill="#8b949e" font-size="12">② 回环默认 + Origin 校验</text><text x="710" y="156" fill="#8b949e" font-size="12">③ 全端点限流</text><text x="710" y="176" fill="#8b949e" font-size="12">④ 模型 SHA256 + 域名白名单</text><text x="710" y="208" fill="#8b949e" font-size="12">打断：VAD 帧级检测 + 回声门控</text><text x="710" y="228" fill="#8b949e" font-size="12">空闲：听与说同权</text><text x="710" y="248" fill="#8b949e" font-size="12">模式：持续聆听 ⇄ 按住说</text><defs><marker id="arr" markerHeight="8" markerWidth="8" orient="auto" refX="7" refY="4"><path fill="#8b949e" d="M0,0 L8,4 L0,8 z"/></marker><marker id="arrR" markerHeight="8" markerWidth="8" orient="auto" refX="7" refY="4"><path fill="#f85149" d="M0,0 L8,4 L0,8 z"/></marker></defs><line x1="230" x2="326" y1="92" y2="92" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="230" x2="326" y1="188" y2="178" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="475" x2="475" y1="124" y2="146" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="230" x2="326" y1="312" y2="264" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="475" x2="475" y1="296" y2="318" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="475" x2="475" y1="386" y2="408" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="620" x2="720" y1="460" y2="460" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="720" x2="720" y1="460" y2="446" stroke="#8b949e" stroke-width="1.5"/><line x1="130" x2="130" y1="410" y2="348" stroke="#8b949e" stroke-width="1.5"/><line x1="330" x2="234" y1="348" y2="348" stroke="#8b949e" stroke-width="1.5"/><line x1="234" x2="234" y1="348" y2="406" stroke="#8b949e" stroke-width="1.5" marker-end="url(#arr)"/><line x1="130" x2="130" y1="486" y2="560" stroke="#f85149" stroke-width="1.5" marker-end="url(#arrR)"/><text x="300" y="556" fill="#f85149" font-size="12">说话打断：本地静音 + 队列作废 + 回合取消（二级 1.5s）</text></svg>
|
package/assets/demo.gif
CHANGED
|
Binary file
|