dsh-voice-mode 0.2.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +28 -15
- package/README.md +69 -32
- package/assets/architecture.svg +1 -159
- package/assets/demo.gif +0 -0
- package/lib/client.js +2252 -359
- package/lib/index.js +1899 -245
- package/lib/sense-worker.mjs +128 -0
- package/lib/tts-vits-worker.cjs +147 -0
- package/package.json +18 -7
- package/scripts/bench-asr.mjs +289 -0
- package/scripts/prefetch.mjs +158 -37
- package/scripts/verify.mjs +34 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// src/sense-worker.ts
|
|
2
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
+
function createSenseWorkerClient(worker) {
|
|
4
|
+
let counter = 0;
|
|
5
|
+
const pending = /* @__PURE__ */ new Map();
|
|
6
|
+
let dead = false;
|
|
7
|
+
const deathFns = /* @__PURE__ */ new Set();
|
|
8
|
+
const die = () => {
|
|
9
|
+
if (dead) return;
|
|
10
|
+
dead = true;
|
|
11
|
+
for (const fn of deathFns) {
|
|
12
|
+
try {
|
|
13
|
+
fn();
|
|
14
|
+
} catch {
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
worker.on?.("message", (msg) => {
|
|
19
|
+
const p = pending.get(msg?.id);
|
|
20
|
+
if (!p) return;
|
|
21
|
+
pending.delete(msg.id);
|
|
22
|
+
if (!msg.ok) {
|
|
23
|
+
p.resolve(null);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
p.resolve(p.op === "create" ? true : msg.text ?? "");
|
|
27
|
+
});
|
|
28
|
+
worker.on?.("error", (e) => {
|
|
29
|
+
die();
|
|
30
|
+
const err = new Error("sense worker error: " + String(e?.message ?? e));
|
|
31
|
+
for (const [, p] of pending) p.reject(err);
|
|
32
|
+
pending.clear();
|
|
33
|
+
});
|
|
34
|
+
worker.on?.("exit", () => {
|
|
35
|
+
die();
|
|
36
|
+
const err = new Error("sense worker exited");
|
|
37
|
+
for (const [, p] of pending) p.reject(err);
|
|
38
|
+
pending.clear();
|
|
39
|
+
});
|
|
40
|
+
const request = (op, samples) => {
|
|
41
|
+
if (dead) return Promise.reject(new Error("sense worker dead"));
|
|
42
|
+
const id = counter++;
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
pending.set(id, { op, resolve, reject });
|
|
45
|
+
const msg = { id, op };
|
|
46
|
+
if (samples) msg.samples = samples;
|
|
47
|
+
try {
|
|
48
|
+
worker.postMessage(msg);
|
|
49
|
+
} catch (e) {
|
|
50
|
+
pending.delete(id);
|
|
51
|
+
reject(e instanceof Error ? e : new Error(String(e)));
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
};
|
|
55
|
+
return {
|
|
56
|
+
request,
|
|
57
|
+
onDeath(fn) {
|
|
58
|
+
deathFns.add(fn);
|
|
59
|
+
},
|
|
60
|
+
terminate: async () => {
|
|
61
|
+
dead = true;
|
|
62
|
+
const err = new Error("sense worker terminated");
|
|
63
|
+
for (const [, p] of pending) p.reject(err);
|
|
64
|
+
pending.clear();
|
|
65
|
+
try {
|
|
66
|
+
await worker.terminate?.();
|
|
67
|
+
} catch {
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function startSenseWorker(data) {
|
|
73
|
+
const port = parentPort;
|
|
74
|
+
if (!port) return;
|
|
75
|
+
let recognizer = null;
|
|
76
|
+
let sherpa = null;
|
|
77
|
+
port.on("message", async (msg) => {
|
|
78
|
+
try {
|
|
79
|
+
if (msg.op === "create" || msg.op === "decode") {
|
|
80
|
+
if (!sherpa) {
|
|
81
|
+
sherpa = await import(data.sherpaModule);
|
|
82
|
+
}
|
|
83
|
+
if (!recognizer) {
|
|
84
|
+
recognizer = sherpa.createOfflineRecognizer({
|
|
85
|
+
featConfig: { sampleRate: 16e3, featureDim: 80 },
|
|
86
|
+
modelConfig: {
|
|
87
|
+
senseVoice: {
|
|
88
|
+
model: data.modelDir + "/model.int8.onnx",
|
|
89
|
+
language: "auto",
|
|
90
|
+
useInverseTextNormalization: 1
|
|
91
|
+
},
|
|
92
|
+
tokens: data.modelDir + "/tokens.txt",
|
|
93
|
+
provider: "cpu",
|
|
94
|
+
debug: 0
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
if (msg.op === "decode" && msg.samples) {
|
|
99
|
+
const stream = recognizer.createStream();
|
|
100
|
+
try {
|
|
101
|
+
stream.acceptWaveform(16e3, msg.samples);
|
|
102
|
+
recognizer.decode(stream);
|
|
103
|
+
const text = recognizer.getResult(stream).text.trim();
|
|
104
|
+
port.postMessage({ id: msg.id, ok: true, text });
|
|
105
|
+
} finally {
|
|
106
|
+
try {
|
|
107
|
+
stream.free();
|
|
108
|
+
} catch {
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
port.postMessage({ id: msg.id, ok: true, text: "" });
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
port.postMessage({ id: msg.id, ok: false, error: "unknown op: " + msg.op });
|
|
117
|
+
} catch (e) {
|
|
118
|
+
port.postMessage({ id: msg.id, ok: false, error: String(e) });
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
if (parentPort) {
|
|
123
|
+
startSenseWorker(workerData);
|
|
124
|
+
}
|
|
125
|
+
export {
|
|
126
|
+
createSenseWorkerClient,
|
|
127
|
+
startSenseWorker
|
|
128
|
+
};
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __copyProps = (to, from, except, desc) => {
|
|
9
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
10
|
+
for (let key of __getOwnPropNames(from))
|
|
11
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
12
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
13
|
+
}
|
|
14
|
+
return to;
|
|
15
|
+
};
|
|
16
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
17
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
18
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
19
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
20
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
21
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
22
|
+
mod
|
|
23
|
+
));
|
|
24
|
+
|
|
25
|
+
// src/tts-vits-worker.ts
|
|
26
|
+
var import_sherpa_onnx = __toESM(require("sherpa-onnx"), 1);
|
|
27
|
+
var import_sherpa_onnx_node = __toESM(require("sherpa-onnx-node"), 1);
|
|
28
|
+
|
|
29
|
+
// src/segmenter.ts
|
|
30
|
+
function sanitizeForTts(text) {
|
|
31
|
+
return String(text).replace(/[*_#>`|^=+~]/g, " ").replace(/\s{2,}/g, " ").replace(/([\u3400-\u9fff])\s+(?=[\u3400-\u9fff])/g, "$1").trim();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// src/tts-vits-worker.ts
|
|
35
|
+
var { createOfflineTts } = import_sherpa_onnx.default;
|
|
36
|
+
var { OfflineTts, GenerationConfig } = import_sherpa_onnx_node.default;
|
|
37
|
+
var send = (msg) => {
|
|
38
|
+
if (typeof process.send === "function") process.send(msg);
|
|
39
|
+
};
|
|
40
|
+
var wasmTts = null;
|
|
41
|
+
var nativeTts = null;
|
|
42
|
+
var initKind = "vits";
|
|
43
|
+
var initPaths = {};
|
|
44
|
+
function createEngine() {
|
|
45
|
+
if (initKind === "kokoro") {
|
|
46
|
+
nativeTts = new OfflineTts({
|
|
47
|
+
model: {
|
|
48
|
+
kokoro: {
|
|
49
|
+
model: initPaths.model,
|
|
50
|
+
voices: initPaths.voices,
|
|
51
|
+
tokens: initPaths.tokens,
|
|
52
|
+
dataDir: initPaths.dataDir,
|
|
53
|
+
lexicon: initPaths.lexicon
|
|
54
|
+
},
|
|
55
|
+
// 2 线程：留核给主进程的 ASR 解码（4 线程满核会饿死识别，
|
|
56
|
+
// 打断后定稿等待分钟级；2 线程 RTF≈1 仍实时）。
|
|
57
|
+
numThreads: 2,
|
|
58
|
+
debug: 0,
|
|
59
|
+
provider: "cpu"
|
|
60
|
+
},
|
|
61
|
+
// 中文数字/日期/电话规范化（与 VITS 同源 FST）：阿拉伯数字按中文读。
|
|
62
|
+
ruleFsts: [initPaths.date, initPaths.phone, initPaths.number].filter(Boolean).join(","),
|
|
63
|
+
maxNumSentences: 1
|
|
64
|
+
});
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
wasmTts = createOfflineTts({
|
|
68
|
+
model: {
|
|
69
|
+
vits: {
|
|
70
|
+
model: initPaths.model,
|
|
71
|
+
lexicon: initPaths.lexicon,
|
|
72
|
+
tokens: initPaths.tokens
|
|
73
|
+
},
|
|
74
|
+
numThreads: 1,
|
|
75
|
+
debug: 0,
|
|
76
|
+
provider: "cpu"
|
|
77
|
+
},
|
|
78
|
+
ruleFsts: [initPaths.date, initPaths.phone, initPaths.number].join(","),
|
|
79
|
+
ruleFars: "",
|
|
80
|
+
maxNumSentences: 1
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
function reply(id, payload) {
|
|
84
|
+
send({ id: typeof id === "number" ? id : 0, ...payload });
|
|
85
|
+
}
|
|
86
|
+
process.on("message", (msg) => {
|
|
87
|
+
try {
|
|
88
|
+
if (msg.type === "init") {
|
|
89
|
+
initKind = msg.kind === "kokoro" ? "kokoro" : "vits";
|
|
90
|
+
initPaths = msg.paths;
|
|
91
|
+
createEngine();
|
|
92
|
+
reply(msg.id, { ok: true });
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (msg.type === "synth") {
|
|
96
|
+
const text = sanitizeForTts(
|
|
97
|
+
String(msg.text ?? "").replace(/[\u{1F000}-\u{1FAFF}\u{2600}-\u{27BF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}]/gu, " ").replace(/(\d)[.．]\s*/g, "$1\u3001")
|
|
98
|
+
);
|
|
99
|
+
if (!text.trim()) {
|
|
100
|
+
const sr = 24e3;
|
|
101
|
+
reply(msg.id, {
|
|
102
|
+
ok: true,
|
|
103
|
+
sampleRate: sr,
|
|
104
|
+
samples: Buffer.alloc(Math.floor(sr * 0.2) * 4).toString("base64")
|
|
105
|
+
});
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const sid = Number(msg.sid ?? 0);
|
|
109
|
+
const speed = Number(msg.speed ?? 1);
|
|
110
|
+
let audio;
|
|
111
|
+
if (initKind === "kokoro") {
|
|
112
|
+
if (!nativeTts) {
|
|
113
|
+
reply(msg.id, { ok: false, error: "tts child not initialized" });
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
const gc = new GenerationConfig({ sid, speed, silenceScale: 0.2 });
|
|
117
|
+
audio = nativeTts.generate({ text, generationConfig: gc });
|
|
118
|
+
} else {
|
|
119
|
+
if (!wasmTts) {
|
|
120
|
+
reply(msg.id, { ok: false, error: "tts child not initialized" });
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
audio = wasmTts.generate({ text, sid, speed });
|
|
124
|
+
}
|
|
125
|
+
const buf = Buffer.from(audio.samples.buffer, audio.samples.byteOffset, audio.samples.byteLength);
|
|
126
|
+
reply(msg.id, { ok: true, sampleRate: audio.sampleRate, samples: buf.toString("base64") });
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (msg.type === "close") {
|
|
130
|
+
try {
|
|
131
|
+
wasmTts?.free();
|
|
132
|
+
} catch {
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
nativeTts?.free();
|
|
136
|
+
} catch {
|
|
137
|
+
}
|
|
138
|
+
wasmTts = null;
|
|
139
|
+
nativeTts = null;
|
|
140
|
+
reply(msg.id, { ok: true });
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
reply(msg.id, { ok: false, error: `unknown message type: ${String(msg.type)}` });
|
|
144
|
+
} catch (e) {
|
|
145
|
+
reply(typeof msg?.id === "number" ? msg.id : 0, { ok: false, error: String(e) });
|
|
146
|
+
}
|
|
147
|
+
});
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-voice-mode",
|
|
3
|
-
"description": "Full-duplex voice mode for DeepSeek Harness: zipformer2 streaming ASR → editable draft, Edge TTS
|
|
4
|
-
"version": "0.
|
|
3
|
+
"description": "Full-duplex voice mode for DeepSeek Harness: zipformer2 streaming ASR → editable draft, Edge TTS (default, fast) or local (VITS / Kokoro int8, privacy-first) read-aloud with live captions, true barge-in; hardened HTTP surface (loopback/origin/rate-limit) + model SHA256 pinning. · DSH 语音双工对话：流式识别入草稿、Edge 默认朗读或本地（VITS / Kokoro int8）朗读+实时字幕、开口即打断；HTTP 面安全加固。",
|
|
4
|
+
"version": "0.5.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"repository": {
|
|
@@ -23,12 +23,16 @@
|
|
|
23
23
|
},
|
|
24
24
|
"files": [
|
|
25
25
|
"lib/index.js",
|
|
26
|
+
"lib/tts-vits-worker.cjs",
|
|
26
27
|
"lib/client.js",
|
|
28
|
+
"lib/sense-worker.mjs",
|
|
27
29
|
"cordis.patch.yml",
|
|
28
30
|
"assets/architecture.svg",
|
|
29
31
|
"assets/demo.gif",
|
|
30
32
|
"scripts/prefetch.mjs",
|
|
31
33
|
"scripts/list-voices.mjs",
|
|
34
|
+
"scripts/bench-asr.mjs",
|
|
35
|
+
"scripts/verify.mjs",
|
|
32
36
|
"README.md",
|
|
33
37
|
"README.en.md",
|
|
34
38
|
"LICENSE"
|
|
@@ -36,10 +40,11 @@
|
|
|
36
40
|
"scripts": {
|
|
37
41
|
"build": "node build.mjs",
|
|
38
42
|
"prepack": "node build.mjs",
|
|
39
|
-
"test": "node test/segmenter.test.mjs && node test/
|
|
43
|
+
"test": "node test/segmenter.test.mjs && node test/aec.test.mjs && node test/download.test.mjs && node test/endpoint.test.mjs && node test/resample.test.mjs && node test/sense-worker.test.mjs && node test/detect-route.test.mjs && node test/verify-client.mjs",
|
|
40
44
|
"verify:client": "node test/verify-client.mjs",
|
|
41
45
|
"prefetch": "node scripts/prefetch.mjs",
|
|
42
46
|
"list-voices": "node scripts/list-voices.mjs",
|
|
47
|
+
"bench-asr": "node scripts/bench-asr.mjs",
|
|
43
48
|
"verify": "node scripts/verify.mjs",
|
|
44
49
|
"typecheck": "node node_modules/typescript/bin/tsc -p tsconfig.json --noEmit && node node_modules/typescript/bin/tsc -p tsconfig.client.json --noEmit"
|
|
45
50
|
},
|
|
@@ -56,9 +61,10 @@
|
|
|
56
61
|
}
|
|
57
62
|
},
|
|
58
63
|
"dependencies": {
|
|
64
|
+
"@deepseek-ai/schemastery": "^3.18.1",
|
|
59
65
|
"msedge-tts": "^2.0.7",
|
|
60
66
|
"sherpa-onnx": "^1.13.6",
|
|
61
|
-
"
|
|
67
|
+
"sherpa-onnx-node": "1.13.6"
|
|
62
68
|
},
|
|
63
69
|
"peerDependencies": {
|
|
64
70
|
"@deepseek-ai/cordis": "^4.0.1",
|
|
@@ -66,10 +72,15 @@
|
|
|
66
72
|
"react": "^18.2.0"
|
|
67
73
|
},
|
|
68
74
|
"devDependencies": {
|
|
69
|
-
"
|
|
70
|
-
"
|
|
75
|
+
"@deepseek-ai/cordis": "^4.0.1",
|
|
76
|
+
"@deepseek-ai/dsh-host-webserver": "^0.1.1-rc.2",
|
|
77
|
+
"@deepseek-ai/dsh-llm": "^0.1.1-rc.2",
|
|
78
|
+
"@deepseek-ai/dsh-settings": "^0.1.1-rc.2",
|
|
79
|
+
"@deepseek-ai/dsh-system-prompt": "^0.1.1-rc.2",
|
|
71
80
|
"@types/node": "^26.0.0",
|
|
72
|
-
"@types/react": "^19.0.0"
|
|
81
|
+
"@types/react": "^19.0.0",
|
|
82
|
+
"esbuild": "^0.25.0",
|
|
83
|
+
"typescript": "^5.9.2"
|
|
73
84
|
},
|
|
74
85
|
"keywords": [
|
|
75
86
|
"dsh",
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* P4-2 在线 ASR 换型离线对照评测（CER/段延迟/体积）。
|
|
4
|
+
*
|
|
5
|
+
* 用法：
|
|
6
|
+
* node scripts/bench-asr.mjs --dir <测试集目录>
|
|
7
|
+
*
|
|
8
|
+
* 测试集目录约定：*.wav（16k 单声道 16bit PCM）+ 同名 *.txt（参考文本，UTF-8）。
|
|
9
|
+
* 模型自动懒下载至平台缓存目录（与插件同一约定：Linux/macOS
|
|
10
|
+
* ~/.cache/dsh-voice-mode/models/，Windows %LOCALAPPDATA%\dsh-voice-mode\models），
|
|
11
|
+
* .part 断点续传，huggingface.co ↗ hf-mirror.com 回退（--host 可指定镜像）。
|
|
12
|
+
*
|
|
13
|
+
* 输出：Markdown 表格（model | CER% | 平均段延迟 ms | 模型体积 MB），
|
|
14
|
+
* 「数据说话」支撑 P4 换型决策（plan.md §3-P4）。
|
|
15
|
+
*/
|
|
16
|
+
import { createWriteStream, readdirSync, readFileSync, statSync, mkdirSync, renameSync } from 'node:fs'
|
|
17
|
+
import { dirname, join } from 'node:path'
|
|
18
|
+
import { fileURLToPath } from 'node:url'
|
|
19
|
+
import { homedir } from 'node:os'
|
|
20
|
+
import sherpa_onnx from 'sherpa-onnx'
|
|
21
|
+
|
|
22
|
+
const here = dirname(fileURLToPath(import.meta.url))
|
|
23
|
+
const root = join(here, '..')
|
|
24
|
+
|
|
25
|
+
// ---------- 模型清单（repo/文件清单/设备配置） ----------
|
|
26
|
+
const MODELS = [
|
|
27
|
+
{
|
|
28
|
+
id: 'zipformer-zh-int8',
|
|
29
|
+
repo: 'csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30',
|
|
30
|
+
files: ['encoder.int8.onnx', 'decoder.onnx', 'joiner.int8.onnx', 'tokens.txt'],
|
|
31
|
+
make: (t) => ({
|
|
32
|
+
modelConfig: {
|
|
33
|
+
transducer: { encoder: t('encoder.int8.onnx'), decoder: t('decoder.onnx'), joiner: t('joiner.int8.onnx') },
|
|
34
|
+
tokens: t('tokens.txt'), numThreads: 4, provider: 'cpu', debug: 0,
|
|
35
|
+
},
|
|
36
|
+
decodingMethod: 'greedy_search',
|
|
37
|
+
}),
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'zipformer-zh-xlarge-int8',
|
|
41
|
+
repo: 'csukuangfj/sherpa-onnx-streaming-zipformer-zh-xlarge-int8-2025-06-30',
|
|
42
|
+
files: ['encoder.int8.onnx', 'decoder.onnx', 'joiner.int8.onnx', 'tokens.txt'],
|
|
43
|
+
make: (t) => ({
|
|
44
|
+
modelConfig: {
|
|
45
|
+
transducer: { encoder: t('encoder.int8.onnx'), decoder: t('decoder.onnx'), joiner: t('joiner.int8.onnx') },
|
|
46
|
+
tokens: t('tokens.txt'), numThreads: 4, provider: 'cpu', debug: 0,
|
|
47
|
+
},
|
|
48
|
+
decodingMethod: 'greedy_search',
|
|
49
|
+
}),
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
id: 'zipformer-small-ctc-zh-int8',
|
|
53
|
+
repo: 'csukuangfj/sherpa-onnx-streaming-zipformer-small-ctc-zh-int8-2025-04-01',
|
|
54
|
+
files: ['model.int8.onnx', 'tokens.txt'],
|
|
55
|
+
make: (t) => ({
|
|
56
|
+
modelConfig: {
|
|
57
|
+
zipformer2Ctc: { model: t('model.int8.onnx') },
|
|
58
|
+
tokens: t('tokens.txt'), numThreads: 4, provider: 'cpu', debug: 0,
|
|
59
|
+
},
|
|
60
|
+
}),
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
id: 'paraformer-bilingual-zh-en',
|
|
64
|
+
repo: 'csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en',
|
|
65
|
+
files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'tokens.txt'],
|
|
66
|
+
make: (t) => ({
|
|
67
|
+
modelConfig: {
|
|
68
|
+
paraformer: { encoder: t('encoder.int8.onnx'), decoder: t('decoder.int8.onnx') },
|
|
69
|
+
tokens: t('tokens.txt'), numThreads: 4, provider: 'cpu', debug: 0,
|
|
70
|
+
},
|
|
71
|
+
}),
|
|
72
|
+
},
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
// ---------- 参数 ----------
|
|
76
|
+
function parseArgs(argv) {
|
|
77
|
+
const args = { dir: null, host: 'https://huggingface.co' }
|
|
78
|
+
for (let i = 0; i < argv.length; i++) {
|
|
79
|
+
if (argv[i] === '--dir') args.dir = argv[i + 1]
|
|
80
|
+
else if (argv[i] === '--host') args.host = argv[i + 1]
|
|
81
|
+
}
|
|
82
|
+
return args
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ---------- 平台缓存目录 ----------
|
|
86
|
+
function cacheDir() {
|
|
87
|
+
return process.platform === 'win32'
|
|
88
|
+
? join(process.env.LOCALAPPDATA ?? join(homedir(), 'AppData', 'Local'), 'dsh-voice-mode', 'models')
|
|
89
|
+
: join(homedir(), '.cache', 'dsh-voice-mode', 'models')
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ---------- 懒下载（.part 续传 + host 回退，与插件 ensureFile 同构） ----------
|
|
93
|
+
async function ensureFile(repoDir, file, hosts) {
|
|
94
|
+
const localPath = join(repoDir, file)
|
|
95
|
+
try {
|
|
96
|
+
if (statSync(localPath).isFile()) return true
|
|
97
|
+
} catch {
|
|
98
|
+
// 缺失
|
|
99
|
+
}
|
|
100
|
+
if (file === 'tokens.txt') console.log(` 下载 ${file}…`)
|
|
101
|
+
else console.log(` 下载 ${file}（可能较大）…`)
|
|
102
|
+
mkdirSync(repoDir, { recursive: true })
|
|
103
|
+
const partPath = localPath + '.part'
|
|
104
|
+
let partSize = 0
|
|
105
|
+
try {
|
|
106
|
+
partSize = statSync(partPath).size
|
|
107
|
+
} catch {
|
|
108
|
+
// 无 .part
|
|
109
|
+
}
|
|
110
|
+
for (const host of hosts) {
|
|
111
|
+
try {
|
|
112
|
+
const url = `${host}/${repoDir.split(/[\\/]/).pop()}/resolve/main/${file}`
|
|
113
|
+
const headers = { 'user-agent': 'dsh-voice-mode-bench' }
|
|
114
|
+
if (partSize > 0) headers.range = `bytes=${partSize}-`
|
|
115
|
+
const res = await fetch(url, { headers })
|
|
116
|
+
if (res.status === 416) {
|
|
117
|
+
renameSync(partPath, localPath)
|
|
118
|
+
return true
|
|
119
|
+
}
|
|
120
|
+
if (res.status !== 200 && res.status !== 206) continue
|
|
121
|
+
const sink = createWriteStream(partPath, partSize > 0 ? { flags: 'a' } : {})
|
|
122
|
+
const reader = res.body.getReader()
|
|
123
|
+
for (;;) {
|
|
124
|
+
const { done, value } = await reader.read()
|
|
125
|
+
if (done) break
|
|
126
|
+
if (!sink.write(value)) await new Promise((r) => sink.once('drain', r))
|
|
127
|
+
}
|
|
128
|
+
await new Promise((resolve, reject) => {
|
|
129
|
+
sink.end(() => resolve())
|
|
130
|
+
sink.on('error', reject)
|
|
131
|
+
})
|
|
132
|
+
renameSync(partPath, localPath)
|
|
133
|
+
return true
|
|
134
|
+
} catch {
|
|
135
|
+
partSize = 0 // 换 host 重来
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return false
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
async function ensureModels(cache, repo, files, hosts) {
|
|
143
|
+
const repoDir = join(cache, repo)
|
|
144
|
+
for (const f of files) {
|
|
145
|
+
if (!(await ensureFile(repoDir, f, hosts))) {
|
|
146
|
+
console.error(` 模型下载失败: ${repo}/${f}`)
|
|
147
|
+
return false
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return true
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ---------- WAV 读取（readWaveFromBinaryData → {samples, sampleRate}） ----------
|
|
154
|
+
async function loadTestSet(dir) {
|
|
155
|
+
const entries = readdirSync(dir)
|
|
156
|
+
const wavs = entries.filter((n) => n.endsWith('.wav')).sort()
|
|
157
|
+
const cases = []
|
|
158
|
+
for (const w of wavs) {
|
|
159
|
+
const txt = w.replace(/\.wav$/, '.txt')
|
|
160
|
+
if (!entries.includes(txt)) {
|
|
161
|
+
console.warn(`跳过 ${w}：无同名 ${txt} 参考文本`)
|
|
162
|
+
continue
|
|
163
|
+
}
|
|
164
|
+
const buf = readFileSync(join(dir, w))
|
|
165
|
+
let wav
|
|
166
|
+
try {
|
|
167
|
+
wav = sherpa_onnx.readWaveFromBinaryData(new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength))
|
|
168
|
+
} catch (e) {
|
|
169
|
+
console.warn(`跳过 ${w}：无法解析 WAV（${String(e).slice(0, 80)}）`)
|
|
170
|
+
continue
|
|
171
|
+
}
|
|
172
|
+
if (wav.sampleRate !== 16000) {
|
|
173
|
+
console.warn(`跳过 ${w}：采样率 ${wav.sampleRate}Hz ≠ 16k（请先重采样）`)
|
|
174
|
+
continue
|
|
175
|
+
}
|
|
176
|
+
const ref = readFileSync(join(dir, txt), 'utf8').trim()
|
|
177
|
+
cases.push({ name: w, samples: wav.samples, ref })
|
|
178
|
+
}
|
|
179
|
+
return cases
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ---------- 编辑距离（字符级，中文按字） ----------
|
|
183
|
+
function cer(hyp, ref) {
|
|
184
|
+
const a = [...hyp]
|
|
185
|
+
const b = [...ref]
|
|
186
|
+
const m = a.length
|
|
187
|
+
const n = b.length
|
|
188
|
+
const dp = new Uint32Array((m + 1) * (n + 1))
|
|
189
|
+
for (let i = 0; i <= m; i++) dp[i * (n + 1)] = i
|
|
190
|
+
for (let j = 0; j <= n; j++) dp[j] = j
|
|
191
|
+
for (let i = 1; i <= m; i++) {
|
|
192
|
+
for (let j = 1; j <= n; j++) {
|
|
193
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1
|
|
194
|
+
dp[i * (n + 1) + j] = Math.min(
|
|
195
|
+
dp[(i - 1) * (n + 1) + j] + 1,
|
|
196
|
+
dp[i * (n + 1) + j - 1] + 1,
|
|
197
|
+
dp[(i - 1) * (n + 1) + j - 1] + cost,
|
|
198
|
+
)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const dist = dp[m * (n + 1) + n]
|
|
202
|
+
const denom = Math.max(n, 1)
|
|
203
|
+
return (dist / denom) * 100
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
// ---------- 单模型评测 ----------
|
|
209
|
+
async function evalModel(model, cases, hosts, cache) {
|
|
210
|
+
const repoDir = join(cache, model.repo)
|
|
211
|
+
if (!(await ensureModels(cache, model.repo, model.files, hosts))) {
|
|
212
|
+
return { id: model.id, cer: null, ms: null, mb: null, error: '模型下载失败' }
|
|
213
|
+
}
|
|
214
|
+
const t = (f) => join(repoDir, f)
|
|
215
|
+
const rec = sherpa_onnx.createOnlineRecognizer(model.make(t))
|
|
216
|
+
let distSum = 0
|
|
217
|
+
let lenSum = 0
|
|
218
|
+
const timings = []
|
|
219
|
+
for (const c of cases) {
|
|
220
|
+
const stream = rec.createStream()
|
|
221
|
+
const t0 = performance.now()
|
|
222
|
+
stream.acceptWaveform(16000, c.samples)
|
|
223
|
+
while (rec.isReady(stream)) rec.decode(stream)
|
|
224
|
+
const text = rec.getResult(stream).text
|
|
225
|
+
timings.push(performance.now() - t0)
|
|
226
|
+
distSum += cer(text, c.ref) * Math.max(c.ref.length, 1)
|
|
227
|
+
lenSum += Math.max(c.ref.length, 1)
|
|
228
|
+
stream.free?.()
|
|
229
|
+
}
|
|
230
|
+
rec.free?.()
|
|
231
|
+
let bytes = 0
|
|
232
|
+
for (const f of model.files) {
|
|
233
|
+
try {
|
|
234
|
+
bytes += statSync(join(repoDir, f)).size
|
|
235
|
+
} catch {
|
|
236
|
+
// ignore
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
id: model.id,
|
|
241
|
+
cer: lenSum > 0 ? distSum / lenSum : null,
|
|
242
|
+
ms: timings.length > 0 ? timings.reduce((a, b) => a + b, 0) / timings.length : null,
|
|
243
|
+
mb: bytes / (1024 * 1024),
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
async function main() {
|
|
248
|
+
const args = parseArgs(process.argv.slice(2))
|
|
249
|
+
if (!args.dir) {
|
|
250
|
+
console.error('用法: node scripts/bench-asr.mjs --dir <测试集目录> [--host 镜像]')
|
|
251
|
+
process.exit(1)
|
|
252
|
+
}
|
|
253
|
+
const cache = cacheDir()
|
|
254
|
+
const hosts = [...new Set([args.host, 'https://huggingface.co', 'https://hf-mirror.com'].filter(Boolean))]
|
|
255
|
+
const cases = await loadTestSet(args.dir)
|
|
256
|
+
if (cases.length === 0) {
|
|
257
|
+
console.error(`测试集为空：${args.dir}（需 16k 单声道 16bit PCM .wav + 同名 .txt）`)
|
|
258
|
+
process.exit(1)
|
|
259
|
+
}
|
|
260
|
+
console.log(`测试集 ${args.dir}：${cases.length} 段（合计约 ${Math.round(cases.reduce((a, c) => a + c.samples.length, 0) / 16000)}s 音频）`)
|
|
261
|
+
console.log('')
|
|
262
|
+
const rows = []
|
|
263
|
+
for (const m of MODELS) {
|
|
264
|
+
const r = await evalModel(m, cases, hosts, cache)
|
|
265
|
+
rows.push(r)
|
|
266
|
+
console.log(
|
|
267
|
+
r.error
|
|
268
|
+
? `- ${m.id}: ${r.error}`
|
|
269
|
+
: `- ${m.id}: CER ${r.cer.toFixed(2)}% · 平均段延迟 ${r.ms.toFixed(0)}ms · 体积 ${r.mb.toFixed(0)}MB`,
|
|
270
|
+
)
|
|
271
|
+
}
|
|
272
|
+
console.log('')
|
|
273
|
+
console.log('| 模型 | CER% | 平均段延迟 ms | 体积 MB |')
|
|
274
|
+
console.log('| --- | --- | --- | --- |')
|
|
275
|
+
for (const r of rows) {
|
|
276
|
+
console.log(
|
|
277
|
+
r.error || r.cer === null
|
|
278
|
+
? `| ${r.id} | — | — | — (${r.error ?? '无数据'}) |`
|
|
279
|
+
: `| ${r.id} | ${r.cer.toFixed(2)} | ${r.ms.toFixed(0)} | ${r.mb.toFixed(0)} |`,
|
|
280
|
+
)
|
|
281
|
+
}
|
|
282
|
+
console.log('')
|
|
283
|
+
console.log('说明：CER = 字符编辑距离/参考长度；段延迟 = 整段喂入到 getResult 的墙钟；体积 = 模型文件磁盘占用。')
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
main().catch((e) => {
|
|
287
|
+
console.error('bench failed:', e)
|
|
288
|
+
process.exit(1)
|
|
289
|
+
})
|