dsh-voice-mode 0.2.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +28 -15
- package/README.md +69 -32
- package/assets/architecture.svg +1 -159
- package/assets/demo.gif +0 -0
- package/lib/client.js +2252 -359
- package/lib/index.js +1899 -245
- package/lib/sense-worker.mjs +128 -0
- package/lib/tts-vits-worker.cjs +147 -0
- package/package.json +18 -7
- package/scripts/bench-asr.mjs +289 -0
- package/scripts/prefetch.mjs +158 -37
- package/scripts/verify.mjs +34 -0
package/lib/index.js
CHANGED
|
@@ -1,45 +1,425 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
2
|
import z from "@deepseek-ai/schemastery";
|
|
3
|
-
import { join as
|
|
3
|
+
import { join as join4 } from "node:path";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
|
+
import { rm } from "node:fs/promises";
|
|
6
|
+
|
|
7
|
+
// src/asr-host.ts
|
|
8
|
+
import { statSync } from "node:fs";
|
|
9
|
+
import { stat as stat2 } from "node:fs/promises";
|
|
10
|
+
import { join as join2 } from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { Worker } from "node:worker_threads";
|
|
13
|
+
|
|
14
|
+
// src/sense-worker.ts
|
|
15
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
16
|
+
function createSenseWorkerClient(worker) {
|
|
17
|
+
let counter = 0;
|
|
18
|
+
const pending = /* @__PURE__ */ new Map();
|
|
19
|
+
let dead = false;
|
|
20
|
+
const deathFns = /* @__PURE__ */ new Set();
|
|
21
|
+
const die = () => {
|
|
22
|
+
if (dead) return;
|
|
23
|
+
dead = true;
|
|
24
|
+
for (const fn of deathFns) {
|
|
25
|
+
try {
|
|
26
|
+
fn();
|
|
27
|
+
} catch {
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
worker.on?.("message", (msg) => {
|
|
32
|
+
const p = pending.get(msg?.id);
|
|
33
|
+
if (!p) return;
|
|
34
|
+
pending.delete(msg.id);
|
|
35
|
+
if (!msg.ok) {
|
|
36
|
+
p.resolve(null);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
p.resolve(p.op === "create" ? true : msg.text ?? "");
|
|
40
|
+
});
|
|
41
|
+
worker.on?.("error", (e) => {
|
|
42
|
+
die();
|
|
43
|
+
const err = new Error("sense worker error: " + String(e?.message ?? e));
|
|
44
|
+
for (const [, p] of pending) p.reject(err);
|
|
45
|
+
pending.clear();
|
|
46
|
+
});
|
|
47
|
+
worker.on?.("exit", () => {
|
|
48
|
+
die();
|
|
49
|
+
const err = new Error("sense worker exited");
|
|
50
|
+
for (const [, p] of pending) p.reject(err);
|
|
51
|
+
pending.clear();
|
|
52
|
+
});
|
|
53
|
+
const request = (op, samples) => {
|
|
54
|
+
if (dead) return Promise.reject(new Error("sense worker dead"));
|
|
55
|
+
const id = counter++;
|
|
56
|
+
return new Promise((resolve, reject) => {
|
|
57
|
+
pending.set(id, { op, resolve, reject });
|
|
58
|
+
const msg = { id, op };
|
|
59
|
+
if (samples) msg.samples = samples;
|
|
60
|
+
try {
|
|
61
|
+
worker.postMessage(msg);
|
|
62
|
+
} catch (e) {
|
|
63
|
+
pending.delete(id);
|
|
64
|
+
reject(e instanceof Error ? e : new Error(String(e)));
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
};
|
|
68
|
+
return {
|
|
69
|
+
request,
|
|
70
|
+
onDeath(fn) {
|
|
71
|
+
deathFns.add(fn);
|
|
72
|
+
},
|
|
73
|
+
terminate: async () => {
|
|
74
|
+
dead = true;
|
|
75
|
+
const err = new Error("sense worker terminated");
|
|
76
|
+
for (const [, p] of pending) p.reject(err);
|
|
77
|
+
pending.clear();
|
|
78
|
+
try {
|
|
79
|
+
await worker.terminate?.();
|
|
80
|
+
} catch {
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function startSenseWorker(data) {
|
|
86
|
+
const port = parentPort;
|
|
87
|
+
if (!port) return;
|
|
88
|
+
let recognizer = null;
|
|
89
|
+
let sherpa = null;
|
|
90
|
+
port.on("message", async (msg) => {
|
|
91
|
+
try {
|
|
92
|
+
if (msg.op === "create" || msg.op === "decode") {
|
|
93
|
+
if (!sherpa) {
|
|
94
|
+
sherpa = await import(data.sherpaModule);
|
|
95
|
+
}
|
|
96
|
+
if (!recognizer) {
|
|
97
|
+
recognizer = sherpa.createOfflineRecognizer({
|
|
98
|
+
featConfig: { sampleRate: 16e3, featureDim: 80 },
|
|
99
|
+
modelConfig: {
|
|
100
|
+
senseVoice: {
|
|
101
|
+
model: data.modelDir + "/model.int8.onnx",
|
|
102
|
+
language: "auto",
|
|
103
|
+
useInverseTextNormalization: 1
|
|
104
|
+
},
|
|
105
|
+
tokens: data.modelDir + "/tokens.txt",
|
|
106
|
+
provider: "cpu",
|
|
107
|
+
debug: 0
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
if (msg.op === "decode" && msg.samples) {
|
|
112
|
+
const stream = recognizer.createStream();
|
|
113
|
+
try {
|
|
114
|
+
stream.acceptWaveform(16e3, msg.samples);
|
|
115
|
+
recognizer.decode(stream);
|
|
116
|
+
const text = recognizer.getResult(stream).text.trim();
|
|
117
|
+
port.postMessage({ id: msg.id, ok: true, text });
|
|
118
|
+
} finally {
|
|
119
|
+
try {
|
|
120
|
+
stream.free();
|
|
121
|
+
} catch {
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
port.postMessage({ id: msg.id, ok: true, text: "" });
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
port.postMessage({ id: msg.id, ok: false, error: "unknown op: " + msg.op });
|
|
130
|
+
} catch (e) {
|
|
131
|
+
port.postMessage({ id: msg.id, ok: false, error: String(e) });
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
if (parentPort) {
|
|
136
|
+
startSenseWorker(workerData);
|
|
137
|
+
}
|
|
5
138
|
|
|
6
139
|
// src/asr-host.ts
|
|
140
|
+
import sherpa_onnx from "sherpa-onnx";
|
|
141
|
+
|
|
142
|
+
// src/models.ts
|
|
143
|
+
import { createHash } from "node:crypto";
|
|
7
144
|
import { createWriteStream } from "node:fs";
|
|
8
145
|
import { mkdir, rename, stat, unlink } from "node:fs/promises";
|
|
9
146
|
import { join } from "node:path";
|
|
10
|
-
|
|
11
|
-
var
|
|
147
|
+
var HOST_PRIMARY = "https://huggingface.co";
|
|
148
|
+
var HOST_FALLBACK = "https://hf-mirror.com";
|
|
149
|
+
var ALLOWED_MODEL_HOSTNAMES = ["huggingface.co", "hf.co", "hf-mirror.com"];
|
|
150
|
+
function validateModelHost(raw, allowCustomHost) {
|
|
151
|
+
if (!raw) return null;
|
|
152
|
+
let u;
|
|
153
|
+
try {
|
|
154
|
+
u = new URL(raw);
|
|
155
|
+
} catch {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
if (u.protocol !== "https:") return null;
|
|
159
|
+
const hostname = u.hostname.toLowerCase();
|
|
160
|
+
if (!ALLOWED_MODEL_HOSTNAMES.includes(hostname) && !allowCustomHost) {
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
return `${u.protocol}//${u.hostname}${u.port ? `:${u.port}` : ""}`;
|
|
164
|
+
}
|
|
165
|
+
function redirectHostAllowed(finalUrl, allowCustomHost) {
|
|
166
|
+
try {
|
|
167
|
+
const u = new URL(finalUrl);
|
|
168
|
+
if (u.protocol !== "https:") return false;
|
|
169
|
+
const hostname = u.hostname.toLowerCase();
|
|
170
|
+
if (allowCustomHost) return true;
|
|
171
|
+
return hostname === "huggingface.co" || hostname.endsWith(".huggingface.co") || hostname === "hf.co" || hostname.endsWith(".hf.co") || hostname === "hf-mirror.com" || hostname.endsWith(".hf-mirror.com");
|
|
172
|
+
} catch {
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
async function sha256OfFile(path) {
|
|
177
|
+
const hash = createHash("sha256");
|
|
178
|
+
const { createReadStream } = await import("node:fs");
|
|
179
|
+
await new Promise((resolve, reject) => {
|
|
180
|
+
const stream = createReadStream(path);
|
|
181
|
+
stream.on("data", (c) => hash.update(c));
|
|
182
|
+
stream.on("error", reject);
|
|
183
|
+
stream.on("end", () => resolve());
|
|
184
|
+
});
|
|
185
|
+
return hash.digest("hex");
|
|
186
|
+
}
|
|
187
|
+
async function ensureModelFile(opts) {
|
|
188
|
+
const { repo, repoDir, spec, primaryHost, allowCustomHost, broadcast } = opts;
|
|
189
|
+
const localPath = join(repoDir, spec.file);
|
|
190
|
+
const partPath = `${localPath}.part`;
|
|
191
|
+
if ((await stat(localPath).catch(() => null))?.isFile()) {
|
|
192
|
+
const ok = await sha256OfFile(localPath).catch(() => "") === spec.sha256;
|
|
193
|
+
if (ok) return true;
|
|
194
|
+
await unlink(localPath).catch(() => void 0);
|
|
195
|
+
}
|
|
196
|
+
await mkdir(join(repoDir, spec.file.includes("/") ? spec.file.slice(0, spec.file.lastIndexOf("/")) : ""), {
|
|
197
|
+
recursive: true
|
|
198
|
+
}).catch(() => void 0);
|
|
199
|
+
const hosts = [...new Set([primaryHost, HOST_PRIMARY, HOST_FALLBACK].filter(Boolean))];
|
|
200
|
+
let lastError = "no upstream reachable";
|
|
201
|
+
for (const host of hosts) {
|
|
202
|
+
try {
|
|
203
|
+
const done = await downloadVerified({ ...opts, host, partPath, localPath });
|
|
204
|
+
if (done) return true;
|
|
205
|
+
} catch (e) {
|
|
206
|
+
lastError = String(e);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
broadcast("asr-error", { file: spec.file, reason: "checksum_or_download_failed", detail: lastError });
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
async function downloadVerified(opts) {
|
|
213
|
+
const { repo, spec, host, allowCustomHost, partPath, localPath, broadcast } = opts;
|
|
214
|
+
const url = `${host}/${repo}/resolve/main/${spec.file}`;
|
|
215
|
+
const partSt = await stat(partPath).catch(() => null);
|
|
216
|
+
const resumeFrom = partSt?.isFile() ? partSt.size : 0;
|
|
217
|
+
const headers = { "user-agent": "dsh-voice-mode" };
|
|
218
|
+
if (resumeFrom > 0) headers.range = `bytes=${resumeFrom}-`;
|
|
219
|
+
const res = await fetch(url, { headers, redirect: "follow" });
|
|
220
|
+
if (!redirectHostAllowed(res.url, allowCustomHost)) return false;
|
|
221
|
+
if (res.status === 416) {
|
|
222
|
+
if (await sha256OfFile(partPath).catch(() => "") === spec.sha256) {
|
|
223
|
+
await rename(partPath, localPath);
|
|
224
|
+
return true;
|
|
225
|
+
}
|
|
226
|
+
await unlink(partPath).catch(() => void 0);
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
if (res.status !== 200 && res.status !== 206) return false;
|
|
230
|
+
const resume = res.status === 206 ? resumeFrom : 0;
|
|
231
|
+
const total = Number(res.headers.get("content-length") ?? 0) + resume;
|
|
232
|
+
const src = res.body;
|
|
233
|
+
if (!src) return false;
|
|
234
|
+
const sink = createWriteStream(partPath, resume > 0 ? { flags: "a" } : {});
|
|
235
|
+
const reader = src.getReader();
|
|
236
|
+
let received = resume;
|
|
237
|
+
await new Promise((resolve, reject) => {
|
|
238
|
+
sink.on("error", (e) => reject(e));
|
|
239
|
+
sink.on("finish", () => resolve());
|
|
240
|
+
void (async () => {
|
|
241
|
+
try {
|
|
242
|
+
for (; ; ) {
|
|
243
|
+
const { done, value } = await reader.read();
|
|
244
|
+
if (done) break;
|
|
245
|
+
received += value.byteLength;
|
|
246
|
+
if (!sink.write(value)) {
|
|
247
|
+
await new Promise((r) => sink.once("drain", r));
|
|
248
|
+
}
|
|
249
|
+
if (total > 0) {
|
|
250
|
+
broadcast("asr-progress", {
|
|
251
|
+
file: spec.file,
|
|
252
|
+
percent: Math.min(100, Math.round(received / total * 100))
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
sink.end();
|
|
257
|
+
} catch (e) {
|
|
258
|
+
sink.destroy(e);
|
|
259
|
+
reject(e);
|
|
260
|
+
}
|
|
261
|
+
})();
|
|
262
|
+
});
|
|
263
|
+
const actual = await sha256OfFile(partPath).catch(() => "");
|
|
264
|
+
if (actual !== spec.sha256) {
|
|
265
|
+
await unlink(partPath).catch(() => void 0);
|
|
266
|
+
return false;
|
|
267
|
+
}
|
|
268
|
+
await rename(partPath, localPath);
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
271
|
+
async function ensureModelTree(opts) {
|
|
272
|
+
const { repo, repoDir, subdir, primaryHost, allowCustomHost, broadcast } = opts;
|
|
273
|
+
const hosts = [...new Set([primaryHost, HOST_PRIMARY, HOST_FALLBACK].filter(Boolean))];
|
|
274
|
+
const subRoot = join(repoDir, subdir);
|
|
275
|
+
await mkdir(subRoot, { recursive: true }).catch(() => void 0);
|
|
276
|
+
let tree = [];
|
|
277
|
+
for (const host of hosts) {
|
|
278
|
+
try {
|
|
279
|
+
const res = await fetch(host + "/api/models/" + repo + "?blobs=true", { headers: { "user-agent": "dsh-voice-mode" } });
|
|
280
|
+
if (res.ok) {
|
|
281
|
+
const j = await res.json();
|
|
282
|
+
tree = (j.siblings ?? []).map((s) => s.rfilename ?? "").filter((f) => f.startsWith(subdir + "/") && f.length > 0);
|
|
283
|
+
if (tree.length > 0) break;
|
|
284
|
+
}
|
|
285
|
+
} catch {
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
if (tree.length === 0) return false;
|
|
289
|
+
let allOk = true;
|
|
290
|
+
let done = 0;
|
|
291
|
+
const queue = [...tree];
|
|
292
|
+
const worker = async () => {
|
|
293
|
+
for (; ; ) {
|
|
294
|
+
const rel = queue.shift();
|
|
295
|
+
if (rel === void 0) return;
|
|
296
|
+
const localPath = join(repoDir, rel);
|
|
297
|
+
const partPath = localPath + ".part";
|
|
298
|
+
if ((await stat(localPath).catch(() => null))?.isFile() && (await stat(localPath)).size > 0) {
|
|
299
|
+
done++;
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
await mkdir(join(repoDir, rel.slice(0, rel.lastIndexOf("/"))), { recursive: true }).catch(() => void 0);
|
|
303
|
+
let ok = false;
|
|
304
|
+
for (const host of hosts) {
|
|
305
|
+
try {
|
|
306
|
+
const url = host + "/" + repo + "/resolve/main/" + encodeURIComponent(rel);
|
|
307
|
+
const res = await fetch(url, { headers: { "user-agent": "dsh-voice-mode" }, redirect: "follow" });
|
|
308
|
+
if (!redirectHostAllowed(res.url, allowCustomHost)) continue;
|
|
309
|
+
if (res.status !== 200) continue;
|
|
310
|
+
const sink = createWriteStream(partPath);
|
|
311
|
+
const reader = res.body?.getReader();
|
|
312
|
+
if (!reader) continue;
|
|
313
|
+
let size = 0;
|
|
314
|
+
await new Promise((resolve, reject) => {
|
|
315
|
+
sink.on("error", reject);
|
|
316
|
+
sink.on("finish", resolve);
|
|
317
|
+
void (async () => {
|
|
318
|
+
try {
|
|
319
|
+
for (; ; ) {
|
|
320
|
+
const r = await reader.read();
|
|
321
|
+
if (r.done) break;
|
|
322
|
+
size += r.value.byteLength;
|
|
323
|
+
if (!sink.write(r.value)) await new Promise((r2) => sink.once("drain", r2));
|
|
324
|
+
}
|
|
325
|
+
sink.end();
|
|
326
|
+
} catch (e) {
|
|
327
|
+
sink.destroy(e);
|
|
328
|
+
reject(e);
|
|
329
|
+
}
|
|
330
|
+
})();
|
|
331
|
+
});
|
|
332
|
+
if (size > 0) {
|
|
333
|
+
await rename(partPath, localPath);
|
|
334
|
+
ok = true;
|
|
335
|
+
break;
|
|
336
|
+
}
|
|
337
|
+
} catch {
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
if (ok) {
|
|
341
|
+
done++;
|
|
342
|
+
broadcast("asr-progress", { file: rel, percent: Math.round(done / tree.length * 100) });
|
|
343
|
+
} else {
|
|
344
|
+
allOk = false;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
await Promise.all(Array.from({ length: 8 }, () => worker()));
|
|
349
|
+
return allOk;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// src/asr-host.ts
|
|
353
|
+
var { createOnlineRecognizer, createVad } = sherpa_onnx;
|
|
12
354
|
var MODEL_REPO = "csukuangfj/sherpa-onnx-streaming-zipformer-zh-int8-2025-06-30";
|
|
13
|
-
var MODEL_FILES = [
|
|
355
|
+
var MODEL_FILES = [
|
|
356
|
+
{ file: "encoder.int8.onnx", sha256: "5ac51e27981bb4dab01bb9be4958453ba50c3b61c063ddda0eab23fd3671aa4f" },
|
|
357
|
+
{ file: "decoder.onnx", sha256: "06522ad63cec0fdf6809f4e1db9bb4f7d710c34582e3b35db62ac60eccafac7e" },
|
|
358
|
+
{ file: "joiner.int8.onnx", sha256: "b34584dc6f561089e1d747fedebb3765f2caa72c927ef54d7ca55e5ae40a814b" },
|
|
359
|
+
{ file: "tokens.txt", sha256: "6193c7ea1c96d0d9a1e9652789b40d13a8a913b434a5451e93158f5a09fd6652" }
|
|
360
|
+
];
|
|
361
|
+
var VAD_REPO = "csukuangfj/vad";
|
|
362
|
+
var VAD_FILES = [
|
|
363
|
+
{ file: "silero_vad.onnx", sha256: "a35ebf52fd3ce5f1469b2a36158dba761bc47b973ea3382b3186ca15b1f5af28" }
|
|
364
|
+
];
|
|
365
|
+
var SENSE_REPO = "csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17";
|
|
366
|
+
var SENSE_FILES = [
|
|
367
|
+
{ file: "model.int8.onnx", sha256: "c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51" },
|
|
368
|
+
{ file: "tokens.txt", sha256: "f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc" }
|
|
369
|
+
];
|
|
14
370
|
function pcmToSamples(buf) {
|
|
15
|
-
if (buf.length % 4 !== 0
|
|
371
|
+
if (buf.length % 4 !== 0) return null;
|
|
16
372
|
return new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4);
|
|
17
373
|
}
|
|
18
374
|
var MAX_ASR_BYTES = 4 * 1024 * 1024;
|
|
375
|
+
var SEGMENT_IDLE_MS = 9e4;
|
|
376
|
+
var VAD_CONTINUE_RMS = 0.02;
|
|
377
|
+
var CONFIRM_CONJUNCTION_MS = 800;
|
|
378
|
+
var CONFIRM_LONG_SENTENCE_MS = 350;
|
|
379
|
+
var CONFIRM_LONG_SENTENCE_S = 8;
|
|
380
|
+
var CONFIRM_MIN_MS = 400;
|
|
381
|
+
var CONJUNCTION_TAIL = /(然后|还有|以及|并且|而且|此外|再说|接着|然后呢|比方说|比如说|比如|例如|等等|或者|或是|还有呢)$/;
|
|
382
|
+
function endpointConfirmMs(text, spokenMs) {
|
|
383
|
+
const tail = text.trimEnd();
|
|
384
|
+
if (CONJUNCTION_TAIL.test(tail)) return CONFIRM_CONJUNCTION_MS;
|
|
385
|
+
if (spokenMs > CONFIRM_LONG_SENTENCE_S * 1e3) return CONFIRM_LONG_SENTENCE_MS;
|
|
386
|
+
return 0;
|
|
387
|
+
}
|
|
388
|
+
function rmsOf(samples) {
|
|
389
|
+
if (samples.length === 0) return 0;
|
|
390
|
+
let sum = 0;
|
|
391
|
+
for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i];
|
|
392
|
+
return Math.sqrt(sum / samples.length);
|
|
393
|
+
}
|
|
19
394
|
function createAsrRuntime(options) {
|
|
20
|
-
const { cacheDir, modelHost, broadcast } = options;
|
|
21
|
-
|
|
395
|
+
const { cacheDir, modelHost, broadcast, senseVoice, allowCustomHost } = options;
|
|
396
|
+
let lastProgress = null;
|
|
397
|
+
const localBroadcast = (event, payload) => {
|
|
398
|
+
if (event === "asr-progress") lastProgress = payload;
|
|
399
|
+
broadcast(event, payload);
|
|
400
|
+
};
|
|
401
|
+
const repoDir = join2(cacheDir, MODEL_REPO);
|
|
402
|
+
const vadDir = join2(cacheDir, VAD_REPO);
|
|
403
|
+
const senseDir = join2(cacheDir, SENSE_REPO);
|
|
404
|
+
const normalizedModelHost = () => validateModelHost(modelHost(), allowCustomHost) ?? HOST_PRIMARY;
|
|
22
405
|
const segments = /* @__PURE__ */ new Map();
|
|
406
|
+
const finalized = /* @__PURE__ */ new Map();
|
|
407
|
+
const finalizing = /* @__PURE__ */ new Map();
|
|
408
|
+
const resetGen = /* @__PURE__ */ new Map();
|
|
23
409
|
let recognizer = null;
|
|
24
410
|
let modelsReady = false;
|
|
25
411
|
let modelsLoading = null;
|
|
26
|
-
|
|
27
|
-
for (const f of MODEL_FILES) {
|
|
28
|
-
const st = await stat(join(repoDir, f)).catch(() => null);
|
|
29
|
-
if (!st?.isFile()) return false;
|
|
30
|
-
}
|
|
31
|
-
return true;
|
|
32
|
-
};
|
|
412
|
+
let asrFailAt = 0;
|
|
33
413
|
const ensureModels = async () => {
|
|
34
414
|
if (modelsReady) return true;
|
|
415
|
+
if (Date.now() < asrFailAt) return false;
|
|
35
416
|
if (!modelsLoading) {
|
|
36
417
|
modelsLoading = (async () => {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
}
|
|
418
|
+
for (const f of MODEL_FILES) {
|
|
419
|
+
if (!await ensureModelFile({ repo: MODEL_REPO, repoDir, spec: f, primaryHost: normalizedModelHost(), allowCustomHost, broadcast: localBroadcast })) {
|
|
420
|
+
asrFailAt = Date.now() + 6e4;
|
|
421
|
+
broadcast("asr-error", { file: f.file });
|
|
422
|
+
return false;
|
|
43
423
|
}
|
|
44
424
|
}
|
|
45
425
|
modelsReady = true;
|
|
@@ -54,7 +434,7 @@ function createAsrRuntime(options) {
|
|
|
54
434
|
const getRecognizer = async () => {
|
|
55
435
|
if (!await ensureModels()) return null;
|
|
56
436
|
if (recognizer) return recognizer;
|
|
57
|
-
const t = (f) =>
|
|
437
|
+
const t = (f) => join2(repoDir, f);
|
|
58
438
|
recognizer = createOnlineRecognizer({
|
|
59
439
|
modelConfig: {
|
|
60
440
|
transducer: {
|
|
@@ -71,97 +451,451 @@ function createAsrRuntime(options) {
|
|
|
71
451
|
});
|
|
72
452
|
return recognizer;
|
|
73
453
|
};
|
|
74
|
-
|
|
454
|
+
let vadModelReady = false;
|
|
455
|
+
let vadLoading = null;
|
|
456
|
+
let vadFailAt = 0;
|
|
457
|
+
const ensureVadModel = async () => {
|
|
458
|
+
if (vadModelReady) return join2(vadDir, VAD_FILES[0].file);
|
|
459
|
+
if (Date.now() < vadFailAt) return null;
|
|
460
|
+
if (!vadLoading) {
|
|
461
|
+
vadLoading = (async () => {
|
|
462
|
+
for (const f of VAD_FILES) {
|
|
463
|
+
if (!await ensureModelFile({ repo: VAD_REPO, repoDir: vadDir, spec: f, primaryHost: normalizedModelHost(), allowCustomHost, broadcast: localBroadcast })) {
|
|
464
|
+
vadFailAt = Date.now() + 6e4;
|
|
465
|
+
return null;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
vadModelReady = true;
|
|
469
|
+
return join2(vadDir, VAD_FILES[0].file);
|
|
470
|
+
})().finally(() => {
|
|
471
|
+
vadLoading = null;
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
return vadLoading;
|
|
475
|
+
};
|
|
476
|
+
const newVad = (vadPath, threshold = 0.5) => createVad({
|
|
477
|
+
sileroVad: {
|
|
478
|
+
model: vadPath,
|
|
479
|
+
threshold,
|
|
480
|
+
minSilenceDuration: 0.5,
|
|
481
|
+
minSpeechDuration: 0.25,
|
|
482
|
+
maxSpeechDuration: 20,
|
|
483
|
+
windowSize: 512
|
|
484
|
+
},
|
|
485
|
+
sampleRate: 16e3,
|
|
486
|
+
numThreads: 1,
|
|
487
|
+
provider: "cpu",
|
|
488
|
+
debug: 0,
|
|
489
|
+
bufferSizeInSeconds: 30
|
|
490
|
+
});
|
|
491
|
+
const ensureSessionVad = async (seg) => {
|
|
492
|
+
if (seg.vad) return seg.vad;
|
|
493
|
+
const vadPath = await ensureVadModel();
|
|
494
|
+
if (!vadPath) return null;
|
|
495
|
+
seg.vad = newVad(vadPath);
|
|
496
|
+
return seg.vad;
|
|
497
|
+
};
|
|
498
|
+
const detectVads = /* @__PURE__ */ new Map();
|
|
499
|
+
const detectVadLastUse = /* @__PURE__ */ new Map();
|
|
500
|
+
const ensureDetectVad = async (sessionId) => {
|
|
501
|
+
const existing = detectVads.get(sessionId);
|
|
502
|
+
if (existing) return existing;
|
|
503
|
+
const vadPath = await ensureVadModel();
|
|
504
|
+
if (!vadPath) return null;
|
|
505
|
+
const vad = newVad(vadPath, 0.35);
|
|
506
|
+
detectVads.set(sessionId, vad);
|
|
507
|
+
return vad;
|
|
508
|
+
};
|
|
509
|
+
let senseModelReady = false;
|
|
510
|
+
let senseLoading = null;
|
|
511
|
+
let senseFailAt = 0;
|
|
512
|
+
const ensureSenseModel = async () => {
|
|
513
|
+
if (senseModelReady) return join2(senseDir, SENSE_FILES[0].file);
|
|
514
|
+
if (Date.now() < senseFailAt) return null;
|
|
515
|
+
if (!senseLoading) {
|
|
516
|
+
senseLoading = (async () => {
|
|
517
|
+
for (const f of SENSE_FILES) {
|
|
518
|
+
if (!await ensureModelFile({ repo: SENSE_REPO, repoDir: senseDir, spec: f, primaryHost: normalizedModelHost(), allowCustomHost, broadcast: localBroadcast })) {
|
|
519
|
+
senseFailAt = Date.now() + 6e4;
|
|
520
|
+
return null;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
senseModelReady = true;
|
|
524
|
+
return join2(senseDir, SENSE_FILES[0].file);
|
|
525
|
+
})().finally(() => {
|
|
526
|
+
senseLoading = null;
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
return senseLoading;
|
|
530
|
+
};
|
|
531
|
+
let senseWorker = null;
|
|
532
|
+
let senseWorkerSyncing = null;
|
|
533
|
+
const getSenseWorker = async () => {
|
|
534
|
+
if (!senseVoice()) return null;
|
|
535
|
+
if (senseWorker) return senseWorker;
|
|
536
|
+
if (senseWorkerSyncing) return senseWorkerSyncing;
|
|
537
|
+
senseWorkerSyncing = (async () => {
|
|
538
|
+
const sensePath = await ensureSenseModel();
|
|
539
|
+
if (!sensePath) return null;
|
|
540
|
+
try {
|
|
541
|
+
const workerPath = fileURLToPath(new URL("./sense-worker.mjs", import.meta.url));
|
|
542
|
+
const w = new Worker(workerPath, {
|
|
543
|
+
workerData: { sherpaModule: "sherpa-onnx", modelDir: senseDir }
|
|
544
|
+
});
|
|
545
|
+
const client = createSenseWorkerClient(w);
|
|
546
|
+
client.onDeath(() => {
|
|
547
|
+
senseWorker = null;
|
|
548
|
+
senseWorkerSyncing = null;
|
|
549
|
+
});
|
|
550
|
+
if (!await client.request("create")) {
|
|
551
|
+
await client.terminate();
|
|
552
|
+
return null;
|
|
553
|
+
}
|
|
554
|
+
senseWorker = client;
|
|
555
|
+
return client;
|
|
556
|
+
} catch (e) {
|
|
557
|
+
console.warn("[dsh-voice-mode] SenseVoice worker init failed: " + String(e));
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
})().finally(() => {
|
|
561
|
+
if (!senseWorker) senseWorkerSyncing = null;
|
|
562
|
+
});
|
|
563
|
+
return senseWorkerSyncing;
|
|
564
|
+
};
|
|
565
|
+
const senseTranscribe = async (allSamples) => {
|
|
566
|
+
try {
|
|
567
|
+
const worker = await getSenseWorker();
|
|
568
|
+
if (!worker) return null;
|
|
569
|
+
const total = allSamples.reduce((acc, c) => acc + c.length, 0);
|
|
570
|
+
if (total === 0) return null;
|
|
571
|
+
const buf = new Float32Array(total);
|
|
572
|
+
let off = 0;
|
|
573
|
+
for (const c of allSamples) {
|
|
574
|
+
buf.set(c, off);
|
|
575
|
+
off += c.length;
|
|
576
|
+
}
|
|
577
|
+
return await worker.request("decode", buf);
|
|
578
|
+
} catch (e) {
|
|
579
|
+
console.warn("[dsh-voice-mode] SenseVoice re-transcribe failed: " + String(e));
|
|
580
|
+
return null;
|
|
581
|
+
}
|
|
582
|
+
};
|
|
583
|
+
const feed = async (sessionId, samples, final, offset = 0, epoch = 0) => {
|
|
75
584
|
const rec = await getRecognizer();
|
|
76
585
|
if (!rec) return { text: "", loading: true };
|
|
77
|
-
|
|
586
|
+
if (!final && senseVoice()) {
|
|
587
|
+
void getSenseWorker().catch(() => {
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
let finMap = finalized.get(sessionId);
|
|
591
|
+
const myGen = resetGen.get(sessionId) ?? 0;
|
|
592
|
+
const cached = finMap?.get(epoch);
|
|
593
|
+
if (cached !== void 0) return { text: cached };
|
|
594
|
+
let sessSegs = segments.get(sessionId);
|
|
595
|
+
if (!sessSegs) {
|
|
596
|
+
sessSegs = /* @__PURE__ */ new Map();
|
|
597
|
+
segments.set(sessionId, sessSegs);
|
|
598
|
+
}
|
|
599
|
+
let seg = sessSegs.get(epoch);
|
|
78
600
|
if (!seg) {
|
|
79
601
|
if (samples.length === 0 && final) return { text: "" };
|
|
80
|
-
seg = { stream: rec.createStream(), fed: 0 };
|
|
81
|
-
|
|
602
|
+
seg = { stream: rec.createStream(), fed: 0, vad: null, pendingEndpoint: null, lastText: "", allSamples: [], lastActivity: Date.now() };
|
|
603
|
+
sessSegs.set(epoch, seg);
|
|
82
604
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
605
|
+
seg.lastActivity = Date.now();
|
|
606
|
+
let endpoint = false;
|
|
607
|
+
let text = "";
|
|
608
|
+
let isSpeech;
|
|
609
|
+
if (offset + samples.length > seg.fed) {
|
|
610
|
+
const skip = Math.max(seg.fed - offset, 0);
|
|
611
|
+
const inc = samples.subarray(skip);
|
|
612
|
+
seg.stream.acceptWaveform(rec.config.featConfig.sampleRate, inc);
|
|
613
|
+
seg.fed = offset + samples.length;
|
|
614
|
+
if (seg.fed <= rec.config.featConfig.sampleRate * 60) seg.allSamples.push(inc);
|
|
86
615
|
while (rec.isReady(seg.stream)) rec.decode(seg.stream);
|
|
616
|
+
text = rec.getResult(seg.stream).text;
|
|
617
|
+
seg.lastText = text;
|
|
618
|
+
if (!final) {
|
|
619
|
+
const vad = await ensureSessionVad(seg);
|
|
620
|
+
if (vad) {
|
|
621
|
+
if (seg.pendingEndpoint) {
|
|
622
|
+
const now = Date.now();
|
|
623
|
+
const rms = rmsOf(inc);
|
|
624
|
+
if (rms > VAD_CONTINUE_RMS) {
|
|
625
|
+
seg.pendingEndpoint = null;
|
|
626
|
+
} else if (now - seg.pendingEndpoint.at >= CONFIRM_MIN_MS && text === seg.pendingEndpoint.textAtPending) {
|
|
627
|
+
seg.pendingEndpoint = null;
|
|
628
|
+
endpoint = true;
|
|
629
|
+
} else if (now - seg.pendingEndpoint.at >= seg.pendingEndpoint.confirmMs) {
|
|
630
|
+
seg.pendingEndpoint = null;
|
|
631
|
+
endpoint = true;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
vad.acceptWaveform(inc);
|
|
635
|
+
isSpeech = vad.isDetected();
|
|
636
|
+
if (!vad.isEmpty()) {
|
|
637
|
+
let spokenMs = 0;
|
|
638
|
+
while (!vad.isEmpty()) {
|
|
639
|
+
const sp = vad.front();
|
|
640
|
+
spokenMs = sp.samples.length / 16e3 * 1e3;
|
|
641
|
+
vad.pop();
|
|
642
|
+
}
|
|
643
|
+
const confirmMs = endpointConfirmMs(seg.lastText, spokenMs);
|
|
644
|
+
if (confirmMs <= 0) {
|
|
645
|
+
endpoint = true;
|
|
646
|
+
} else {
|
|
647
|
+
seg.pendingEndpoint = { at: Date.now(), confirmMs, textAtPending: seg.lastText };
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (!final) return { text, endpoint, isSpeech };
|
|
654
|
+
const inflightMap = finalizing.get(sessionId);
|
|
655
|
+
const inflightP = inflightMap?.get(epoch);
|
|
656
|
+
if (inflightP) return { text: await inflightP };
|
|
657
|
+
sessSegs.delete(epoch);
|
|
658
|
+
if (sessSegs.size === 0) segments.delete(sessionId);
|
|
659
|
+
const finalizeP = (async () => {
|
|
660
|
+
const all = seg.allSamples;
|
|
661
|
+
const senseP = all.length > 0 ? Promise.race([
|
|
662
|
+
senseTranscribe(all),
|
|
663
|
+
new Promise((resolve) => setTimeout(() => resolve(null), 1e4))
|
|
664
|
+
]) : Promise.resolve(null);
|
|
665
|
+
const pad = new Float32Array(rec.config.featConfig.sampleRate / 2);
|
|
666
|
+
seg.stream.acceptWaveform(rec.config.featConfig.sampleRate, pad);
|
|
667
|
+
while (rec.isReady(seg.stream)) rec.decode(seg.stream);
|
|
668
|
+
const settled = rec.getResult(seg.stream).text;
|
|
669
|
+
try {
|
|
670
|
+
seg.vad?.free?.();
|
|
671
|
+
} catch {
|
|
672
|
+
}
|
|
673
|
+
seg.stream.free();
|
|
674
|
+
const sense = await senseP;
|
|
675
|
+
return (sense && sense.trim() ? sense : settled) || "";
|
|
676
|
+
})().then((finalText) => {
|
|
677
|
+
if ((resetGen.get(sessionId) ?? 0) !== myGen) return finalText;
|
|
678
|
+
let fm = finalized.get(sessionId);
|
|
679
|
+
if (!fm) {
|
|
680
|
+
fm = /* @__PURE__ */ new Map();
|
|
681
|
+
finalized.set(sessionId, fm);
|
|
682
|
+
}
|
|
683
|
+
fm.set(epoch, finalText);
|
|
684
|
+
if (fm.size > 32) {
|
|
685
|
+
const first = fm.keys().next().value;
|
|
686
|
+
if (first !== void 0) fm.delete(first);
|
|
687
|
+
}
|
|
688
|
+
const ff = finalizing.get(sessionId);
|
|
689
|
+
ff?.delete(epoch);
|
|
690
|
+
if (ff && ff.size === 0) finalizing.delete(sessionId);
|
|
691
|
+
return finalText;
|
|
692
|
+
}).catch((e) => {
|
|
693
|
+
const ff = finalizing.get(sessionId);
|
|
694
|
+
ff?.delete(epoch);
|
|
695
|
+
if (ff && ff.size === 0) finalizing.delete(sessionId);
|
|
696
|
+
console.warn("[dsh-voice-mode] finalize failed: " + String(e));
|
|
697
|
+
return "";
|
|
698
|
+
});
|
|
699
|
+
if (!inflightMap) {
|
|
700
|
+
finalizing.set(sessionId, /* @__PURE__ */ new Map());
|
|
87
701
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
const pad = new Float32Array(rec.config.featConfig.sampleRate / 2);
|
|
91
|
-
seg.stream.acceptWaveform(rec.config.featConfig.sampleRate, pad);
|
|
92
|
-
while (rec.isReady(seg.stream)) rec.decode(seg.stream);
|
|
93
|
-
const settled = rec.getResult(seg.stream).text;
|
|
94
|
-
seg.stream.free();
|
|
95
|
-
segments.delete(sessionId);
|
|
96
|
-
return { text: settled };
|
|
702
|
+
finalizing.get(sessionId).set(epoch, finalizeP);
|
|
703
|
+
return { text: await finalizeP };
|
|
97
704
|
};
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
705
|
+
const sweep = () => {
|
|
706
|
+
const now = Date.now();
|
|
707
|
+
for (const [sid, sessSegs] of segments) {
|
|
708
|
+
for (const [epoch, s] of sessSegs) {
|
|
709
|
+
if (now - s.lastActivity > SEGMENT_IDLE_MS) {
|
|
710
|
+
try {
|
|
711
|
+
s.vad?.free?.();
|
|
712
|
+
} catch {
|
|
713
|
+
}
|
|
714
|
+
try {
|
|
715
|
+
s.stream.free();
|
|
716
|
+
} catch {
|
|
717
|
+
}
|
|
718
|
+
sessSegs.delete(epoch);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
if (sessSegs.size === 0) {
|
|
722
|
+
segments.delete(sid);
|
|
723
|
+
finalized.delete(sid);
|
|
114
724
|
}
|
|
115
|
-
} catch {
|
|
116
725
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
726
|
+
for (const [sid, at] of detectVadLastUse) {
|
|
727
|
+
if (now - at > SEGMENT_IDLE_MS) {
|
|
728
|
+
try {
|
|
729
|
+
detectVads.get(sid)?.free?.();
|
|
730
|
+
} catch {
|
|
731
|
+
}
|
|
732
|
+
detectVads.delete(sid);
|
|
733
|
+
detectVadLastUse.delete(sid);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
};
|
|
737
|
+
const sweepTimer = setInterval(sweep, 3e4);
|
|
738
|
+
return {
|
|
739
|
+
feed,
|
|
740
|
+
detect: async (sessionId, samples) => {
|
|
741
|
+
const vad = await ensureDetectVad(sessionId);
|
|
742
|
+
if (!vad) return { isSpeech: false };
|
|
743
|
+
detectVadLastUse.set(sessionId, Date.now());
|
|
744
|
+
if (samples.length > 0) vad.acceptWaveform(samples);
|
|
745
|
+
const speech = vad.isDetected();
|
|
746
|
+
while (!vad.isEmpty()) vad.pop();
|
|
747
|
+
return { isSpeech: speech };
|
|
748
|
+
},
|
|
749
|
+
reset: (sessionId) => {
|
|
750
|
+
const sessSegs = segments.get(sessionId);
|
|
751
|
+
if (sessSegs) {
|
|
752
|
+
for (const [, s] of sessSegs) {
|
|
753
|
+
try {
|
|
754
|
+
s.vad?.free?.();
|
|
755
|
+
} catch {
|
|
756
|
+
}
|
|
757
|
+
try {
|
|
758
|
+
s.stream.free();
|
|
759
|
+
} catch {
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
segments.delete(sessionId);
|
|
763
|
+
}
|
|
764
|
+
finalized.delete(sessionId);
|
|
765
|
+
resetGen.set(sessionId, (resetGen.get(sessionId) ?? 0) + 1);
|
|
766
|
+
finalizing.delete(sessionId);
|
|
767
|
+
const dv = detectVads.get(sessionId);
|
|
768
|
+
if (dv) {
|
|
769
|
+
try {
|
|
770
|
+
dv.free?.();
|
|
771
|
+
} catch {
|
|
772
|
+
}
|
|
773
|
+
detectVads.delete(sessionId);
|
|
774
|
+
}
|
|
775
|
+
detectVadLastUse.delete(sessionId);
|
|
776
|
+
},
|
|
777
|
+
dispose: () => {
|
|
778
|
+
clearInterval(sweepTimer);
|
|
779
|
+
let w = senseWorker;
|
|
780
|
+
senseWorker = null;
|
|
781
|
+
senseWorkerSyncing = null;
|
|
782
|
+
if (w) void w.terminate();
|
|
783
|
+
for (const [, sessSegs] of segments) {
|
|
784
|
+
for (const [, s] of sessSegs) {
|
|
785
|
+
try {
|
|
786
|
+
s.vad?.free?.();
|
|
787
|
+
} catch {
|
|
788
|
+
}
|
|
789
|
+
try {
|
|
790
|
+
s.stream.free();
|
|
791
|
+
} catch {
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
segments.clear();
|
|
796
|
+
finalized.clear();
|
|
797
|
+
finalizing.clear();
|
|
798
|
+
resetGen.clear();
|
|
141
799
|
try {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
800
|
+
recognizer?.free?.();
|
|
801
|
+
} catch {
|
|
802
|
+
}
|
|
803
|
+
recognizer = null;
|
|
804
|
+
for (const [, dv] of detectVads) {
|
|
805
|
+
try {
|
|
806
|
+
dv.free?.();
|
|
807
|
+
} catch {
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
detectVads.clear();
|
|
811
|
+
detectVadLastUse.clear();
|
|
812
|
+
},
|
|
813
|
+
warmup: () => {
|
|
814
|
+
void getRecognizer().catch(() => void 0);
|
|
815
|
+
void ensureVadModel().catch(() => void 0);
|
|
816
|
+
if (senseVoice()) void getSenseWorker().catch(() => void 0);
|
|
817
|
+
},
|
|
818
|
+
modelStatus: () => {
|
|
819
|
+
const statFile = async (dir, repo, name2) => {
|
|
820
|
+
const st = await stat2(join2(dir, repo, name2)).catch(() => null);
|
|
821
|
+
return { exists: !!st?.isFile(), size: st?.size ?? 0 };
|
|
822
|
+
};
|
|
823
|
+
const asrFiles = MODEL_FILES.map((n) => ({
|
|
824
|
+
name: n.file,
|
|
825
|
+
exists: (() => {
|
|
826
|
+
try {
|
|
827
|
+
return statSync(join2(repoDir, n.file)).isFile();
|
|
828
|
+
} catch {
|
|
829
|
+
return false;
|
|
148
830
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
831
|
+
})(),
|
|
832
|
+
size: (() => {
|
|
833
|
+
try {
|
|
834
|
+
return statSync(join2(repoDir, n.file)).size;
|
|
835
|
+
} catch {
|
|
836
|
+
return 0;
|
|
154
837
|
}
|
|
838
|
+
})()
|
|
839
|
+
}));
|
|
840
|
+
const vadSize = (() => {
|
|
841
|
+
try {
|
|
842
|
+
return statSync(join2(vadDir, VAD_FILES[0].file)).size;
|
|
843
|
+
} catch {
|
|
844
|
+
return 0;
|
|
155
845
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
846
|
+
})();
|
|
847
|
+
const senseSize = (() => {
|
|
848
|
+
try {
|
|
849
|
+
return statSync(join2(senseDir, SENSE_FILES[0].file)).size;
|
|
850
|
+
} catch {
|
|
851
|
+
return 0;
|
|
852
|
+
}
|
|
853
|
+
})();
|
|
854
|
+
return {
|
|
855
|
+
// ready 语义 = 文件可用(exists),而非进程内是否已实例化——
|
|
856
|
+
// 重启后文件齐全却显示「未下载」会误导用户(体验修复)。
|
|
857
|
+
asr: {
|
|
858
|
+
repo: MODEL_REPO,
|
|
859
|
+
ready: asrFiles.every((f) => f.exists),
|
|
860
|
+
files: asrFiles,
|
|
861
|
+
failLatchMs: Math.max(0, asrFailAt - Date.now())
|
|
862
|
+
},
|
|
863
|
+
vad: {
|
|
864
|
+
repo: VAD_REPO,
|
|
865
|
+
ready: vadSize > 0,
|
|
866
|
+
size: vadSize,
|
|
867
|
+
failLatchMs: Math.max(0, vadFailAt - Date.now())
|
|
868
|
+
},
|
|
869
|
+
sense: {
|
|
870
|
+
repo: SENSE_REPO,
|
|
871
|
+
ready: senseSize > 0,
|
|
872
|
+
size: senseSize,
|
|
873
|
+
failLatchMs: Math.max(0, senseFailAt - Date.now()),
|
|
874
|
+
enabled: senseVoice()
|
|
875
|
+
},
|
|
876
|
+
progress: lastProgress
|
|
877
|
+
};
|
|
878
|
+
},
|
|
879
|
+
retryModel: async (kind) => {
|
|
880
|
+
if (kind === "vad") {
|
|
881
|
+
vadFailAt = 0;
|
|
882
|
+
return !!await ensureVadModel();
|
|
160
883
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
884
|
+
if (kind === "sense") {
|
|
885
|
+
if (!senseVoice()) return false;
|
|
886
|
+
senseFailAt = 0;
|
|
887
|
+
return !!await ensureSenseModel();
|
|
888
|
+
}
|
|
889
|
+
if (modelsReady) return true;
|
|
890
|
+
asrFailAt = 0;
|
|
891
|
+
return await ensureModels();
|
|
892
|
+
}
|
|
893
|
+
};
|
|
164
894
|
}
|
|
895
|
+
var respondJson = (res, status, payload) => {
|
|
896
|
+
res.writeHead(status, { "content-type": "application/json" });
|
|
897
|
+
res.end(JSON.stringify(payload));
|
|
898
|
+
};
|
|
165
899
|
function handleAsrRequest(asr, activeSessionId, req, res) {
|
|
166
900
|
const chunks = [];
|
|
167
901
|
let received = 0;
|
|
@@ -171,45 +905,66 @@ function handleAsrRequest(asr, activeSessionId, req, res) {
|
|
|
171
905
|
received += c.length;
|
|
172
906
|
if (received > MAX_ASR_BYTES) {
|
|
173
907
|
tooLarge = true;
|
|
174
|
-
res
|
|
175
|
-
res.end(JSON.stringify({ error: "pcm payload too large" }));
|
|
908
|
+
respondJson(res, 413, { error: "pcm payload too large" });
|
|
176
909
|
return;
|
|
177
910
|
}
|
|
178
911
|
chunks.push(c);
|
|
179
912
|
});
|
|
180
913
|
req.on("end", () => {
|
|
181
914
|
if (tooLarge) return;
|
|
182
|
-
res.setHeader("content-type", "application/json");
|
|
183
915
|
const url = new URL(req.url ?? "/", "http://localhost");
|
|
184
916
|
const sessionId = url.searchParams.get("sessionId") ?? "";
|
|
185
917
|
const final = url.searchParams.get("final") === "1";
|
|
186
918
|
const reset = url.searchParams.get("reset") === "1";
|
|
919
|
+
const epochParam = url.searchParams.get("epoch");
|
|
920
|
+
const epochN = Number(epochParam);
|
|
921
|
+
const epochOK = epochParam === null || Number.isFinite(epochN) && epochN >= 0 && Number.isInteger(epochN);
|
|
922
|
+
const offsetParam = url.searchParams.get("offset");
|
|
923
|
+
const offsetOK = offsetParam === null || Number.isFinite(Number(offsetParam)) && Number(offsetParam) >= 0 && Number(offsetParam) <= MAX_ASR_BYTES / 4;
|
|
924
|
+
if (!offsetOK) {
|
|
925
|
+
respondJson(res, 400, { error: "invalid offset" });
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
if (!epochOK) {
|
|
929
|
+
respondJson(res, 400, { error: "invalid epoch" });
|
|
930
|
+
return;
|
|
931
|
+
}
|
|
932
|
+
const epoch = epochParam === null ? 0 : Math.floor(epochN);
|
|
933
|
+
const offset = offsetParam === null ? 0 : Math.floor(Number(offsetParam));
|
|
187
934
|
if (!sessionId || sessionId !== activeSessionId) {
|
|
188
|
-
res
|
|
189
|
-
res.end(JSON.stringify({ error: "not the active voice session" }));
|
|
935
|
+
respondJson(res, 403, { error: "not the active voice session" });
|
|
190
936
|
return;
|
|
191
937
|
}
|
|
192
938
|
if (reset) {
|
|
193
939
|
asr.reset(sessionId);
|
|
194
|
-
res
|
|
940
|
+
respondJson(res, 200, { ok: true });
|
|
195
941
|
return;
|
|
196
942
|
}
|
|
197
|
-
const
|
|
943
|
+
const raw = Buffer.concat(chunks);
|
|
944
|
+
const samples = raw.length === 0 ? final ? new Float32Array(0) : null : pcmToSamples(raw);
|
|
198
945
|
if (!samples) {
|
|
199
|
-
res
|
|
200
|
-
res.end(JSON.stringify({ error: "invalid pcm payload" }));
|
|
946
|
+
respondJson(res, 400, { error: "invalid pcm payload" });
|
|
201
947
|
return;
|
|
202
948
|
}
|
|
203
|
-
|
|
949
|
+
if (url.searchParams.get("vadOnly") === "1") {
|
|
950
|
+
void asr.detect(sessionId, samples).then((out) => {
|
|
951
|
+
respondJson(res, 200, { isSpeech: out.isSpeech });
|
|
952
|
+
}).catch((e) => {
|
|
953
|
+
respondJson(res, 500, { error: String(e) });
|
|
954
|
+
});
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
void asr.feed(sessionId, samples, final, offset, epoch).then((out) => {
|
|
204
958
|
if (out.loading) {
|
|
205
|
-
res
|
|
206
|
-
res.end(JSON.stringify({ loading: true }));
|
|
959
|
+
respondJson(res, 202, { loading: true });
|
|
207
960
|
return;
|
|
208
961
|
}
|
|
209
|
-
|
|
962
|
+
const body = { text: out.text };
|
|
963
|
+
if (out.endpoint) body.endpoint = true;
|
|
964
|
+
if (out.isSpeech !== void 0) body.isSpeech = out.isSpeech;
|
|
965
|
+
respondJson(res, 200, body);
|
|
210
966
|
}).catch((e) => {
|
|
211
|
-
res
|
|
212
|
-
res.end(JSON.stringify({ error: String(e) }));
|
|
967
|
+
respondJson(res, 500, { error: String(e) });
|
|
213
968
|
});
|
|
214
969
|
});
|
|
215
970
|
}
|
|
@@ -219,6 +974,9 @@ var SKIP_PREFIX = /^[\s.,,、::;;!?!?)\])"'”’〉》】]+$/;
|
|
|
219
974
|
function plainText(text) {
|
|
220
975
|
return String(text).replace(/```[\s\S]*?```/g, " ").replace(/`([^`]*)`/g, "$1").replace(/!\[[^\]]*\]\([^)]*\)/g, " ").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/\*\*([^*]+)\*\*/g, "$1").replace(/\*([^*]+)\*/g, "$1").replace(/^[-*+]\s+/gm, "").replace(/^\d+\.\s+/gm, "").replace(/<\/?[a-zA-Z][^>]*>/g, " ");
|
|
221
976
|
}
|
|
977
|
+
function sanitizeForTts(text) {
|
|
978
|
+
return String(text).replace(/[*_#>`|^=+~]/g, " ").replace(/\s{2,}/g, " ").replace(/([\u3400-\u9fff])\s+(?=[\u3400-\u9fff])/g, "$1").trim();
|
|
979
|
+
}
|
|
222
980
|
function splitSentences(chunk) {
|
|
223
981
|
const sentences = [];
|
|
224
982
|
let start = 0;
|
|
@@ -248,13 +1006,13 @@ var SentenceSegmenter = class {
|
|
|
248
1006
|
this.buffer = tail;
|
|
249
1007
|
const out = [];
|
|
250
1008
|
for (const s of sentences) {
|
|
251
|
-
const t = s.trim();
|
|
1009
|
+
const t = sanitizeForTts(s).trim();
|
|
252
1010
|
if (t && !SKIP_PREFIX.test(t)) out.push(t);
|
|
253
1011
|
}
|
|
254
1012
|
if (this.buffer.length > this.maxChars) {
|
|
255
1013
|
const cut = this.buffer.search(/[,,、\s]/);
|
|
256
1014
|
const idx = cut > 0 ? cut : Math.floor(this.maxChars / 2);
|
|
257
|
-
const head = this.buffer.slice(0, idx).trim();
|
|
1015
|
+
const head = sanitizeForTts(this.buffer.slice(0, idx)).trim();
|
|
258
1016
|
this.buffer = this.buffer.slice(idx);
|
|
259
1017
|
if (head) out.push(head);
|
|
260
1018
|
}
|
|
@@ -262,7 +1020,7 @@ var SentenceSegmenter = class {
|
|
|
262
1020
|
}
|
|
263
1021
|
/** 收尾:flush 剩余缓冲(流结束)。 */
|
|
264
1022
|
flush() {
|
|
265
|
-
const t = this.buffer.trim();
|
|
1023
|
+
const t = sanitizeForTts(this.buffer).trim();
|
|
266
1024
|
this.buffer = "";
|
|
267
1025
|
if (t && !SKIP_PREFIX.test(t)) return [t];
|
|
268
1026
|
return [];
|
|
@@ -279,38 +1037,28 @@ function prosodyFromRate(rate) {
|
|
|
279
1037
|
}
|
|
280
1038
|
function isValidMp3(buf) {
|
|
281
1039
|
return buf.length > 0 && buf[0] === MP3_MAGIC;
|
|
282
|
-
}
|
|
283
|
-
var
|
|
284
|
-
tts = new MsEdgeTTS();
|
|
285
|
-
queues = /* @__PURE__ */ new Map();
|
|
286
|
-
listeners = /* @__PURE__ */ new Set();
|
|
1040
|
+
}
|
|
1041
|
+
var EdgeTtsEngine = class {
|
|
287
1042
|
voice;
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
constructor(options = {}) {
|
|
293
|
-
this.voice = options.voice ?? "zh-CN-XiaoxiaoNeural";
|
|
294
|
-
this.prosody = prosodyFromRate(options.rate);
|
|
295
|
-
this.onError = options.onError;
|
|
1043
|
+
rate;
|
|
1044
|
+
constructor(voice = "zh-CN-XiaoxiaoNeural", rate) {
|
|
1045
|
+
this.voice = voice;
|
|
1046
|
+
this.rate = rate;
|
|
296
1047
|
}
|
|
297
|
-
|
|
1048
|
+
mime = "audio/mpeg";
|
|
298
1049
|
updateVoice(voice, rate) {
|
|
299
|
-
const nextProsody = prosodyFromRate(rate);
|
|
300
|
-
if (voice === this.voice && nextProsody?.rate === this.prosody?.rate) return;
|
|
301
1050
|
this.voice = voice;
|
|
302
|
-
this.
|
|
303
|
-
this.ready = null;
|
|
1051
|
+
if (rate !== void 0 && Number.isFinite(rate)) this.rate = rate;
|
|
304
1052
|
}
|
|
305
|
-
/**
|
|
306
|
-
* 一次性合成(设置卡「试听」用):独立连接,不干扰朗读队列的在途合成;
|
|
307
|
-
* 音色/语速可指定,缺省用当前队列参数。失败(含非法 ShortName)抛错。
|
|
308
|
-
*/
|
|
309
1053
|
async synthesize(text, options = {}) {
|
|
310
1054
|
const tts = new MsEdgeTTS();
|
|
311
1055
|
try {
|
|
312
|
-
await tts.setMetadata(
|
|
313
|
-
|
|
1056
|
+
await tts.setMetadata(
|
|
1057
|
+
options.voice ?? this.voice,
|
|
1058
|
+
OUTPUT_FORMAT.AUDIO_24KHZ_48KBITRATE_MONO_MP3,
|
|
1059
|
+
TTS_METADATA
|
|
1060
|
+
);
|
|
1061
|
+
const { audioStream } = tts.toStream(text, prosodyFromRate(options.rate ?? this.rate));
|
|
314
1062
|
const chunks = [];
|
|
315
1063
|
for await (const chunk of audioStream) chunks.push(chunk);
|
|
316
1064
|
const buf = Buffer.concat(chunks);
|
|
@@ -323,14 +1071,70 @@ var TtsQueue = class {
|
|
|
323
1071
|
}
|
|
324
1072
|
}
|
|
325
1073
|
}
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
1074
|
+
async close() {
|
|
1075
|
+
}
|
|
1076
|
+
};
|
|
1077
|
+
var edgeVoicesCache = null;
|
|
1078
|
+
async function listEdgeVoices(force = false) {
|
|
1079
|
+
if (edgeVoicesCache && !force) return edgeVoicesCache;
|
|
1080
|
+
const tts = new MsEdgeTTS();
|
|
1081
|
+
try {
|
|
1082
|
+
const voices = await tts.getVoices();
|
|
1083
|
+
edgeVoicesCache = voices.map((v) => ({
|
|
1084
|
+
ShortName: v.ShortName,
|
|
1085
|
+
Locale: v.Locale,
|
|
1086
|
+
Gender: v.Gender,
|
|
1087
|
+
FriendlyName: v.FriendlyName
|
|
1088
|
+
}));
|
|
1089
|
+
return edgeVoicesCache;
|
|
1090
|
+
} finally {
|
|
1091
|
+
try {
|
|
1092
|
+
await tts.close();
|
|
1093
|
+
} catch {
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
var TtsQueue = class {
|
|
1098
|
+
queues = /* @__PURE__ */ new Map();
|
|
1099
|
+
listeners = /* @__PURE__ */ new Set();
|
|
1100
|
+
engine;
|
|
1101
|
+
/** TTS 全体不可达通知(每会话去重,成功后复位)。 */
|
|
1102
|
+
onError;
|
|
1103
|
+
constructor(options) {
|
|
1104
|
+
this.engine = options.engine;
|
|
1105
|
+
this.onError = options.onError;
|
|
1106
|
+
}
|
|
1107
|
+
/** 当前引擎音频 MIME(/preview 的 Content-Type 也用它)。 */
|
|
1108
|
+
get mime() {
|
|
1109
|
+
return this.engine.mime;
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* 运行时切换引擎(设置面板「朗读引擎」即时生效):
|
|
1113
|
+
* 关闭旧引擎、清空所有会话队列;新句子用新引擎合成。
|
|
1114
|
+
*/
|
|
1115
|
+
setEngine(engine) {
|
|
1116
|
+
const old = this.engine;
|
|
1117
|
+
this.engine = engine;
|
|
1118
|
+
this.queues.clear();
|
|
1119
|
+
void old.close().catch(() => {
|
|
332
1120
|
});
|
|
333
|
-
|
|
1121
|
+
}
|
|
1122
|
+
/** 动态更换音色/语速(设置即时生效;正在合成的句子不受影响)。 */
|
|
1123
|
+
updateVoice(voice, rate) {
|
|
1124
|
+
this.engine.updateVoice(voice, rate);
|
|
1125
|
+
}
|
|
1126
|
+
/** 当前引擎/模型现状(设置面板状态区轮询)。 */
|
|
1127
|
+
status() {
|
|
1128
|
+
const s = this.engine.status?.();
|
|
1129
|
+
if (s) return s;
|
|
1130
|
+
return { engine: "edge", ready: true, loading: false };
|
|
1131
|
+
}
|
|
1132
|
+
/**
|
|
1133
|
+
* 一次性合成(设置卡「试听」用):委托当前引擎;不干扰朗读队列的在途合成。
|
|
1134
|
+
* 失败(含非法音色)抛错。
|
|
1135
|
+
*/
|
|
1136
|
+
async synthesize(text, options = {}) {
|
|
1137
|
+
return this.engine.synthesize(text, options);
|
|
334
1138
|
}
|
|
335
1139
|
subscribe(listener) {
|
|
336
1140
|
this.listeners.add(listener);
|
|
@@ -345,12 +1149,13 @@ var TtsQueue = class {
|
|
|
345
1149
|
q = { pending: [], busy: false, seq: 0, epoch: 0, errorNotified: false, backoff: 0 };
|
|
346
1150
|
this.queues.set(sessionId, q);
|
|
347
1151
|
}
|
|
1152
|
+
if (q.pending.length >= 20) q.pending.shift();
|
|
348
1153
|
q.pending.push({ text, epoch: q.epoch });
|
|
349
1154
|
void this.pump(sessionId, q);
|
|
350
1155
|
}
|
|
351
1156
|
/**
|
|
352
1157
|
* 弃掉某会话的所有积压并作废正在合成的句子(打断)。之后入队的句子
|
|
353
|
-
* 获得新 epoch
|
|
1158
|
+
* 获得新 epoch 正常播放。同时立刻中止在途合成(本地引擎杀子进程释放 CPU)。
|
|
354
1159
|
*/
|
|
355
1160
|
cancel(sessionId) {
|
|
356
1161
|
const q = this.queues.get(sessionId);
|
|
@@ -358,37 +1163,56 @@ var TtsQueue = class {
|
|
|
358
1163
|
q.epoch++;
|
|
359
1164
|
q.pending.length = 0;
|
|
360
1165
|
}
|
|
1166
|
+
this.engine.interrupt?.();
|
|
361
1167
|
}
|
|
362
1168
|
/** 会话退出/被抢占时彻底清理其队列(防止 Map 长期累积)。 */
|
|
363
1169
|
prune(sessionId) {
|
|
1170
|
+
const q = this.queues.get(sessionId);
|
|
1171
|
+
if (q) {
|
|
1172
|
+
q.epoch++;
|
|
1173
|
+
q.pending.length = 0;
|
|
1174
|
+
}
|
|
364
1175
|
this.queues.delete(sessionId);
|
|
365
1176
|
}
|
|
366
1177
|
async pump(sessionId, q) {
|
|
367
1178
|
if (q.busy) return;
|
|
368
1179
|
q.busy = true;
|
|
369
1180
|
try {
|
|
370
|
-
await this.ensureReady();
|
|
371
1181
|
while (q.pending.length > 0) {
|
|
372
1182
|
const item = q.pending.shift();
|
|
373
1183
|
try {
|
|
374
|
-
const
|
|
375
|
-
const chunks = [];
|
|
376
|
-
for await (const chunk of audioStream) {
|
|
377
|
-
chunks.push(chunk);
|
|
378
|
-
}
|
|
379
|
-
const buf = Buffer.concat(chunks);
|
|
380
|
-
if (!isValidMp3(buf)) continue;
|
|
1184
|
+
const buf = await this.engine.synthesize(item.text);
|
|
381
1185
|
if (item.epoch !== q.epoch) continue;
|
|
382
1186
|
q.errorNotified = false;
|
|
383
|
-
|
|
1187
|
+
q.backoff = 0;
|
|
1188
|
+
const sentenceId = q.seq++;
|
|
1189
|
+
const mime = this.engine.mime;
|
|
1190
|
+
const dataFrame = {
|
|
1191
|
+
sessionId,
|
|
1192
|
+
sentenceId,
|
|
1193
|
+
chunkId: 0,
|
|
1194
|
+
final: false,
|
|
1195
|
+
audio: buf.toString("base64"),
|
|
1196
|
+
mime
|
|
1197
|
+
};
|
|
1198
|
+
for (const fn of this.listeners) {
|
|
1199
|
+
try {
|
|
1200
|
+
fn(dataFrame);
|
|
1201
|
+
} catch {
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
const finalFrame = {
|
|
384
1205
|
sessionId,
|
|
385
|
-
|
|
1206
|
+
sentenceId,
|
|
1207
|
+
chunkId: 1,
|
|
1208
|
+
final: true,
|
|
386
1209
|
text: item.text,
|
|
387
|
-
audio:
|
|
1210
|
+
audio: "",
|
|
1211
|
+
mime
|
|
388
1212
|
};
|
|
389
1213
|
for (const fn of this.listeners) {
|
|
390
1214
|
try {
|
|
391
|
-
fn(
|
|
1215
|
+
fn(finalFrame);
|
|
392
1216
|
} catch {
|
|
393
1217
|
}
|
|
394
1218
|
}
|
|
@@ -404,6 +1228,7 @@ var TtsQueue = class {
|
|
|
404
1228
|
}
|
|
405
1229
|
} finally {
|
|
406
1230
|
q.busy = false;
|
|
1231
|
+
if (this.queues.get(sessionId) !== q) return;
|
|
407
1232
|
if (q.pending.length > 0) {
|
|
408
1233
|
const delay = q.errorNotified ? q.backoff : 0;
|
|
409
1234
|
q.backoff = Math.min(8e3, delay + 1e3);
|
|
@@ -413,8 +1238,495 @@ var TtsQueue = class {
|
|
|
413
1238
|
}
|
|
414
1239
|
}
|
|
415
1240
|
async close() {
|
|
416
|
-
await this.
|
|
417
|
-
|
|
1241
|
+
await this.engine.close();
|
|
1242
|
+
}
|
|
1243
|
+
};
|
|
1244
|
+
|
|
1245
|
+
// src/tts-local.ts
|
|
1246
|
+
import { fork } from "node:child_process";
|
|
1247
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
1248
|
+
import { join as join3 } from "node:path";
|
|
1249
|
+
import { statSync as statSync2 } from "node:fs";
|
|
1250
|
+
var TTS_MODEL_REPO = "csukuangfj/sherpa-onnx-vits-zh-ll";
|
|
1251
|
+
var KOKORO_MODEL_DIR = "csukuangfj/kokoro-int8-multi-lang-v1_1";
|
|
1252
|
+
var TTS_MODEL_FILES = [
|
|
1253
|
+
{ file: "model.onnx", sha256: "6c349bdd73dc928234dd7bc86929748bba32cd5264d32d915bf7b7aa0595965b" },
|
|
1254
|
+
{ file: "lexicon.txt", sha256: "b3a82f16b286c424953dea3686039e7ab465fa8e15d87ef8abd0ec69175beb21" },
|
|
1255
|
+
{ file: "tokens.txt", sha256: "34b035b9aeb070df6188b022f29c00e0e142c7ade9f25611ced65db5e9cc8402" },
|
|
1256
|
+
{ file: "G_multisperaker_latest.json", sha256: "f31e4bf23827c3528fdf090fd7b6fb8e63333709b80670d40fa864f1fa9fadf3" },
|
|
1257
|
+
{ file: "date.fst", sha256: "eb8aa079ae3cb81d8f4404992f39d61a0cb990947512b5b8d1e54d1f6980e718" },
|
|
1258
|
+
{ file: "phone.fst", sha256: "1ac2b6fa56b1442320c4de7db08353bab8963a2b57f365eebcdd3a2d3562f8d7" },
|
|
1259
|
+
{ file: "number.fst", sha256: "743f402181fcfebf76cc2f0546b71fa26476e626fbe4e460fb7b4c3a7a8bd5bd" }
|
|
1260
|
+
];
|
|
1261
|
+
var VITS_SPEAKERS = [
|
|
1262
|
+
{ name: "suyingxue", sid: 0, label: "\u7D20\u6620\u96EA \xB7 \u5973" },
|
|
1263
|
+
{ name: "gunian", sid: 1, label: "\u987E\u5FF5 \xB7 \u7537" },
|
|
1264
|
+
{ name: "fushiyu", sid: 2, label: "\u5085\u65AF\u9047 \xB7 \u5973" },
|
|
1265
|
+
{ name: "bingjiao", sid: 3, label: "\u51B0\u5A07 \xB7 \u7537" },
|
|
1266
|
+
{ name: "bazong", sid: 4, label: "\u9738\u603B \xB7 \u7537" }
|
|
1267
|
+
];
|
|
1268
|
+
var KOKORO_F0 = [
|
|
1269
|
+
224,
|
|
1270
|
+
189,
|
|
1271
|
+
154,
|
|
1272
|
+
261,
|
|
1273
|
+
226,
|
|
1274
|
+
222,
|
|
1275
|
+
220,
|
|
1276
|
+
229,
|
|
1277
|
+
198,
|
|
1278
|
+
186,
|
|
1279
|
+
212,
|
|
1280
|
+
293,
|
|
1281
|
+
233,
|
|
1282
|
+
161,
|
|
1283
|
+
247,
|
|
1284
|
+
207,
|
|
1285
|
+
218,
|
|
1286
|
+
216,
|
|
1287
|
+
220,
|
|
1288
|
+
238,
|
|
1289
|
+
242,
|
|
1290
|
+
229,
|
|
1291
|
+
198,
|
|
1292
|
+
286,
|
|
1293
|
+
211,
|
|
1294
|
+
190,
|
|
1295
|
+
264,
|
|
1296
|
+
261,
|
|
1297
|
+
226,
|
|
1298
|
+
147,
|
|
1299
|
+
216,
|
|
1300
|
+
240,
|
|
1301
|
+
233,
|
|
1302
|
+
188,
|
|
1303
|
+
222,
|
|
1304
|
+
247,
|
|
1305
|
+
253,
|
|
1306
|
+
270,
|
|
1307
|
+
276,
|
|
1308
|
+
276,
|
|
1309
|
+
279,
|
|
1310
|
+
320,
|
|
1311
|
+
247,
|
|
1312
|
+
296,
|
|
1313
|
+
276,
|
|
1314
|
+
235,
|
|
1315
|
+
139,
|
|
1316
|
+
240,
|
|
1317
|
+
282,
|
|
1318
|
+
282,
|
|
1319
|
+
238,
|
|
1320
|
+
226,
|
|
1321
|
+
273,
|
|
1322
|
+
216,
|
|
1323
|
+
286,
|
|
1324
|
+
270,
|
|
1325
|
+
198,
|
|
1326
|
+
179,
|
|
1327
|
+
117,
|
|
1328
|
+
130,
|
|
1329
|
+
114,
|
|
1330
|
+
128,
|
|
1331
|
+
108,
|
|
1332
|
+
106,
|
|
1333
|
+
122,
|
|
1334
|
+
136,
|
|
1335
|
+
190,
|
|
1336
|
+
112,
|
|
1337
|
+
108,
|
|
1338
|
+
128,
|
|
1339
|
+
131,
|
|
1340
|
+
111,
|
|
1341
|
+
110,
|
|
1342
|
+
132,
|
|
1343
|
+
138,
|
|
1344
|
+
189,
|
|
1345
|
+
137,
|
|
1346
|
+
148,
|
|
1347
|
+
151,
|
|
1348
|
+
127,
|
|
1349
|
+
135,
|
|
1350
|
+
111,
|
|
1351
|
+
138,
|
|
1352
|
+
114,
|
|
1353
|
+
125,
|
|
1354
|
+
158,
|
|
1355
|
+
128,
|
|
1356
|
+
156,
|
|
1357
|
+
132,
|
|
1358
|
+
162,
|
|
1359
|
+
131,
|
|
1360
|
+
136,
|
|
1361
|
+
142,
|
|
1362
|
+
124,
|
|
1363
|
+
129,
|
|
1364
|
+
136,
|
|
1365
|
+
126,
|
|
1366
|
+
135,
|
|
1367
|
+
161,
|
|
1368
|
+
150,
|
|
1369
|
+
124,
|
|
1370
|
+
104,
|
|
1371
|
+
124
|
|
1372
|
+
];
|
|
1373
|
+
var KOKORO_NAMED = {
|
|
1374
|
+
48: { name: "zf_xiaobei", label: "\u5C0F\u5317 \xB7 \u4E2D\u6587\u5973" },
|
|
1375
|
+
49: { name: "zf_xiaoni", label: "\u5C0F\u59AE \xB7 \u4E2D\u6587\u5973" },
|
|
1376
|
+
50: { name: "zf_xiaoxiao", label: "\u5C0F\u5C0F \xB7 \u4E2D\u6587\u5973" },
|
|
1377
|
+
51: { name: "zf_xiaoyi", label: "\u5C0F\u827A \xB7 \u4E2D\u6587\u5973" }
|
|
1378
|
+
};
|
|
1379
|
+
var KOKORO_LABEL_OVERRIDES = {
|
|
1380
|
+
62: "62 \xB7 \u6DF1\u6C89 \xB7 \u5E38\u7528\u7537\u58F0",
|
|
1381
|
+
68: "68 \xB7 \u6D51\u539A \xB7 \u5E38\u7528\u7537\u58F0",
|
|
1382
|
+
75: "75 \xB7 \u6E05\u4EAE \xB7 \u5E38\u7528\u7537\u58F0",
|
|
1383
|
+
76: "76 \xB7 \u78C1\u6027 \xB7 \u5E38\u7528\u7537\u58F0"
|
|
1384
|
+
};
|
|
1385
|
+
var KOKORO_PINNED = [62, 68, 75, 76];
|
|
1386
|
+
function kokoroVoice(sid) {
|
|
1387
|
+
const custom = KOKORO_LABEL_OVERRIDES[sid];
|
|
1388
|
+
if (custom) return { name: String(sid), sid, label: custom };
|
|
1389
|
+
const named = KOKORO_NAMED[sid];
|
|
1390
|
+
if (named) return { name: named.name, sid, label: named.label };
|
|
1391
|
+
const hz = KOKORO_F0[sid] ?? null;
|
|
1392
|
+
if (hz === null) return { name: String(sid), sid, label: `${sid} \xB7 \u97F3\u8272` };
|
|
1393
|
+
return { name: String(sid), sid, label: `${sid} \xB7 ${hz < 180 ? "\u7537\u58F0" : "\u5973\u58F0"} \xB7 ${hz}Hz` };
|
|
1394
|
+
}
|
|
1395
|
+
var KOKORO_VOICES = [
|
|
1396
|
+
...KOKORO_PINNED.map((sid) => kokoroVoice(sid)),
|
|
1397
|
+
...KOKORO_F0.map((_, sid) => kokoroVoice(sid)).filter((v) => !KOKORO_PINNED.includes(v.sid))
|
|
1398
|
+
];
|
|
1399
|
+
function voiceToSid(voice) {
|
|
1400
|
+
const v = String(voice ?? "").trim().toLowerCase();
|
|
1401
|
+
if (/^\d+$/.test(v)) {
|
|
1402
|
+
const n = Number(v);
|
|
1403
|
+
if (n >= 0 && n < VITS_SPEAKERS.length) return n;
|
|
1404
|
+
}
|
|
1405
|
+
const hit = VITS_SPEAKERS.find((s) => s.name.toLowerCase() === v);
|
|
1406
|
+
return hit ? hit.sid : 0;
|
|
1407
|
+
}
|
|
1408
|
+
function kokoroVoiceToSid(voice) {
|
|
1409
|
+
const v = String(voice ?? "").trim().toLowerCase();
|
|
1410
|
+
if (/^\d+$/.test(v)) {
|
|
1411
|
+
const n = Number(v);
|
|
1412
|
+
if (n >= 0 && n <= KOKORO_VOICES.length - 1) return n;
|
|
1413
|
+
}
|
|
1414
|
+
const hit = KOKORO_VOICES.find((s) => s.name.toLowerCase() === v);
|
|
1415
|
+
return hit ? hit.sid : 48;
|
|
1416
|
+
}
|
|
1417
|
+
function floatToPcm16(samples) {
|
|
1418
|
+
const buf = Buffer.alloc(samples.length * 2);
|
|
1419
|
+
for (let i = 0; i < samples.length; i++) {
|
|
1420
|
+
const s = Math.max(-1, Math.min(1, samples[i]));
|
|
1421
|
+
buf.writeInt16LE(s < 0 ? s * 32768 : s * 32767, i * 2);
|
|
1422
|
+
}
|
|
1423
|
+
return buf;
|
|
1424
|
+
}
|
|
1425
|
+
function pcmToWav(pcm, sampleRate) {
|
|
1426
|
+
const header = Buffer.alloc(44);
|
|
1427
|
+
header.write("RIFF", 0);
|
|
1428
|
+
header.writeUInt32LE(36 + pcm.length, 4);
|
|
1429
|
+
header.write("WAVE", 8);
|
|
1430
|
+
header.write("fmt ", 12);
|
|
1431
|
+
header.writeUInt32LE(16, 16);
|
|
1432
|
+
header.writeUInt16LE(1, 20);
|
|
1433
|
+
header.writeUInt16LE(1, 22);
|
|
1434
|
+
header.writeUInt32LE(sampleRate, 24);
|
|
1435
|
+
header.writeUInt32LE(sampleRate * 2, 28);
|
|
1436
|
+
header.writeUInt16LE(2, 32);
|
|
1437
|
+
header.writeUInt16LE(16, 34);
|
|
1438
|
+
header.write("data", 36);
|
|
1439
|
+
header.writeUInt32LE(pcm.length, 40);
|
|
1440
|
+
return Buffer.concat([header, pcm]);
|
|
1441
|
+
}
|
|
1442
|
+
var VITS_SPEC = {
|
|
1443
|
+
files: TTS_MODEL_FILES,
|
|
1444
|
+
workerPaths: (dir) => ({
|
|
1445
|
+
model: join3(dir, "model.onnx"),
|
|
1446
|
+
lexicon: join3(dir, "lexicon.txt"),
|
|
1447
|
+
tokens: join3(dir, "tokens.txt"),
|
|
1448
|
+
date: join3(dir, "date.fst"),
|
|
1449
|
+
phone: join3(dir, "phone.fst"),
|
|
1450
|
+
number: join3(dir, "number.fst")
|
|
1451
|
+
}),
|
|
1452
|
+
defaultVoice: "suyingxue",
|
|
1453
|
+
toSid: voiceToSid
|
|
1454
|
+
};
|
|
1455
|
+
var KOKORO_SPEC = {
|
|
1456
|
+
files: [
|
|
1457
|
+
{ file: "model.int8.onnx", sha256: "bda15858163726a492d02a9a727bc263551b86ac77f90812c4b30ff41d380e26" },
|
|
1458
|
+
{ file: "voices.bin", sha256: "e64a5a581d8c2a350d848f51c3121657cd83aa07ed6109172177345874a7244c" },
|
|
1459
|
+
{ file: "tokens.txt", sha256: "931ab2df2400cd65d580a22402024c2347ced8ae9ea300e545144b1aacc48e14" },
|
|
1460
|
+
{ file: "lexicon-us-en.txt", sha256: "7daaab53a181be9885b853a8582bf1838186317e5dadacbcef9c426d6fa0da14" },
|
|
1461
|
+
{ file: "lexicon-zh.txt", sha256: "11111d8cd695fba2ace1367a1d0a708b586e6ef5c1f9be91da5d7eef129b651c" },
|
|
1462
|
+
{ file: "espeak-ng-data/phontab", sha256: "886f3fa402cb0ba73d483aa8ad000af47a6b7cc06293c75a97913fba68a530f6" },
|
|
1463
|
+
{ file: "date-zh.fst", sha256: "eb8aa079ae3cb81d8f4404992f39d61a0cb990947512b5b8d1e54d1f6980e718" },
|
|
1464
|
+
{ file: "number-zh.fst", sha256: "743f402181fcfebf76cc2f0546b71fa26476e626fbe4e460fb7b4c3a7a8bd5bd" },
|
|
1465
|
+
{ file: "phone-zh.fst", sha256: "1ac2b6fa56b1442320c4de7db08353bab8963a2b57f365eebcdd3a2d3562f8d7" }
|
|
1466
|
+
],
|
|
1467
|
+
workerPaths: (dir) => ({
|
|
1468
|
+
model: join3(dir, "model.int8.onnx"),
|
|
1469
|
+
voices: join3(dir, "voices.bin"),
|
|
1470
|
+
tokens: join3(dir, "tokens.txt"),
|
|
1471
|
+
dataDir: join3(dir, "espeak-ng-data"),
|
|
1472
|
+
lexicon: [join3(dir, "lexicon-us-en.txt"), join3(dir, "lexicon-zh.txt")].join(","),
|
|
1473
|
+
date: join3(dir, "date-zh.fst"),
|
|
1474
|
+
phone: join3(dir, "phone-zh.fst"),
|
|
1475
|
+
number: join3(dir, "number-zh.fst"),
|
|
1476
|
+
lang: ""
|
|
1477
|
+
}),
|
|
1478
|
+
defaultVoice: "zf_xiaobei",
|
|
1479
|
+
toSid: kokoroVoiceToSid
|
|
1480
|
+
};
|
|
1481
|
+
function createSherpaLocalEngine(options) {
|
|
1482
|
+
const { cacheDir, modelHost, allowCustomHost, broadcast } = options;
|
|
1483
|
+
const spec = options.kind === "kokoro" ? KOKORO_SPEC : VITS_SPEC;
|
|
1484
|
+
const repoName = options.kind === "kokoro" ? KOKORO_MODEL_DIR : TTS_MODEL_REPO;
|
|
1485
|
+
const repoDir = join3(cacheDir, repoName);
|
|
1486
|
+
const workerPath = fileURLToPath2(new URL("./tts-vits-worker.cjs", import.meta.url));
|
|
1487
|
+
let child = null;
|
|
1488
|
+
let childInit = false;
|
|
1489
|
+
const respawnChild = async () => {
|
|
1490
|
+
if (child) {
|
|
1491
|
+
child.kill();
|
|
1492
|
+
child = null;
|
|
1493
|
+
}
|
|
1494
|
+
childInit = false;
|
|
1495
|
+
ready = null;
|
|
1496
|
+
};
|
|
1497
|
+
let voice = spec.defaultVoice;
|
|
1498
|
+
let speed = 1;
|
|
1499
|
+
let ready = null;
|
|
1500
|
+
let engineLoading = false;
|
|
1501
|
+
let engineError;
|
|
1502
|
+
let nextId = 1;
|
|
1503
|
+
const pending = /* @__PURE__ */ new Map();
|
|
1504
|
+
const call = (msg) => new Promise((resolve, reject) => {
|
|
1505
|
+
if (!child) {
|
|
1506
|
+
reject(new Error("tts child not running"));
|
|
1507
|
+
return;
|
|
1508
|
+
}
|
|
1509
|
+
const id = nextId++;
|
|
1510
|
+
pending.set(id, { resolve, reject });
|
|
1511
|
+
child.send({ id, ...msg });
|
|
1512
|
+
});
|
|
1513
|
+
const rejectAll = (e) => {
|
|
1514
|
+
for (const p of pending.values()) p.reject(e);
|
|
1515
|
+
pending.clear();
|
|
1516
|
+
};
|
|
1517
|
+
const ensureReady = () => {
|
|
1518
|
+
if (ready && child) return ready;
|
|
1519
|
+
if (!ready) {
|
|
1520
|
+
ready = (async () => {
|
|
1521
|
+
engineLoading = true;
|
|
1522
|
+
engineError = void 0;
|
|
1523
|
+
try {
|
|
1524
|
+
if (!child || !childInit) {
|
|
1525
|
+
for (const f of spec.files) {
|
|
1526
|
+
const ok = await ensureModelFile({
|
|
1527
|
+
repo: repoName,
|
|
1528
|
+
repoDir,
|
|
1529
|
+
spec: f,
|
|
1530
|
+
primaryHost: modelHost(),
|
|
1531
|
+
allowCustomHost,
|
|
1532
|
+
broadcast
|
|
1533
|
+
});
|
|
1534
|
+
if (!ok) throw new Error("local TTS model download/verify failed: " + f.file);
|
|
1535
|
+
}
|
|
1536
|
+
if (options.kind === "kokoro") {
|
|
1537
|
+
const treeOk = await ensureModelTree({
|
|
1538
|
+
repo: repoName,
|
|
1539
|
+
repoDir,
|
|
1540
|
+
subdir: "espeak-ng-data",
|
|
1541
|
+
primaryHost: modelHost(),
|
|
1542
|
+
allowCustomHost,
|
|
1543
|
+
broadcast
|
|
1544
|
+
});
|
|
1545
|
+
if (!treeOk) throw new Error("local TTS model download failed: espeak-ng-data");
|
|
1546
|
+
}
|
|
1547
|
+
if (!child) {
|
|
1548
|
+
child = fork(workerPath, [], { stdio: ["ignore", "ignore", "pipe", "ipc"] });
|
|
1549
|
+
let stderrTail = "";
|
|
1550
|
+
child.stderr?.on("data", (chunk) => {
|
|
1551
|
+
stderrTail += String(chunk);
|
|
1552
|
+
const lines = stderrTail.split("\n");
|
|
1553
|
+
stderrTail = lines.pop() ?? "";
|
|
1554
|
+
for (const line of lines) {
|
|
1555
|
+
const s = line.trim();
|
|
1556
|
+
if (!s) continue;
|
|
1557
|
+
if (/Skip unknown phonemes/.test(s)) continue;
|
|
1558
|
+
console.error(`[tts-worker:${options.kind}] ${s}`);
|
|
1559
|
+
}
|
|
1560
|
+
});
|
|
1561
|
+
child.on("message", (m) => {
|
|
1562
|
+
const p = pending.get(m.id);
|
|
1563
|
+
if (!p) return;
|
|
1564
|
+
pending.delete(m.id);
|
|
1565
|
+
if (m.ok) p.resolve(m);
|
|
1566
|
+
else p.reject(new Error(m.error ?? "tts child error"));
|
|
1567
|
+
});
|
|
1568
|
+
child.on("error", (e) => {
|
|
1569
|
+
rejectAll(e instanceof Error ? e : new Error(String(e)));
|
|
1570
|
+
child = null;
|
|
1571
|
+
childInit = false;
|
|
1572
|
+
ready = null;
|
|
1573
|
+
});
|
|
1574
|
+
child.on("exit", (code) => {
|
|
1575
|
+
rejectAll(new Error(`tts child exited with code ${code}`));
|
|
1576
|
+
child = null;
|
|
1577
|
+
childInit = false;
|
|
1578
|
+
ready = null;
|
|
1579
|
+
});
|
|
1580
|
+
}
|
|
1581
|
+
if (!childInit) {
|
|
1582
|
+
const init = await call({ type: "init", kind: options.kind, paths: spec.workerPaths(repoDir) });
|
|
1583
|
+
if (!init.ok) throw new Error(init.error ?? "tts child init failed");
|
|
1584
|
+
childInit = true;
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
broadcast("tts-ready", { engine: options.kind, worker: true });
|
|
1588
|
+
} catch (e) {
|
|
1589
|
+
engineError = e instanceof Error ? e.message : String(e);
|
|
1590
|
+
throw e;
|
|
1591
|
+
} finally {
|
|
1592
|
+
engineLoading = false;
|
|
1593
|
+
}
|
|
1594
|
+
})().finally(() => {
|
|
1595
|
+
ready = null;
|
|
1596
|
+
});
|
|
1597
|
+
}
|
|
1598
|
+
return ready;
|
|
1599
|
+
};
|
|
1600
|
+
return {
|
|
1601
|
+
mime: "audio/wav",
|
|
1602
|
+
updateVoice(nextVoice, nextRate) {
|
|
1603
|
+
voice = nextVoice || voice;
|
|
1604
|
+
if (typeof nextRate === "number" && Number.isFinite(nextRate)) {
|
|
1605
|
+
speed = Math.min(2, Math.max(0.5, nextRate));
|
|
1606
|
+
}
|
|
1607
|
+
},
|
|
1608
|
+
status() {
|
|
1609
|
+
const files = spec.files.map((f) => {
|
|
1610
|
+
const p = join3(repoDir, f.file);
|
|
1611
|
+
let exists = false;
|
|
1612
|
+
let size = 0;
|
|
1613
|
+
try {
|
|
1614
|
+
const st = statSync2(p);
|
|
1615
|
+
exists = st.isFile();
|
|
1616
|
+
size = st.size;
|
|
1617
|
+
} catch {
|
|
1618
|
+
}
|
|
1619
|
+
return { name: f.file, exists, size };
|
|
1620
|
+
});
|
|
1621
|
+
return {
|
|
1622
|
+
engine: options.kind,
|
|
1623
|
+
ready: childInit === true,
|
|
1624
|
+
loading: engineLoading,
|
|
1625
|
+
error: engineError,
|
|
1626
|
+
local: {
|
|
1627
|
+
repo: repoName,
|
|
1628
|
+
ready: files.every((f) => f.exists),
|
|
1629
|
+
loading: engineLoading,
|
|
1630
|
+
error: engineError,
|
|
1631
|
+
files
|
|
1632
|
+
}
|
|
1633
|
+
};
|
|
1634
|
+
},
|
|
1635
|
+
async synthesize(text, opts = {}) {
|
|
1636
|
+
await ensureReady();
|
|
1637
|
+
const sid = spec.toSid(opts.voice ?? voice);
|
|
1638
|
+
const spd = typeof opts.rate === "number" && Number.isFinite(opts.rate) ? Math.min(2, Math.max(0.5, opts.rate)) : speed;
|
|
1639
|
+
let res = await call({ type: "synth", text, sid, speed: spd });
|
|
1640
|
+
if (!res.ok && /Aborted/.test(res.error ?? "")) {
|
|
1641
|
+
await respawnChild();
|
|
1642
|
+
res = await call({ type: "synth", text, sid, speed: spd });
|
|
1643
|
+
}
|
|
1644
|
+
if (!res.ok) throw new Error(res.error ?? "local TTS synthesis failed");
|
|
1645
|
+
if (typeof res.samples !== "string" || res.samples.length === 0) {
|
|
1646
|
+
throw new Error("local TTS produced empty audio");
|
|
1647
|
+
}
|
|
1648
|
+
const bytes = Buffer.from(res.samples, "base64");
|
|
1649
|
+
const samples = new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
1650
|
+
if (samples.length === 0) {
|
|
1651
|
+
throw new Error("local TTS produced empty audio");
|
|
1652
|
+
}
|
|
1653
|
+
return pcmToWav(floatToPcm16(samples), res.sampleRate || 16e3);
|
|
1654
|
+
},
|
|
1655
|
+
async close() {
|
|
1656
|
+
const c = child;
|
|
1657
|
+
if (c) {
|
|
1658
|
+
try {
|
|
1659
|
+
await call({ type: "close" });
|
|
1660
|
+
} catch {
|
|
1661
|
+
}
|
|
1662
|
+
try {
|
|
1663
|
+
c.kill();
|
|
1664
|
+
} catch {
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
child = null;
|
|
1668
|
+
childInit = false;
|
|
1669
|
+
ready = null;
|
|
1670
|
+
},
|
|
1671
|
+
interrupt() {
|
|
1672
|
+
void respawnChild();
|
|
1673
|
+
}
|
|
1674
|
+
};
|
|
1675
|
+
}
|
|
1676
|
+
function createSherpaVitsEngine(options) {
|
|
1677
|
+
return createSherpaLocalEngine({ ...options, kind: "vits" });
|
|
1678
|
+
}
|
|
1679
|
+
function createSherpaKokoroEngine(options) {
|
|
1680
|
+
return createSherpaLocalEngine({ ...options, kind: "kokoro" });
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
// src/security.ts
|
|
1684
|
+
var LOOPBACK_ADDRESSES = /* @__PURE__ */ new Set(["127.0.0.1", "::1", "::ffff:127.0.0.1"]);
|
|
1685
|
+
function isLoopbackRequest(req) {
|
|
1686
|
+
const addr = req.socket.remoteAddress ?? "";
|
|
1687
|
+
return LOOPBACK_ADDRESSES.has(addr);
|
|
1688
|
+
}
|
|
1689
|
+
function sameOriginRequest(req) {
|
|
1690
|
+
const origin = req.headers.origin;
|
|
1691
|
+
if (!origin) return true;
|
|
1692
|
+
try {
|
|
1693
|
+
const originHost = new URL(origin).host;
|
|
1694
|
+
const xfh = req.headers["x-forwarded-host"];
|
|
1695
|
+
const host = (typeof xfh === "string" && xfh ? xfh.split(",")[0].trim() : "") || req.headers.host;
|
|
1696
|
+
if (!host) return false;
|
|
1697
|
+
return originHost === host;
|
|
1698
|
+
} catch {
|
|
1699
|
+
return false;
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
var RateLimiter = class {
|
|
1703
|
+
buckets = /* @__PURE__ */ new Map();
|
|
1704
|
+
maxKeys;
|
|
1705
|
+
constructor(maxKeys = 1e4) {
|
|
1706
|
+
this.maxKeys = maxKeys;
|
|
1707
|
+
}
|
|
1708
|
+
/** 命中一次;返回是否允许。maxHits 次 / windowMs 毫秒。 */
|
|
1709
|
+
hit(key, maxHits, windowMs) {
|
|
1710
|
+
const now = Date.now();
|
|
1711
|
+
const cutoff = now - windowMs;
|
|
1712
|
+
let bucket = this.buckets.get(key);
|
|
1713
|
+
if (!bucket) {
|
|
1714
|
+
if (this.buckets.size >= this.maxKeys) return false;
|
|
1715
|
+
bucket = [];
|
|
1716
|
+
this.buckets.set(key, bucket);
|
|
1717
|
+
}
|
|
1718
|
+
while (bucket.length > 0 && bucket[0] <= cutoff) bucket.shift();
|
|
1719
|
+
if (bucket.length >= maxHits) return false;
|
|
1720
|
+
bucket.push(now);
|
|
1721
|
+
return true;
|
|
1722
|
+
}
|
|
1723
|
+
/** 定期清理(由调用方在低频路径触发即可)。 */
|
|
1724
|
+
prune(now = Date.now(), windowMs) {
|
|
1725
|
+
const cutoff = now - windowMs;
|
|
1726
|
+
for (const [key, bucket] of this.buckets) {
|
|
1727
|
+
while (bucket.length > 0 && bucket[0] <= cutoff) bucket.shift();
|
|
1728
|
+
if (bucket.length === 0) this.buckets.delete(key);
|
|
1729
|
+
}
|
|
418
1730
|
}
|
|
419
1731
|
};
|
|
420
1732
|
|
|
@@ -422,37 +1734,57 @@ var TtsQueue = class {
|
|
|
422
1734
|
var name = "voice-mode";
|
|
423
1735
|
var NS_VOICE_MODE = "voice-mode";
|
|
424
1736
|
var BASE_PATH = "/voice-mode";
|
|
1737
|
+
var respondJson2 = (res, status, payload) => {
|
|
1738
|
+
res.writeHead(status, { "content-type": "application/json" });
|
|
1739
|
+
res.end(JSON.stringify(payload));
|
|
1740
|
+
};
|
|
425
1741
|
var VOICE_SPOKEN_PROMPT = "\u3010\u8BED\u97F3\u6A21\u5F0F\u3011\u5F53\u524D\u56DE\u590D\u4F1A\u88AB\u8BED\u97F3\u6717\u8BFB\uFF0C\u8BF7\u59CB\u7EC8\u7528\u7528\u6237\u6240\u7528\u8BED\u8A00\u3001\u4EE5\u53E3\u8BED\u5316\u7684\u77ED\u53E5\u76F4\u63A5\u56DE\u7B54\uFF0C\u50CF\u9762\u5BF9\u9762\u804A\u5929\u4E00\u6837\u81EA\u7136\uFF0C\u907F\u514D\u4E66\u9762\u8BED\u548C\u957F\u96BE\u53E5\u3002\u4E0D\u8981\u4F7F\u7528\u4EFB\u4F55 Markdown \u6216\u6392\u7248\u7B26\u53F7\uFF08\u661F\u53F7\u3001\u4E0B\u5212\u7EBF\u3001\u53CD\u5F15\u53F7\u3001\u4E95\u53F7\u3001\u5217\u8868\u4E0E\u8868\u683C\u6807\u8BB0\u3001\u4EE3\u7801\u5757\u7B49\uFF09\u3002\u9700\u8981\u5206\u70B9\u8BF4\u660E\u65F6\u7528\u300C\u7B2C\u4E00\u3001\u7B2C\u4E8C\u300D\u6216\u8FDE\u8D2F\u7684\u77ED\u53E5\u8868\u8FBE\uFF1B\u9664\u975E\u7528\u6237\u660E\u786E\u8981\u6C42\uFF0C\u4E0D\u8981\u8F93\u51FA\u4EE3\u7801\u7247\u6BB5\u3001\u5B8C\u6574 URL \u6216\u5197\u957F\u5B9A\u4E49\uFF0C\u7528\u4E00\u4E24\u53E5\u8BDD\u6982\u62EC\u542B\u4E49\u5373\u53EF\u3002\u56DE\u7B54\u7B80\u6D01\u76F4\u63A5\uFF0C\u4E0D\u8981\u91CD\u590D\u548C\u5BD2\u6684\u3002";
|
|
426
1742
|
var VOICE_SPOKEN_SECTION = "voice-mode:spoken-format";
|
|
427
|
-
var inject = ["webServer", "settings"];
|
|
428
|
-
var defaultModelCacheDir = () => process.platform === "win32" ?
|
|
1743
|
+
var inject = ["webServer", "settings", "sessions"];
|
|
1744
|
+
var defaultModelCacheDir = () => process.platform === "win32" ? join4(process.env.LOCALAPPDATA ?? join4(homedir(), "AppData", "Local"), "dsh-voice-mode", "models") : join4(homedir(), ".cache", "dsh-voice-mode", "models");
|
|
429
1745
|
var VOICE_SETTINGS_DEFAULTS = {
|
|
1746
|
+
ttsEngine: "edge",
|
|
430
1747
|
voice: "zh-CN-XiaoxiaoNeural",
|
|
431
1748
|
rate: 1,
|
|
432
1749
|
interruptLevel: 0,
|
|
433
|
-
silenceMs:
|
|
1750
|
+
silenceMs: 700,
|
|
434
1751
|
idleTimeoutMinutes: 10,
|
|
435
1752
|
modelHost: "",
|
|
436
1753
|
autoSend: true,
|
|
1754
|
+
autoResume: false,
|
|
437
1755
|
mode: "toggle",
|
|
1756
|
+
bargeInMode: "auto",
|
|
1757
|
+
echoGateDb: 6,
|
|
1758
|
+
shortcut: "Ctrl+Shift+V",
|
|
1759
|
+
spokenFormat: true,
|
|
1760
|
+
senseVoice: true,
|
|
438
1761
|
wakeWord: "",
|
|
439
|
-
|
|
1762
|
+
toolBeep: false
|
|
440
1763
|
};
|
|
441
1764
|
function createVoiceSettingsSchema(defs) {
|
|
442
1765
|
const d = { ...VOICE_SETTINGS_DEFAULTS, ...defs };
|
|
443
1766
|
return z.object({
|
|
1767
|
+
ttsEngine: z.union([z.const("vits"), z.const("kokoro"), z.const("edge")]).default(d.ttsEngine).description(
|
|
1768
|
+
"\u6717\u8BFB\u5F15\u64CE\uFF1Aedge \u5FAE\u8F6F\u4E91\u7AEF\uFF08\u9ED8\u8BA4\uFF0C\u5FEB\u3001\u97F3\u8D28\u81EA\u7136\uFF0C\u88AB\u6717\u8BFB\u6587\u672C\u4F1A\u53D1\u9001\u5230\u5FAE\u8F6F\uFF09/ vits \u672C\u5730\u4E2D\u6587 / kokoro \u672C\u5730\u4E2D\u82F1\uFF08\u56DE\u590D\u6587\u672C\u4E0D\u51FA\u672C\u673A\uFF09\uFF1B\u5207\u6362\u5373\u65F6\u751F\u6548"
|
|
1769
|
+
),
|
|
444
1770
|
voice: z.string().default(d.voice).description(
|
|
445
|
-
"
|
|
1771
|
+
"\u6717\u8BFB\u97F3\u8272\uFF08\u6309 ttsEngine \u53D6\u503C\uFF1Avits \u7528\u8BF4\u8BDD\u4EBA\u540D suyingxue/gunian/fushiyu/bingjiao/bazong\uFF1Bkokoro \u7528 0-102 \u7F16\u53F7\u6216\u4E2D\u6587\u540D zf_xiaobei/zf_xiaoni/zf_xiaoxiao/zf_xiaoyi\uFF1Bedge \u7528 Edge ShortName \u5982 zh-CN-XiaoxiaoNeural \u6653\u6653\xB7\u5973\uFF0C\u5B8C\u6574\u6E05\u5355\u89C1 scripts/list-voices.mjs\uFF09"
|
|
446
1772
|
),
|
|
447
1773
|
rate: z.number().min(0.5).max(2).default(d.rate).description("\u6717\u8BFB\u8BED\u901F\u500D\u7387\uFF080.5 = \u6162\u901F\uFF0C2.0 = \u5FEB\u901F\uFF0C1.0 = \u6B63\u5E38\uFF09"),
|
|
448
1774
|
interruptLevel: z.union([z.const(0), z.const(1), z.const(2)]).default(d.interruptLevel).description("\u53D1\u58F0\u6253\u65AD\u7075\u654F\u5EA6\uFF1A0 \u9AD8\u95E8\u69DB\uFF08\u5B89\u9759\u73AF\u5883\uFF0C\u9ED8\u8BA4\uFF09/ 1 \u4E2D / 2 \u4F4E\uFF08\u5608\u6742\u73AF\u5883\u66F4\u5BB9\u6613\u6253\u65AD\uFF09"),
|
|
449
|
-
silenceMs: z.number().min(500).max(3e4).default(d.silenceMs).description("\u8BF4\u5B8C\u6574\u4E00\u53E5\u7684\u9759\u97F3\u505C\u987F\u6BEB\u79D2\u6570\uFF08\u9ED8\u8BA4
|
|
1775
|
+
silenceMs: z.number().min(500).max(3e4).default(d.silenceMs).description("\u8BF4\u5B8C\u6574\u4E00\u53E5\u7684\u9759\u97F3\u505C\u987F\u6BEB\u79D2\u6570\uFF08\u9ED8\u8BA4 700 \u6BEB\u79D2\uFF1B\u81F3\u5C11 250ms \u8BED\u97F3\u624D\u5224\u53E5\uFF0C\u9632\u77ED\u4FC3\u566A\u58F0\u8BEF\u89E6\u53D1\uFF09"),
|
|
450
1776
|
idleTimeoutMinutes: z.number().min(1).max(120).default(d.idleTimeoutMinutes).description("\u65E0\u6D3B\u52A8\u81EA\u52A8\u9000\u51FA\u8BED\u97F3\u6A21\u5F0F\u7684\u5206\u949F\u6570\uFF08\u9ED8\u8BA4 10\uFF09"),
|
|
451
1777
|
modelHost: z.string().default(d.modelHost).description("ASR \u6A21\u578B\u4E0B\u8F7D\u6E90\uFF08\u7559\u7A7A\u7528\u9ED8\u8BA4\u6E90\uFF1B\u56FD\u5185\u7F51\u7EDC\u53EF\u586B https://hf-mirror.com\uFF09"),
|
|
452
1778
|
autoSend: z.boolean().default(d.autoSend).description("\u8BC6\u522B\u5B9A\u7A3F\u540E\u81EA\u52A8\u53D1\u9001\uFF08\u5173\u95ED\u5219\u53EA\u8FDB\u8349\u7A3F\u4F9B\u7F16\u8F91\uFF1B\u6309\u4F4F Ctrl / hold \u677E\u624B\u4ECD\u4F1A\u53D1\u9001\uFF09"),
|
|
1779
|
+
autoResume: z.boolean().default(d.autoResume).description("\u5207\u6362\u56DE\u4E0A\u6B21\u8BED\u97F3\u4F1A\u8BDD\u65F6\u81EA\u52A8\u6062\u590D\u8BED\u97F3\u6A21\u5F0F\uFF08\u9ED8\u8BA4\u5173\uFF0C\u9700\u9EA6\u514B\u98CE\u6743\u9650\u5DF2\u6388\u4E88\uFF1B\u5173\u95ED\u5219\u6BCF\u6B21\u5207\u6362\u4F1A\u8BDD\u540E\u9700\u91CD\u65B0\u70B9\u9EA6\u514B\u98CE\uFF09"),
|
|
453
1780
|
mode: z.union([z.const("toggle"), z.const("hold")]).default(d.mode).description("\u4EA4\u4E92\u6A21\u5F0F\uFF1Atoggle \u6301\u7EED\u8046\u542C + \u9759\u97F3\u81EA\u52A8\u65AD\u53E5\uFF08\u9ED8\u8BA4\uFF09\uFF1Bhold \u6309\u4F4F\u8BF4\u8BDD\u3001\u677E\u624B\u53D1\u9001\uFF08\u77ED\u6309\u9000\u51FA\uFF09"),
|
|
1781
|
+
bargeInMode: z.union([z.const("auto"), z.const("manual")]).default(d.bargeInMode).description("\u6253\u65AD\u65B9\u5F0F\uFF1Aauto \u81EA\u52A8\u6253\u65AD\uFF08\u5F00\u53E3\u5373\u6253\u65AD\uFF0C\u8033\u673A/\u5B89\u9759\u73AF\u5883\u63A8\u8350\uFF09\uFF1Bmanual \u624B\u52A8\u6253\u65AD\uFF08\u5916\u653E\u63A8\u8350\u2014\u2014\u5916\u653E\u56DE\u58F0\u4F1A\u8BEF\u89E6\u53D1\u81EA\u52A8\u6253\u65AD\uFF0C\u6539\u6309\u4F4F\u9EA6\u514B\u98CE/Ctrl \u663E\u5F0F\u6253\u65AD\uFF0C\u6C38\u4E0D\u81EA\u6253\u65AD\uFF09"),
|
|
1782
|
+
echoGateDb: z.number().min(3).max(12).default(d.echoGateDb).description("\u56DE\u58F0\u95E8\u63A7\u9608\u503C\uFF08dB\uFF0C\u9ED8\u8BA4 6\uFF09\uFF1A\u81EA\u52A8\u6253\u65AD\u8981\u6C42\u6B8B\u5DEE\u9AD8\u4E8E\u56DE\u58F0\u5730\u677F\u6B64\u503C\uFF1B\u5916\u653E\u4ECD\u8BEF\u6253\u65AD\u8C03\u5927\uFF088~10\uFF09\uFF0C\u592A\u96BE\u6253\u65AD\u8C03\u5C0F\uFF083~4\uFF09"),
|
|
1783
|
+
shortcut: z.string().default(d.shortcut).description("\u8FDB\u5165/\u9000\u51FA\u8BED\u97F3\u6A21\u5F0F\u7684\u5FEB\u6377\u952E\uFF08\u5F62\u5982 Ctrl+Shift+V\uFF0C\u4FEE\u9970\u952E Ctrl/Shift/Alt/Meta + \u4E00\u4E2A\u5B57\u6BCD\u952E\uFF1B\u7559\u7A7A\u7981\u7528\u5FEB\u6377\u952E\uFF0C\u7528\u9EA6\u514B\u98CE\u6309\u94AE\uFF09"),
|
|
1784
|
+
spokenFormat: z.boolean().default(d.spokenFormat).description("\u8BED\u97F3\u4F1A\u8BDD\u6CE8\u5165\u53E3\u8BED\u5316\u63D0\u793A\u8BCD\uFF08\u53E3\u8BED\u5316\u77ED\u53E5\u3001\u4E0D\u7528 Markdown \u6392\u7248\u7B26\u53F7\uFF0C\u6717\u8BFB\u66F4\u987A\u66F4\u5FEB\uFF1B\u9ED8\u8BA4\u5F00\uFF0C\u6539\u52A8\u5373\u65F6\u751F\u6548\uFF09"),
|
|
1785
|
+
senseVoice: z.boolean().default(d.senseVoice).description("\u5B9A\u7A3F\u7528 SenseVoice \u91CD\u8BD1\uFF08\u5E26\u6807\u70B9+\u6570\u5B57\u5F52\u4E00\u5316\u3001\u8BC6\u522B\u66F4\u51C6\uFF1B\u9ED8\u8BA4\u5F00\u3002\u5173\u95ED\u53EF\u7701 228MB \u6A21\u578B\uFF0C\u53EA\u8D70\u6D41\u5F0F\u8BC6\u522B\uFF09"),
|
|
454
1786
|
wakeWord: z.string().default(d.wakeWord).description("\u5524\u9192\u8BCD\uFF1A\u5728\u5F85\u673A\u6001\u8BF4\u51FA\u540E\u5F00\u59CB\u8BC6\u522B\uFF08\u9ED8\u8BA4\u5173\uFF1B\u5982\u300C\u4F60\u597D\u5C0FD\u300D\uFF09"),
|
|
455
|
-
|
|
1787
|
+
toolBeep: z.boolean().default(d.toolBeep).description('\u5DE5\u5177\u8C03\u7528\u63D0\u793A\u97F3\uFF08\u9ED8\u8BA4\u5173\uFF09\uFF1A\u5F00\u542F\u540E AI \u8C03\u7528\u5DE5\u5177\u65F6"\u6EF4"\u4E00\u58F0\uFF0C\u5173\u95ED\u5219\u5168\u7A0B\u9759\u9ED8')
|
|
456
1788
|
});
|
|
457
1789
|
}
|
|
458
1790
|
var VoiceSettingsSchema = createVoiceSettingsSchema();
|
|
@@ -460,19 +1792,55 @@ var Config = z.object({
|
|
|
460
1792
|
enabled: z.boolean().default(true),
|
|
461
1793
|
cacheDir: z.string().default(defaultModelCacheDir()),
|
|
462
1794
|
modelHost: z.string().default("https://huggingface.co"),
|
|
1795
|
+
ttsEngine: z.union([z.const("edge"), z.const("vits"), z.const("kokoro")]).default("edge"),
|
|
1796
|
+
allowLan: z.boolean().default(false),
|
|
1797
|
+
allowCustomModelHost: z.boolean().default(false),
|
|
463
1798
|
voice: z.string().default("zh-CN-XiaoxiaoNeural"),
|
|
464
1799
|
rate: z.number().default(1),
|
|
465
1800
|
interruptLevel: z.union([z.const(0), z.const(1), z.const(2)]).default(0),
|
|
466
|
-
silenceMs: z.number().default(
|
|
1801
|
+
silenceMs: z.number().default(700),
|
|
467
1802
|
idleTimeoutMinutes: z.number().default(10)
|
|
468
1803
|
});
|
|
469
1804
|
function apply(ctx, config) {
|
|
470
1805
|
let activeVoiceSession = null;
|
|
1806
|
+
let activeTabId = null;
|
|
1807
|
+
let ownerYieldTimer = null;
|
|
1808
|
+
const turnStates = /* @__PURE__ */ new Map();
|
|
1809
|
+
const setTurn = (sessionId, state) => {
|
|
1810
|
+
if (turnStates.get(sessionId) === state) return;
|
|
1811
|
+
turnStates.set(sessionId, state);
|
|
1812
|
+
broadcast("turn", { sessionId, state });
|
|
1813
|
+
};
|
|
1814
|
+
const turnGen = /* @__PURE__ */ new Map();
|
|
1815
|
+
const sessions = ctx.get("sessions");
|
|
1816
|
+
const limiter = new RateLimiter();
|
|
1817
|
+
const limiterPrune = setInterval(() => limiter.prune(Date.now(), 6e4), 6e4);
|
|
1818
|
+
ctx.effect(() => () => clearInterval(limiterPrune));
|
|
1819
|
+
const normalizedModelHost = () => validateModelHost(vset.modelHost, config.allowCustomModelHost) ?? HOST_PRIMARY;
|
|
1820
|
+
const denyNonLoopback = (req, res) => {
|
|
1821
|
+
if (!config.allowLan && !isLoopbackRequest(req)) {
|
|
1822
|
+
res.statusCode = 403;
|
|
1823
|
+
res.setHeader("content-type", "application/json");
|
|
1824
|
+
res.end(JSON.stringify({ error: "loopback only (allowLan=false)" }));
|
|
1825
|
+
return true;
|
|
1826
|
+
}
|
|
1827
|
+
return false;
|
|
1828
|
+
};
|
|
1829
|
+
const denyCrossOrigin = (req, res) => {
|
|
1830
|
+
if (!sameOriginRequest(req)) {
|
|
1831
|
+
res.statusCode = 403;
|
|
1832
|
+
res.setHeader("content-type", "application/json");
|
|
1833
|
+
res.end(JSON.stringify({ error: "cross-origin request denied" }));
|
|
1834
|
+
return true;
|
|
1835
|
+
}
|
|
1836
|
+
return false;
|
|
1837
|
+
};
|
|
471
1838
|
const sseClients = /* @__PURE__ */ new Set();
|
|
1839
|
+
const latestConnByTab = /* @__PURE__ */ new Map();
|
|
472
1840
|
const broadcast = (event, payload) => {
|
|
473
|
-
for (const
|
|
1841
|
+
for (const c of sseClients) {
|
|
474
1842
|
try {
|
|
475
|
-
send(event, payload);
|
|
1843
|
+
c.send(event, payload);
|
|
476
1844
|
} catch {
|
|
477
1845
|
}
|
|
478
1846
|
}
|
|
@@ -482,6 +1850,7 @@ function apply(ctx, config) {
|
|
|
482
1850
|
createVoiceSettingsSchema(),
|
|
483
1851
|
{
|
|
484
1852
|
base: {
|
|
1853
|
+
ttsEngine: config.ttsEngine,
|
|
485
1854
|
voice: config.voice,
|
|
486
1855
|
rate: config.rate,
|
|
487
1856
|
interruptLevel: config.interruptLevel,
|
|
@@ -495,25 +1864,66 @@ function apply(ctx, config) {
|
|
|
495
1864
|
const asr = createAsrRuntime({
|
|
496
1865
|
cacheDir: config.cacheDir,
|
|
497
1866
|
modelHost: () => vset.modelHost,
|
|
1867
|
+
// P4:SenseVoice 定稿重译开关(实时读取,关闭则不下载/不创建模型)。
|
|
1868
|
+
senseVoice: () => vset.senseVoice,
|
|
1869
|
+
allowCustomHost: config.allowCustomModelHost,
|
|
498
1870
|
broadcast
|
|
499
1871
|
});
|
|
1872
|
+
ctx.effect(() => () => asr.dispose());
|
|
1873
|
+
void asr.warmup();
|
|
1874
|
+
const makeEngine = (kind) => {
|
|
1875
|
+
if (kind === "edge") return new EdgeTtsEngine(config.voice, config.rate);
|
|
1876
|
+
if (kind === "kokoro") {
|
|
1877
|
+
return createSherpaKokoroEngine({
|
|
1878
|
+
cacheDir: config.cacheDir,
|
|
1879
|
+
modelHost: normalizedModelHost,
|
|
1880
|
+
allowCustomHost: config.allowCustomModelHost,
|
|
1881
|
+
broadcast
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
return createSherpaVitsEngine({
|
|
1885
|
+
cacheDir: config.cacheDir,
|
|
1886
|
+
modelHost: normalizedModelHost,
|
|
1887
|
+
allowCustomHost: config.allowCustomModelHost,
|
|
1888
|
+
broadcast
|
|
1889
|
+
});
|
|
1890
|
+
};
|
|
1891
|
+
let engineKind = vset.ttsEngine ?? config.ttsEngine;
|
|
500
1892
|
const queue = new TtsQueue({
|
|
501
|
-
|
|
502
|
-
rate: vset.rate,
|
|
1893
|
+
engine: makeEngine(engineKind),
|
|
503
1894
|
onError: (sessionId) => broadcast("tts-error", { sessionId })
|
|
504
1895
|
});
|
|
1896
|
+
queue.updateVoice(vset.voice, vset.rate);
|
|
505
1897
|
const unsubscribe = queue.subscribe((frame) => broadcast("audio", frame));
|
|
506
1898
|
ctx.effect(() => unsubscribe);
|
|
507
1899
|
ctx.effect(() => () => void queue.close());
|
|
508
1900
|
ctx.effect(
|
|
509
1901
|
() => settingsScope.watch((next) => {
|
|
510
1902
|
vset = next;
|
|
1903
|
+
if (next.ttsEngine !== engineKind) {
|
|
1904
|
+
engineKind = next.ttsEngine;
|
|
1905
|
+
queue.setEngine(makeEngine(engineKind));
|
|
1906
|
+
}
|
|
511
1907
|
queue.updateVoice(next.voice, next.rate);
|
|
512
1908
|
})
|
|
513
1909
|
);
|
|
514
1910
|
const currentVoice = () => vset.voice;
|
|
515
1911
|
const currentRate = () => vset.rate;
|
|
516
1912
|
const currentInterrupt = () => vset.interruptLevel;
|
|
1913
|
+
const currentEngine = () => engineKind;
|
|
1914
|
+
const yieldActiveSession = (expectedSid) => {
|
|
1915
|
+
ownerYieldTimer = null;
|
|
1916
|
+
const sid = activeVoiceSession;
|
|
1917
|
+
if (!sid) return;
|
|
1918
|
+
if (expectedSid !== void 0 && expectedSid !== sid) return;
|
|
1919
|
+
activeVoiceSession = null;
|
|
1920
|
+
activeTabId = null;
|
|
1921
|
+
queue.cancel(sid);
|
|
1922
|
+
asr.reset(sid);
|
|
1923
|
+
setTurn(sid, "idle");
|
|
1924
|
+
turnStates.delete(sid);
|
|
1925
|
+
broadcast("mode", { active: null, ownerTabId: activeTabId });
|
|
1926
|
+
};
|
|
517
1927
|
ctx.on("system-prompt/assemble", (assembly, context, next) => {
|
|
518
1928
|
if (!config.enabled || !vset.spokenFormat) return next();
|
|
519
1929
|
const agentId = context.agent?.id;
|
|
@@ -526,24 +1936,31 @@ function apply(ctx, config) {
|
|
|
526
1936
|
const sessionId = options.sessionId;
|
|
527
1937
|
if (!config.enabled || sessionId === void 0 || options.purpose !== void 0) return next();
|
|
528
1938
|
if (activeVoiceSession !== sessionId) return next();
|
|
529
|
-
|
|
1939
|
+
const gen = (turnGen.get(sessionId) ?? 0) + 1;
|
|
1940
|
+
turnGen.set(sessionId, gen);
|
|
1941
|
+
return tapActiveStream(
|
|
1942
|
+
sessionId,
|
|
1943
|
+
next(),
|
|
1944
|
+
queue,
|
|
1945
|
+
broadcast,
|
|
1946
|
+
(state) => {
|
|
1947
|
+
if ((turnGen.get(sessionId) ?? 0) === gen) setTurn(sessionId, state);
|
|
1948
|
+
}
|
|
1949
|
+
);
|
|
530
1950
|
});
|
|
531
1951
|
const base = BASE_PATH;
|
|
532
1952
|
ctx.effect(
|
|
533
1953
|
() => ctx.webServer.register({
|
|
534
1954
|
kind: "prefix",
|
|
535
1955
|
path: base,
|
|
536
|
-
handler: (
|
|
537
|
-
res
|
|
538
|
-
res
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
active: activeVoiceSession
|
|
545
|
-
})
|
|
546
|
-
);
|
|
1956
|
+
handler: (req, res) => {
|
|
1957
|
+
if (denyNonLoopback(req, res)) return;
|
|
1958
|
+
respondJson2(res, 200, {
|
|
1959
|
+
ok: true,
|
|
1960
|
+
name: "dsh-voice-mode",
|
|
1961
|
+
enabled: config.enabled,
|
|
1962
|
+
active: activeVoiceSession
|
|
1963
|
+
});
|
|
547
1964
|
}
|
|
548
1965
|
})
|
|
549
1966
|
);
|
|
@@ -551,24 +1968,30 @@ function apply(ctx, config) {
|
|
|
551
1968
|
() => ctx.webServer.register({
|
|
552
1969
|
kind: "exact",
|
|
553
1970
|
path: `${base}/config`,
|
|
554
|
-
handler: (
|
|
555
|
-
res
|
|
556
|
-
res
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
1971
|
+
handler: (req, res) => {
|
|
1972
|
+
if (denyNonLoopback(req, res)) return;
|
|
1973
|
+
respondJson2(res, 200, {
|
|
1974
|
+
basePath: base,
|
|
1975
|
+
rate: currentRate(),
|
|
1976
|
+
voice: currentVoice(),
|
|
1977
|
+
senseVoice: vset.senseVoice,
|
|
1978
|
+
interruptLevel: currentInterrupt(),
|
|
1979
|
+
silenceMs: vset.silenceMs,
|
|
1980
|
+
idleTimeoutMinutes: vset.idleTimeoutMinutes,
|
|
1981
|
+
modelHost: vset.modelHost,
|
|
1982
|
+
autoSend: vset.autoSend,
|
|
1983
|
+
autoResume: vset.autoResume,
|
|
1984
|
+
mode: vset.mode,
|
|
1985
|
+
bargeInMode: vset.bargeInMode,
|
|
1986
|
+
echoGateDb: vset.echoGateDb,
|
|
1987
|
+
shortcut: vset.shortcut,
|
|
1988
|
+
wakeWord: vset.wakeWord,
|
|
1989
|
+
toolBeep: vset.toolBeep,
|
|
1990
|
+
cacheDir: config.cacheDir,
|
|
1991
|
+
ttsEngine: currentEngine(),
|
|
1992
|
+
audioMime: queue.mime,
|
|
1993
|
+
allowLan: config.allowLan
|
|
1994
|
+
});
|
|
572
1995
|
}
|
|
573
1996
|
})
|
|
574
1997
|
);
|
|
@@ -577,10 +2000,16 @@ function apply(ctx, config) {
|
|
|
577
2000
|
kind: "exact",
|
|
578
2001
|
path: `${base}/preview`,
|
|
579
2002
|
handler: (req, res) => {
|
|
580
|
-
if (
|
|
581
|
-
|
|
2003
|
+
if (denyNonLoopback(req, res)) return;
|
|
2004
|
+
if (denyCrossOrigin(req, res)) return;
|
|
2005
|
+
if (!limiter.hit(`preview:${req.socket.remoteAddress ?? "unknown"}`, 20, 6e4)) {
|
|
2006
|
+
res.statusCode = 429;
|
|
582
2007
|
res.setHeader("content-type", "application/json");
|
|
583
|
-
res.end(JSON.stringify({ error: "
|
|
2008
|
+
res.end(JSON.stringify({ error: "rate limited" }));
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
if (!config.enabled) {
|
|
2012
|
+
respondJson2(res, 403, { error: "voice mode disabled" });
|
|
584
2013
|
return;
|
|
585
2014
|
}
|
|
586
2015
|
collectBody(req, res, MAX_JSON_BODY, async (body) => {
|
|
@@ -595,31 +2024,23 @@ function apply(ctx, config) {
|
|
|
595
2024
|
} catch {
|
|
596
2025
|
}
|
|
597
2026
|
if (voice.length > 128) {
|
|
598
|
-
res
|
|
599
|
-
res.setHeader("content-type", "application/json");
|
|
600
|
-
res.end(JSON.stringify({ error: "voice too long" }));
|
|
2027
|
+
respondJson2(res, 400, { error: "voice too long" });
|
|
601
2028
|
return;
|
|
602
2029
|
}
|
|
603
2030
|
if (!voice) {
|
|
604
|
-
res
|
|
605
|
-
res.setHeader("content-type", "application/json");
|
|
606
|
-
res.end(JSON.stringify({ error: "voice required" }));
|
|
2031
|
+
respondJson2(res, 400, { error: "voice required" });
|
|
607
2032
|
return;
|
|
608
2033
|
}
|
|
609
|
-
const sample = voice.startsWith("zh-") ? "\u4F60\u597D\uFF0C\u6B22\u8FCE\u4F7F\u7528\u8BED\u97F3\u6A21\u5F0F\u3002" : "Hello, welcome to voice mode.";
|
|
2034
|
+
const sample = currentEngine() === "kokoro" ? "\u4F60\u597D\uFF0C\u6B22\u8FCE\u4F7F\u7528\u8BED\u97F3\u6A21\u5F0F\u3002Hello, welcome to voice mode." : currentEngine() === "vits" || voice.startsWith("zh-") ? "\u4F60\u597D\uFF0C\u6B22\u8FCE\u4F7F\u7528\u8BED\u97F3\u6A21\u5F0F\u3002" : "Hello, welcome to voice mode.";
|
|
610
2035
|
let buf;
|
|
611
2036
|
try {
|
|
612
2037
|
buf = await queue.synthesize(sample, { voice, rate });
|
|
613
2038
|
} catch (e) {
|
|
614
2039
|
console.warn(`[dsh-voice-mode] preview synthesis failed: ${String(e)}`);
|
|
615
|
-
res
|
|
616
|
-
res.setHeader("content-type", "application/json");
|
|
617
|
-
res.end(JSON.stringify({ error: "\u9884\u89C8\u5408\u6210\u5931\u8D25\uFF1A\u8BF7\u68C0\u67E5\u7F51\u7EDC\u6216\u97F3\u8272\u540D\uFF08ShortName\uFF09\u662F\u5426\u6B63\u786E" }));
|
|
2040
|
+
respondJson2(res, 502, { error: "\u9884\u89C8\u5408\u6210\u5931\u8D25\uFF1A\u8BF7\u68C0\u67E5\u7F51\u7EDC\u6216\u97F3\u8272\u540D\uFF08ShortName\uFF09\u662F\u5426\u6B63\u786E" });
|
|
618
2041
|
return;
|
|
619
2042
|
}
|
|
620
|
-
res.
|
|
621
|
-
res.setHeader("content-type", "audio/mpeg");
|
|
622
|
-
res.setHeader("cache-control", "no-store");
|
|
2043
|
+
res.writeHead(200, { "content-type": queue.mime, "cache-control": "no-store" });
|
|
623
2044
|
res.end(buf);
|
|
624
2045
|
});
|
|
625
2046
|
}
|
|
@@ -630,48 +2051,195 @@ function apply(ctx, config) {
|
|
|
630
2051
|
kind: "exact",
|
|
631
2052
|
path: `${base}/toggle`,
|
|
632
2053
|
handler: (req, res) => {
|
|
2054
|
+
if (denyNonLoopback(req, res)) return;
|
|
2055
|
+
if (denyCrossOrigin(req, res)) return;
|
|
633
2056
|
collectBody(req, res, MAX_JSON_BODY, (body) => {
|
|
634
2057
|
let sessionId;
|
|
635
2058
|
let on;
|
|
2059
|
+
let tabId;
|
|
636
2060
|
try {
|
|
637
2061
|
const parsed = JSON.parse(body || "{}");
|
|
638
2062
|
sessionId = parsed.sessionId;
|
|
639
2063
|
on = parsed.on;
|
|
2064
|
+
tabId = typeof parsed.tabId === "string" && parsed.tabId.length <= 64 ? parsed.tabId : void 0;
|
|
640
2065
|
} catch {
|
|
641
2066
|
}
|
|
642
2067
|
if (!sessionId) {
|
|
643
|
-
res
|
|
644
|
-
|
|
2068
|
+
respondJson2(res, 400, { error: "sessionId required" });
|
|
2069
|
+
return;
|
|
2070
|
+
}
|
|
2071
|
+
if (on !== void 0 && typeof on !== "boolean") {
|
|
2072
|
+
respondJson2(res, 400, { error: "invalid on" });
|
|
2073
|
+
return;
|
|
2074
|
+
}
|
|
2075
|
+
if (!limiter.hit(`toggle:${sessionId}`, 1, 2e3)) {
|
|
2076
|
+
res.statusCode = 429;
|
|
2077
|
+
res.setHeader("content-type", "application/json");
|
|
2078
|
+
res.end(JSON.stringify({ error: "rate limited" }));
|
|
645
2079
|
return;
|
|
646
2080
|
}
|
|
647
2081
|
if (on === true) {
|
|
648
2082
|
if (!config.enabled) {
|
|
649
|
-
res
|
|
650
|
-
|
|
2083
|
+
respondJson2(res, 403, { error: "voice mode disabled" });
|
|
2084
|
+
return;
|
|
2085
|
+
}
|
|
2086
|
+
if (sessions && !sessions.get(sessionId)) {
|
|
2087
|
+
respondJson2(res, 403, { error: "unknown session" });
|
|
651
2088
|
return;
|
|
652
2089
|
}
|
|
2090
|
+
asr.reset(sessionId);
|
|
2091
|
+
queue.cancel(sessionId);
|
|
653
2092
|
const previous = activeVoiceSession;
|
|
654
2093
|
activeVoiceSession = sessionId;
|
|
655
|
-
|
|
656
|
-
|
|
2094
|
+
activeTabId = tabId ?? null;
|
|
2095
|
+
if (ownerYieldTimer) {
|
|
2096
|
+
clearTimeout(ownerYieldTimer);
|
|
2097
|
+
ownerYieldTimer = null;
|
|
2098
|
+
}
|
|
2099
|
+
if (previous && previous !== sessionId) {
|
|
2100
|
+
queue.cancel(previous);
|
|
2101
|
+
asr.reset(previous);
|
|
2102
|
+
setTurn(previous, "idle");
|
|
2103
|
+
turnStates.delete(previous);
|
|
2104
|
+
}
|
|
2105
|
+
broadcast("mode", { active: activeVoiceSession, ownerTabId: activeTabId });
|
|
657
2106
|
} else {
|
|
658
2107
|
if (activeVoiceSession === sessionId) {
|
|
659
2108
|
activeVoiceSession = null;
|
|
660
|
-
|
|
661
|
-
|
|
2109
|
+
activeTabId = null;
|
|
2110
|
+
if (ownerYieldTimer) {
|
|
2111
|
+
clearTimeout(ownerYieldTimer);
|
|
2112
|
+
ownerYieldTimer = null;
|
|
2113
|
+
}
|
|
2114
|
+
queue.cancel(sessionId);
|
|
2115
|
+
asr.reset(sessionId);
|
|
2116
|
+
setTurn(sessionId, "idle");
|
|
2117
|
+
turnStates.delete(sessionId);
|
|
2118
|
+
broadcast("mode", { active: null, ownerTabId: null });
|
|
662
2119
|
}
|
|
663
2120
|
}
|
|
664
|
-
res
|
|
665
|
-
|
|
2121
|
+
respondJson2(res, 200, { active: activeVoiceSession });
|
|
2122
|
+
});
|
|
2123
|
+
}
|
|
2124
|
+
})
|
|
2125
|
+
);
|
|
2126
|
+
ctx.effect(
|
|
2127
|
+
() => ctx.webServer.register({
|
|
2128
|
+
kind: "exact",
|
|
2129
|
+
path: `${base}/models/status`,
|
|
2130
|
+
handler: (req, res) => {
|
|
2131
|
+
if (denyNonLoopback(req, res)) return;
|
|
2132
|
+
respondJson2(res, 200, { ...asr.modelStatus(), tts: queue.status() });
|
|
2133
|
+
}
|
|
2134
|
+
})
|
|
2135
|
+
);
|
|
2136
|
+
ctx.effect(
|
|
2137
|
+
() => ctx.webServer.register({
|
|
2138
|
+
kind: "exact",
|
|
2139
|
+
path: `${base}/models/retry`,
|
|
2140
|
+
handler: (req, res) => {
|
|
2141
|
+
if (denyNonLoopback(req, res)) return;
|
|
2142
|
+
if (!config.enabled) {
|
|
2143
|
+
respondJson2(res, 403, { error: "voice mode disabled" });
|
|
2144
|
+
return;
|
|
2145
|
+
}
|
|
2146
|
+
collectBody(req, res, MAX_JSON_BODY, (body) => {
|
|
2147
|
+
let kind = "asr";
|
|
2148
|
+
try {
|
|
2149
|
+
const p = JSON.parse(body || "{}");
|
|
2150
|
+
if (p.kind === void 0) {
|
|
2151
|
+
} else if (p.kind === "vad" || p.kind === "sense" || p.kind === "asr") {
|
|
2152
|
+
kind = p.kind;
|
|
2153
|
+
} else {
|
|
2154
|
+
respondJson2(res, 400, { error: "invalid kind" });
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
2157
|
+
} catch {
|
|
2158
|
+
respondJson2(res, 400, { error: "invalid json" });
|
|
2159
|
+
return;
|
|
2160
|
+
}
|
|
2161
|
+
void asr.retryModel(kind).then((done) => {
|
|
2162
|
+
respondJson2(res, 200, { ok: done, kind });
|
|
2163
|
+
});
|
|
2164
|
+
});
|
|
2165
|
+
}
|
|
2166
|
+
})
|
|
2167
|
+
);
|
|
2168
|
+
ctx.effect(
|
|
2169
|
+
() => ctx.webServer.register({
|
|
2170
|
+
kind: "exact",
|
|
2171
|
+
path: `${base}/models/clean`,
|
|
2172
|
+
handler: (req, res) => {
|
|
2173
|
+
if (denyNonLoopback(req, res)) return;
|
|
2174
|
+
if (denyCrossOrigin(req, res)) return;
|
|
2175
|
+
if (!config.enabled) {
|
|
2176
|
+
respondJson2(res, 403, { error: "voice mode disabled" });
|
|
2177
|
+
return;
|
|
2178
|
+
}
|
|
2179
|
+
collectBody(req, res, MAX_JSON_BODY, (body) => {
|
|
2180
|
+
let engine = "vits";
|
|
2181
|
+
try {
|
|
2182
|
+
const p = JSON.parse(body || "{}");
|
|
2183
|
+
if (p.engine === "kokoro" || p.engine === "vits") engine = p.engine;
|
|
2184
|
+
else {
|
|
2185
|
+
respondJson2(res, 400, { error: "invalid engine" });
|
|
2186
|
+
return;
|
|
2187
|
+
}
|
|
2188
|
+
} catch {
|
|
2189
|
+
respondJson2(res, 400, { error: "invalid json" });
|
|
2190
|
+
return;
|
|
2191
|
+
}
|
|
2192
|
+
const dir = join4(config.cacheDir, engine === "kokoro" ? KOKORO_MODEL_DIR : TTS_MODEL_REPO);
|
|
2193
|
+
void rm(dir, { recursive: true, force: true }).then(() => {
|
|
2194
|
+
if (engineKind === engine) {
|
|
2195
|
+
queue.setEngine(makeEngine(engine));
|
|
2196
|
+
queue.updateVoice(vset.voice, vset.rate);
|
|
2197
|
+
}
|
|
2198
|
+
respondJson2(res, 200, { ok: true, engine });
|
|
2199
|
+
}).catch((e) => respondJson2(res, 500, { error: String(e) }));
|
|
666
2200
|
});
|
|
667
2201
|
}
|
|
668
2202
|
})
|
|
669
2203
|
);
|
|
2204
|
+
ctx.effect(
|
|
2205
|
+
() => ctx.webServer.register({
|
|
2206
|
+
kind: "exact",
|
|
2207
|
+
path: `${base}/voices`,
|
|
2208
|
+
handler: async (req, res) => {
|
|
2209
|
+
if (denyNonLoopback(req, res)) return;
|
|
2210
|
+
if (denyCrossOrigin(req, res)) return;
|
|
2211
|
+
try {
|
|
2212
|
+
const voices = await listEdgeVoices();
|
|
2213
|
+
respondJson2(res, 200, { voices });
|
|
2214
|
+
} catch (e) {
|
|
2215
|
+
respondJson2(res, 502, { error: String(e) });
|
|
2216
|
+
}
|
|
2217
|
+
}
|
|
2218
|
+
})
|
|
2219
|
+
);
|
|
670
2220
|
ctx.effect(
|
|
671
2221
|
() => ctx.webServer.register({
|
|
672
2222
|
kind: "exact",
|
|
673
2223
|
path: `${base}/asr`,
|
|
674
2224
|
handler: (req, res) => {
|
|
2225
|
+
if (denyNonLoopback(req, res)) return;
|
|
2226
|
+
let sid = "";
|
|
2227
|
+
try {
|
|
2228
|
+
const url = new URL(req.url ?? "/", "http://localhost");
|
|
2229
|
+
sid = url.searchParams.get("sessionId") ?? "";
|
|
2230
|
+
} catch {
|
|
2231
|
+
}
|
|
2232
|
+
if (!limiter.hit(`asr:${sid || "unknown"}`, 60, 1e3)) {
|
|
2233
|
+
respondJson2(res, 429, { error: "rate limited" });
|
|
2234
|
+
return;
|
|
2235
|
+
}
|
|
2236
|
+
if (sid && sid === activeVoiceSession) {
|
|
2237
|
+
try {
|
|
2238
|
+
const url = new URL(req.url ?? "/", "http://localhost");
|
|
2239
|
+
setTurn(sid, url.searchParams.get("final") === "1" ? "finalizing" : "listening");
|
|
2240
|
+
} catch {
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
675
2243
|
handleAsrRequest(asr, activeVoiceSession, req, res);
|
|
676
2244
|
}
|
|
677
2245
|
})
|
|
@@ -681,19 +2249,60 @@ function apply(ctx, config) {
|
|
|
681
2249
|
kind: "exact",
|
|
682
2250
|
path: `${base}/cancel`,
|
|
683
2251
|
handler: (req, res) => {
|
|
2252
|
+
if (denyNonLoopback(req, res)) return;
|
|
2253
|
+
if (denyCrossOrigin(req, res)) return;
|
|
684
2254
|
collectBody(req, res, MAX_JSON_BODY, (body) => {
|
|
685
2255
|
let sessionId;
|
|
2256
|
+
let keepAsr = false;
|
|
686
2257
|
try {
|
|
687
2258
|
const parsed = JSON.parse(body || "{}");
|
|
688
2259
|
sessionId = parsed.sessionId;
|
|
2260
|
+
keepAsr = parsed.keepAsr === true;
|
|
689
2261
|
} catch {
|
|
690
2262
|
}
|
|
691
|
-
if (sessionId) {
|
|
2263
|
+
if (sessionId && sessionId === activeVoiceSession) {
|
|
2264
|
+
if (!limiter.hit(`cancel:${sessionId}`, 2, 1e3)) {
|
|
2265
|
+
respondJson2(res, 429, { error: "rate limited" });
|
|
2266
|
+
return;
|
|
2267
|
+
}
|
|
692
2268
|
queue.cancel(sessionId);
|
|
693
|
-
asr.reset(sessionId);
|
|
2269
|
+
if (!keepAsr) asr.reset(sessionId);
|
|
694
2270
|
}
|
|
695
|
-
res
|
|
696
|
-
|
|
2271
|
+
respondJson2(res, 200, { ok: true });
|
|
2272
|
+
});
|
|
2273
|
+
}
|
|
2274
|
+
})
|
|
2275
|
+
);
|
|
2276
|
+
ctx.effect(
|
|
2277
|
+
() => ctx.webServer.register({
|
|
2278
|
+
kind: "exact",
|
|
2279
|
+
path: `${base}/mode`,
|
|
2280
|
+
handler: (req, res) => {
|
|
2281
|
+
if (denyNonLoopback(req, res)) return;
|
|
2282
|
+
if (denyCrossOrigin(req, res)) return;
|
|
2283
|
+
collectBody(req, res, MAX_JSON_BODY, (body) => {
|
|
2284
|
+
let mode;
|
|
2285
|
+
try {
|
|
2286
|
+
const parsed = JSON.parse(body || "{}");
|
|
2287
|
+
mode = parsed.mode === "toggle" || parsed.mode === "hold" ? parsed.mode : void 0;
|
|
2288
|
+
} catch {
|
|
2289
|
+
}
|
|
2290
|
+
if (!mode) {
|
|
2291
|
+
res.statusCode = 400;
|
|
2292
|
+
res.setHeader("content-type", "application/json");
|
|
2293
|
+
res.end(JSON.stringify({ error: "mode must be toggle or hold" }));
|
|
2294
|
+
return;
|
|
2295
|
+
}
|
|
2296
|
+
void settingsScope.update({ mode }).then(() => {
|
|
2297
|
+
res.statusCode = 200;
|
|
2298
|
+
res.setHeader("content-type", "application/json");
|
|
2299
|
+
res.end(JSON.stringify({ ok: true, mode }));
|
|
2300
|
+
}).catch((e) => {
|
|
2301
|
+
console.warn(`[dsh-voice-mode] mode update failed: ${String(e)}`);
|
|
2302
|
+
res.statusCode = 500;
|
|
2303
|
+
res.setHeader("content-type", "application/json");
|
|
2304
|
+
res.end(JSON.stringify({ error: "mode update failed" }));
|
|
2305
|
+
});
|
|
697
2306
|
});
|
|
698
2307
|
}
|
|
699
2308
|
})
|
|
@@ -702,7 +2311,19 @@ function apply(ctx, config) {
|
|
|
702
2311
|
() => ctx.webServer.register({
|
|
703
2312
|
kind: "exact",
|
|
704
2313
|
path: `${base}/stream`,
|
|
705
|
-
handler: (
|
|
2314
|
+
handler: (req, res) => {
|
|
2315
|
+
if (denyNonLoopback(req, res)) return;
|
|
2316
|
+
if (sseClients.size >= 4) {
|
|
2317
|
+
respondJson2(res, 429, { error: "too many streams" });
|
|
2318
|
+
return;
|
|
2319
|
+
}
|
|
2320
|
+
let tabId = null;
|
|
2321
|
+
try {
|
|
2322
|
+
const u = new URL(req.url ?? "/", "http://localhost");
|
|
2323
|
+
tabId = u.searchParams.get("tabId");
|
|
2324
|
+
} catch {
|
|
2325
|
+
}
|
|
2326
|
+
if (tabId !== null && tabId.length > 64) tabId = null;
|
|
706
2327
|
res.writeHead(200, {
|
|
707
2328
|
"content-type": "text/event-stream; charset=utf-8",
|
|
708
2329
|
"cache-control": "no-cache, no-transform",
|
|
@@ -715,16 +2336,35 @@ data: ${JSON.stringify(payload)}
|
|
|
715
2336
|
|
|
716
2337
|
`);
|
|
717
2338
|
};
|
|
718
|
-
|
|
719
|
-
|
|
2339
|
+
const client = { tabId, send };
|
|
2340
|
+
sseClients.add(client);
|
|
2341
|
+
if (tabId !== null) latestConnByTab.set(tabId, client);
|
|
2342
|
+
if (tabId !== null && tabId === activeTabId && ownerYieldTimer) {
|
|
2343
|
+
clearTimeout(ownerYieldTimer);
|
|
2344
|
+
ownerYieldTimer = null;
|
|
2345
|
+
}
|
|
2346
|
+
send("mode", { active: activeVoiceSession, ownerTabId: activeTabId });
|
|
720
2347
|
const heartbeat = setInterval(() => {
|
|
721
|
-
|
|
2348
|
+
try {
|
|
2349
|
+
res.write(": hb\n");
|
|
2350
|
+
} catch {
|
|
2351
|
+
}
|
|
722
2352
|
}, 25e3);
|
|
2353
|
+
let cleaned = false;
|
|
723
2354
|
const cleanup = () => {
|
|
2355
|
+
if (cleaned) return;
|
|
2356
|
+
cleaned = true;
|
|
724
2357
|
clearInterval(heartbeat);
|
|
725
|
-
sseClients.delete(
|
|
2358
|
+
sseClients.delete(client);
|
|
2359
|
+
if (tabId !== null && latestConnByTab.get(tabId) === client) {
|
|
2360
|
+
latestConnByTab.delete(tabId);
|
|
2361
|
+
if (tabId === activeTabId) {
|
|
2362
|
+
if (ownerYieldTimer) clearTimeout(ownerYieldTimer);
|
|
2363
|
+
ownerYieldTimer = setTimeout(() => yieldActiveSession(activeVoiceSession), 8e3);
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
726
2366
|
};
|
|
727
|
-
|
|
2367
|
+
req.on("close", cleanup);
|
|
728
2368
|
res.on("close", cleanup);
|
|
729
2369
|
}
|
|
730
2370
|
})
|
|
@@ -732,20 +2372,22 @@ data: ${JSON.stringify(payload)}
|
|
|
732
2372
|
}
|
|
733
2373
|
var MAX_JSON_BODY = 16 * 1024;
|
|
734
2374
|
function collectBody(req, res, maxBytes, onBody) {
|
|
735
|
-
|
|
2375
|
+
const chunks = [];
|
|
2376
|
+
let received = 0;
|
|
736
2377
|
let tooLarge = false;
|
|
737
2378
|
req.on("data", (c) => {
|
|
738
2379
|
if (tooLarge) return;
|
|
739
|
-
|
|
740
|
-
if (
|
|
2380
|
+
received += c.length;
|
|
2381
|
+
if (received > maxBytes) {
|
|
741
2382
|
tooLarge = true;
|
|
742
|
-
res
|
|
743
|
-
|
|
744
|
-
res.end(JSON.stringify({ error: "request body too large" }));
|
|
2383
|
+
respondJson2(res, 413, { error: "request body too large" });
|
|
2384
|
+
return;
|
|
745
2385
|
}
|
|
2386
|
+
chunks.push(c);
|
|
746
2387
|
});
|
|
747
2388
|
req.on("end", () => {
|
|
748
2389
|
if (tooLarge) return;
|
|
2390
|
+
const body = Buffer.concat(chunks).toString("utf8");
|
|
749
2391
|
try {
|
|
750
2392
|
const r = onBody(body);
|
|
751
2393
|
if (r && typeof r.then === "function") r.catch(() => {
|
|
@@ -756,8 +2398,10 @@ function collectBody(req, res, maxBytes, onBody) {
|
|
|
756
2398
|
req.on("error", () => {
|
|
757
2399
|
});
|
|
758
2400
|
}
|
|
759
|
-
async function* tapActiveStream(sessionId, inner, queue, broadcast) {
|
|
2401
|
+
async function* tapActiveStream(sessionId, inner, queue, broadcast, onTurn) {
|
|
760
2402
|
const segmenter = new SentenceSegmenter();
|
|
2403
|
+
let firstTokenBroadcast = false;
|
|
2404
|
+
let firstSentenceBroadcast = false;
|
|
761
2405
|
let flushed = false;
|
|
762
2406
|
let finishReason = null;
|
|
763
2407
|
const flushOnce = () => {
|
|
@@ -770,7 +2414,16 @@ async function* tapActiveStream(sessionId, inner, queue, broadcast) {
|
|
|
770
2414
|
try {
|
|
771
2415
|
for await (const chunk of inner) {
|
|
772
2416
|
if (chunk.type === "text-delta" && chunk.text) {
|
|
2417
|
+
if (!firstTokenBroadcast) {
|
|
2418
|
+
firstTokenBroadcast = true;
|
|
2419
|
+
broadcast("latency", { sessionId, stage: "first-llm-token" });
|
|
2420
|
+
onTurn("agent-speaking");
|
|
2421
|
+
}
|
|
773
2422
|
for (const s of segmenter.feed(chunk.text)) {
|
|
2423
|
+
if (!firstSentenceBroadcast) {
|
|
2424
|
+
firstSentenceBroadcast = true;
|
|
2425
|
+
broadcast("latency", { sessionId, stage: "first-sentence-text" });
|
|
2426
|
+
}
|
|
774
2427
|
queue.enqueue(sessionId, s);
|
|
775
2428
|
}
|
|
776
2429
|
}
|
|
@@ -785,6 +2438,7 @@ async function* tapActiveStream(sessionId, inner, queue, broadcast) {
|
|
|
785
2438
|
} finally {
|
|
786
2439
|
const aborted = finishReason !== null && typeof finishReason === "object" && finishReason.kind === "aborted";
|
|
787
2440
|
if (!aborted) flushOnce();
|
|
2441
|
+
onTurn("listening");
|
|
788
2442
|
}
|
|
789
2443
|
}
|
|
790
2444
|
export {
|