@bojackduy/opencode-voice 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -19
- package/index.js +8 -1
- package/lib/chatterbox-server.js +9 -1
- package/lib/conversation-status.js +269 -0
- package/lib/conversation.js +139 -1
- package/lib/streaming-stt.js +101 -0
- package/lib/streaming-transcript.js +59 -6
- package/lib/stt-log.js +39 -0
- package/lib/stt.js +474 -27
- package/lib/tts-player.js +356 -0
- package/lib/tts-status.js +266 -0
- package/lib/tts.js +856 -173
- package/lib/v2-compat.js +41 -0
- package/lib/v2-preview.js +242 -22
- package/lib/vieneu-server.js +287 -0
- package/package.json +12 -1
package/lib/tts.js
CHANGED
|
@@ -7,6 +7,10 @@ import { spawn } from "node:child_process";
|
|
|
7
7
|
import { getSessionTitle } from "./session.js";
|
|
8
8
|
import { clearProcessingToast, showProcessingToast, updateProcessingToast } from "./stt.js";
|
|
9
9
|
import { createChatterboxClient, resolveTtsEngine } from "./chatterbox-server.js";
|
|
10
|
+
import { createVieneuClient } from "./vieneu-server.js";
|
|
11
|
+
import { createTtsPlayer, measurePcmDurationSec, measureWavDurationSec } from "./tts-player.js";
|
|
12
|
+
import { nextRate as nextTtsRate } from "./tts-status.js";
|
|
13
|
+
import { sttLog } from "./stt-log.js";
|
|
10
14
|
|
|
11
15
|
const VOICES_DIR = path.join(os.homedir(), ".local", "share", "piper-voices");
|
|
12
16
|
|
|
@@ -24,7 +28,79 @@ const PIPER_RATE = 22050;
|
|
|
24
28
|
const PIPER_BITS = 16;
|
|
25
29
|
const PIPER_CHANNELS = 1;
|
|
26
30
|
|
|
31
|
+
// ---- Structured content: tables and diagrams are not speakable literally ----
|
|
32
|
+
// A literal read ("pipe name pipe status pipe...") is noise, so both the
|
|
33
|
+
// local cleanup and the LLM narrator summarize instead: what the table is
|
|
34
|
+
// about, then the few rows that matter. Pure functions, tested in tts.test.js.
|
|
35
|
+
|
|
36
|
+
// Structural-only guard: diagrams, tables, fences. Safe on clean text.
|
|
37
|
+
const MERMAID_FENCE_RE = /```mermaid[^\S\n]*\r?\n?([\s\S]*?)```/gi;
|
|
38
|
+
const MERMAID_DECL_RE =
|
|
39
|
+
/^(graph|flowchart|sequenceDiagram|classDiagram|stateDiagram|erDiagram|gantt|pie|mindmap|journey|gitGraph)\b/i;
|
|
40
|
+
const MERMAID_LINE_RE = /-->|---|==>|-\.->|==>|\[|\]|\(|\)/;
|
|
41
|
+
const MERMAID_DECL_MAX_LEN = 32;
|
|
42
|
+
|
|
43
|
+
const TABLE_ROW_CAP = 3;
|
|
44
|
+
const TABLE_TEXT_CAP = 320;
|
|
45
|
+
const MERMAID_LABEL_CAP = 6;
|
|
46
|
+
|
|
47
|
+
function splitTableCells(row) {
|
|
48
|
+
return row
|
|
49
|
+
.trim()
|
|
50
|
+
.replace(/^\||\|$/g, "")
|
|
51
|
+
.split("|")
|
|
52
|
+
.map((cell) => cell.trim());
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function isTableSeparator(line) {
|
|
56
|
+
const cells = splitTableCells(line);
|
|
57
|
+
return cells.length > 0 && cells.every((cell) => /^:?-+:?$/.test(cell));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function summarizeMarkdownTable(block) {
|
|
61
|
+
const lines = (block || "")
|
|
62
|
+
.split("\n")
|
|
63
|
+
.map((line) => line.trim())
|
|
64
|
+
.filter((line) => line.startsWith("|"));
|
|
65
|
+
const headerIdx = lines.findIndex((line) => !isTableSeparator(line));
|
|
66
|
+
if (headerIdx === -1) return "Table.";
|
|
67
|
+
const headers = splitTableCells(lines[headerIdx]).filter(Boolean);
|
|
68
|
+
const rows = lines
|
|
69
|
+
.slice(headerIdx + 1)
|
|
70
|
+
.filter((line) => !isTableSeparator(line))
|
|
71
|
+
.map((line) => splitTableCells(line).filter(Boolean))
|
|
72
|
+
.filter((cells) => cells.length > 0);
|
|
73
|
+
if (headers.length === 0 && rows.length === 0) return "Table.";
|
|
74
|
+
const shown = rows.slice(0, TABLE_ROW_CAP).map((cells) => cells.join(", "));
|
|
75
|
+
let out = headers.length > 0 ? `Table with columns ${headers.join(", ")}.` : "Table.";
|
|
76
|
+
if (rows.length > 0) {
|
|
77
|
+
out += ` ${rows.length} row${rows.length === 1 ? "" : "s"}: ${shown.join("; ")}.`;
|
|
78
|
+
if (rows.length > shown.length) out += ` And ${rows.length - shown.length} more.`;
|
|
79
|
+
}
|
|
80
|
+
return out.length > TABLE_TEXT_CAP ? `${out.slice(0, TABLE_TEXT_CAP - 1)}…` : out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function summarizeMermaidForSpeech(code) {
|
|
84
|
+
const text = code || "";
|
|
85
|
+
const firstLine =
|
|
86
|
+
text
|
|
87
|
+
.split("\n")
|
|
88
|
+
.map((line) => line.trim())
|
|
89
|
+
.find(Boolean) || "";
|
|
90
|
+
const kindMatch = /^(graph|flowchart)\b/i.exec(firstLine);
|
|
91
|
+
const kind = kindMatch ? kindMatch[1].toLowerCase() : "";
|
|
92
|
+
const labels = [...text.matchAll(/\[([^\[\]]+)\]/g)]
|
|
93
|
+
.map((m) => m[1].trim())
|
|
94
|
+
.filter(Boolean)
|
|
95
|
+
.slice(0, MERMAID_LABEL_CAP);
|
|
96
|
+
const what = kind ? `Diagram, ${kind}` : "Diagram";
|
|
97
|
+
if (labels.length >= 2) return `${what}: ${labels.join(", ")}.`;
|
|
98
|
+
if (labels.length === 1) return `${what}: ${labels[0]}.`;
|
|
99
|
+
return `${what}.`;
|
|
100
|
+
}
|
|
101
|
+
|
|
27
102
|
// ---- Local (no-LLM) speech cleanup ----
|
|
103
|
+
|
|
28
104
|
// Deterministic markdown -> spoken-text transform. Zero network latency, so it
|
|
29
105
|
// backs streaming speech and ttsNormalizeMode "local". Less polished than the
|
|
30
106
|
// LLM narrator (no summarization), but instant.
|
|
@@ -36,9 +112,61 @@ function splitIdentifier(token) {
|
|
|
36
112
|
.replace(/[_-]+/g, " ");
|
|
37
113
|
}
|
|
38
114
|
|
|
115
|
+
function summarizeUnfencedMermaid(lines, start) {
|
|
116
|
+
if (lines[start].trim().length > MERMAID_DECL_MAX_LEN) return null;
|
|
117
|
+
const body = [];
|
|
118
|
+
let i = start + 1;
|
|
119
|
+
while (i < lines.length && lines[i].trim() === "") i++;
|
|
120
|
+
for (; i < lines.length; i++) {
|
|
121
|
+
const line = lines[i];
|
|
122
|
+
if (line.trim() === "") break;
|
|
123
|
+
if (!MERMAID_LINE_RE.test(line)) return null;
|
|
124
|
+
body.push(line);
|
|
125
|
+
}
|
|
126
|
+
if (body.length === 0) return null;
|
|
127
|
+
return {
|
|
128
|
+
text: summarizeMermaidForSpeech([lines[start], ...body].join("\n")),
|
|
129
|
+
end: i,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export function sanitizeSpokenText(text) {
|
|
134
|
+
if (!text) return "";
|
|
135
|
+
let out = String(text);
|
|
136
|
+
out = out.replace(MERMAID_FENCE_RE, (_, code) => ` ${summarizeMermaidForSpeech(code)} `);
|
|
137
|
+
{
|
|
138
|
+
const lines = out.split("\n");
|
|
139
|
+
const kept = [];
|
|
140
|
+
for (let i = 0; i < lines.length; i++) {
|
|
141
|
+
if (MERMAID_DECL_RE.test(lines[i].trim())) {
|
|
142
|
+
const found = summarizeUnfencedMermaid(lines, i);
|
|
143
|
+
if (found) {
|
|
144
|
+
kept.push(found.text);
|
|
145
|
+
i = found.end - 1;
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
kept.push(lines[i]);
|
|
150
|
+
}
|
|
151
|
+
out = kept.join("\n");
|
|
152
|
+
}
|
|
153
|
+
out = out.replace(
|
|
154
|
+
/((?:^[ \t]*\|.*\|\s*$\n?){2,})/gm,
|
|
155
|
+
(block) => ` ${summarizeMarkdownTable(block)} `,
|
|
156
|
+
);
|
|
157
|
+
out = out.replace(/```[\s\S]*?```/g, " code snippet ");
|
|
158
|
+
return out;
|
|
159
|
+
}
|
|
160
|
+
|
|
39
161
|
export function localSpeechCleanup(text) {
|
|
40
162
|
if (!text) return "";
|
|
41
|
-
let out = text;
|
|
163
|
+
let out = sanitizeSpokenText(text);
|
|
164
|
+
// (Diagrams, tables and generic fences were already handled by sanitizeSpokenText above.)
|
|
165
|
+
// Tables next: runs of pipe lines become a short spoken summary.
|
|
166
|
+
out = out.replace(
|
|
167
|
+
/((?:^[ \t]*\|.*\|\s*$\n?){2,})/gm,
|
|
168
|
+
(block) => ` ${summarizeMarkdownTable(block)} `,
|
|
169
|
+
);
|
|
42
170
|
out = out.replace(/```[\s\S]*?```/g, " code snippet ");
|
|
43
171
|
out = out.replace(/`([^`]+)`/g, (_, code) => ` ${splitIdentifier(code)} `);
|
|
44
172
|
out = out.replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1");
|
|
@@ -77,7 +205,7 @@ export function localSpeechCleanup(text) {
|
|
|
77
205
|
// Pure so it is testable without a TUI, a model, or piper.
|
|
78
206
|
|
|
79
207
|
export function resolveSpeechText(rawText, llmResult) {
|
|
80
|
-
if (llmResult?.text) return { text: llmResult.text };
|
|
208
|
+
if (llmResult?.text) return { text: sanitizeSpokenText(llmResult.text) };
|
|
81
209
|
return { text: localSpeechCleanup(rawText), error: llmResult?.error, fellBack: true };
|
|
82
210
|
}
|
|
83
211
|
|
|
@@ -111,7 +239,11 @@ async function normalizeWithinBound(promise) {
|
|
|
111
239
|
|
|
112
240
|
export function splitSpokenSentences(text) {
|
|
113
241
|
const flat = (text || "").replace(/\s+/g, " ");
|
|
114
|
-
|
|
242
|
+
// Only split where a new sentence plausibly starts (uppercase letter,
|
|
243
|
+
// quote, or bracket). This keeps version numbers ("v1.2.3"), decimals,
|
|
244
|
+
// and abbreviations ("e.g. this") inside one span so a fragment can't
|
|
245
|
+
// be voice-routed on its own.
|
|
246
|
+
const parts = flat.split(/(?<=[.!?…])\s+(?=[\p{Lu}"“'‘(\[])/u);
|
|
115
247
|
if (parts.length === 0) return { sentences: [], rest: "" };
|
|
116
248
|
const lastComplete = /[.!?…]\s*$/.test(flat);
|
|
117
249
|
if (lastComplete) return { sentences: parts.filter(Boolean), rest: "" };
|
|
@@ -125,6 +257,27 @@ export function isSpeakableSentence(sentence) {
|
|
|
125
257
|
return true;
|
|
126
258
|
}
|
|
127
259
|
|
|
260
|
+
// Per-sentence language routing: Vietnamese and English need different
|
|
261
|
+
// engines/voices (Chatterbox cannot speak vi; Piper vi mangles English),
|
|
262
|
+
// so an utterance is split into sentences and consecutive same-language
|
|
263
|
+
// sentences are grouped. Word order is preserved exactly; grouping only
|
|
264
|
+
// decides which voice synthesizes which span. Pure export for tests.
|
|
265
|
+
export function groupSentencesByLang(text) {
|
|
266
|
+
const flat = (text || "").replace(/\s+/g, " ").trim();
|
|
267
|
+
if (!flat) return [];
|
|
268
|
+
const { sentences, rest } = splitSpokenSentences(flat);
|
|
269
|
+
const parts = [...sentences];
|
|
270
|
+
if (rest.trim()) parts.push(rest.trim());
|
|
271
|
+
const groups = [];
|
|
272
|
+
for (const part of parts) {
|
|
273
|
+
const lang = detectLang(part);
|
|
274
|
+
const last = groups[groups.length - 1];
|
|
275
|
+
if (last && last.lang === lang) last.text = `${last.text} ${part}`;
|
|
276
|
+
else groups.push({ lang, text: part });
|
|
277
|
+
}
|
|
278
|
+
return groups;
|
|
279
|
+
}
|
|
280
|
+
|
|
128
281
|
// Decide ONE language for the whole call (utterance or, during streaming,
|
|
129
282
|
// one completed sentence) and speak all of it with that single voice - no
|
|
130
283
|
// mid-utterance voice switching, since splicing separately synthesized clips
|
|
@@ -189,6 +342,25 @@ You have three modes depending on the content complexity:
|
|
|
189
342
|
|
|
190
343
|
Choose the appropriate mode based on the content. Most responses with code blocks should use SUMMARIZE mode. Simple Q&A or short explanations use NARRATE. Build results, "done", confirmations use NOTIFY.
|
|
191
344
|
|
|
345
|
+
STRUCTURED CONTENT RULES (apply in every mode):
|
|
346
|
+
- TABLES: never read a markdown table cell by cell. First understand what the
|
|
347
|
+
table lists, then say one short intro (the topic) plus the 2-3 most
|
|
348
|
+
important rows in plain sentences. Skip long tables after the gist.
|
|
349
|
+
- DIAGRAMS: mermaid and other fenced diagrams are never read as syntax.
|
|
350
|
+
Describe the flow in one or two sentences (what leads to what). Say the
|
|
351
|
+
word "diagram" once so the listener knows what you summarized.
|
|
352
|
+
- CODE DUMPS: covered by SUMMARIZE above - never spell out code.
|
|
353
|
+
|
|
354
|
+
CODE-SWITCHING (Vietnamese matrix language with English technical terms):
|
|
355
|
+
- Keep technical terms, commands, identifiers, and proper nouns in English.
|
|
356
|
+
Never translate them into Vietnamese.
|
|
357
|
+
- When a sentence mixes Vietnamese and English, prefer keeping the English
|
|
358
|
+
term at the end or beginning of its own clause so it can be spoken with
|
|
359
|
+
the right voice. Do not embed English terms mid-Vietnamese-phrase.
|
|
360
|
+
- Expand an acronym on first use only when it aids understanding, then use
|
|
361
|
+
the term as-is. Never spell acronyms letter by letter unprompted.
|
|
362
|
+
- Do not narrate markdown table pipes or diagram arrows - see above.
|
|
363
|
+
|
|
192
364
|
Output ONLY the spoken text. Nothing else. No mode labels. No commentary.`;
|
|
193
365
|
|
|
194
366
|
const SYSTEM_MANUAL = `You are a text-to-speech reader for a coding assistant. The user has explicitly requested this text be read aloud. Read the prose content faithfully and in detail.
|
|
@@ -203,6 +375,13 @@ Rules:
|
|
|
203
375
|
- Error codes: expand naturally (ECONNREFUSED -> "connection refused")
|
|
204
376
|
- Shell commands: read them naturally (npm test -> "npm test")
|
|
205
377
|
- List items: read each item
|
|
378
|
+
- Tables: read compactly, not cell by cell - headers once, then each row as a
|
|
379
|
+
short phrase (max 5 rows), e.g. "Table with columns Name, Status. build,
|
|
380
|
+
pass. test, fail."
|
|
381
|
+
- Diagrams (mermaid): summarize the flow in one or two sentences, never read
|
|
382
|
+
the syntax. Say the word "diagram" once.
|
|
383
|
+
- Code-switched terms: keep English technical terms, commands, and identifiers
|
|
384
|
+
in English. Never translate them.
|
|
206
385
|
- Remove markdown formatting but preserve all the informational content
|
|
207
386
|
- Do NOT summarize. Do NOT say "check the screen". Read everything that is prose.
|
|
208
387
|
- Output ONLY the spoken text`;
|
|
@@ -255,6 +434,17 @@ export async function getTurnAssistantText(client, api) {
|
|
|
255
434
|
// ---- Public API for TUI plugin ----
|
|
256
435
|
|
|
257
436
|
export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {}) {
|
|
437
|
+
// Seed engine from opts if the host delivered it (same pattern as sttMode).
|
|
438
|
+
try {
|
|
439
|
+
if (opts?.ttsEngine && kv.get("tts.engine") == null) {
|
|
440
|
+
kv.set("tts.engine", opts.ttsEngine);
|
|
441
|
+
}
|
|
442
|
+
} catch {}
|
|
443
|
+
logger?.log?.(
|
|
444
|
+
"TTS",
|
|
445
|
+
`Engine=${kv.get("tts.engine", "piper")} opts.engine=${opts?.ttsEngine ?? "none"}`,
|
|
446
|
+
"debug",
|
|
447
|
+
);
|
|
258
448
|
if (deps.isConversationActive || deps.onConversationStopKey) {
|
|
259
449
|
__setConversationTtsHooks({
|
|
260
450
|
isActive: deps.isConversationActive,
|
|
@@ -317,124 +507,247 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
317
507
|
}
|
|
318
508
|
|
|
319
509
|
// ---- Audio pipeline ----
|
|
510
|
+
// Measured media controller (lib/tts-player.js): every utterance is
|
|
511
|
+
// synthesized to a complete buffered asset first, its total measured from
|
|
512
|
+
// real audio frames, then played through ONE owned `play` process. Totals
|
|
513
|
+
// are exact; the cursor is wall-clock derived and explicitly approximate.
|
|
514
|
+
// Pause/resume use SIGSTOP/SIGCONT (no restart), speed uses sox `tempo`,
|
|
515
|
+
// seeks respawn with `trim`. Stale completions are fenced twice: speechGen
|
|
516
|
+
// here for synthesis, the player's generation token for playback.
|
|
320
517
|
|
|
321
518
|
let piperProc = null;
|
|
322
|
-
|
|
323
|
-
//
|
|
324
|
-
|
|
519
|
+
// A Piper utterance is "speaking" from the moment synthesis starts, even
|
|
520
|
+
// when synthesis is injected (tests) and owns no OS process handle.
|
|
521
|
+
let piperSynthPending = false;
|
|
522
|
+
// Invalidated by stopSpeech (and by a preempting speak): a cancel landing
|
|
523
|
+
// between LLM normalization/synthesis and playback must prevent the stale
|
|
524
|
+
// speech from starting afterwards.
|
|
325
525
|
let speechGen = 0;
|
|
326
526
|
|
|
327
|
-
function
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
} catch {}
|
|
332
|
-
piperProc = null;
|
|
333
|
-
}
|
|
334
|
-
if (playProc) {
|
|
335
|
-
try {
|
|
336
|
-
playProc.kill("SIGKILL");
|
|
337
|
-
} catch {}
|
|
338
|
-
playProc = null;
|
|
339
|
-
}
|
|
527
|
+
function currentRate() {
|
|
528
|
+
const raw = Number(kv.get("tts.rate", 1));
|
|
529
|
+
if (!Number.isFinite(raw)) return 1;
|
|
530
|
+
return Math.min(2, Math.max(0.75, raw));
|
|
340
531
|
}
|
|
341
532
|
|
|
342
|
-
//
|
|
343
|
-
//
|
|
344
|
-
//
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
533
|
+
// Real process adapters for the player core. `feed` hands the buffered WAV
|
|
534
|
+
// to the owned play process; stderr is tapped so failed launches stay
|
|
535
|
+
// visible in the log exactly like the previous inline pipeline.
|
|
536
|
+
function realPlayerHooks() {
|
|
537
|
+
return {
|
|
538
|
+
spawn(argv) {
|
|
539
|
+
let child = null;
|
|
540
|
+
let closeCb = null;
|
|
541
|
+
const handle = {
|
|
542
|
+
pid: -1,
|
|
543
|
+
stdin: null,
|
|
544
|
+
onClose: (cb) => {
|
|
545
|
+
closeCb = cb;
|
|
546
|
+
},
|
|
547
|
+
};
|
|
548
|
+
const finish = (code) => {
|
|
549
|
+
try {
|
|
550
|
+
closeCb?.(code);
|
|
551
|
+
} catch {}
|
|
552
|
+
};
|
|
553
|
+
try {
|
|
554
|
+
child = spawn("play", argv, { stdio: ["pipe", "ignore", "pipe"] });
|
|
555
|
+
} catch (err) {
|
|
556
|
+
logger?.log?.("TTS", `play error: ${err.message}`, "error");
|
|
557
|
+
sttLog("tts.play", `spawn throw: ${err.message}`);
|
|
558
|
+
queueMicrotask(() => finish(-1));
|
|
559
|
+
return handle;
|
|
560
|
+
}
|
|
561
|
+
handle.pid = child.pid;
|
|
562
|
+
handle.stdin = child.stdin;
|
|
563
|
+
let playStderr = "";
|
|
564
|
+
try {
|
|
565
|
+
child.stderr.on("data", (chunk) => {
|
|
566
|
+
playStderr += chunk.toString();
|
|
567
|
+
});
|
|
568
|
+
} catch {}
|
|
569
|
+
// A failed launch emits 'error' and may never emit 'close': settle
|
|
570
|
+
// on both so playback never hangs. The player's generation token
|
|
571
|
+
// makes a second settle a harmless no-op.
|
|
572
|
+
child.on("error", (err) => {
|
|
573
|
+
logger?.log?.("TTS", `play error: ${err.message}`, "error");
|
|
574
|
+
sttLog("tts.play", `spawn error event: ${err.message}`);
|
|
575
|
+
finish(-1);
|
|
576
|
+
});
|
|
577
|
+
child.on("close", (code) => {
|
|
578
|
+
if (code !== 0 && code !== null) {
|
|
579
|
+
logger?.log?.("TTS", `play exited code=${code} stderr=${playStderr.trim()}`, "error");
|
|
580
|
+
} else {
|
|
581
|
+
logger?.log?.("TTS", "playback finished", "debug");
|
|
582
|
+
}
|
|
583
|
+
sttLog("tts.play", `close code=${code} stderr=${playStderr.trim().slice(-160)}`);
|
|
584
|
+
finish(code);
|
|
585
|
+
});
|
|
586
|
+
return handle;
|
|
587
|
+
},
|
|
588
|
+
signal: (pid, sig) => {
|
|
589
|
+
try {
|
|
590
|
+
process.kill(pid, sig);
|
|
591
|
+
} catch {}
|
|
592
|
+
},
|
|
593
|
+
now: () => Date.now(),
|
|
594
|
+
feed: (handle, { wav }) => {
|
|
595
|
+
try {
|
|
596
|
+
if (!handle.stdin || handle.stdin.destroyed) {
|
|
597
|
+
sttLog("tts.play", "feed skipped: stdin missing/destroyed");
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
sttLog("tts.play", `feed bytes=${wav?.length ?? 0}`);
|
|
601
|
+
handle.stdin.end(wav);
|
|
602
|
+
} catch (err) {
|
|
603
|
+
sttLog("tts.play", `feed throw: ${err?.message || err}`);
|
|
604
|
+
}
|
|
605
|
+
},
|
|
606
|
+
};
|
|
376
607
|
}
|
|
377
608
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
piperProc = spawn("piper", ["-m", voiceModel, "--output_raw"], {
|
|
400
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
401
|
-
});
|
|
402
|
-
|
|
403
|
-
piperProc.stderr.on("data", (chunk) => {
|
|
404
|
-
piperStderr += chunk.toString();
|
|
405
|
-
});
|
|
609
|
+
const player = createTtsPlayer(deps.playerHooks ?? realPlayerHooks());
|
|
610
|
+
|
|
611
|
+
// Utterance-level progress across language groups: the player holds one
|
|
612
|
+
// group asset at a time, so completed-group seconds accumulate here. The
|
|
613
|
+
// media snapshot drives api.ttsStatus (v2 slot) and stays null when idle
|
|
614
|
+
// so v1 toast progress is untouched. Manual transport ops (replay/forward/
|
|
615
|
+
// rewind) collapse back to the current single asset.
|
|
616
|
+
let utterDoneSec = 0;
|
|
617
|
+
let utterTotalSec = 0;
|
|
618
|
+
let utterActive = false;
|
|
619
|
+
// True from speak() until the first measured total exists. The slot shows
|
|
620
|
+
// a "Loading voice" line during sidecar startup and long synthesis so a
|
|
621
|
+
// slow engine never looks dead.
|
|
622
|
+
let utterLoading = false;
|
|
623
|
+
|
|
624
|
+
function utterReset() {
|
|
625
|
+
utterDoneSec = 0;
|
|
626
|
+
utterTotalSec = 0;
|
|
627
|
+
utterActive = false;
|
|
628
|
+
utterLoading = false;
|
|
629
|
+
}
|
|
406
630
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
631
|
+
function utteranceState() {
|
|
632
|
+
const s = player.getState();
|
|
633
|
+
if (!utterActive || !(utterTotalSec > 0)) return s;
|
|
634
|
+
const live = s.status === "playing" || s.status === "paused" ? s.cursorSec : 0;
|
|
635
|
+
return {
|
|
636
|
+
...s,
|
|
637
|
+
status: s.status === "idle" ? "playing" : s.status,
|
|
638
|
+
cursorSec: Math.min(utterDoneSec + live, utterTotalSec),
|
|
639
|
+
totalSec: utterTotalSec,
|
|
640
|
+
};
|
|
641
|
+
}
|
|
412
642
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
}
|
|
420
|
-
});
|
|
643
|
+
function collapseUtteranceToPlayer() {
|
|
644
|
+
const s = player.getState();
|
|
645
|
+
utterDoneSec = 0;
|
|
646
|
+
utterTotalSec = s.totalSec;
|
|
647
|
+
utterActive = s.totalSec > 0;
|
|
648
|
+
}
|
|
421
649
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
650
|
+
const ttsStatus = api.ttsStatus || null;
|
|
651
|
+
if (api.setTtsStatusReader) {
|
|
652
|
+
try {
|
|
653
|
+
api.setTtsStatusReader(() => {
|
|
654
|
+
if (utterLoading)
|
|
655
|
+
return { status: "loading", cursorSec: 0, totalSec: 0, rate: currentRate() };
|
|
656
|
+
if (!utterActive || !(utterTotalSec > 0)) return null;
|
|
657
|
+
const s = utteranceState();
|
|
658
|
+
return { status: s.status, cursorSec: s.cursorSec, totalSec: s.totalSec, rate: s.rate };
|
|
659
|
+
});
|
|
660
|
+
} catch {}
|
|
661
|
+
}
|
|
662
|
+
if (api.setTtsStatusActions) {
|
|
663
|
+
try {
|
|
664
|
+
api.setTtsStatusActions({
|
|
665
|
+
toggle: () => {
|
|
666
|
+
const s = player.getState();
|
|
667
|
+
if (s.status === "playing") controller.pause();
|
|
668
|
+
else if (s.status === "paused") controller.resume();
|
|
669
|
+
},
|
|
670
|
+
rewind: () => controller.rewind(SEEK_STEP_SEC),
|
|
671
|
+
forward: () => controller.forward(SEEK_STEP_SEC),
|
|
672
|
+
rate: () => controller.setRate(nextTtsRate(currentRate())),
|
|
673
|
+
});
|
|
674
|
+
} catch {}
|
|
675
|
+
}
|
|
676
|
+
function statusBegin() {
|
|
677
|
+
try {
|
|
678
|
+
sttLog("tts.status", "begin");
|
|
679
|
+
ttsStatus?.begin?.();
|
|
680
|
+
} catch {}
|
|
681
|
+
}
|
|
682
|
+
function statusUpdate() {
|
|
683
|
+
try {
|
|
684
|
+
ttsStatus?.update?.();
|
|
685
|
+
} catch {}
|
|
686
|
+
}
|
|
687
|
+
function statusClear() {
|
|
688
|
+
try {
|
|
689
|
+
sttLog("tts.status", "clear");
|
|
690
|
+
ttsStatus?.clear?.();
|
|
691
|
+
} catch {}
|
|
427
692
|
}
|
|
428
693
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
694
|
+
function killProcs() {
|
|
695
|
+
if (piperProc) {
|
|
696
|
+
try {
|
|
697
|
+
piperProc.kill("SIGKILL");
|
|
698
|
+
} catch {}
|
|
699
|
+
piperProc = null;
|
|
435
700
|
}
|
|
701
|
+
player.stop();
|
|
436
702
|
}
|
|
437
703
|
|
|
704
|
+
// Buffered Piper synthesis: run piper to completion, collect raw PCM, and
|
|
705
|
+
// hand the bytes back. Playback starts only after measurement, so the
|
|
706
|
+
// total is exact before the first sample sounds. Returns null on failure.
|
|
707
|
+
// Test seam: deps.synthPiperToPcm replaces the real piper spawn.
|
|
708
|
+
const synthPiperToPcm =
|
|
709
|
+
deps.synthPiperToPcm ??
|
|
710
|
+
((voiceModel, line) =>
|
|
711
|
+
new Promise((resolve) => {
|
|
712
|
+
let chunks = [];
|
|
713
|
+
let piperStderr = "";
|
|
714
|
+
let child = null;
|
|
715
|
+
try {
|
|
716
|
+
child = spawn("piper", ["-m", voiceModel, "--output_raw"], {
|
|
717
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
718
|
+
});
|
|
719
|
+
} catch (err) {
|
|
720
|
+
logger?.log?.("TTS", `piper error: ${err.message}`, "error");
|
|
721
|
+
resolve(null);
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
piperProc = child;
|
|
725
|
+
child.stdout.on("data", (chunk) => chunks.push(chunk));
|
|
726
|
+
child.stderr.on("data", (chunk) => {
|
|
727
|
+
piperStderr += chunk.toString();
|
|
728
|
+
});
|
|
729
|
+
child.on("close", (code) => {
|
|
730
|
+
if (child === piperProc) piperProc = null;
|
|
731
|
+
if (code !== 0 && code !== null) {
|
|
732
|
+
logger?.log?.("TTS", `piper exited code=${code} stderr=${piperStderr.trim()}`, "error");
|
|
733
|
+
resolve(null);
|
|
734
|
+
} else {
|
|
735
|
+
resolve(Buffer.concat(chunks));
|
|
736
|
+
}
|
|
737
|
+
});
|
|
738
|
+
child.on("error", (err) => {
|
|
739
|
+
logger?.log?.("TTS", `piper error: ${err.message}`, "error");
|
|
740
|
+
if (child === piperProc) piperProc = null;
|
|
741
|
+
resolve(null);
|
|
742
|
+
});
|
|
743
|
+
try {
|
|
744
|
+
child.stdin.write(line + "\n");
|
|
745
|
+
child.stdin.end();
|
|
746
|
+
} catch {
|
|
747
|
+
resolve(null);
|
|
748
|
+
}
|
|
749
|
+
}));
|
|
750
|
+
|
|
438
751
|
function resolveVoiceOrWarn(line) {
|
|
439
752
|
const voiceModel = getVoiceModel(line);
|
|
440
753
|
if (!piperOnPath()) {
|
|
@@ -526,81 +839,271 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
526
839
|
return chatterboxStarting;
|
|
527
840
|
}
|
|
528
841
|
|
|
842
|
+
// ---- Optional VieNeu-TTS engine (lib/vieneu-server.js) ----
|
|
843
|
+
// Bilingual Vietnamese-English with code-switching. Unlike Chatterbox
|
|
844
|
+
// (which cannot speak vi) and Piper (which needs separate voices), VieNeu
|
|
845
|
+
// handles mixed-language text in a single pass. Runs as a Python sidecar
|
|
846
|
+
// serving an OpenAI-compatible API at /v1/audio/speech.
|
|
847
|
+
const createVieneu = deps.createVieneuClient ?? createVieneuClient;
|
|
848
|
+
let vieneuClient = null;
|
|
849
|
+
let vieneuStarting = null;
|
|
850
|
+
let vieneuToastShown = false;
|
|
851
|
+
|
|
852
|
+
function vieneuWarn(message) {
|
|
853
|
+
logger?.log?.("TTS", message, "warn");
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
function noteVieneuFailure(message) {
|
|
857
|
+
logger?.log?.("TTS", message, "warn");
|
|
858
|
+
if (vieneuToastShown) return;
|
|
859
|
+
vieneuToastShown = true;
|
|
860
|
+
toast(`${message} - using Piper`);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
function noteVieneuOk() {
|
|
864
|
+
vieneuToastShown = false;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
function getVieneuClient() {
|
|
868
|
+
if (!vieneuClient) {
|
|
869
|
+
vieneuClient = createVieneu({
|
|
870
|
+
python: opts?.ttsVieneuPython,
|
|
871
|
+
voice: opts?.ttsVieneuVoice,
|
|
872
|
+
logger,
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
return vieneuClient;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
async function ensureVieneuReady() {
|
|
879
|
+
if (vieneuClient?.isRunning()) return true;
|
|
880
|
+
if (!vieneuStarting) {
|
|
881
|
+
vieneuStarting = getVieneuClient()
|
|
882
|
+
.start()
|
|
883
|
+
.then((ok) => {
|
|
884
|
+
if (ok) {
|
|
885
|
+
noteVieneuOk();
|
|
886
|
+
return true;
|
|
887
|
+
}
|
|
888
|
+
const err = vieneuClient.getLastError?.();
|
|
889
|
+
noteVieneuFailure(
|
|
890
|
+
`VieNeu-TTS unavailable (${err?.code || "START_FAILED"}: ${err?.message || "unknown"})`,
|
|
891
|
+
);
|
|
892
|
+
return false;
|
|
893
|
+
})
|
|
894
|
+
.catch((err) => {
|
|
895
|
+
noteVieneuFailure(`VieNeu-TTS unavailable (${err.message})`);
|
|
896
|
+
return false;
|
|
897
|
+
})
|
|
898
|
+
.finally(() => {
|
|
899
|
+
vieneuStarting = null;
|
|
900
|
+
});
|
|
901
|
+
}
|
|
902
|
+
return vieneuStarting;
|
|
903
|
+
}
|
|
904
|
+
|
|
529
905
|
function speak(text) {
|
|
530
|
-
const myGen = speechGen;
|
|
531
906
|
if (!text) return Promise.resolve();
|
|
532
907
|
const line = text.replace(/\n/g, " ").trim();
|
|
533
908
|
if (!line) return Promise.resolve();
|
|
534
909
|
|
|
910
|
+
// Preempt: a new utterance invalidates any in-flight synthesis so stale
|
|
911
|
+
// audio can never reach the speaker after this call.
|
|
912
|
+
const takingOver =
|
|
913
|
+
player.getState().status !== "idle" ||
|
|
914
|
+
piperProc !== null ||
|
|
915
|
+
piperSynthPending ||
|
|
916
|
+
synthPending;
|
|
917
|
+
speechGen += 1;
|
|
918
|
+
const myGen = speechGen;
|
|
535
919
|
killProcs();
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
logger?.log?.("TTS", `Speak requested chars=${line.length} voice=${voiceModel}`, "debug");
|
|
554
|
-
return new Promise((resolve) => {
|
|
555
|
-
spawnPlayback(voiceModel, resolve);
|
|
556
|
-
if (piperProc?.stdin && !piperProc.stdin.destroyed) {
|
|
557
|
-
piperProc.stdin.write(line + "\n");
|
|
558
|
-
piperProc.stdin.end();
|
|
559
|
-
}
|
|
560
|
-
});
|
|
920
|
+
utterReset();
|
|
921
|
+
if (takingOver) statusClear();
|
|
922
|
+
|
|
923
|
+
// VieNeu-TTS handles bilingual code-switching in one pass, so the
|
|
924
|
+
// whole utterance goes as a single group (no per-sentence routing).
|
|
925
|
+
const engine = kv.get("tts.engine", opts?.ttsEngine);
|
|
926
|
+
const groups = engine === "vieneu" ? [{ lang: "vi", text: line }] : groupSentencesByLang(line);
|
|
927
|
+
if (groups.length === 0) return Promise.resolve();
|
|
928
|
+
sttLog(
|
|
929
|
+
"tts.speak",
|
|
930
|
+
`chars=${line.length} groups=${groups.length} langs=${groups.map((g) => g.lang).join(",")}`,
|
|
931
|
+
);
|
|
932
|
+
// Mount the status line before the slow part (sidecar startup, long
|
|
933
|
+
// synthesis) so the utterance never looks dead while working.
|
|
934
|
+
utterLoading = true;
|
|
935
|
+
statusBegin();
|
|
936
|
+
return speakGroups(groups, myGen);
|
|
561
937
|
}
|
|
562
938
|
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
939
|
+
// Synthesize every group BEFORE any audio sounds: totals are exact up
|
|
940
|
+
// front, a stop during synthesis drops the whole utterance, and playback
|
|
941
|
+
// order always matches text order. One voice per group (vi groups get the
|
|
942
|
+
// vi voice, en groups the configured engine) - no mid-sentence switching.
|
|
943
|
+
async function speakGroups(groups, myGen) {
|
|
567
944
|
synthPending = true;
|
|
945
|
+
const voiceCache = new Map();
|
|
946
|
+
const resolveGroupVoice = (group) => {
|
|
947
|
+
if (voiceCache.has(group.lang)) return voiceCache.get(group.lang);
|
|
948
|
+
const voice = (deps.resolveVoice ?? resolveVoiceOrWarn)(group.text);
|
|
949
|
+
voiceCache.set(group.lang, voice || null);
|
|
950
|
+
return voiceCache.get(group.lang);
|
|
951
|
+
};
|
|
952
|
+
let assets = null;
|
|
568
953
|
try {
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
try {
|
|
576
|
-
result = await getChatterboxClient().synthesize(line, lang);
|
|
577
|
-
} catch (err) {
|
|
578
|
-
// The client already returns typed errors; this only guards a thrown
|
|
579
|
-
// client bug so speech still falls back instead of rejecting.
|
|
580
|
-
result = { ok: false, code: "REQUEST_FAILED", message: err.message };
|
|
954
|
+
assets = [];
|
|
955
|
+
for (const group of groups) {
|
|
956
|
+
if (myGen !== speechGen) return;
|
|
957
|
+
const asset = await synthGroupAsset(group, resolveGroupVoice, myGen);
|
|
958
|
+
if (myGen !== speechGen) return;
|
|
959
|
+
if (asset) assets.push(asset);
|
|
581
960
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
961
|
+
} finally {
|
|
962
|
+
synthPending = false;
|
|
963
|
+
}
|
|
964
|
+
if (myGen !== speechGen) {
|
|
965
|
+
sttLog("tts.synth", "dropped: preempted before playback");
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
if (!assets || assets.length === 0) {
|
|
969
|
+
sttLog("tts.synth", "dropped: no speakable assets");
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
utterLoading = false;
|
|
973
|
+
utterTotalSec = assets.reduce((sum, a) => sum + a.totalSec, 0);
|
|
974
|
+
utterDoneSec = 0;
|
|
975
|
+
utterActive = true;
|
|
976
|
+
sttLog("tts.play", `groups=${assets.length} totalSec=${utterTotalSec.toFixed(1)}`);
|
|
977
|
+
try {
|
|
978
|
+
for (const asset of assets) {
|
|
979
|
+
if (myGen !== speechGen) {
|
|
980
|
+
sttLog("tts.play", "dropped: preempted mid-chain");
|
|
981
|
+
return;
|
|
982
|
+
}
|
|
983
|
+
if (!player.load(asset.loadArg)) {
|
|
984
|
+
logger?.log?.("TTS", "Skipping unloadable group asset", "debug");
|
|
985
|
+
sttLog("tts.play", "skipped: unloadable group asset");
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
statusUpdate();
|
|
989
|
+
await player.play();
|
|
990
|
+
if (myGen !== speechGen) return;
|
|
991
|
+
utterDoneSec = Math.min(utterDoneSec + asset.totalSec, utterTotalSec);
|
|
992
|
+
sttLog("tts.play", `group done cursor=${utterDoneSec.toFixed(1)}`);
|
|
993
|
+
statusUpdate();
|
|
589
994
|
}
|
|
995
|
+
} finally {
|
|
996
|
+
utterReset();
|
|
997
|
+
statusClear();
|
|
998
|
+
}
|
|
999
|
+
}
|
|
590
1000
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
1001
|
+
// Buffer one language group to a measured asset. Chatterbox failure falls
|
|
1002
|
+
// back to Piper for THIS group only; other groups are unaffected. Returns
|
|
1003
|
+
// null when stopped, when no voice exists, or when nothing is speakable -
|
|
1004
|
+
// callers skip null groups instead of going silent.
|
|
1005
|
+
async function synthGroupAsset(group, resolveGroupVoice, myGen) {
|
|
1006
|
+
const effectiveOpts = { ...opts, ttsEngine: kv.get("tts.engine", opts?.ttsEngine) };
|
|
1007
|
+
const route = resolveTtsEngine(effectiveOpts, group.lang, chatterboxWarn);
|
|
1008
|
+
sttLog("tts.synth", `lang=${group.lang} engine=${route.engine} chars=${group.text.length}`);
|
|
1009
|
+
if (route.engine === "chatterbox") {
|
|
1010
|
+
const ready = await ensureChatterboxReady();
|
|
1011
|
+
if (myGen !== speechGen) return null;
|
|
1012
|
+
if (ready) {
|
|
1013
|
+
let result;
|
|
1014
|
+
try {
|
|
1015
|
+
result = await getChatterboxClient().synthesize(group.text, group.lang);
|
|
1016
|
+
} catch (err) {
|
|
1017
|
+
result = { ok: false, code: "REQUEST_FAILED", message: err.message };
|
|
1018
|
+
}
|
|
1019
|
+
if (myGen !== speechGen) return null;
|
|
1020
|
+
if (result?.ok) {
|
|
1021
|
+
const totalSec = measureWavDurationSec(result.wav);
|
|
1022
|
+
sttLog("tts.synth", `chatterbox ok=${!!result?.ok} wavBytes=${result?.wav?.length ?? 0}`);
|
|
1023
|
+
if (totalSec !== null && totalSec > 0) {
|
|
1024
|
+
noteChatterboxOk();
|
|
1025
|
+
logger?.log?.(
|
|
1026
|
+
"TTS",
|
|
1027
|
+
`Speak via Chatterbox chars=${group.text.length} lang=${group.lang} wavBytes=${result.wav.length}`,
|
|
1028
|
+
"debug",
|
|
1029
|
+
);
|
|
1030
|
+
return {
|
|
1031
|
+
totalSec,
|
|
1032
|
+
loadArg: { wav: result.wav, rate: currentRate() },
|
|
1033
|
+
};
|
|
1034
|
+
}
|
|
1035
|
+
noteChatterboxFailure(`Chatterbox synthesis failed (BAD_WAV: unreadable audio)`);
|
|
1036
|
+
} else {
|
|
1037
|
+
noteChatterboxFailure(
|
|
1038
|
+
`Chatterbox synthesis failed (${result?.code}: ${result?.message})`,
|
|
1039
|
+
);
|
|
1040
|
+
}
|
|
1041
|
+
// Fall through to Piper for this group.
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
if (route.engine === "vieneu") {
|
|
1045
|
+
const ready = await ensureVieneuReady();
|
|
1046
|
+
if (myGen !== speechGen) return null;
|
|
1047
|
+
if (ready) {
|
|
1048
|
+
let result;
|
|
1049
|
+
try {
|
|
1050
|
+
result = await getVieneuClient().synthesize(group.text);
|
|
1051
|
+
} catch (err) {
|
|
1052
|
+
result = { ok: false, code: "REQUEST_FAILED", message: err.message };
|
|
1053
|
+
}
|
|
1054
|
+
if (myGen !== speechGen) return null;
|
|
1055
|
+
if (result?.ok) {
|
|
1056
|
+
const totalSec = measureWavDurationSec(result.wav);
|
|
1057
|
+
sttLog("tts.synth", `vieneu ok=${!!result?.ok} wavBytes=${result?.wav?.length ?? 0}`);
|
|
1058
|
+
if (totalSec !== null && totalSec > 0) {
|
|
1059
|
+
noteVieneuOk();
|
|
1060
|
+
return {
|
|
1061
|
+
totalSec,
|
|
1062
|
+
loadArg: { wav: result.wav, rate: currentRate() },
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
noteVieneuFailure(`VieNeu-TTS synthesis failed (BAD_WAV: unreadable audio)`);
|
|
1066
|
+
} else {
|
|
1067
|
+
noteVieneuFailure(`VieNeu-TTS synthesis failed (${result?.code}: ${result?.message})`);
|
|
1068
|
+
}
|
|
1069
|
+
// Fall through to Piper for this group.
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
// Piper path (default, vi, and Chatterbox fallback): buffer the whole
|
|
1073
|
+
// group, measure the exact total from real PCM frames, then play.
|
|
1074
|
+
// Test seam: resolveVoice replaces the PATH/model-filesystem probe so
|
|
1075
|
+
// buffering is exercisable without installed voices.
|
|
1076
|
+
const voiceModel = resolveGroupVoice(group);
|
|
1077
|
+
sttLog("tts.synth", `piper voice=${voiceModel ? "resolved" : "MISSING"}`);
|
|
1078
|
+
if (!voiceModel) return null;
|
|
1079
|
+
if (myGen !== speechGen) return null;
|
|
1080
|
+
logger?.log?.("TTS", `Speak requested chars=${group.text.length} voice=${voiceModel}`, "debug");
|
|
1081
|
+
piperSynthPending = true;
|
|
1082
|
+
let pcm;
|
|
1083
|
+
try {
|
|
1084
|
+
pcm = await synthPiperToPcm(voiceModel, group.text);
|
|
600
1085
|
} finally {
|
|
601
|
-
|
|
602
|
-
synthPending = false;
|
|
1086
|
+
piperSynthPending = false;
|
|
603
1087
|
}
|
|
1088
|
+
// Stopped (or preempted) during synthesis: the bytes are stale, drop them.
|
|
1089
|
+
if (myGen !== speechGen) return null;
|
|
1090
|
+
if (!pcm || pcm.length === 0) {
|
|
1091
|
+
sttLog("tts.synth", "piper empty audio");
|
|
1092
|
+
return null;
|
|
1093
|
+
}
|
|
1094
|
+
const rate = getVoiceRate(voiceModel);
|
|
1095
|
+
const totalSec = measurePcmDurationSec({ bytes: pcm }, rate, PIPER_CHANNELS, PIPER_BITS);
|
|
1096
|
+
if (!(totalSec > 0)) return null;
|
|
1097
|
+
return {
|
|
1098
|
+
totalSec,
|
|
1099
|
+
loadArg: {
|
|
1100
|
+
pcm,
|
|
1101
|
+
sampleRate: rate,
|
|
1102
|
+
channels: PIPER_CHANNELS,
|
|
1103
|
+
bits: PIPER_BITS,
|
|
1104
|
+
rate: currentRate(),
|
|
1105
|
+
},
|
|
1106
|
+
};
|
|
604
1107
|
}
|
|
605
1108
|
|
|
606
1109
|
// ---- Session-prefixed announcements ----
|
|
@@ -617,11 +1120,19 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
617
1120
|
function stopSpeech() {
|
|
618
1121
|
// Always invalidate: a pending normalize handoff must not speak after
|
|
619
1122
|
// the cancel even when nothing is currently playing.
|
|
620
|
-
speechGen += 1;
|
|
621
|
-
//
|
|
622
|
-
//
|
|
623
|
-
const wasPlaying =
|
|
1123
|
+
speechGen += 1; // A waiting Chatterbox synthesis counts as playing, so /tts-stop and the
|
|
1124
|
+
// global cancel kill it exactly like Piper playback. Paused playback
|
|
1125
|
+
// counts too: stopping it is a real state change.
|
|
1126
|
+
const wasPlaying =
|
|
1127
|
+
player.getState().status !== "idle" ||
|
|
1128
|
+
piperProc !== null ||
|
|
1129
|
+
piperSynthPending ||
|
|
1130
|
+
synthPending;
|
|
1131
|
+
sttLog("tts.stop", `wasPlaying=${wasPlaying}`);
|
|
624
1132
|
synthPending = false;
|
|
1133
|
+
piperSynthPending = false;
|
|
1134
|
+
utterReset();
|
|
1135
|
+
statusClear();
|
|
625
1136
|
killProcs();
|
|
626
1137
|
return wasPlaying;
|
|
627
1138
|
}
|
|
@@ -806,8 +1317,27 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
806
1317
|
|
|
807
1318
|
const DEFAULT_KEYBINDS = {
|
|
808
1319
|
"tts.speak-last": "<leader>]",
|
|
809
|
-
|
|
1320
|
+
// No default bind for tts.stop: the global voice-cancel (<leader>.)
|
|
1321
|
+
// already stops TTS playback, and pause (<leader>p) freezes it. The
|
|
1322
|
+
// command stays available via palette/slash for explicit use or
|
|
1323
|
+
// custom rebinding through the `keybinds` option.
|
|
1324
|
+
"tts.pause": "<leader>p",
|
|
1325
|
+
"tts.rewind": "<leader>-",
|
|
1326
|
+
"tts.forward": "<leader>=",
|
|
1327
|
+
"tts.replay": "<leader>0",
|
|
810
1328
|
};
|
|
1329
|
+
// Media-style seek step for the rewind/forward commands.
|
|
1330
|
+
const SEEK_STEP_SEC = 5;
|
|
1331
|
+
|
|
1332
|
+
function fmtClock(totalSec) {
|
|
1333
|
+
const s = Math.max(0, Math.floor(Number(totalSec) || 0));
|
|
1334
|
+
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
function progressLabel() {
|
|
1338
|
+
const state = utteranceState();
|
|
1339
|
+
return `${fmtClock(state.cursorSec)} / ${fmtClock(state.totalSec)}${state.rate !== 1 ? ` @${state.rate}x` : ""}`;
|
|
1340
|
+
}
|
|
811
1341
|
function kb(value) {
|
|
812
1342
|
const kb = opts?.keybinds;
|
|
813
1343
|
if (!kb || typeof kb !== "object" || Array.isArray(kb)) return DEFAULT_KEYBINDS[value];
|
|
@@ -821,9 +1351,51 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
821
1351
|
speak: (text) => speak(text),
|
|
822
1352
|
speakAssistantTurn,
|
|
823
1353
|
stop: () => stopSpeech(),
|
|
824
|
-
// A Chatterbox utterance is "speaking" from the moment it is handed to
|
|
825
|
-
// sidecar, not only once audio reaches sox.
|
|
826
|
-
|
|
1354
|
+
// A Chatterbox utterance is "speaking" from the moment it is handed to
|
|
1355
|
+
// the sidecar, not only once audio reaches sox. A Piper utterance is
|
|
1356
|
+
// "speaking" from synthesis start (piperProc alive) for the same reason.
|
|
1357
|
+
isSpeaking: () =>
|
|
1358
|
+
player.getState().status !== "idle" ||
|
|
1359
|
+
piperProc !== null ||
|
|
1360
|
+
piperSynthPending ||
|
|
1361
|
+
synthPending,
|
|
1362
|
+
pause: () => {
|
|
1363
|
+
const ok = player.pause();
|
|
1364
|
+
if (ok) statusUpdate();
|
|
1365
|
+
return ok;
|
|
1366
|
+
},
|
|
1367
|
+
resume: () => {
|
|
1368
|
+
const ok = player.resume();
|
|
1369
|
+
if (ok) statusUpdate();
|
|
1370
|
+
return ok;
|
|
1371
|
+
},
|
|
1372
|
+
replay: () => {
|
|
1373
|
+
const ok = player.replay();
|
|
1374
|
+
if (ok) {
|
|
1375
|
+
collapseUtteranceToPlayer();
|
|
1376
|
+
statusUpdate();
|
|
1377
|
+
}
|
|
1378
|
+
return ok;
|
|
1379
|
+
},
|
|
1380
|
+
forward: (sec = SEEK_STEP_SEC) => {
|
|
1381
|
+
const at = player.forward(sec);
|
|
1382
|
+
collapseUtteranceToPlayer();
|
|
1383
|
+
statusUpdate();
|
|
1384
|
+
return at;
|
|
1385
|
+
},
|
|
1386
|
+
rewind: (sec = SEEK_STEP_SEC) => {
|
|
1387
|
+
const at = player.rewind(sec);
|
|
1388
|
+
collapseUtteranceToPlayer();
|
|
1389
|
+
statusUpdate();
|
|
1390
|
+
return at;
|
|
1391
|
+
},
|
|
1392
|
+
setRate: (rate) => {
|
|
1393
|
+
const applied = player.setRate(rate);
|
|
1394
|
+
kv.set("tts.rate", applied);
|
|
1395
|
+
statusUpdate();
|
|
1396
|
+
return applied;
|
|
1397
|
+
},
|
|
1398
|
+
getState: () => player.getState(),
|
|
827
1399
|
setConversationActive: (v) => {
|
|
828
1400
|
conversationActive = !!v;
|
|
829
1401
|
},
|
|
@@ -850,6 +1422,8 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
850
1422
|
} catch {}
|
|
851
1423
|
chatterboxClient?.stop();
|
|
852
1424
|
chatterboxClient = null;
|
|
1425
|
+
vieneuClient?.stop();
|
|
1426
|
+
vieneuClient = null;
|
|
853
1427
|
});
|
|
854
1428
|
|
|
855
1429
|
// ---- Commands ----
|
|
@@ -905,6 +1479,85 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
905
1479
|
if (stopSpeech()) toast("TTS stopped");
|
|
906
1480
|
},
|
|
907
1481
|
},
|
|
1482
|
+
{
|
|
1483
|
+
title: "TTS: pause / resume playback",
|
|
1484
|
+
value: "tts.pause",
|
|
1485
|
+
category: "opencode-voice",
|
|
1486
|
+
description: "Pause speaking mid-speech, resume when paused",
|
|
1487
|
+
...(kb("tts.pause") ? { keybind: kb("tts.pause") } : {}),
|
|
1488
|
+
slash: { name: "tts-pause" },
|
|
1489
|
+
onSelect() {
|
|
1490
|
+
const state = player.getState();
|
|
1491
|
+
if (state.status === "playing" && player.pause()) toast("Paused");
|
|
1492
|
+
else if (state.status === "paused" && player.resume()) toast("Resumed");
|
|
1493
|
+
else toast("Nothing playing", "warning");
|
|
1494
|
+
},
|
|
1495
|
+
},
|
|
1496
|
+
{
|
|
1497
|
+
title: "TTS: replay from start",
|
|
1498
|
+
value: "tts.replay",
|
|
1499
|
+
category: "opencode-voice",
|
|
1500
|
+
description: "Restart the current utterance from zero",
|
|
1501
|
+
...(kb("tts.replay") ? { keybind: kb("tts.replay") } : {}),
|
|
1502
|
+
slash: { name: "tts-replay" },
|
|
1503
|
+
onSelect() {
|
|
1504
|
+
if (player.replay()) toast(`Replaying from start (${progressLabel()})`);
|
|
1505
|
+
else toast("Nothing to replay", "warning");
|
|
1506
|
+
},
|
|
1507
|
+
},
|
|
1508
|
+
{
|
|
1509
|
+
title: "TTS: rewind 5 seconds",
|
|
1510
|
+
value: "tts.rewind",
|
|
1511
|
+
category: "opencode-voice",
|
|
1512
|
+
description: "Jump back 5 seconds in the current utterance",
|
|
1513
|
+
...(kb("tts.rewind") ? { keybind: kb("tts.rewind") } : {}),
|
|
1514
|
+
slash: { name: "tts-rewind" },
|
|
1515
|
+
onSelect() {
|
|
1516
|
+
if (player.getState().exactTotal) {
|
|
1517
|
+
player.rewind(SEEK_STEP_SEC);
|
|
1518
|
+
toast(`Rewound (${progressLabel()})`);
|
|
1519
|
+
} else toast("Nothing to rewind", "warning");
|
|
1520
|
+
},
|
|
1521
|
+
},
|
|
1522
|
+
{
|
|
1523
|
+
title: "TTS: forward 5 seconds",
|
|
1524
|
+
value: "tts.forward",
|
|
1525
|
+
category: "opencode-voice",
|
|
1526
|
+
description: "Jump ahead 5 seconds in the current utterance",
|
|
1527
|
+
...(kb("tts.forward") ? { keybind: kb("tts.forward") } : {}),
|
|
1528
|
+
slash: { name: "tts-forward" },
|
|
1529
|
+
onSelect() {
|
|
1530
|
+
if (player.getState().exactTotal) {
|
|
1531
|
+
player.forward(SEEK_STEP_SEC);
|
|
1532
|
+
toast(`Forwarded (${progressLabel()})`);
|
|
1533
|
+
} else toast("Nothing to skip", "warning");
|
|
1534
|
+
},
|
|
1535
|
+
},
|
|
1536
|
+
{
|
|
1537
|
+
title: "TTS: playback speed",
|
|
1538
|
+
value: "tts.rate",
|
|
1539
|
+
category: "opencode-voice",
|
|
1540
|
+
description: "Choose playback speed 0.75x-2x (persisted)",
|
|
1541
|
+
slash: { name: "tts-rate" },
|
|
1542
|
+
onSelect() {
|
|
1543
|
+
const current = String(currentRate());
|
|
1544
|
+
api.ui.dialog.replace(() =>
|
|
1545
|
+
api.ui.DialogSelect({
|
|
1546
|
+
title: "Select playback speed",
|
|
1547
|
+
current,
|
|
1548
|
+
options: ["0.75", "1", "1.25", "1.5", "1.75", "2"].map((label) => ({
|
|
1549
|
+
title: `${label}x`,
|
|
1550
|
+
value: label,
|
|
1551
|
+
onSelect() {
|
|
1552
|
+
const applied = controller.setRate(Number(label));
|
|
1553
|
+
toast(`Playback speed: ${applied}x`);
|
|
1554
|
+
api.ui.dialog.clear();
|
|
1555
|
+
},
|
|
1556
|
+
})),
|
|
1557
|
+
}),
|
|
1558
|
+
);
|
|
1559
|
+
},
|
|
1560
|
+
},
|
|
908
1561
|
{
|
|
909
1562
|
title: "TTS: select voice",
|
|
910
1563
|
value: "tts.voice",
|
|
@@ -931,6 +1584,36 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
931
1584
|
);
|
|
932
1585
|
},
|
|
933
1586
|
},
|
|
1587
|
+
{
|
|
1588
|
+
title: "TTS: select engine",
|
|
1589
|
+
value: "tts.engine",
|
|
1590
|
+
category: "opencode-voice",
|
|
1591
|
+
description: "Choose TTS engine (piper, chatterbox, vieneu) - persisted",
|
|
1592
|
+
slash: { name: "tts-engine" },
|
|
1593
|
+
onSelect() {
|
|
1594
|
+
const current = kv.get("tts.engine", opts?.ttsEngine || "piper");
|
|
1595
|
+
api.ui.dialog.replace(() =>
|
|
1596
|
+
api.ui.DialogSelect({
|
|
1597
|
+
title: "Select TTS engine",
|
|
1598
|
+
current,
|
|
1599
|
+
options: [
|
|
1600
|
+
{ title: "Piper (default, local, vi+en separate voices)", value: "piper" },
|
|
1601
|
+
{ title: "Chatterbox (English, voice cloning)", value: "chatterbox" },
|
|
1602
|
+
{ title: "VieNeu-TTS (bilingual vi+en code-switching)", value: "vieneu" },
|
|
1603
|
+
].map((o) => ({
|
|
1604
|
+
title: o.title,
|
|
1605
|
+
value: o.value,
|
|
1606
|
+
onSelect() {
|
|
1607
|
+
kv.set("tts.engine", o.value);
|
|
1608
|
+
sttLog("tts.engine.set", o.value);
|
|
1609
|
+
toast(`TTS engine: ${o.value}`);
|
|
1610
|
+
api.ui.dialog.clear();
|
|
1611
|
+
},
|
|
1612
|
+
})),
|
|
1613
|
+
}),
|
|
1614
|
+
);
|
|
1615
|
+
},
|
|
1616
|
+
},
|
|
934
1617
|
];
|
|
935
1618
|
|
|
936
1619
|
return { commands, controller };
|