@bojackduy/opencode-voice 0.11.1 → 0.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/tts.js CHANGED
@@ -7,6 +7,9 @@ import { spawn } from "node:child_process";
7
7
  import { getSessionTitle } from "./session.js";
8
8
  import { clearProcessingToast, showProcessingToast, updateProcessingToast } from "./stt.js";
9
9
  import { createChatterboxClient, resolveTtsEngine } from "./chatterbox-server.js";
10
+ import { createTtsPlayer, measurePcmDurationSec, measureWavDurationSec } from "./tts-player.js";
11
+ import { nextRate as nextTtsRate } from "./tts-status.js";
12
+ import { sttLog } from "./stt-log.js";
10
13
 
11
14
  const VOICES_DIR = path.join(os.homedir(), ".local", "share", "piper-voices");
12
15
 
@@ -24,7 +27,72 @@ const PIPER_RATE = 22050;
24
27
  const PIPER_BITS = 16;
25
28
  const PIPER_CHANNELS = 1;
26
29
 
30
+ // ---- Structured content: tables and diagrams are not speakable literally ----
31
+ // A literal read ("pipe name pipe status pipe...") is noise, so both the
32
+ // local cleanup and the LLM narrator summarize instead: what the table is
33
+ // about, then the few rows that matter. Pure functions, tested in tts.test.js.
34
+
35
+ const TABLE_ROW_CAP = 3;
36
+ const TABLE_TEXT_CAP = 320;
37
+ const MERMAID_LABEL_CAP = 6;
38
+
39
+ function splitTableCells(row) {
40
+ return row
41
+ .trim()
42
+ .replace(/^\||\|$/g, "")
43
+ .split("|")
44
+ .map((cell) => cell.trim());
45
+ }
46
+
47
+ function isTableSeparator(line) {
48
+ const cells = splitTableCells(line);
49
+ return cells.length > 0 && cells.every((cell) => /^:?-+:?$/.test(cell));
50
+ }
51
+
52
+ export function summarizeMarkdownTable(block) {
53
+ const lines = (block || "")
54
+ .split("\n")
55
+ .map((line) => line.trim())
56
+ .filter((line) => line.startsWith("|"));
57
+ const headerIdx = lines.findIndex((line) => !isTableSeparator(line));
58
+ if (headerIdx === -1) return "Table.";
59
+ const headers = splitTableCells(lines[headerIdx]).filter(Boolean);
60
+ const rows = lines
61
+ .slice(headerIdx + 1)
62
+ .filter((line) => !isTableSeparator(line))
63
+ .map((line) => splitTableCells(line).filter(Boolean))
64
+ .filter((cells) => cells.length > 0);
65
+ if (headers.length === 0 && rows.length === 0) return "Table.";
66
+ const shown = rows.slice(0, TABLE_ROW_CAP).map((cells) => cells.join(", "));
67
+ let out = headers.length > 0 ? `Table with columns ${headers.join(", ")}.` : "Table.";
68
+ if (rows.length > 0) {
69
+ out += ` ${rows.length} row${rows.length === 1 ? "" : "s"}: ${shown.join("; ")}.`;
70
+ if (rows.length > shown.length) out += ` And ${rows.length - shown.length} more.`;
71
+ }
72
+ return out.length > TABLE_TEXT_CAP ? `${out.slice(0, TABLE_TEXT_CAP - 1)}…` : out;
73
+ }
74
+
75
+ export function summarizeMermaidForSpeech(code) {
76
+ const text = code || "";
77
+ const firstLine =
78
+ text
79
+ .split("\n")
80
+ .map((line) => line.trim())
81
+ .find(Boolean) || "";
82
+ const kindMatch = /^(graph|flowchart)\b/i.exec(firstLine);
83
+ const kind = kindMatch ? kindMatch[1].toLowerCase() : "";
84
+ const labels = [...text.matchAll(/\[([^\[\]]+)\]/g)]
85
+ .map((m) => m[1].trim())
86
+ .filter(Boolean)
87
+ .slice(0, MERMAID_LABEL_CAP);
88
+ const what = kind ? `Diagram, ${kind}` : "Diagram";
89
+ if (labels.length >= 2) return `${what}: ${labels.join(", ")}.`;
90
+ if (labels.length === 1) return `${what}: ${labels[0]}.`;
91
+ return `${what}.`;
92
+ }
93
+
27
94
  // ---- Local (no-LLM) speech cleanup ----
95
+
28
96
  // Deterministic markdown -> spoken-text transform. Zero network latency, so it
29
97
  // backs streaming speech and ttsNormalizeMode "local". Less polished than the
30
98
  // LLM narrator (no summarization), but instant.
@@ -39,6 +107,17 @@ function splitIdentifier(token) {
39
107
  export function localSpeechCleanup(text) {
40
108
  if (!text) return "";
41
109
  let out = text;
110
+ // Diagrams first: a mermaid fence would otherwise collapse into the
111
+ // generic "code snippet" below and lose the flow.
112
+ out = out.replace(
113
+ /```mermaid\n([\s\S]*?)```/gi,
114
+ (_, code) => ` ${summarizeMermaidForSpeech(code)} `,
115
+ );
116
+ // Tables next: runs of pipe lines become a short spoken summary.
117
+ out = out.replace(
118
+ /((?:^[ \t]*\|.*\|\s*$\n?){2,})/gm,
119
+ (block) => ` ${summarizeMarkdownTable(block)} `,
120
+ );
42
121
  out = out.replace(/```[\s\S]*?```/g, " code snippet ");
43
122
  out = out.replace(/`([^`]+)`/g, (_, code) => ` ${splitIdentifier(code)} `);
44
123
  out = out.replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1");
@@ -125,6 +204,27 @@ export function isSpeakableSentence(sentence) {
125
204
  return true;
126
205
  }
127
206
 
207
+ // Per-sentence language routing: Vietnamese and English need different
208
+ // engines/voices (Chatterbox cannot speak vi; Piper vi mangles English),
209
+ // so an utterance is split into sentences and consecutive same-language
210
+ // sentences are grouped. Word order is preserved exactly; grouping only
211
+ // decides which voice synthesizes which span. Pure export for tests.
212
+ export function groupSentencesByLang(text) {
213
+ const flat = (text || "").replace(/\s+/g, " ").trim();
214
+ if (!flat) return [];
215
+ const { sentences, rest } = splitSpokenSentences(flat);
216
+ const parts = [...sentences];
217
+ if (rest.trim()) parts.push(rest.trim());
218
+ const groups = [];
219
+ for (const part of parts) {
220
+ const lang = detectLang(part);
221
+ const last = groups[groups.length - 1];
222
+ if (last && last.lang === lang) last.text = `${last.text} ${part}`;
223
+ else groups.push({ lang, text: part });
224
+ }
225
+ return groups;
226
+ }
227
+
128
228
  // Decide ONE language for the whole call (utterance or, during streaming,
129
229
  // one completed sentence) and speak all of it with that single voice - no
130
230
  // mid-utterance voice switching, since splicing separately synthesized clips
@@ -189,6 +289,22 @@ You have three modes depending on the content complexity:
189
289
 
190
290
  Choose the appropriate mode based on the content. Most responses with code blocks should use SUMMARIZE mode. Simple Q&A or short explanations use NARRATE. Build results, "done", confirmations use NOTIFY.
191
291
 
292
+ STRUCTURED CONTENT RULES (apply in every mode):
293
+ - TABLES: never read a markdown table cell by cell. First understand what the
294
+ table lists, then say one short intro (the topic) plus the 2-3 most
295
+ important rows in plain sentences. Skip long tables after the gist.
296
+ - DIAGRAMS: mermaid and other fenced diagrams are never read as syntax.
297
+ Describe the flow in one or two sentences (what leads to what). Say the
298
+ word "diagram" once so the listener knows what you summarized.
299
+ - CODE DUMPS: covered by SUMMARIZE above - never spell out code.
300
+
301
+ CODE-SWITCHING (Vietnamese matrix language with English technical terms):
302
+ - Keep technical terms, commands, identifiers, and proper nouns in English.
303
+ Never translate them into Vietnamese.
304
+ - Expand an acronym on first use only when it aids understanding, then use
305
+ the term as-is. Never spell acronyms letter by letter unprompted.
306
+ - Do not narrate markdown table pipes or diagram arrows - see above.
307
+
192
308
  Output ONLY the spoken text. Nothing else. No mode labels. No commentary.`;
193
309
 
194
310
  const SYSTEM_MANUAL = `You are a text-to-speech reader for a coding assistant. The user has explicitly requested this text be read aloud. Read the prose content faithfully and in detail.
@@ -203,6 +319,13 @@ Rules:
203
319
  - Error codes: expand naturally (ECONNREFUSED -> "connection refused")
204
320
  - Shell commands: read them naturally (npm test -> "npm test")
205
321
  - List items: read each item
322
+ - Tables: read compactly, not cell by cell - headers once, then each row as a
323
+ short phrase (max 5 rows), e.g. "Table with columns Name, Status. build,
324
+ pass. test, fail."
325
+ - Diagrams (mermaid): summarize the flow in one or two sentences, never read
326
+ the syntax. Say the word "diagram" once.
327
+ - Code-switched terms: keep English technical terms, commands, and identifiers
328
+ in English. Never translate them.
206
329
  - Remove markdown formatting but preserve all the informational content
207
330
  - Do NOT summarize. Do NOT say "check the screen". Read everything that is prose.
208
331
  - Output ONLY the spoken text`;
@@ -317,124 +440,247 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
317
440
  }
318
441
 
319
442
  // ---- Audio pipeline ----
443
+ // Measured media controller (lib/tts-player.js): every utterance is
444
+ // synthesized to a complete buffered asset first, its total measured from
445
+ // real audio frames, then played through ONE owned `play` process. Totals
446
+ // are exact; the cursor is wall-clock derived and explicitly approximate.
447
+ // Pause/resume use SIGSTOP/SIGCONT (no restart), speed uses sox `tempo`,
448
+ // seeks respawn with `trim`. Stale completions are fenced twice: speechGen
449
+ // here for synthesis, the player's generation token for playback.
320
450
 
321
451
  let piperProc = null;
322
- let playProc = null;
323
- // Invalidated by stopSpeech: a cancel landing between LLM normalization
324
- // and playback must prevent the stale speech from starting afterwards.
452
+ // A Piper utterance is "speaking" from the moment synthesis starts, even
453
+ // when synthesis is injected (tests) and owns no OS process handle.
454
+ let piperSynthPending = false;
455
+ // Invalidated by stopSpeech (and by a preempting speak): a cancel landing
456
+ // between LLM normalization/synthesis and playback must prevent the stale
457
+ // speech from starting afterwards.
325
458
  let speechGen = 0;
326
459
 
327
- function killProcs() {
328
- if (piperProc) {
329
- try {
330
- piperProc.kill("SIGKILL");
331
- } catch {}
332
- piperProc = null;
333
- }
334
- if (playProc) {
335
- try {
336
- playProc.kill("SIGKILL");
337
- } catch {}
338
- playProc = null;
339
- }
460
+ function currentRate() {
461
+ const raw = Number(kv.get("tts.rate", 1));
462
+ if (!Number.isFinite(raw)) return 1;
463
+ return Math.min(2, Math.max(0.75, raw));
340
464
  }
341
465
 
342
- // Spawn the sox `play` process every engine speaks through.
343
- //
344
- // Piper streams raw PCM, so the sample format must be declared up front.
345
- // Chatterbox returns a complete WAV, which sox detects from the RIFF header -
346
- // so it passes no formatArgs and we reuse this exact process rather than
347
- // building a second audio pipeline.
348
- function startPlayProcess(formatArgs, onDone) {
349
- let playStderr = "";
350
- playProc = spawn("play", [...formatArgs, "-q", "-"], {
351
- stdio: ["pipe", "ignore", "pipe"],
352
- });
353
-
354
- playProc.stderr.on("data", (chunk) => {
355
- playStderr += chunk.toString();
356
- });
357
-
358
- playProc.on("close", (code) => {
359
- if (code !== 0 && code !== null) {
360
- logger?.log?.("TTS", `play exited code=${code} stderr=${playStderr.trim()}`, "error");
361
- } else {
362
- logger?.log?.("TTS", "playback finished", "debug");
363
- }
364
- piperProc = null;
365
- playProc = null;
366
- onDone();
367
- });
368
-
369
- playProc.on("error", (err) => {
370
- logger?.log?.("TTS", `play error: ${err.message}`, "error");
371
- killProcs();
372
- onDone();
373
- });
374
-
375
- return playProc;
466
+ // Real process adapters for the player core. `feed` hands the buffered WAV
467
+ // to the owned play process; stderr is tapped so failed launches stay
468
+ // visible in the log exactly like the previous inline pipeline.
469
+ function realPlayerHooks() {
470
+ return {
471
+ spawn(argv) {
472
+ let child = null;
473
+ let closeCb = null;
474
+ const handle = {
475
+ pid: -1,
476
+ stdin: null,
477
+ onClose: (cb) => {
478
+ closeCb = cb;
479
+ },
480
+ };
481
+ const finish = (code) => {
482
+ try {
483
+ closeCb?.(code);
484
+ } catch {}
485
+ };
486
+ try {
487
+ child = spawn("play", argv, { stdio: ["pipe", "ignore", "pipe"] });
488
+ } catch (err) {
489
+ logger?.log?.("TTS", `play error: ${err.message}`, "error");
490
+ sttLog("tts.play", `spawn throw: ${err.message}`);
491
+ queueMicrotask(() => finish(-1));
492
+ return handle;
493
+ }
494
+ handle.pid = child.pid;
495
+ handle.stdin = child.stdin;
496
+ let playStderr = "";
497
+ try {
498
+ child.stderr.on("data", (chunk) => {
499
+ playStderr += chunk.toString();
500
+ });
501
+ } catch {}
502
+ // A failed launch emits 'error' and may never emit 'close': settle
503
+ // on both so playback never hangs. The player's generation token
504
+ // makes a second settle a harmless no-op.
505
+ child.on("error", (err) => {
506
+ logger?.log?.("TTS", `play error: ${err.message}`, "error");
507
+ sttLog("tts.play", `spawn error event: ${err.message}`);
508
+ finish(-1);
509
+ });
510
+ child.on("close", (code) => {
511
+ if (code !== 0 && code !== null) {
512
+ logger?.log?.("TTS", `play exited code=${code} stderr=${playStderr.trim()}`, "error");
513
+ } else {
514
+ logger?.log?.("TTS", "playback finished", "debug");
515
+ }
516
+ sttLog("tts.play", `close code=${code} stderr=${playStderr.trim().slice(-160)}`);
517
+ finish(code);
518
+ });
519
+ return handle;
520
+ },
521
+ signal: (pid, sig) => {
522
+ try {
523
+ process.kill(pid, sig);
524
+ } catch {}
525
+ },
526
+ now: () => Date.now(),
527
+ feed: (handle, { wav }) => {
528
+ try {
529
+ if (!handle.stdin || handle.stdin.destroyed) {
530
+ sttLog("tts.play", "feed skipped: stdin missing/destroyed");
531
+ return;
532
+ }
533
+ sttLog("tts.play", `feed bytes=${wav?.length ?? 0}`);
534
+ handle.stdin.end(wav);
535
+ } catch (err) {
536
+ sttLog("tts.play", `feed throw: ${err?.message || err}`);
537
+ }
538
+ },
539
+ };
376
540
  }
377
541
 
378
- // Spawn piper -> play and wire them. Resolves onDone when playback ends.
379
-
380
- function spawnPlayback(voiceModel, onDone) {
381
- let piperStderr = "";
382
- const rate = getVoiceRate(voiceModel);
383
- startPlayProcess(
384
- [
385
- "-t",
386
- "raw",
387
- "-r",
388
- String(rate),
389
- "-e",
390
- "signed",
391
- "-b",
392
- String(PIPER_BITS),
393
- "-c",
394
- String(PIPER_CHANNELS),
395
- ],
396
- onDone,
397
- );
398
-
399
- piperProc = spawn("piper", ["-m", voiceModel, "--output_raw"], {
400
- stdio: ["pipe", "pipe", "pipe"],
401
- });
402
-
403
- piperProc.stderr.on("data", (chunk) => {
404
- piperStderr += chunk.toString();
405
- });
542
+ const player = createTtsPlayer(deps.playerHooks ?? realPlayerHooks());
543
+
544
+ // Utterance-level progress across language groups: the player holds one
545
+ // group asset at a time, so completed-group seconds accumulate here. The
546
+ // media snapshot drives api.ttsStatus (v2 slot) and stays null when idle
547
+ // so v1 toast progress is untouched. Manual transport ops (replay/forward/
548
+ // rewind) collapse back to the current single asset.
549
+ let utterDoneSec = 0;
550
+ let utterTotalSec = 0;
551
+ let utterActive = false;
552
+ // True from speak() until the first measured total exists. The slot shows
553
+ // a "Loading voice" line during sidecar startup and long synthesis so a
554
+ // slow engine never looks dead.
555
+ let utterLoading = false;
556
+
557
+ function utterReset() {
558
+ utterDoneSec = 0;
559
+ utterTotalSec = 0;
560
+ utterActive = false;
561
+ utterLoading = false;
562
+ }
406
563
 
407
- piperProc.stdout.on("data", (chunk) => {
408
- if (playProc?.stdin && !playProc.stdin.destroyed) {
409
- playProc.stdin.write(chunk);
410
- }
411
- });
564
+ function utteranceState() {
565
+ const s = player.getState();
566
+ if (!utterActive || !(utterTotalSec > 0)) return s;
567
+ const live = s.status === "playing" || s.status === "paused" ? s.cursorSec : 0;
568
+ return {
569
+ ...s,
570
+ status: s.status === "idle" ? "playing" : s.status,
571
+ cursorSec: Math.min(utterDoneSec + live, utterTotalSec),
572
+ totalSec: utterTotalSec,
573
+ };
574
+ }
412
575
 
413
- piperProc.on("close", (code) => {
414
- if (code !== 0 && code !== null) {
415
- logger?.log?.("TTS", `piper exited code=${code} stderr=${piperStderr.trim()}`, "error");
416
- }
417
- if (playProc?.stdin && !playProc.stdin.destroyed) {
418
- playProc.stdin.end();
419
- }
420
- });
576
+ function collapseUtteranceToPlayer() {
577
+ const s = player.getState();
578
+ utterDoneSec = 0;
579
+ utterTotalSec = s.totalSec;
580
+ utterActive = s.totalSec > 0;
581
+ }
421
582
 
422
- piperProc.on("error", (err) => {
423
- logger?.log?.("TTS", `piper error: ${err.message}`, "error");
424
- killProcs();
425
- onDone();
426
- });
583
+ const ttsStatus = api.ttsStatus || null;
584
+ if (api.setTtsStatusReader) {
585
+ try {
586
+ api.setTtsStatusReader(() => {
587
+ if (utterLoading)
588
+ return { status: "loading", cursorSec: 0, totalSec: 0, rate: currentRate() };
589
+ if (!utterActive || !(utterTotalSec > 0)) return null;
590
+ const s = utteranceState();
591
+ return { status: s.status, cursorSec: s.cursorSec, totalSec: s.totalSec, rate: s.rate };
592
+ });
593
+ } catch {}
594
+ }
595
+ if (api.setTtsStatusActions) {
596
+ try {
597
+ api.setTtsStatusActions({
598
+ toggle: () => {
599
+ const s = player.getState();
600
+ if (s.status === "playing") controller.pause();
601
+ else if (s.status === "paused") controller.resume();
602
+ },
603
+ rewind: () => controller.rewind(SEEK_STEP_SEC),
604
+ forward: () => controller.forward(SEEK_STEP_SEC),
605
+ rate: () => controller.setRate(nextTtsRate(currentRate())),
606
+ });
607
+ } catch {}
608
+ }
609
+ function statusBegin() {
610
+ try {
611
+ sttLog("tts.status", "begin");
612
+ ttsStatus?.begin?.();
613
+ } catch {}
614
+ }
615
+ function statusUpdate() {
616
+ try {
617
+ ttsStatus?.update?.();
618
+ } catch {}
619
+ }
620
+ function statusClear() {
621
+ try {
622
+ sttLog("tts.status", "clear");
623
+ ttsStatus?.clear?.();
624
+ } catch {}
427
625
  }
428
626
 
429
- // Chatterbox playback: the sidecar already produced whole WAV bytes, so this
430
- // is just the same play process fed the buffer and closed.
431
- function spawnWavPlayback(wavBuffer, onDone) {
432
- const play = startPlayProcess([], onDone);
433
- if (play?.stdin && !play.stdin.destroyed) {
434
- play.stdin.end(wavBuffer);
627
+ function killProcs() {
628
+ if (piperProc) {
629
+ try {
630
+ piperProc.kill("SIGKILL");
631
+ } catch {}
632
+ piperProc = null;
435
633
  }
634
+ player.stop();
436
635
  }
437
636
 
637
+ // Buffered Piper synthesis: run piper to completion, collect raw PCM, and
638
+ // hand the bytes back. Playback starts only after measurement, so the
639
+ // total is exact before the first sample sounds. Returns null on failure.
640
+ // Test seam: deps.synthPiperToPcm replaces the real piper spawn.
641
+ const synthPiperToPcm =
642
+ deps.synthPiperToPcm ??
643
+ ((voiceModel, line) =>
644
+ new Promise((resolve) => {
645
+ let chunks = [];
646
+ let piperStderr = "";
647
+ let child = null;
648
+ try {
649
+ child = spawn("piper", ["-m", voiceModel, "--output_raw"], {
650
+ stdio: ["pipe", "pipe", "pipe"],
651
+ });
652
+ } catch (err) {
653
+ logger?.log?.("TTS", `piper error: ${err.message}`, "error");
654
+ resolve(null);
655
+ return;
656
+ }
657
+ piperProc = child;
658
+ child.stdout.on("data", (chunk) => chunks.push(chunk));
659
+ child.stderr.on("data", (chunk) => {
660
+ piperStderr += chunk.toString();
661
+ });
662
+ child.on("close", (code) => {
663
+ if (child === piperProc) piperProc = null;
664
+ if (code !== 0 && code !== null) {
665
+ logger?.log?.("TTS", `piper exited code=${code} stderr=${piperStderr.trim()}`, "error");
666
+ resolve(null);
667
+ } else {
668
+ resolve(Buffer.concat(chunks));
669
+ }
670
+ });
671
+ child.on("error", (err) => {
672
+ logger?.log?.("TTS", `piper error: ${err.message}`, "error");
673
+ if (child === piperProc) piperProc = null;
674
+ resolve(null);
675
+ });
676
+ try {
677
+ child.stdin.write(line + "\n");
678
+ child.stdin.end();
679
+ } catch {
680
+ resolve(null);
681
+ }
682
+ }));
683
+
438
684
  function resolveVoiceOrWarn(line) {
439
685
  const voiceModel = getVoiceModel(line);
440
686
  if (!piperOnPath()) {
@@ -527,80 +773,177 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
527
773
  }
528
774
 
529
775
  function speak(text) {
530
- const myGen = speechGen;
531
776
  if (!text) return Promise.resolve();
532
777
  const line = text.replace(/\n/g, " ").trim();
533
778
  if (!line) return Promise.resolve();
534
779
 
780
+ // Preempt: a new utterance invalidates any in-flight synthesis so stale
781
+ // audio can never reach the speaker after this call.
782
+ const takingOver =
783
+ player.getState().status !== "idle" ||
784
+ piperProc !== null ||
785
+ piperSynthPending ||
786
+ synthPending;
787
+ speechGen += 1;
788
+ const myGen = speechGen;
535
789
  killProcs();
536
-
537
- const lang = detectLang(line);
538
- const route = resolveTtsEngine(opts, lang, chatterboxWarn);
539
- if (route.engine === "chatterbox") {
540
- return speakViaChatterbox(line, lang, route.variant, myGen);
541
- }
542
- // One voice for the whole call (see detectLang) - no mid-utterance
543
- // switching.
544
- return speakViaPiper(line, myGen);
545
- }
546
-
547
- // Piper synthesis path, shared by the default route and by the Chatterbox
548
- // fallback so both play identically.
549
- function speakViaPiper(line, myGen = speechGen) {
550
- const voiceModel = resolveVoiceOrWarn(line);
551
- if (!voiceModel) return Promise.resolve();
552
- if (myGen !== speechGen) return Promise.resolve();
553
- logger?.log?.("TTS", `Speak requested chars=${line.length} voice=${voiceModel}`, "debug");
554
- return new Promise((resolve) => {
555
- spawnPlayback(voiceModel, resolve);
556
- if (piperProc?.stdin && !piperProc.stdin.destroyed) {
557
- piperProc.stdin.write(line + "\n");
558
- piperProc.stdin.end();
559
- }
560
- });
790
+ utterReset();
791
+ if (takingOver) statusClear();
792
+
793
+ const groups = groupSentencesByLang(line);
794
+ if (groups.length === 0) return Promise.resolve();
795
+ sttLog(
796
+ "tts.speak",
797
+ `chars=${line.length} groups=${groups.length} langs=${groups.map((g) => g.lang).join(",")}`,
798
+ );
799
+ // Mount the status line before the slow part (sidecar startup, long
800
+ // synthesis) so the utterance never looks dead while working.
801
+ utterLoading = true;
802
+ statusBegin();
803
+ return speakGroups(groups, myGen);
561
804
  }
562
805
 
563
- async function speakViaChatterbox(line, lang, variant, myGen) {
564
- // Claim "speaking" for the WHOLE attempt, including the sidecar startup
565
- // wait: set before the first await, so /tts-stop and the global cancel act
566
- // on a Chatterbox utterance the instant it is requested.
806
+ // Synthesize every group BEFORE any audio sounds: totals are exact up
807
+ // front, a stop during synthesis drops the whole utterance, and playback
808
+ // order always matches text order. One voice per group (vi groups get the
809
+ // vi voice, en groups the configured engine) - no mid-sentence switching.
810
+ async function speakGroups(groups, myGen) {
567
811
  synthPending = true;
812
+ const voiceCache = new Map();
813
+ const resolveGroupVoice = (group) => {
814
+ if (voiceCache.has(group.lang)) return voiceCache.get(group.lang);
815
+ const voice = (deps.resolveVoice ?? resolveVoiceOrWarn)(group.text);
816
+ voiceCache.set(group.lang, voice || null);
817
+ return voiceCache.get(group.lang);
818
+ };
819
+ let assets = null;
568
820
  try {
569
- const ready = await ensureChatterboxReady();
570
- // A cancel that landed while the sidecar was loading must not speak.
571
- if (myGen !== speechGen) return;
572
- if (!ready) return speakViaPiper(line, myGen);
573
-
574
- let result;
575
- try {
576
- result = await getChatterboxClient().synthesize(line, lang);
577
- } catch (err) {
578
- // The client already returns typed errors; this only guards a thrown
579
- // client bug so speech still falls back instead of rejecting.
580
- result = { ok: false, code: "REQUEST_FAILED", message: err.message };
821
+ assets = [];
822
+ for (const group of groups) {
823
+ if (myGen !== speechGen) return;
824
+ const asset = await synthGroupAsset(group, resolveGroupVoice, myGen);
825
+ if (myGen !== speechGen) return;
826
+ if (asset) assets.push(asset);
581
827
  }
582
-
583
- // Same generation guard as Piper: a stop during synthesis must prevent
584
- // the audio from ever reaching sox.
585
- if (myGen !== speechGen) return;
586
- if (!result?.ok) {
587
- noteChatterboxFailure(`Chatterbox synthesis failed (${result?.code}: ${result?.message})`);
588
- return speakViaPiper(line, myGen);
828
+ } finally {
829
+ synthPending = false;
830
+ }
831
+ if (myGen !== speechGen) {
832
+ sttLog("tts.synth", "dropped: preempted before playback");
833
+ return;
834
+ }
835
+ if (!assets || assets.length === 0) {
836
+ sttLog("tts.synth", "dropped: no speakable assets");
837
+ return;
838
+ }
839
+ utterLoading = false;
840
+ utterTotalSec = assets.reduce((sum, a) => sum + a.totalSec, 0);
841
+ utterDoneSec = 0;
842
+ utterActive = true;
843
+ sttLog("tts.play", `groups=${assets.length} totalSec=${utterTotalSec.toFixed(1)}`);
844
+ try {
845
+ for (const asset of assets) {
846
+ if (myGen !== speechGen) {
847
+ sttLog("tts.play", "dropped: preempted mid-chain");
848
+ return;
849
+ }
850
+ if (!player.load(asset.loadArg)) {
851
+ logger?.log?.("TTS", "Skipping unloadable group asset", "debug");
852
+ sttLog("tts.play", "skipped: unloadable group asset");
853
+ continue;
854
+ }
855
+ statusUpdate();
856
+ await player.play();
857
+ if (myGen !== speechGen) return;
858
+ utterDoneSec = Math.min(utterDoneSec + asset.totalSec, utterTotalSec);
859
+ sttLog("tts.play", `group done cursor=${utterDoneSec.toFixed(1)}`);
860
+ statusUpdate();
589
861
  }
862
+ } finally {
863
+ utterReset();
864
+ statusClear();
865
+ }
866
+ }
590
867
 
591
- noteChatterboxOk();
592
- logger?.log?.(
593
- "TTS",
594
- `Speak via Chatterbox chars=${line.length} lang=${lang} variant=${variant} wavBytes=${result.wav.length}`,
595
- "debug",
596
- );
597
- return new Promise((resolve) => {
598
- spawnWavPlayback(result.wav, resolve);
599
- });
868
+ // Buffer one language group to a measured asset. Chatterbox failure falls
869
+ // back to Piper for THIS group only; other groups are unaffected. Returns
870
+ // null when stopped, when no voice exists, or when nothing is speakable -
871
+ // callers skip null groups instead of going silent.
872
+ async function synthGroupAsset(group, resolveGroupVoice, myGen) {
873
+ const route = resolveTtsEngine(opts, group.lang, chatterboxWarn);
874
+ sttLog("tts.synth", `lang=${group.lang} engine=${route.engine} chars=${group.text.length}`);
875
+ if (route.engine === "chatterbox") {
876
+ const ready = await ensureChatterboxReady();
877
+ if (myGen !== speechGen) return null;
878
+ if (ready) {
879
+ let result;
880
+ try {
881
+ result = await getChatterboxClient().synthesize(group.text, group.lang);
882
+ } catch (err) {
883
+ // The client already returns typed errors; this only guards a
884
+ // thrown client bug so speech still falls back instead of rejecting.
885
+ result = { ok: false, code: "REQUEST_FAILED", message: err.message };
886
+ }
887
+ if (myGen !== speechGen) return null;
888
+ if (result?.ok) {
889
+ const totalSec = measureWavDurationSec(result.wav);
890
+ sttLog("tts.synth", `chatterbox ok=${!!result?.ok} wavBytes=${result?.wav?.length ?? 0}`);
891
+ if (totalSec !== null && totalSec > 0) {
892
+ noteChatterboxOk();
893
+ logger?.log?.(
894
+ "TTS",
895
+ `Speak via Chatterbox chars=${group.text.length} lang=${group.lang} variant=${route.variant} wavBytes=${result.wav.length}`,
896
+ "debug",
897
+ );
898
+ return {
899
+ totalSec,
900
+ loadArg: { wav: result.wav, rate: currentRate() },
901
+ };
902
+ }
903
+ noteChatterboxFailure(`Chatterbox synthesis failed (BAD_WAV: unreadable audio)`);
904
+ } else {
905
+ noteChatterboxFailure(
906
+ `Chatterbox synthesis failed (${result?.code}: ${result?.message})`,
907
+ );
908
+ }
909
+ // Fall through to Piper for this group.
910
+ }
911
+ }
912
+ // Piper path (default, vi, and Chatterbox fallback): buffer the whole
913
+ // group, measure the exact total from real PCM frames, then play.
914
+ // Test seam: resolveVoice replaces the PATH/model-filesystem probe so
915
+ // buffering is exercisable without installed voices.
916
+ const voiceModel = resolveGroupVoice(group);
917
+ sttLog("tts.synth", `piper voice=${voiceModel ? "resolved" : "MISSING"}`);
918
+ if (!voiceModel) return null;
919
+ if (myGen !== speechGen) return null;
920
+ logger?.log?.("TTS", `Speak requested chars=${group.text.length} voice=${voiceModel}`, "debug");
921
+ piperSynthPending = true;
922
+ let pcm;
923
+ try {
924
+ pcm = await synthPiperToPcm(voiceModel, group.text);
600
925
  } finally {
601
- // Playback, if it started, is tracked by playProc from here on.
602
- synthPending = false;
926
+ piperSynthPending = false;
603
927
  }
928
+ // Stopped (or preempted) during synthesis: the bytes are stale, drop them.
929
+ if (myGen !== speechGen) return null;
930
+ if (!pcm || pcm.length === 0) {
931
+ sttLog("tts.synth", "piper empty audio");
932
+ return null;
933
+ }
934
+ const rate = getVoiceRate(voiceModel);
935
+ const totalSec = measurePcmDurationSec({ bytes: pcm }, rate, PIPER_CHANNELS, PIPER_BITS);
936
+ if (!(totalSec > 0)) return null;
937
+ return {
938
+ totalSec,
939
+ loadArg: {
940
+ pcm,
941
+ sampleRate: rate,
942
+ channels: PIPER_CHANNELS,
943
+ bits: PIPER_BITS,
944
+ rate: currentRate(),
945
+ },
946
+ };
604
947
  }
605
948
 
606
949
  // ---- Session-prefixed announcements ----
@@ -617,11 +960,19 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
617
960
  function stopSpeech() {
618
961
  // Always invalidate: a pending normalize handoff must not speak after
619
962
  // the cancel even when nothing is currently playing.
620
- speechGen += 1;
621
- // A waiting Chatterbox synthesis counts as playing, so /tts-stop and the
622
- // global cancel kill it exactly like Piper playback.
623
- const wasPlaying = piperProc !== null || playProc !== null || synthPending;
963
+ speechGen += 1; // A waiting Chatterbox synthesis counts as playing, so /tts-stop and the
964
+ // global cancel kill it exactly like Piper playback. Paused playback
965
+ // counts too: stopping it is a real state change.
966
+ const wasPlaying =
967
+ player.getState().status !== "idle" ||
968
+ piperProc !== null ||
969
+ piperSynthPending ||
970
+ synthPending;
971
+ sttLog("tts.stop", `wasPlaying=${wasPlaying}`);
624
972
  synthPending = false;
973
+ piperSynthPending = false;
974
+ utterReset();
975
+ statusClear();
625
976
  killProcs();
626
977
  return wasPlaying;
627
978
  }
@@ -806,8 +1157,27 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
806
1157
 
807
1158
  const DEFAULT_KEYBINDS = {
808
1159
  "tts.speak-last": "<leader>]",
809
- "tts.stop": "<leader>;",
1160
+ // No default bind for tts.stop: the global voice-cancel (<leader>.)
1161
+ // already stops TTS playback, and pause (<leader>p) freezes it. The
1162
+ // command stays available via palette/slash for explicit use or
1163
+ // custom rebinding through the `keybinds` option.
1164
+ "tts.pause": "<leader>p",
1165
+ "tts.rewind": "<leader>-",
1166
+ "tts.forward": "<leader>=",
1167
+ "tts.replay": "<leader>0",
810
1168
  };
1169
+ // Media-style seek step for the rewind/forward commands.
1170
+ const SEEK_STEP_SEC = 5;
1171
+
1172
+ function fmtClock(totalSec) {
1173
+ const s = Math.max(0, Math.floor(Number(totalSec) || 0));
1174
+ return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
1175
+ }
1176
+
1177
+ function progressLabel() {
1178
+ const state = utteranceState();
1179
+ return `${fmtClock(state.cursorSec)} / ${fmtClock(state.totalSec)}${state.rate !== 1 ? ` @${state.rate}x` : ""}`;
1180
+ }
811
1181
  function kb(value) {
812
1182
  const kb = opts?.keybinds;
813
1183
  if (!kb || typeof kb !== "object" || Array.isArray(kb)) return DEFAULT_KEYBINDS[value];
@@ -821,9 +1191,51 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
821
1191
  speak: (text) => speak(text),
822
1192
  speakAssistantTurn,
823
1193
  stop: () => stopSpeech(),
824
- // A Chatterbox utterance is "speaking" from the moment it is handed to the
825
- // sidecar, not only once audio reaches sox.
826
- isSpeaking: () => piperProc !== null || playProc !== null || synthPending,
1194
+ // A Chatterbox utterance is "speaking" from the moment it is handed to
1195
+ // the sidecar, not only once audio reaches sox. A Piper utterance is
1196
+ // "speaking" from synthesis start (piperProc alive) for the same reason.
1197
+ isSpeaking: () =>
1198
+ player.getState().status !== "idle" ||
1199
+ piperProc !== null ||
1200
+ piperSynthPending ||
1201
+ synthPending,
1202
+ pause: () => {
1203
+ const ok = player.pause();
1204
+ if (ok) statusUpdate();
1205
+ return ok;
1206
+ },
1207
+ resume: () => {
1208
+ const ok = player.resume();
1209
+ if (ok) statusUpdate();
1210
+ return ok;
1211
+ },
1212
+ replay: () => {
1213
+ const ok = player.replay();
1214
+ if (ok) {
1215
+ collapseUtteranceToPlayer();
1216
+ statusUpdate();
1217
+ }
1218
+ return ok;
1219
+ },
1220
+ forward: (sec = SEEK_STEP_SEC) => {
1221
+ const at = player.forward(sec);
1222
+ collapseUtteranceToPlayer();
1223
+ statusUpdate();
1224
+ return at;
1225
+ },
1226
+ rewind: (sec = SEEK_STEP_SEC) => {
1227
+ const at = player.rewind(sec);
1228
+ collapseUtteranceToPlayer();
1229
+ statusUpdate();
1230
+ return at;
1231
+ },
1232
+ setRate: (rate) => {
1233
+ const applied = player.setRate(rate);
1234
+ kv.set("tts.rate", applied);
1235
+ statusUpdate();
1236
+ return applied;
1237
+ },
1238
+ getState: () => player.getState(),
827
1239
  setConversationActive: (v) => {
828
1240
  conversationActive = !!v;
829
1241
  },
@@ -905,6 +1317,85 @@ export function registerTTS(api, kv, complete, prompts, opts, logger, deps = {})
905
1317
  if (stopSpeech()) toast("TTS stopped");
906
1318
  },
907
1319
  },
1320
+ {
1321
+ title: "TTS: pause / resume playback",
1322
+ value: "tts.pause",
1323
+ category: "opencode-voice",
1324
+ description: "Pause speaking mid-speech, resume when paused",
1325
+ ...(kb("tts.pause") ? { keybind: kb("tts.pause") } : {}),
1326
+ slash: { name: "tts-pause" },
1327
+ onSelect() {
1328
+ const state = player.getState();
1329
+ if (state.status === "playing" && player.pause()) toast("Paused");
1330
+ else if (state.status === "paused" && player.resume()) toast("Resumed");
1331
+ else toast("Nothing playing", "warning");
1332
+ },
1333
+ },
1334
+ {
1335
+ title: "TTS: replay from start",
1336
+ value: "tts.replay",
1337
+ category: "opencode-voice",
1338
+ description: "Restart the current utterance from zero",
1339
+ ...(kb("tts.replay") ? { keybind: kb("tts.replay") } : {}),
1340
+ slash: { name: "tts-replay" },
1341
+ onSelect() {
1342
+ if (player.replay()) toast(`Replaying from start (${progressLabel()})`);
1343
+ else toast("Nothing to replay", "warning");
1344
+ },
1345
+ },
1346
+ {
1347
+ title: "TTS: rewind 5 seconds",
1348
+ value: "tts.rewind",
1349
+ category: "opencode-voice",
1350
+ description: "Jump back 5 seconds in the current utterance",
1351
+ ...(kb("tts.rewind") ? { keybind: kb("tts.rewind") } : {}),
1352
+ slash: { name: "tts-rewind" },
1353
+ onSelect() {
1354
+ if (player.getState().exactTotal) {
1355
+ player.rewind(SEEK_STEP_SEC);
1356
+ toast(`Rewound (${progressLabel()})`);
1357
+ } else toast("Nothing to rewind", "warning");
1358
+ },
1359
+ },
1360
+ {
1361
+ title: "TTS: forward 5 seconds",
1362
+ value: "tts.forward",
1363
+ category: "opencode-voice",
1364
+ description: "Jump ahead 5 seconds in the current utterance",
1365
+ ...(kb("tts.forward") ? { keybind: kb("tts.forward") } : {}),
1366
+ slash: { name: "tts-forward" },
1367
+ onSelect() {
1368
+ if (player.getState().exactTotal) {
1369
+ player.forward(SEEK_STEP_SEC);
1370
+ toast(`Forwarded (${progressLabel()})`);
1371
+ } else toast("Nothing to skip", "warning");
1372
+ },
1373
+ },
1374
+ {
1375
+ title: "TTS: playback speed",
1376
+ value: "tts.rate",
1377
+ category: "opencode-voice",
1378
+ description: "Choose playback speed 0.75x-2x (persisted)",
1379
+ slash: { name: "tts-rate" },
1380
+ onSelect() {
1381
+ const current = String(currentRate());
1382
+ api.ui.dialog.replace(() =>
1383
+ api.ui.DialogSelect({
1384
+ title: "Select playback speed",
1385
+ current,
1386
+ options: ["0.75", "1", "1.25", "1.5", "1.75", "2"].map((label) => ({
1387
+ title: `${label}x`,
1388
+ value: label,
1389
+ onSelect() {
1390
+ const applied = controller.setRate(Number(label));
1391
+ toast(`Playback speed: ${applied}x`);
1392
+ api.ui.dialog.clear();
1393
+ },
1394
+ })),
1395
+ }),
1396
+ );
1397
+ },
1398
+ },
908
1399
  {
909
1400
  title: "TTS: select voice",
910
1401
  value: "tts.voice",