@bojackduy/opencode-voice 0.11.2 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -12,6 +12,8 @@ spoken aloud via Piper TTS. Both directions use an LLM to normalize text for
12
12
  natural speech (fixing homophones, splitting camelCase identifiers, summarizing
13
13
  code-heavy responses, etc.).
14
14
 
15
+ ![Demo: browsing voice commands in OpenCode](assets/demo.gif)
16
+
15
17
  ## OpenCode v1 and v2
16
18
 
17
19
  One package runs on both OpenCode 1.x and 2.x. The plugin detects the host
@@ -96,7 +98,8 @@ capability detection already picks the right path per host.
96
98
  in that dialog instead of being sent as a prompt.
97
99
  - **STT (streaming dictation):** enable with `"sttMode": "streaming"` in plugin
98
100
  options. Local whisper streaming displays a bounded live transcript above
99
- the prompt footer on home and session screens. Stable words are plain;
101
+ the prompt footer on home and session screens, with the live input waveform
102
+ beside it. Stable words are plain;
100
103
  the tentative tail is bracketed and may change. Press the record key again
101
104
  to finalize: the full text is inserted once at the focused cursor (and
102
105
  submitted when `sttAutoSubmit` is set), or cancel to clear the preview
@@ -552,14 +555,15 @@ If a path is not set, the built-in default prompt is used.
552
555
 
553
556
  ### Speech-to-text
554
557
 
555
- | Command | Keybind | Description |
556
- | --------------- | ---------- | -------------------------------------- |
557
- | `/stt-record` | `ctrl+r` | Start/stop recording + transcribe |
558
- | `/stt-submit` | `leader+r` | Stop recording, transcribe, and submit |
559
- | `/stt-stop` | | Cancel recording |
560
- | `/stt-model` | | Select whisper model |
561
- | `/stt-language` | | Select transcription language |
562
- | `/stt-mic` | | Select microphone |
558
+ | Command | Keybind | Description |
559
+ | --------------- | ---------- | -------------------------------------------------------------------------- |
560
+ | `/stt-record` | `ctrl+r` | Start/stop recording + transcribe |
561
+ | `/stt-submit` | `leader+r` | Stop recording, transcribe, and submit |
562
+ | `/stt-stop` | | Cancel recording |
563
+ | `/stt-pause` | | Pause/resume live dictation (streaming only; mic frozen, nothing recorded) |
564
+ | `/stt-model` | | Select whisper model |
565
+ | `/stt-language` | | Select transcription language |
566
+ | `/stt-mic` | | Select microphone |
563
567
 
564
568
  `/stt-mic` lists CoreAudio input devices on macOS, and PulseAudio sources on
565
569
  Linux (via `pactl`, monitor sources excluded). On systems without a supported
@@ -573,18 +577,27 @@ In `streaming` mode (`sttMode: "streaming"`), `/stt-record` toggles
573
577
  start/finalize of a live dictation session instead of one-shot recording
574
578
  (see "Streaming dictation" below); `/stt-submit` finalizes and submits via
575
579
  the dictated field only; `/stt-stop` cancels and removes only dictated text.
580
+ `/stt-pause` (or the ⏸ button in the live preview) freezes the mic
581
+ mid-dictation so noise never enters the transcript; pressing record while
582
+ paused resumes instead of finalizing.
576
583
 
577
584
  ### Text-to-speech
578
585
 
579
586
  The `leader` key in OpenCode is `ctrl+x`. So `leader+s` means press `ctrl+x`
580
587
  then `s`.
581
588
 
582
- | Command | Keybind | Description |
583
- | ------------ | ---------- | ------------------------ |
584
- | `/tts-speak` | `leader+s` | Read last response aloud |
585
- | `/tts-mode` | | Toggle auto TTS on/off |
586
- | `/tts-stop` | `escape` | Stop playback |
587
- | `/tts-voice` | | Select TTS voice |
589
+ | Command | Keybind | Description |
590
+ | -------------- | ---------- | ----------------------------------------------------- |
591
+ | `/tts-speak` | `leader+s` | Read last response aloud |
592
+ | `/tts-mode` | | Toggle auto TTS on/off |
593
+ | `/tts-stop` | | Stop playback (palette; global `leader+.` also stops) |
594
+ | `/tts-voice` | | Select TTS voice |
595
+ | `/tts-engine` | | Select TTS engine (piper / chatterbox / vieneu) |
596
+ | `/tts-pause` | | Pause/resume TTS playback (mid-speech, no restart) |
597
+ | `/tts-replay` | `leader+0` | Replay from start |
598
+ | `/tts-rewind` | `leader+-` | Rewind 5 seconds |
599
+ | `/tts-forward` | `leader+=` | Forward 5 seconds |
600
+ | `/tts-rate` | | Cycle/select playback speed 0.75x-2x (persisted) |
588
601
 
589
602
  ### Voice conversation
590
603
 
@@ -597,7 +610,7 @@ One key drives the whole loop. It always means the same thing - "I want the
597
610
  floor" - and what it does follows the toast on screen:
598
611
 
599
612
  ```
600
- press -> record -> press -> transcribe -> normalize -> submit -> speak -> press ...
613
+ press -> record -> press -> transcribe -> normalize -> submit -> speak -> auto-listen ...
601
614
  ```
602
615
 
603
616
  | Toast shows | Pressing the key does |
@@ -613,6 +626,10 @@ paused, so it cannot record the room while nobody is talking. Barge-in drops
613
626
  the pending reply on purpose - pressing means talking now, not hearing the
614
627
  rest.
615
628
 
629
+ **Hands-free mode:** In streaming mode, recording auto-finalizes after 1.5s of
630
+ silence (no keypress needed). After the reply is spoken, the mic re-opens
631
+ automatically after a short delay. Press the key to barge in at any time.
632
+
616
633
  Answering a permission or a question is the one thing the key cannot do,
617
634
  because that answer is given on screen. The mode announces the gate, keeps the
618
635
  mic shut, then speaks the agent's answer once the agent resumes.
@@ -767,9 +784,10 @@ there is never a silent gap — `Loading speech model…` during cold start,
767
784
  `● Streaming dictation — listening` with an elapsed timer while live
768
785
  (plus a `(listening…)` note when inference runs slow), and `Finalizing…`
769
786
  between the second keypress and finalization. On v2, a bounded transcript
770
- above the prompt footer updates during recording: stable words are plain,
771
- tentative words are bracketed. Finalization inserts the full text once at
772
- the focused cursor; cancel clears the preview without sending.
787
+ above the prompt footer updates during recording, with the live input
788
+ waveform beside it: stable words are plain, tentative words are bracketed.
789
+ Finalization inserts the full text once at the focused cursor; cancel clears
790
+ the preview without sending.
773
791
  The OpenCode renderer only
774
792
  exposes `insertText`/`submit`, with no live range-replacement API, so
775
793
  partials render as preview toasts (`🎙 ...`) alongside (not instead of)
@@ -868,6 +886,46 @@ When enabled (`/tts-mode`), the plugin automatically speaks:
868
886
  - Permission requests
869
887
  - Questions that need your answer
870
888
 
889
+ ## Demo
890
+
891
+ The GIF above was recorded with `asciinema` + `agg` driving the OpenCode TUI
892
+ via `tmux` (playback sped up 2x, idle capped at 0.5s). To regenerate it:
893
+
894
+ ```bash
895
+ scripts/record-fullflow.sh assets/demo.cast /path/to/project
896
+ agg assets/demo.cast assets/demo.gif --speed 2 --idle-time-limit 0.5
897
+ ```
898
+
899
+ Requires `tmux`, `asciinema`, and `agg` on PATH.
900
+
901
+ ### Manual full-voice demo (real audio + waveform)
902
+
903
+ For a demo with real microphone audio and the recording waveform, use the
904
+ full-stack recorder (terminal + mic → MP4 with audio):
905
+
906
+ ```bash
907
+ scripts/record-with-audio.sh demo /path/to/project
908
+ ```
909
+
910
+ Speak during the `stt record` step; the script muxes your voice into the
911
+ final MP4. Requires `tmux`, `asciinema`, `agg`, `sox`, and `ffmpeg`.
912
+
913
+ Or record by hand:
914
+
915
+ 1. `asciinema rec -c 'opencode' assets/voice.cast` (in your project dir).
916
+ 2. Open command palette (`Ctrl+P`) and run, in order, with ~3–5s pauses:
917
+ - `tts engine` → pick piper / chatterbox / vieneu
918
+ - `tts voice` → pick a voice
919
+ - `stt mic` → pick your mic
920
+ - `tts rate` → pick a speed
921
+ - `stt record` → **speak** (waveform shows) → `stt submit`
922
+ - wait for the agent reply to finish, then `tts speak`
923
+ - `voice conversation` → speak → `voice conversation stop`
924
+ 3. `Ctrl+C` to end, then `agg assets/voice.cast assets/voice.gif --speed 2 --idle-time-limit 0.5`.
925
+
926
+ The provided `scripts/demo-voice.exp` automates the keystrokes (no audio);
927
+ run it inside an asciinema session if you only need a UI walkthrough.
928
+
871
929
  ## Contributing
872
930
 
873
931
  opencode-voice is open to contributions and ideas!
package/index.js CHANGED
@@ -21,13 +21,20 @@
21
21
  // /stt-record - start/stop recording + transcribe (default: <leader>[ = ctrl+x, [)
22
22
  // /stt-submit - stop recording + transcribe + submit (palette-only)
23
23
  // /stt-stop - cancel recording (palette-only)
24
+ // /stt-pause - pause/resume live dictation, mic frozen (palette-only)
24
25
  // /stt-model - select whisper model
25
26
  // /stt-language - select transcription language
26
27
  // /stt-mic - select microphone
27
28
  // /tts-speak - read last response aloud (default: <leader>] = ctrl+x, ])
28
29
  // /tts-mode - toggle auto TTS on/off (palette-only)
29
- // /tts-stop - stop playback (default: <leader>; , also palette)
30
+ // /tts-stop - stop playback (palette-only; global <leader>. also stops TTS)
30
31
  // /tts-voice - select TTS voice
32
+ // /tts-engine - select TTS engine (piper/chatterbox/vieneu) (palette-only)
33
+ // /tts-pause - pause/resume TTS playback (palette-only)
34
+ // /tts-replay - replay from start (default: <leader>0)
35
+ // /tts-rewind - rewind 5s (default: <leader>-)
36
+ // /tts-forward - forward 5s (default: <leader>=)
37
+ // /tts-rate - cycle/select playback speed 0.75x-2x (palette-only)
31
38
  // /voice-conversation - toggle hands-free voice conversation (default: <leader>v = ctrl+x, v)
32
39
  // /voice-conversation-stop - exit voice conversation mode (palette-only)
33
40
  // /voice-notes-start - start continuous live-notes recording (palette-only)
@@ -26,6 +26,7 @@ import fs from "node:fs";
26
26
  import os from "node:os";
27
27
  import { fileURLToPath } from "node:url";
28
28
  import { spawn } from "node:child_process";
29
+ import { vieneuSupportsLang } from "./vieneu-server.js";
29
30
 
30
31
  const DEFAULT_HOST = "127.0.0.1";
31
32
  const DEFAULT_PORT = 8120;
@@ -100,8 +101,15 @@ export function chatterboxSupportsLang(variant, lang) {
100
101
  export function resolveTtsEngine(opts, lang, warn) {
101
102
  const configured = opts?.ttsEngine;
102
103
  if (!configured || configured === "piper") return { engine: "piper", reason: "default" };
104
+ if (configured === "vieneu") {
105
+ if (vieneuSupportsLang(lang)) return { engine: "vieneu", reason: "ok" };
106
+ warn?.(`VieNeu-TTS cannot speak "${lang}" - using Piper for this utterance`);
107
+ return { engine: "piper", reason: "unsupported-language" };
108
+ }
103
109
  if (configured !== "chatterbox") {
104
- warn?.(`Unknown ttsEngine "${configured}" - expected "piper" or "chatterbox"; using Piper`);
110
+ warn?.(
111
+ `Unknown ttsEngine "${configured}" - expected "piper", "chatterbox", or "vieneu"; using Piper`,
112
+ );
105
113
  return { engine: "piper", reason: "unknown-engine" };
106
114
  }
107
115
  const requested = opts?.ttsChatterboxVariant;
@@ -0,0 +1,269 @@
1
+ // Persistent conversation status line for v2 (mirrors lib/tts-status.js).
2
+ //
3
+ // Why a slot and not a toast: v2 Toast.show is fire-and-forget text with no
4
+ // update or buttons, so polling would stack. This is one bounded line showing
5
+ // the current phase and the 2-3 actions that matter right now.
6
+ //
7
+ // Phase-aware buttons (see README): recording -> cancel/send-now; processing ->
8
+ // cancel; waiting -> cancel/barge-in; speaking -> stop/talk-now/pause-TTS;
9
+ // paused -> talk/end. Only the relevant subset renders per phase.
10
+
11
+ export function phaseGlyph(phase) {
12
+ switch (phase) {
13
+ case "recording":
14
+ return "●";
15
+ case "processing":
16
+ return "…";
17
+ case "waiting":
18
+ return "…";
19
+ case "speaking":
20
+ return "▶";
21
+ case "paused":
22
+ return "⏸";
23
+ default:
24
+ return "○";
25
+ }
26
+ }
27
+
28
+ export function phaseLabel(phase) {
29
+ switch (phase) {
30
+ case "recording":
31
+ return "Listening";
32
+ case "processing":
33
+ return "Transcribing";
34
+ case "waiting":
35
+ return "Agent working";
36
+ case "speaking":
37
+ return "Speaking";
38
+ case "paused":
39
+ return "Paused";
40
+ default:
41
+ return "Conversation off";
42
+ }
43
+ }
44
+
45
+ // Returns the segments for the current phase. Static segments carry no
46
+ // action; button segments do. The renderer maps each to its own text node
47
+ // so mouse hits land on exactly one action.
48
+ export function conversationSegments(phase) {
49
+ const glyph = phaseGlyph(phase);
50
+ const label = phaseLabel(phase);
51
+ const base = [{ key: "status", text: `${glyph} ${label}`, action: null }];
52
+ switch (phase) {
53
+ case "recording":
54
+ return [
55
+ ...base,
56
+ { key: "cancel", text: "[cancel]", action: "cancel" },
57
+ { key: "send", text: "[send now]", action: "send" },
58
+ ];
59
+ case "processing":
60
+ return [...base, { key: "cancel", text: "[cancel]", action: "cancel" }];
61
+ case "waiting":
62
+ return [
63
+ ...base,
64
+ { key: "cancel", text: "[cancel]", action: "cancel" },
65
+ { key: "barge", text: "[talk now]", action: "barge" },
66
+ ];
67
+ case "speaking":
68
+ return [
69
+ ...base,
70
+ { key: "stop", text: "[stop]", action: "stop" },
71
+ { key: "talk", text: "[talk now]", action: "barge" },
72
+ { key: "pause", text: "[pause]", action: "pauseTts" },
73
+ ];
74
+ case "paused":
75
+ return [
76
+ ...base,
77
+ { key: "talk", text: "[talk]", action: "talk" },
78
+ { key: "end", text: "[end]", action: "end" },
79
+ ];
80
+ default:
81
+ return base;
82
+ }
83
+ }
84
+
85
+ export function formatConversationStatus(phase) {
86
+ if (!phase || phase === "idle") return "";
87
+ return conversationSegments(phase)
88
+ .map((seg) => seg.text)
89
+ .join(" ");
90
+ }
91
+
92
+ export function createConversationStatus(
93
+ ctx,
94
+ readPhase,
95
+ actions = {},
96
+ load = async () => {
97
+ const [solid, tui] = await Promise.all([import("solid-js"), import("@opentui/solid")]);
98
+ return { ...solid, ...tui };
99
+ },
100
+ ) {
101
+ let generation = 0;
102
+ let active = false;
103
+ let lastTexts = new Map();
104
+ let renderErrorShown = false;
105
+ let renderCalls = 0;
106
+ let timer = null;
107
+ const nodes = new Set();
108
+ let setContent = null;
109
+
110
+ function segments() {
111
+ return conversationSegments(readPhase?.() || "idle");
112
+ }
113
+
114
+ function redraw() {
115
+ const segMap = new Map(segments().map((seg) => [seg.key, seg.text]));
116
+ let changed = false;
117
+ for (const [key, text] of segMap) {
118
+ if (lastTexts.get(key) !== text) {
119
+ changed = true;
120
+ break;
121
+ }
122
+ }
123
+ if (!changed) {
124
+ for (const entry of nodes) {
125
+ if (entry.node.isDestroyed) nodes.delete(entry);
126
+ }
127
+ return;
128
+ }
129
+ for (const [key, text] of segMap) {
130
+ lastTexts.set(key, text);
131
+ for (const entry of nodes) {
132
+ if (entry.key !== key || entry.node.isDestroyed) continue;
133
+ setContent?.(entry.node, text);
134
+ }
135
+ }
136
+ }
137
+
138
+ function loudRenderError(err) {
139
+ if (renderErrorShown) return;
140
+ renderErrorShown = true;
141
+ try {
142
+ ctx.ui?.toast?.show?.({
143
+ message: `Conversation status failed: ${err?.message || err}`,
144
+ variant: "error",
145
+ duration: 5000,
146
+ });
147
+ } catch {}
148
+ }
149
+
150
+ function stopTimer() {
151
+ if (timer) {
152
+ clearInterval(timer);
153
+ timer = null;
154
+ }
155
+ }
156
+
157
+ let unclaims = [];
158
+ function clear() {
159
+ generation += 1;
160
+ active = false;
161
+ lastTexts = new Map();
162
+ renderCalls = 0;
163
+ stopTimer();
164
+ nodes.clear();
165
+ setContent = null;
166
+ for (const unclaim of unclaims.splice(0)) unclaim();
167
+ }
168
+
169
+ function press(action) {
170
+ if (!active) return;
171
+ try {
172
+ actions?.[action]?.();
173
+ } catch {}
174
+ }
175
+
176
+ return {
177
+ async begin() {
178
+ clear();
179
+ active = true;
180
+ renderErrorShown = false;
181
+ const mine = generation;
182
+ let createElement;
183
+ let setProp;
184
+ let insertNode;
185
+ let onCleanup;
186
+ try {
187
+ ({ createElement, setProp, insertNode, onCleanup } = await load());
188
+ } catch (loadErr) {
189
+ loudRenderError(loadErr);
190
+ throw loadErr;
191
+ }
192
+ if (!active || mine !== generation) return false;
193
+ setContent = (node, value) => setProp(node, "content", value);
194
+ const themeText = ctx.theme?.text;
195
+ const fg =
196
+ typeof themeText === "string"
197
+ ? themeText
198
+ : themeText?.base || themeText?.muted || "#eeeeee";
199
+ const render = () => {
200
+ if (!active || mine !== generation) return null;
201
+ renderCalls += 1;
202
+ try {
203
+ const box = createElement("box");
204
+ setProp(box, "height", 1);
205
+ setProp(box, "flexShrink", 0);
206
+ setProp(box, "flexDirection", "row");
207
+ setProp(box, "width", "100%");
208
+ const segMap = new Map(segments().map((seg) => [seg.key, seg.text]));
209
+ const mountEntries = [];
210
+ for (const key of ["status", "cancel", "send", "barge", "stop", "talk", "end", "pause"]) {
211
+ const node = createElement("text");
212
+ setProp(node, "maxHeight", 1);
213
+ setProp(node, "minHeight", 1);
214
+ setProp(node, "fg", fg);
215
+ const text = segMap.has(key) ? segMap.get(key) : "";
216
+ setProp(node, "content", text);
217
+ lastTexts.set(key, text);
218
+ const action = segMap.has(key)
219
+ ? (segments().find((s) => s.key === key)?.action ?? null)
220
+ : null;
221
+ if (action) {
222
+ try {
223
+ setProp(node, "onMouseUp", () => press(action));
224
+ } catch {}
225
+ }
226
+ const entry = { key, node };
227
+ nodes.add(entry);
228
+ mountEntries.push(entry);
229
+ insertNode(box, node);
230
+ }
231
+ onCleanup?.(() => {
232
+ for (const entry of mountEntries) nodes.delete(entry);
233
+ });
234
+ redraw();
235
+ return box;
236
+ } catch (renderErr) {
237
+ loudRenderError(renderErr);
238
+ return null;
239
+ }
240
+ };
241
+ unclaims = [
242
+ ctx.ui.slot({ before: "session.composer.top", render }),
243
+ ctx.ui.slot({ before: "home.footer", render }),
244
+ ];
245
+ stopTimer();
246
+ timer = setInterval(() => {
247
+ if (!active) return;
248
+ try {
249
+ redraw();
250
+ } catch {}
251
+ }, 500);
252
+ if (typeof timer.unref === "function") timer.unref();
253
+ return true;
254
+ },
255
+ update() {
256
+ if (!active) return;
257
+ redraw();
258
+ },
259
+ renderCalls: () => renderCalls,
260
+ nodes: () => {
261
+ const keys = new Set();
262
+ for (const entry of nodes) {
263
+ if (!entry.node.isDestroyed) keys.add(entry.key);
264
+ }
265
+ return keys.size;
266
+ },
267
+ clear,
268
+ };
269
+ }