@vaibhavgupta0786/opencode-voice 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +12 -0
- package/README.md +138 -2
- package/bin/SHA256SUMS +3 -0
- package/bin/darwin-arm64/opencode-voice-recorder +0 -0
- package/bin/darwin-x64/opencode-voice-recorder +0 -0
- package/bin/linux-x64/opencode-voice-recorder +0 -0
- package/docs/STT.md +64 -0
- package/docs/TROUBLESHOOTING.md +82 -0
- package/docs/VOICE-MODE.md +42 -0
- package/logic.ts +171 -0
- package/package.json +36 -4
- package/servers/mlx_server.py +94 -0
- package/skills/voice-check/SKILL.md +68 -0
- package/tui.tsx +732 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vaibhavs18
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
NOTICE — third-party binaries in bin/
|
|
2
|
+
|
|
3
|
+
The recorders in bin/darwin-arm64, bin/darwin-x64 and bin/linux-x64 were
|
|
4
|
+
built from recorder/recorder.c of opencode-dictate
|
|
5
|
+
(https://github.com/rodri45l/opencode-dictate), which is MIT-licensed:
|
|
6
|
+
|
|
7
|
+
MIT License, Copyright (c) 2026 Rodrigo
|
|
8
|
+
|
|
9
|
+
The recorder is a thin wrapper around miniaudio (MIT-0 / public-domain
|
|
10
|
+
style licence, https://github.com/mackron/miniaudio) talking directly to
|
|
11
|
+
CoreAudio (macOS) / ALSA / PulseAudio (Linux). It streams raw 16 kHz mono
|
|
12
|
+
PCM to stdout and stops itself at a deadline; it performs no network I/O.
|
package/README.md
CHANGED
|
@@ -1,3 +1,139 @@
|
|
|
1
|
-
#
|
|
1
|
+
# opencode-voice
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Local-only push-to-talk dictation for [OpenCode](https://opencode.ai) v2.
|
|
4
|
+
Press `f9`, talk (pauses welcome), press `f9` again — the transcript lands
|
|
5
|
+
in a draft you can edit; takes pile up until you send them as one prompt.
|
|
6
|
+
|
|
7
|
+
No cloud, no temp audio files, no auto-send, no permission answering.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
Requirements: OpenCode v2, a local STT server (see `docs/STT.md`),
|
|
12
|
+
macOS Microphone permission for your terminal.
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
mkdir -p ~/.config/opencode/plugins
|
|
16
|
+
git clone https://github.com/vaibhavgupta0786/opencode-voice ~/.config/opencode/plugins/voice
|
|
17
|
+
# restart the OpenCode TUI, open a session, press f9
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Or via npm (after `v0.3.0` is published):
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
opencode plugin add @vaibhavgupta0786/opencode-voice
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`f9` may be hijacked by macOS Fn keys — `Ctrl+P` → `dictate` (or
|
|
27
|
+
`/dictate`) works the same.
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
| Input | Action |
|
|
32
|
+
|---|---|
|
|
33
|
+
| `f9` | dictate a take (press to start, press to stop) — appended to the draft |
|
|
34
|
+
| `f9` again | stop → transcribe → draft editor opens with all takes |
|
|
35
|
+
| `f10` | open the draft editor anytime (empty or not — hand-compose works too) |
|
|
36
|
+
| `f12` | wipe the draft (confirmation dialog) |
|
|
37
|
+
| `/dictate`, `/voice-send`, `/voice-clear` | same via slash command / palette |
|
|
38
|
+
|
|
39
|
+
On MacBooks press F-keys **with Fn**. Plain `F10` is the hardware mic-mute
|
|
40
|
+
key and plain `F11` is Show Desktop (macOS reserves it — which is why wipe
|
|
41
|
+
lives on `F12`).
|
|
42
|
+
|
|
43
|
+
> **Windows**: not yet supported — the bundled recorders target macOS
|
|
44
|
+
> (CoreAudio) and Linux (ALSA/PulseAudio). PRs welcome; the recorder source
|
|
45
|
+
> is [opencode-dictate](https://github.com/rodri45l/opencode-dictate)'s
|
|
46
|
+
> `recorder.c` (miniaudio supports WASAPI).
|
|
47
|
+
|
|
48
|
+
**Editor semantics** (one dialog, always the whole draft):
|
|
49
|
+
|
|
50
|
+
- Opens after every take, with the cursor at the end and the view scrolled
|
|
51
|
+
to the bottom — long drafts scroll while you type.
|
|
52
|
+
- `Ctrl+Enter` sends everything as one prompt (`Cmd+Enter` also works where
|
|
53
|
+
the terminal delivers it). `Esc` closes and **keeps your edits** — they
|
|
54
|
+
are folded back into the draft automatically, whichever way the dialog
|
|
55
|
+
was closed.
|
|
56
|
+
- A failed send keeps the draft and lets you retry; switching sessions
|
|
57
|
+
with the editor open refuses to deliver to the wrong chat.
|
|
58
|
+
- The draft is **per session**: dictate in chat A, switch to B, and each
|
|
59
|
+
keeps its own takes. Switching back shows A's draft intact, take count
|
|
60
|
+
and all.
|
|
61
|
+
- Switching chats **mid-take discards that take**: the recorder stops
|
|
62
|
+
within half a second, nothing is transcribed or saved, and a toast says
|
|
63
|
+
so. The drafts themselves are never touched.
|
|
64
|
+
|
|
65
|
+
A mic indicator lives in the prompt footer: 🎤 idle, 🔴 recording, 🟡
|
|
66
|
+
transcribing, `🎤·N` = N takes in the draft. With a local STT server the
|
|
67
|
+
mic carries a health dot — `🟢🎤` server up, `🔴🎤` server down (checked
|
|
68
|
+
every 30 s, refreshed on every take; cloud STT endpoints show no dot).
|
|
69
|
+
Takes cap at 60 s (raise via the `maxMs` option). Debug log:
|
|
70
|
+
/tmp/opencode/voice-plugin.log.
|
|
71
|
+
|
|
72
|
+
## Keeping the STT server alive
|
|
73
|
+
|
|
74
|
+
Dictation needs the local whisper server on `127.0.0.1:8080`. If it ever
|
|
75
|
+
dies (machine restart, crash), recovery is:
|
|
76
|
+
|
|
77
|
+
1. **One-time opt-in:** run `/voice-setup` (palette: "Install voice-check
|
|
78
|
+
recovery skill"). This copies the bundled skill into your global
|
|
79
|
+
skills directory — nothing is installed silently, ever.
|
|
80
|
+
2. Whenever voice breaks: type `@voice-check` (or just say "voice is
|
|
81
|
+
dead") and the agent verifies the server, restarts it if needed, and
|
|
82
|
+
reports back.
|
|
83
|
+
|
|
84
|
+
Manual recovery is one health check plus a restart — see below.
|
|
85
|
+
|
|
86
|
+
## How it works
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
f9 -> bundled mic recorder (16 kHz PCM, stdout, in memory)
|
|
90
|
+
-> silence/VAD state machine (toggle: pauses never end the take)
|
|
91
|
+
-> POST WAV to local whisper at 127.0.0.1:8080
|
|
92
|
+
-> draft (takes accumulate) -> editor -> session.prompt
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
See `docs/STT.md` (server + model choice) and `docs/TROUBLESHOOTING.md`.
|
|
96
|
+
|
|
97
|
+
## Development
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
npm ci
|
|
101
|
+
npm test # 12 unit tests, no mic needed
|
|
102
|
+
npm run typecheck
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Pure logic in `logic.ts`, host/IO in `tui.ts`. See `CONTRIBUTING.md`.
|
|
106
|
+
|
|
107
|
+
## Recorder binaries
|
|
108
|
+
|
|
109
|
+
`bin/` ships prebuilt mic recorders built from
|
|
110
|
+
[opencode-dictate](https://github.com/rodri45l/opencode-dictate)'s
|
|
111
|
+
`recorder/recorder.c` (miniaudio, MIT) — see NOTICE. SHA256 pinned in
|
|
112
|
+
`bin/SHA256SUMS` and verified in CI. To rebuild from source, fetch
|
|
113
|
+
`recorder.c` + `miniaudio.h` upstream and compile, e.g. on macOS:
|
|
114
|
+
`cc -O2 -o recorder recorder.c -framework CoreAudio -framework AudioToolbox -framework CoreFoundation`.
|
|
115
|
+
|
|
116
|
+
## Why this instead of X?
|
|
117
|
+
|
|
118
|
+
| | This | [renjfk/opencode-voice](https://github.com/renjfk/opencode-voice) | [opencode-dictate](https://github.com/rodri45l/opencode-dictate) |
|
|
119
|
+
|---|---|---|---|
|
|
120
|
+
| Input | `F9` press / `F9` press (toggle) | hold `ctrl+r` | hands-free, open mic |
|
|
121
|
+
| Review before send | always (editable draft) | optional | auto-sends in conversation mode |
|
|
122
|
+
| Draft accumulation | per session | single | n/a (immediate send) |
|
|
123
|
+
| LLM cleanup pass | none needed | required (cloud API) | required for voice commands |
|
|
124
|
+
| TTS voice-out | roadmap | yes (Piper) | no |
|
|
125
|
+
| Answers permissions by voice | never | via LLM | yes (opt-out) |
|
|
126
|
+
| External audio deps | none (bundled recorders) | `sox`, `whisper-cli` | none (bundled) |
|
|
127
|
+
|
|
128
|
+
**Choose this if** you want push-to-talk with edit-before-send, zero
|
|
129
|
+
surprises (no auto-send, no permission answering), and nothing to install
|
|
130
|
+
beyond a local whisper server.
|
|
131
|
+
|
|
132
|
+
## Roadmap
|
|
133
|
+
|
|
134
|
+
- 🎤 mic button in the prompt footer
|
|
135
|
+
- Voice-out: spoken digest of long replies (not verbatim readout) + local TTS
|
|
136
|
+
|
|
137
|
+
## License
|
|
138
|
+
|
|
139
|
+
MIT — see LICENSE.
|
package/bin/SHA256SUMS
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
f703bed975f03bbf78fb0f4ba02754722af55aae83edab7df28281dbc1a28117 darwin-arm64/opencode-voice-recorder
|
|
2
|
+
ac164390aa4d208d735dc1389160ab1ea679c4818f385b5a4b69b9f3d7d8bd79 darwin-x64/opencode-voice-recorder
|
|
3
|
+
3b5fa64ff354893f191586ae59ee7e5e693919de7434624678d4ad82053d5148 linux-x64/opencode-voice-recorder
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/docs/STT.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Local speech-to-text
|
|
2
|
+
|
|
3
|
+
opencode-voice sends audio to an OpenAI-compatible
|
|
4
|
+
`/audio/transcriptions` endpoint. Default: `http://127.0.0.1:8080/v1`
|
|
5
|
+
(localhost — audio never leaves your machine).
|
|
6
|
+
|
|
7
|
+
## Install location (canonical)
|
|
8
|
+
|
|
9
|
+
Keep the server install in one place so the recovery skill can find it:
|
|
10
|
+
|
|
11
|
+
~/.local/share/opencode/voice-stt/
|
|
12
|
+
mlx_server.py # Apple Silicon GPU (preferred)
|
|
13
|
+
mlx-env/ # its virtualenv
|
|
14
|
+
mlx-server.log # server log
|
|
15
|
+
|
|
16
|
+
(A legacy location, `~/.local/share/opencode/opencode-dictate/`, is also
|
|
17
|
+
checked by the skill if the canonical one is missing.)
|
|
18
|
+
|
|
19
|
+
## Quick health check
|
|
20
|
+
|
|
21
|
+
curl -s -m 3 http://127.0.0.1:8080/health # expect {"status": "ok"}
|
|
22
|
+
|
|
23
|
+
If it is down, restart it detached:
|
|
24
|
+
|
|
25
|
+
cd ~/.local/share/opencode/voice-stt
|
|
26
|
+
nohup ./mlx-env/bin/python3 mlx_server.py > mlx-server.log 2>&1 &
|
|
27
|
+
|
|
28
|
+
then re-run the health check every 5 s — the model loads in ~10–25 s on a
|
|
29
|
+
cold start. The `@voice-check` skill automates exactly this.
|
|
30
|
+
|
|
31
|
+
## mlx-whisper (recommended, Apple Silicon)
|
|
32
|
+
|
|
33
|
+
Dependency-free server using
|
|
34
|
+
[faster-whisper](https://github.com/SYSTRAN/faster-whisper)
|
|
35
|
+
(CTranslate2 port of OpenAI Whisper, 2–4× faster, CPU-friendly):
|
|
36
|
+
|
|
37
|
+
```sh
|
|
38
|
+
python3 -m venv stt-env
|
|
39
|
+
stt-env/bin/pip install faster-whisper
|
|
40
|
+
WHISPER_MODEL=small VOICE_STT_PORT=8080 stt-env/bin/python3 server.py
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
`server.py` sketch (stdlib only, plus `faster-whisper`): serve
|
|
44
|
+
`POST /v1/audio/transcriptions` with multipart `file` + `model` fields,
|
|
45
|
+
return `{"text": ...}`. `GET /health` must return `{"status": "ok"}`.
|
|
46
|
+
|
|
47
|
+
## Model choice (mac CPU, int8)
|
|
48
|
+
|
|
49
|
+
| Model | Size | Speed/take | Notes |
|
|
50
|
+
|---|---|---|---|
|
|
51
|
+
| `base` | ~150 MB | fastest | noticeably worse |
|
|
52
|
+
| `small` | ~500 MB | ~1–3 s | good default |
|
|
53
|
+
| `small.en` | ~500 MB | ~1–3 s | English-only, slightly better at English |
|
|
54
|
+
| `medium` | ~1.5 GB | ~5–10 s | big accuracy jump |
|
|
55
|
+
| `large-v3` | ~3 GB | very slow on CPU | best; needs GPU to be practical |
|
|
56
|
+
|
|
57
|
+
Switch with `WHISPER_MODEL=medium ...` and restart the server.
|
|
58
|
+
The plugin needs no changes — it just POSTs audio to the endpoint.
|
|
59
|
+
|
|
60
|
+
## Alternatives
|
|
61
|
+
|
|
62
|
+
Any OpenAI-compatible endpoint works: whisper.cpp server, LM Studio /
|
|
63
|
+
Ollama with STT support, or a cloud API (note: cloud uploads your voice).
|
|
64
|
+
Point the plugin at it via the `stt` option.
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Troubleshooting
|
|
2
|
+
|
|
3
|
+
Debug log (if enabled): `/tmp/opencode/voice-plugin.log`.
|
|
4
|
+
Every take logs one line:
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
capture <reason> voiced=700ms loudest=0.058 bytes=96000
|
|
8
|
+
transcript="..."
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## `f9` does nothing at all
|
|
12
|
+
|
|
13
|
+
- The command needs a session view. Dictating on the launch/home view of a
|
|
14
|
+
brand-new session starts that session automatically; other non-session
|
|
15
|
+
views refuse with a toast.
|
|
16
|
+
- `f9` may be hijacked by macOS (Fn keys). Use `Ctrl+P` → `dictate`,
|
|
17
|
+
or `/dictate`, instead.
|
|
18
|
+
|
|
19
|
+
## `voiced=0ms loudest=0.000` — pure silence
|
|
20
|
+
|
|
21
|
+
Two causes, same symptom:
|
|
22
|
+
|
|
23
|
+
1. macOS Microphone permission. **System Settings → Privacy & Security →
|
|
24
|
+
Microphone** → enable your terminal app, then retry. (CoreAudio feeds
|
|
25
|
+
zeros when permission is denied — the recorder runs fine, the mic is muted.)
|
|
26
|
+
2. The hardware mic was muted by a **plain `F10` press** (no Fn) — macOS
|
|
27
|
+
maps it to mic mute, and external monitors display the mute state.
|
|
28
|
+
Unmute via the keyboard/control-center, then retry.
|
|
29
|
+
|
|
30
|
+
## `STT failed ... Is the local server up?`
|
|
31
|
+
|
|
32
|
+
- `curl http://127.0.0.1:8080/health` should return `{"status": "ok"}`.
|
|
33
|
+
- First start downloads the model (small ≈ 500 MB) — watch the server log
|
|
34
|
+
for `ready` before dictating.
|
|
35
|
+
|
|
36
|
+
## Takes cut off mid-thought
|
|
37
|
+
|
|
38
|
+
Toggle mode (default) never ends on silence — only the second `f9` (or the
|
|
39
|
+
60 s cap) ends a take. If takes still split, you are on `toggle: false`;
|
|
40
|
+
either re-enable toggle or raise `silenceMs` (900 → 2500). Very long
|
|
41
|
+
monologues hit the 60 s cap — raise `maxMs` if you want longer takes.
|
|
42
|
+
|
|
43
|
+
## "Take discarded — you switched chats"
|
|
44
|
+
|
|
45
|
+
By design: switching sessions mid-take aborts that take (recording stops
|
|
46
|
+
within half a second, nothing is transcribed or saved) so a recording
|
|
47
|
+
started in one chat can never land in another. The drafts themselves are
|
|
48
|
+
untouched — dictate again in the chat you are now viewing.
|
|
49
|
+
|
|
50
|
+
## Footer mic shows a red dot
|
|
51
|
+
|
|
52
|
+
The STT server is unreachable: run `@voice-check` (or say "voice is
|
|
53
|
+
dead") to verify and restart it. The dot refreshes every 30 s and on
|
|
54
|
+
every take, so it should flip green on its own once the server is back.
|
|
55
|
+
No dot at all means your `stt` endpoint is not localhost — cloud STT has
|
|
56
|
+
no health route to check, so the mic renders dotless by design.
|
|
57
|
+
|
|
58
|
+
## F-keys don't reach the app
|
|
59
|
+
|
|
60
|
+
Press them **with Fn** on MacBooks. Known macOS hardware/system claims:
|
|
61
|
+
plain `F10` = mic mute, plain `F11` = Show Desktop (which is why wipe is
|
|
62
|
+
on `F12`). Leader chords (`Ctrl+X …`) never register from plugin layers —
|
|
63
|
+
that is an OpenCode plugin API limitation, not a bug here.
|
|
64
|
+
|
|
65
|
+
## Review / wipe do nothing (`f10`, `f12`)
|
|
66
|
+
|
|
67
|
+
- Slash commands always work: `/voice-send`, `/voice-clear`.
|
|
68
|
+
- If `/voice-send` shows no toast and no editor, check the debug log —
|
|
69
|
+
an empty draft only toasts; a non-empty one must open the editor.
|
|
70
|
+
|
|
71
|
+
## Phantom / hallucinated transcripts
|
|
72
|
+
|
|
73
|
+
Whisper invents phrases ("Thank you.") from noise. The plugin drops takes
|
|
74
|
+
below `minSpeechMs` (default 300 ms) of voiced audio, but in a noisy room
|
|
75
|
+
raise `vadThreshold` (0.03 → 0.06). The log's `loudest=` shows your levels.
|
|
76
|
+
|
|
77
|
+
## Plugin shows "failed" in the sidebar
|
|
78
|
+
|
|
79
|
+
- `Cannot find package '@opencode/plugin'` (server): you have a stray
|
|
80
|
+
server entrypoint importing it — this plugin is TUI-only by design.
|
|
81
|
+
- `Keymap.Provider is missing`: keymap registration must happen inside a
|
|
82
|
+
component/slot render, not bare in `setup()`.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Voice mode (PARKED — future separate project)
|
|
2
|
+
|
|
3
|
+
Goal: full ChatGPT-style voice mode for OpenCode — always-on mic plus
|
|
4
|
+
spoken answers. Parked until voice input (this repo) is solid.
|
|
5
|
+
|
|
6
|
+
## Background (from OpenAI's realtime/TTS/VAD docs)
|
|
7
|
+
|
|
8
|
+
Three architectures exist: Realtime API (one audio→audio model), chained
|
|
9
|
+
pipeline (STT → text agent → TTS), GPT-Live (voice frontend + text backend).
|
|
10
|
+
Locally only the **chained pipeline** is viable — and it's the right one:
|
|
11
|
+
we want to see and control the text between stages.
|
|
12
|
+
|
|
13
|
+
The five moving parts: VAD/turn-taking, STT (✅ done here, mlx GPU),
|
|
14
|
+
brain, streaming TTS, barge-in + audio plumbing.
|
|
15
|
+
|
|
16
|
+
References: `server_vad` vs `semantic_vad` (OpenAI judges "are they done?"
|
|
17
|
+
from words, not just silence); TTS streams PCM before generation finishes;
|
|
18
|
+
first-audio latency is the metric; `gpt-4o-mini-tts` with 11+ voices.
|
|
19
|
+
|
|
20
|
+
## Why coding voice mode ≠ ChatGPT voice
|
|
21
|
+
|
|
22
|
+
1. **Outputs are 1000-word audits, not sentences.** A digest layer
|
|
23
|
+
(verdict first, 2 things that matter, then ask) is core, not a feature.
|
|
24
|
+
2. **Work takes minutes.** Need status whispers while tools run, and
|
|
25
|
+
interruption mid-tool-run (`session.interrupt` — verify TUI access).
|
|
26
|
+
3. **Permissions by voice are dangerous.** Never auto-answer; read aloud,
|
|
27
|
+
require explicit confirm (possibly typed, not spoken).
|
|
28
|
+
|
|
29
|
+
## Proposed shape
|
|
30
|
+
|
|
31
|
+
Separate project with three independently toggleable modes: push-to-talk
|
|
32
|
+
(this repo), conversation (open mic, auto-send), voice-out (spoken
|
|
33
|
+
digests). Reuse recorder + mlx STT + session APIs. New: Silero VAD,
|
|
34
|
+
digest LLM prompt, TTS engine + player, barge-in wiring.
|
|
35
|
+
|
|
36
|
+
## Suggested order
|
|
37
|
+
|
|
38
|
+
1. TTS bake-off on Apple Silicon (Piper vs Kokoro vs mlx-audio vs `say`):
|
|
39
|
+
naturalness vs latency, same paragraph.
|
|
40
|
+
2. Digest prompt design (the real product work).
|
|
41
|
+
3. Always-on loop with barge-in.
|
|
42
|
+
4. Permission-safety rules.
|
package/logic.ts
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
// Pure, host-independent logic for opencode-voice: VAD endpointing, PCM
|
|
2
|
+
// helpers, WAV encoding and STT URL building. No imports — unit-testable
|
|
3
|
+
// with plain node. tui.ts owns all host/IO concerns.
|
|
4
|
+
|
|
5
|
+
export const RATE = 16_000 // recorder streams 16-bit mono PCM at 16 kHz
|
|
6
|
+
export const TICK_MS = 50 // VAD evaluated in 50 ms windows (1600 bytes)
|
|
7
|
+
|
|
8
|
+
export interface VoiceOptions {
|
|
9
|
+
stt?: string // base URL of OpenAI-compatible STT
|
|
10
|
+
model?: string // STT model name
|
|
11
|
+
silenceMs?: number // quiet time ending the utterance (auto mode only)
|
|
12
|
+
maxMs?: number // hard cap per utterance
|
|
13
|
+
startTimeoutMs?: number // give up if silent this long
|
|
14
|
+
minSpeechMs?: number // voiced audio required
|
|
15
|
+
vadThreshold?: number // peak amplitude counting as voice
|
|
16
|
+
toggle?: boolean // f9 starts, f9 stops; pauses never end the take
|
|
17
|
+
accumulate?: boolean // takes pile into a draft; send once via /voice-send
|
|
18
|
+
debug?: boolean
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const DEFAULTS: {
|
|
22
|
+
stt: string
|
|
23
|
+
model: string
|
|
24
|
+
silenceMs: number
|
|
25
|
+
maxMs: number
|
|
26
|
+
startTimeoutMs: number
|
|
27
|
+
minSpeechMs: number
|
|
28
|
+
vadThreshold: number
|
|
29
|
+
toggle: boolean
|
|
30
|
+
accumulate: boolean
|
|
31
|
+
debug: boolean
|
|
32
|
+
} = {
|
|
33
|
+
stt: "http://127.0.0.1:8080/v1",
|
|
34
|
+
model: "whisper-1",
|
|
35
|
+
silenceMs: 900,
|
|
36
|
+
maxMs: 60_000,
|
|
37
|
+
startTimeoutMs: 4_000,
|
|
38
|
+
minSpeechMs: 300,
|
|
39
|
+
vadThreshold: 0.03,
|
|
40
|
+
toggle: true,
|
|
41
|
+
accumulate: true,
|
|
42
|
+
debug: false,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export type RequiredVoiceOptions = typeof DEFAULTS
|
|
46
|
+
|
|
47
|
+
export function mergeOptions(raw: VoiceOptions = {}): RequiredVoiceOptions {
|
|
48
|
+
return {
|
|
49
|
+
stt: raw.stt ?? DEFAULTS.stt,
|
|
50
|
+
model: raw.model ?? DEFAULTS.model,
|
|
51
|
+
silenceMs: raw.silenceMs ?? DEFAULTS.silenceMs,
|
|
52
|
+
maxMs: raw.maxMs ?? DEFAULTS.maxMs,
|
|
53
|
+
startTimeoutMs: raw.startTimeoutMs ?? DEFAULTS.startTimeoutMs,
|
|
54
|
+
minSpeechMs: raw.minSpeechMs ?? DEFAULTS.minSpeechMs,
|
|
55
|
+
vadThreshold: raw.vadThreshold ?? DEFAULTS.vadThreshold,
|
|
56
|
+
toggle: raw.toggle ?? DEFAULTS.toggle,
|
|
57
|
+
accumulate: raw.accumulate ?? DEFAULTS.accumulate,
|
|
58
|
+
debug: raw.debug ?? DEFAULTS.debug,
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function peakOf(chunk: Uint8Array): number {
|
|
63
|
+
const view = new DataView(chunk.buffer, chunk.byteOffset, chunk.byteLength)
|
|
64
|
+
let peak = 0
|
|
65
|
+
for (let i = 0; i + 1 < chunk.length; i += 2) {
|
|
66
|
+
const sample = Math.abs(view.getInt16(i, true)) / 32768
|
|
67
|
+
if (sample > peak) peak = sample
|
|
68
|
+
}
|
|
69
|
+
return peak
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function wavFromPcm(pcm: Uint8Array): Buffer<ArrayBuffer> {
|
|
73
|
+
const header = Buffer.alloc(44)
|
|
74
|
+
header.write("RIFF", 0)
|
|
75
|
+
header.writeUInt32LE(36 + pcm.length, 4)
|
|
76
|
+
header.write("WAVE", 8)
|
|
77
|
+
header.write("fmt ", 12)
|
|
78
|
+
header.writeUInt32LE(16, 16)
|
|
79
|
+
header.writeUInt16LE(1, 20) // PCM
|
|
80
|
+
header.writeUInt16LE(1, 22) // mono
|
|
81
|
+
header.writeUInt32LE(RATE, 24)
|
|
82
|
+
header.writeUInt32LE(RATE * 2, 28)
|
|
83
|
+
header.writeUInt16LE(2, 32)
|
|
84
|
+
header.writeUInt16LE(16, 34)
|
|
85
|
+
header.write("data", 36)
|
|
86
|
+
header.writeUInt32LE(pcm.length, 40)
|
|
87
|
+
return Buffer.concat([header, pcm])
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface Capture {
|
|
91
|
+
wav: Buffer
|
|
92
|
+
voicedMs: number
|
|
93
|
+
loudest: number
|
|
94
|
+
hadSpeech: boolean
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// --- utterance-endpointing state machine (per 50 ms tick) ---
|
|
98
|
+
|
|
99
|
+
export interface VadState {
|
|
100
|
+
spoken: boolean
|
|
101
|
+
voicedMs: number
|
|
102
|
+
loudest: number
|
|
103
|
+
quietFor: number
|
|
104
|
+
elapsed: number
|
|
105
|
+
done: boolean
|
|
106
|
+
doneReason: string | null
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function initialVadState(): VadState {
|
|
110
|
+
return { spoken: false, voicedMs: 0, loudest: 0, quietFor: 0, elapsed: 0, done: false, doneReason: null }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function vadStep(state: VadState, peak: number, opts: RequiredVoiceOptions): VadState {
|
|
114
|
+
if (state.done) return state
|
|
115
|
+
const next: VadState = { ...state, elapsed: state.elapsed + TICK_MS }
|
|
116
|
+
const finish = (reason: string): VadState => ({ ...next, done: true, doneReason: reason })
|
|
117
|
+
|
|
118
|
+
if (peak > opts.vadThreshold) {
|
|
119
|
+
next.voicedMs += TICK_MS
|
|
120
|
+
if (peak > next.loudest) next.loudest = peak
|
|
121
|
+
next.quietFor = 0
|
|
122
|
+
next.spoken = true
|
|
123
|
+
} else if (next.spoken) {
|
|
124
|
+
next.quietFor += TICK_MS
|
|
125
|
+
// Toggle mode: pauses are thinking time — only manual stop or max cap ends it.
|
|
126
|
+
if (!opts.toggle && next.quietFor >= opts.silenceMs) return finish("silence")
|
|
127
|
+
} else if (next.elapsed >= opts.startTimeoutMs) {
|
|
128
|
+
return finish("start-timeout")
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (next.elapsed >= opts.maxMs) return finish("max-length")
|
|
132
|
+
return next
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Enough voiced audio to be worth transcribing (noise must not reach Whisper). */
|
|
136
|
+
export function hadSpeech(state: VadState, opts: RequiredVoiceOptions): boolean {
|
|
137
|
+
return state.spoken && state.voicedMs >= opts.minSpeechMs
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function sttEndpoint(base: string): string {
|
|
141
|
+
const trimmed = base.replace(/\/+$/, "")
|
|
142
|
+
return trimmed.includes("/audio/transcriptions") ? trimmed : `${trimmed}/audio/transcriptions`
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Health-check candidates, root-first: our mlx server answers /health at
|
|
146
|
+
* the origin; some OpenAI-compatible servers only answer under the API
|
|
147
|
+
* base (e.g. /v1/health). Duplicates removed, order kept. */
|
|
148
|
+
export function healthEndpoints(base: string): string[] {
|
|
149
|
+
const trimmed = base.replace(/\/+$/, "")
|
|
150
|
+
let origin = trimmed
|
|
151
|
+
try {
|
|
152
|
+
origin = new URL(trimmed).origin
|
|
153
|
+
} catch {
|
|
154
|
+
// not a parseable URL — probe the base only
|
|
155
|
+
}
|
|
156
|
+
const candidates = [`${origin}/health`, `${trimmed}/health`]
|
|
157
|
+
return [...new Set(candidates)]
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The health dot is a localhost-server feature: cloud STT endpoints expose
|
|
161
|
+
* no health route, so a red dot there would always lie. */
|
|
162
|
+
export function isLocalEndpoint(base: string): boolean {
|
|
163
|
+
try {
|
|
164
|
+
const raw = new URL(base.replace(/\/+$/, "")).hostname.toLowerCase()
|
|
165
|
+
// URL keeps IPv6 brackets: "[::1]" -> "::1".
|
|
166
|
+
const host = raw.startsWith("[") && raw.endsWith("]") ? raw.slice(1, -1) : raw
|
|
167
|
+
return host === "localhost" || host === "127.0.0.1" || host === "::1"
|
|
168
|
+
} catch {
|
|
169
|
+
return false
|
|
170
|
+
}
|
|
171
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,38 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vaibhavgupta0786/opencode-voice",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Local-only push-to-talk dictation for OpenCode v2 \u2014 press F9, speak, press F9.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"exports": {
|
|
8
|
+
"./tui": "./tui.tsx"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"tui.tsx",
|
|
12
|
+
"logic.ts",
|
|
13
|
+
"bin",
|
|
14
|
+
"servers",
|
|
15
|
+
"docs",
|
|
16
|
+
"skills",
|
|
17
|
+
"NOTICE"
|
|
18
|
+
],
|
|
19
|
+
"scripts": {
|
|
20
|
+
"test": "node --test 'test/*.test.ts'",
|
|
21
|
+
"typecheck": "tsc --noEmit"
|
|
22
|
+
},
|
|
23
|
+
"dependencies": {
|
|
24
|
+
"@opencode/plugin": "2.0.12"
|
|
25
|
+
},
|
|
26
|
+
"peerDependencies": {
|
|
27
|
+
"@opentui/core": ">=0.5.0",
|
|
28
|
+
"@opentui/solid": ">=0.5.0",
|
|
29
|
+
"solid-js": ">=1.9.0"
|
|
30
|
+
},
|
|
31
|
+
"devDependencies": {
|
|
32
|
+
"@opencode/client": "2.0.12",
|
|
33
|
+
"@opentui/solid": "^0.5.0",
|
|
34
|
+
"@types/node": "^22.0.0",
|
|
35
|
+
"solid-js": "^1.9.0",
|
|
36
|
+
"typescript": "^5.4.0"
|
|
37
|
+
}
|
|
38
|
+
}
|