@todoforai/voiceloop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 TODO for AI
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,239 @@
1
+ # voiceloop
2
+
3
+ **The fastest way to put a real, interruptible voice agent in the browser.**
4
+
5
+ A zero-dependency JavaScript library that runs the full voice loop — **VAD → STT → LLM → TTS** — with the hard parts already solved:
6
+
7
+ - **Real barge-in** — interrupt the agent mid-sentence. Triggers on *transcribed novel words*, not raw mic energy, so the agent's own voice leaking into the mic never cuts it off.
8
+ - **Self-echo filtering** — a word-match filter compares what the mic hears against what the agent is currently saying. Benchmarked under real speaker→mic coupling with echo cancellation fully off: 0 self-interruptions in 30 turns, at full speed — while Pipecat (default config) cut its own reply on 20/30 turns and OpenAI Realtime on 17/30 ([echo results](bench/results/RESULTS.md#scenario-echo-smalltalk--speakermic-coupling-15db--30ms)).
9
+ - **First-sentence streaming TTS** — the LLM streams text, TTS synthesizes the first sentence *while the rest is still generating*, then stays one sentence ahead. First audio in well under a second.
10
+ - **Speculative prefetch** — the LLM call starts while you're still finishing your sentence, overlapping model latency with the end-of-turn pause. Replies feel instant.
11
+ - **Tap-to-seek** — playback keeps a tape of synthesized clips; tap anywhere in the transcript to jump the voice there, backward or forward.
12
+ - **Turn serialization** — rapid-fire turns, tool results, holds and replays can never overlap or talk over each other. Locked in by a 100-test regression suite.
13
+ - **Local-first defaults** — Silero VAD (WASM) and Piper TTS (WASM) run in the browser. Free, no cloud round-trip for voice output (assets load from CDN, then cache).
14
+
15
+ Everything is pluggable: bring your own LLM (any OpenAI-compatible endpoint or a custom async generator), pick an STT provider (Web Speech, ElevenLabs Scribe, Deepgram Flux, Speechmatics), swap the TTS.
16
+
17
+ ## Quick start
18
+
19
+ ```sh
20
+ npm i @todoforai/voiceloop
21
+ ```
22
+
23
+ ```js
24
+ import { VoiceAgent, unlockAudio, prewarmVoice } from '@todoforai/voiceloop';
25
+
26
+ prewarmVoice(); // optional but recommended: preload VAD/WASM while your UI sits idle
27
+
28
+ const agent = new VoiceAgent({
29
+ // Any OpenAI-compatible /chat/completions endpoint. In production point this at
30
+ // YOUR proxy route — never ship a provider secret key to the page.
31
+ llmUrl: '/api/chat/completions',
32
+ model: 'gpt-4o-mini',
33
+
34
+ persona: 'You are a friendly cooking assistant.',
35
+ sysmsg: 'The user is on the recipes page.', // live context, update via setSysmsg()
36
+
37
+ onEvent: (e) => {
38
+ // stt: e.committed = locked prefix, e.text = live tail (on turnComplete, e.text is the whole turn)
39
+ if (e.type === 'stt') render.user(e.turnComplete ? e.text : `${e.committed ?? ''} ${e.text}`.trim(), e.turnComplete);
40
+ if (e.type === 'assistant') render.agent(e.text, e.full); // e.text = spoken so far, e.full = whole answer
41
+ if (e.type === 'state') render.state(e.state); // idle|listening|thinking|speaking
42
+ if (e.type === 'error') console.error(e.error);
43
+ },
44
+ });
45
+
46
+ // From a click handler (browsers require a gesture for audio + mic):
47
+ button.onclick = async () => {
48
+ unlockAudio();
49
+ await agent.start();
50
+ };
51
+ ```
52
+
53
+ That's it. Speak, get spoken answers, interrupt at will.
54
+
55
+ ## How the latency adds up
56
+
57
+ ```
58
+ you stop speaking ──┐
59
+ │ end-of-turn debounce (~600ms)
60
+ LLM prefetch ────┤ ← already streaming since your interim transcript stabilized
61
+ │
62
+ first LLM tokens ───┤ first sentence boundary (~8+ chars)
63
+ │ Piper synthesizes sentence 1 while sentence 2 streams in
64
+ first audio ────────┘ next sentences pre-synthesized during playback → zero gaps
65
+ ```
66
+
67
+ **Measured, not claimed**: [voice-agent-bench](https://github.com/todoforai/voice-agent-bench)
68
+ (developed here at [`bench/`](bench/)) is a black-box rig — audio in → audio out, zero
69
+ integration — that scores voiceloop and competing stacks on the same scripted conversations
70
+ and the same fixed mock LLM. Reply latency (voice→voice median), and how often the system
71
+ misbehaves — talks over a hesitating user, or cuts its own reply when its voice echoes back
72
+ into the mic (no AEC):
73
+
74
+ | system | latency (clean) | talked through user | cut itself under echo |
75
+ |---|---|---|---|
76
+ | OpenAI Realtime (own LLM)* | 870ms | **0/30** | **17/30** |
77
+ | **voiceloop** (deepgram + ElevenLabs flash) | **980ms** | **0/30** | **0/30** |
78
+ | **voiceloop** (deepgram + Piper, free local TTS) | **970ms** | **0/30** | — |
79
+ | Pipecat 1.8.1 (same providers) | 1050ms | 2/30 | **20/30** |
80
+ | ElevenLabs ConvAI | 1450ms | **0/30** | 0/30 |
81
+
82
+ Overlapping a hesitating user is fine — people do it, and every stack here except Pipecat backs
83
+ off the moment the user keeps talking (voiceloop yields in 420ms, and only enters 2 of 30 turns
84
+ at all). The failure that still separates them is echo: fed its own voice through the mic with
85
+ no AEC, the other fast stacks hear themselves as the user and cut their own replies.
86
+ voiceloop's word-level echo filter runs echo-coupled turns at full speed (930ms, parity with
87
+ clean).
88
+
89
+ \* Realtime is speech-to-speech — it can't use the bench's fixed mock LLM, so its row isn't
90
+ fully apples-to-apples.
91
+
92
+ Full tables, method and reproduction steps: [`bench/results/RESULTS.md`](bench/results/RESULTS.md).
93
+
94
+ ## STT providers
95
+
96
+ ```js
97
+ new VoiceAgent({ sttProvider: 'webspeech' }) // default: free, browser-native (Chrome/Safari)
98
+ new VoiceAgent({ sttProvider: 'elevenlabs', sttTokenUrl: '/api/stt/token' })
99
+ new VoiceAgent({ sttProvider: 'deepgram', sttTokenUrl: '/api/stt/token' })
100
+ new VoiceAgent({ sttProvider: 'speechmatics', sttTokenUrl: '/api/stt/token' })
101
+ ```
102
+
103
+ Cloud providers authenticate with a **short-TTL token minted by your backend** so the raw API key never reaches the page. Either:
104
+
105
+ - `sttTokenUrl` — a POST route on your server that returns `{ token }` (mint it against the provider's temp-token API with your secret key), or
106
+ - `getSttToken` — an async callback `() => ({ token })` when your auth doesn't fit a bare POST.
107
+
108
+ On browsers without SpeechRecognition (Firefox, WebKitGTK) `webspeech` falls back to `elevenlabs` — which needs a token route configured, so pass `sttTokenUrl`/`getSttToken` if you want the fallback to work.
109
+
110
+ | Provider | End-of-turn | Notes |
111
+ |---|---|---|
112
+ | `webspeech` | Native (browser-managed) | Free, on-device, Chrome/Safari only |
113
+ | `elevenlabs` | VAD-gated | Scribe v2 realtime, great accuracy |
114
+ | `deepgram` | **Native** (Flux) | Provider's own turn model hears the full stream |
115
+ | `speechmatics` | VAD-gated | Cheapest per second, locked-words-only finals |
116
+
117
+ ## Bring your own LLM
118
+
119
+ The default adapter speaks OpenAI's streaming wire format (OpenAI, Groq, Cerebras, OpenRouter, Ollama, vLLM, LiteLLM, ...). For anything else, pass an async generator:
120
+
121
+ ```js
122
+ const agent = new VoiceAgent({
123
+ llm: async function* (history, system, signal, { toolGate } = {}) {
124
+ // history: [{ role: 'user'|'assistant', content }], system: composed persona+context
125
+ // toolGate: only for adapters that run tools themselves — see below; ignore it otherwise
126
+ for await (const delta of myProvider.stream({ history, system, signal })) {
127
+ yield { text: delta }; // speech text → streamed into TTS
128
+ // yield { tool: 'name', args: {...} }; // tool call → fired during playback
129
+ }
130
+ },
131
+ });
132
+ ```
133
+
134
+ ## Tools
135
+
136
+ ```js
137
+ const agent = new VoiceAgent({
138
+ tools: {
139
+ get_weather: {
140
+ description: 'Current weather for a city',
141
+ params: { city: { type: 'string' } }, // JSON-schema properties
142
+ required: ['city'], // optional; defaults to all params
143
+ run: async ({ city }) => fetchWeather(city), // fires DURING speech, in parallel
144
+ },
145
+ },
146
+ });
147
+ ```
148
+
149
+ Tool calls execute while the agent is still talking. Results are recorded in a per-turn ledger so the model never re-fires the same call, and `agent.notify('[TOOL RESULT get_weather] 22°C sunny')` relays an async outcome back for a spoken follow-up — bursts of results collapse into one reply instead of three interrupting monologues.
150
+
151
+ Tools are **dispatched, not awaited**: the turn completes when the agent stops speaking, whatever the tool is still doing. A slow or hung tool can never stall the conversation, and a tool is free to `await agent.notify(...)` with its own result. Long work belongs in a tool that returns promptly ("checking that now") and delivers the outcome later via `notify()`.
152
+
153
+ ### Adapters that run tools themselves
154
+
155
+ Some adapters (agent loops) execute tools **inside** the generator, feeding results back to the model so it keeps talking with the answer in hand — all within one turn. Declare two flags so voiceloop adapts:
156
+
157
+ ```js
158
+ async function* myAgentLoop(history, system, signal, { toolGate } = {}) {
159
+ for await (const ev of loop({ history, system, signal, beforeToolCall: () => toolGate })) {
160
+ if (ev.type === 'text') yield { text: ev.delta };
161
+ if (ev.type === 'tool_start') yield { tool: ev.name, id: ev.id, args: ev.args, running: true };
162
+ if (ev.type === 'tool_end') yield { tool: ev.name, id: ev.id, args: ev.args, result: ev.result };
163
+ }
164
+ }
165
+ myAgentLoop.executesTools = true; // chunks carry the outcome — voiceloop reports, never re-runs
166
+ myAgentLoop.acceptsToolGate = true; // the loop awaits `toolGate` before ANY tool executes
167
+ ```
168
+
169
+ - **`executesTools`** — a tool chunk carrying `result` (or `error`) is *reported*, not executed again. Emitting a `running: true` chunk first opens a live spinner chip that the outcome chunk replaces in place (matched by `id`); if the turn dies before the outcome arrives, voiceloop resolves the chip as `interrupted — did not finish` so a spinner is never left stuck.
170
+ - **`acceptsToolGate`** — required to keep [speculative prefetch](#how-the-latency-adds-up) enabled. Prefetch starts the LLM on a *stable interim*, before the user has finished the sentence — so a tool firing there could send an email the user was still amending, and abort cannot undo it. The gate is a promise voiceloop resolves only when the turn **commits**: text streams speculatively, tools wait at the door, and a discarded speculation **rejects** the gate instead — it always settles, so a loop awaiting it unwinds rather than hanging. Without the flag, voiceloop refuses to speculate on that adapter at all (correct, but slower).
171
+
172
+ ## API surface
173
+
174
+ ```js
175
+ await agent.start(deviceId?) // acquire mic, begin listening (needs a user gesture)
176
+ agent.stop() // pause — pipeline stays warm for instant resume
177
+ agent.destroy() // full teardown
178
+ agent.sendUserText(text) // typed input, same turn pipeline as speech
179
+ agent.notify(text) // soft turn: reply only if nothing newer is queued
180
+ agent.setHeld(bool) // hold: queue utterances, reply over all of them on release
181
+ agent.setMuted(bool) // mic mute
182
+ agent.setTtsMuted(bool) // silent mode: transcript + tools still run
183
+ agent.setSysmsg(text) // update live context mid-session
184
+ agent.seek(nwIndex) // tap-to-seek within the current reply (nwIndex = count of
185
+ // NON-WHITESPACE chars before the tap — whitespace doesn't count)
186
+ agent.replay(text, fromNw?, onProgress?) // re-voice a past reply; onProgress(spokenText, done)
187
+ agent.dumpAudio() // last 30s of mic audio as WAV + stall report (debugging)
188
+ ```
189
+
190
+ Events via `onEvent(e)`: `state`, `stt`, `assistant`, `tool`, `vad`, `echo`, `error`, `diag`.
191
+ A `tool` event carries `{ name, id, args }` plus either `result` (settled) or `running: true` (an [adapter-announced](#adapters-that-run-tools-themselves) call still executing) — render the pair as one chip keyed by `id`.
192
+
193
+ ## Tuning
194
+
195
+ Every latency/sensitivity knob lives in [`src/tuning.js`](src/tuning.js) — VAD thresholds, barge-in minimum characters, sentence-break aggressiveness, prefetch stability window, echo-match threshold. Constructor options (`bargeInMinChars`, `vadOptions`, `turnDetector`, `maxPauseMs`) override per-agent.
196
+
197
+ Liveness: the turn loop is strictly serialized, so host code it awaits could stall it. Tools are dispatched and never joined (a hung tool costs its own result, nothing more), an aborted LLM stream is detached rather than drained (a generator that ignores its `AbortSignal` can't wedge the next turn), and `maxPauseMs` is a single wall-clock deadline bounding how long a `turnDetector` holds a turn open — including one that never returns. Synthesis is the exception: a custom `_synth()` that never settles does block the reply that needs it, since there's nothing to speak without it.
198
+
199
+ ## TTS
200
+
201
+ Default is **Piper** (local WASM, free, many languages and voices):
202
+
203
+ ```js
204
+ import { PiperTTS } from '@todoforai/voiceloop';
205
+ new VoiceAgent({ tts: new PiperTTS('en_US-amy-medium') });
206
+ ```
207
+
208
+ **ElevenLabs** for human-grade voices (cloud; key stays server-side behind your proxy route):
209
+
210
+ ```js
211
+ import { ElevenLabsTTS } from '@todoforai/voiceloop';
212
+ new VoiceAgent({ tts: new ElevenLabsTTS({ ttsUrl: '/api/tts' }) }); // your backend adds xi-api-key
213
+ // dev only: new ElevenLabsTTS({ apiKey: 'xi-…' }) — direct browser→ElevenLabs
214
+ ```
215
+
216
+ Same behavior (sentence streaming, barge-in, seek, speculative presynth) either way — only the voice
217
+ and the cost change: Piper is free and offline, ElevenLabs is paid and sounds human.
218
+
219
+ Custom engines subclass `StreamingTTS` and implement one method:
220
+
221
+ ```js
222
+ class MyTTS extends StreamingTTS {
223
+ async _synth(text) { return wavBlob; } // sentence in, audio Blob out
224
+ }
225
+ ```
226
+
227
+ Sentence chunking, the playback tape, tap-to-seek, barge-in and progress reporting all come from the base class.
228
+
229
+ ## Testing
230
+
231
+ ```
232
+ npm test # 101 tests, no browser needed
233
+ ```
234
+
235
+ The suite locks in turn serialization, hold/release semantics, tool dedup, self-echo classification, prefetch adoption rules, tape seeking, and each STT provider's turn-boundary state machine.
236
+
237
+ ## License
238
+
239
+ MIT
@@ -0,0 +1,100 @@
1
+ <!doctype html>
2
+ <!-- voiceloop minimal example: mic → agent → speaker. No bundler, no framework.
3
+ Serve the repo root (e.g. `npx serve` or `python3 -m http.server`) and open
4
+ /examples/simple-browser.html — file:// won't work (mic + module imports need http). -->
5
+ <html lang="en">
6
+ <head>
7
+ <meta charset="utf-8" />
8
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
9
+ <title>voiceloop — minimal example</title>
10
+ <style>
11
+ body { font: 16px/1.5 system-ui, sans-serif; max-width: 640px; margin: 3rem auto; padding: 0 1rem; }
12
+ #talk { font-size: 1.1rem; padding: .6rem 1.6rem; border-radius: 2rem; border: 1px solid #888; cursor: pointer; }
13
+ #talk.on { background: #e33; color: #fff; border-color: #e33; }
14
+ #state { color: #888; margin-left: .8rem; }
15
+ .msg { margin: .8rem 0; }
16
+ .user { color: #06c; }
17
+ .agent .unspoken { color: #aaa; }
18
+ </style>
19
+ </head>
20
+ <body>
21
+ <h1>voiceloop</h1>
22
+ <p>
23
+ <button id="talk">Start talking</button>
24
+ <span id="state">idle</span>
25
+ </p>
26
+ <div id="log"></div>
27
+
28
+ <script type="module">
29
+ import { VoiceAgent, unlockAudio, prewarmVoice } from '../src/index.js';
30
+
31
+ prewarmVoice(); // preload VAD/WASM while the page sits idle
32
+
33
+ const $ = (id) => document.getElementById(id);
34
+ const log = $('log');
35
+ let userEl = null, agentEl = null;
36
+
37
+ const bubble = (cls) => {
38
+ const el = document.createElement('div');
39
+ el.className = `msg ${cls}`;
40
+ log.appendChild(el);
41
+ return el;
42
+ };
43
+
44
+ const agent = new VoiceAgent({
45
+ // Point at any OpenAI-compatible /chat/completions endpoint.
46
+ // Quick local test with Ollama (from the repo root):
47
+ // ollama pull llama3.2
48
+ // OLLAMA_ORIGINS='*' ollama serve
49
+ // npx serve . → open http://localhost:3000/examples/simple-browser.html
50
+ // (OLLAMA_ORIGINS allows this page's origin; mic access needs http://localhost or https.)
51
+ llmUrl: 'http://localhost:11434/v1/chat/completions',
52
+ model: 'llama3.2',
53
+
54
+ persona: 'You are a concise, friendly voice assistant.',
55
+
56
+ onEvent: (e) => {
57
+ if (e.type === 'state') {
58
+ $('state').textContent = e.state;
59
+ } else if (e.type === 'stt') {
60
+ userEl ??= bubble('user');
61
+ // partial: committed prefix + live tail; turnComplete: e.text is the whole turn
62
+ userEl.textContent = e.turnComplete ? e.text : `${e.committed ?? ''} ${e.text}`.trim();
63
+ if (e.turnComplete) userEl = null;
64
+ } else if (e.type === 'assistant') {
65
+ const full = e.full ?? e.text;
66
+ if (!full) { agentEl?.remove(); agentEl = null; return; } // aborted before any audio → drop the draft
67
+ agentEl ??= bubble('agent');
68
+ agentEl.innerHTML = '';
69
+ agentEl.append(e.text); // spoken so far
70
+ if (full.length > e.text.length) { // not yet spoken → dimmed
71
+ const rest = document.createElement('span');
72
+ rest.className = 'unspoken';
73
+ rest.textContent = full.slice(e.text.length);
74
+ agentEl.appendChild(rest);
75
+ }
76
+ if (e.final) agentEl = null;
77
+ } else if (e.type === 'error') {
78
+ bubble('user').textContent = `⚠ ${e.error}`;
79
+ }
80
+ },
81
+ });
82
+
83
+ const setBtn = (on) => {
84
+ $('talk').textContent = on ? 'Stop' : 'Start talking';
85
+ $('talk').classList.toggle('on', on);
86
+ };
87
+ $('talk').onclick = async () => {
88
+ if (agent.state === 'idle') {
89
+ unlockAudio(); // must happen inside the click gesture
90
+ setBtn(true);
91
+ try { await agent.start(); } catch { /* error surfaced via onEvent */ }
92
+ if (agent.state === 'idle') setBtn(false); // start failed (mic denied, …)
93
+ } else {
94
+ agent.stop();
95
+ setBtn(false);
96
+ }
97
+ };
98
+ </script>
99
+ </body>
100
+ </html>
package/package.json ADDED
@@ -0,0 +1,48 @@
1
+ {
2
+ "name": "@todoforai/voiceloop",
3
+ "version": "0.1.0",
4
+ "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
+ "type": "module",
6
+ "main": "src/index.js",
7
+ "exports": {
8
+ ".": "./src/index.js",
9
+ "./stt": "./src/stt.js",
10
+ "./tuning": "./src/tuning.js",
11
+ "./llm-openai": "./src/llm-openai.js"
12
+ },
13
+ "files": [
14
+ "src",
15
+ "!src/*.test.js",
16
+ "examples"
17
+ ],
18
+ "scripts": {
19
+ "test": "node --test src/*.test.js"
20
+ },
21
+ "keywords": [
22
+ "voice",
23
+ "voice-agent",
24
+ "speech",
25
+ "stt",
26
+ "tts",
27
+ "vad",
28
+ "barge-in",
29
+ "llm",
30
+ "realtime",
31
+ "conversational-ai",
32
+ "elevenlabs",
33
+ "deepgram",
34
+ "speechmatics",
35
+ "piper"
36
+ ],
37
+ "repository": {
38
+ "type": "git",
39
+ "url": "git+https://github.com/todoforai/voiceloop.git"
40
+ },
41
+ "license": "MIT",
42
+ "engines": {
43
+ "node": ">=20"
44
+ },
45
+ "publishConfig": {
46
+ "access": "public"
47
+ }
48
+ }
package/src/deps.js ADDED
@@ -0,0 +1,49 @@
1
+ // ── Browser dependency loader ───────────────────────────────────────────────────────────────────
2
+ // The VAD stack ships as UMD globals (window.ort, window.vad) and Piper imports a bare
3
+ // "onnxruntime-web" specifier resolved via an importmap. `loadVoiceDeps()` injects all of it once
4
+ // from jsDelivr — call it before constructing a VoiceAgent (or just call `prewarmVoice()` while
5
+ // your mic button sits idle, and the cold start only owes getUserMedia + the STT connect).
6
+ //
7
+ // If you bundle these deps yourself (Vite/webpack), skip this file entirely: expose the vad-web
8
+ // UMD bundle as `window.vad` and you're done.
9
+
10
+ import { warmVad } from './voice-agent.js';
11
+
12
+ const CDN = 'https://cdn.jsdelivr.net/npm/';
13
+ const DEPS = [
14
+ { id: 'vl-ort', src: `${CDN}onnxruntime-web@1.22.0/dist/ort.js` },
15
+ { id: 'vl-vad', src: `${CDN}@ricky0123/vad-web@0.0.29/dist/bundle.min.js` },
16
+ ];
17
+
18
+ let depsPromise = null;
19
+ export function loadVoiceDeps() {
20
+ return (depsPromise ??= new Promise((resolve, reject) => {
21
+ if (!document.getElementById('vl-importmap')) {
22
+ const im = document.createElement('script');
23
+ im.type = 'importmap';
24
+ im.id = 'vl-importmap';
25
+ im.textContent = JSON.stringify({ imports: { 'onnxruntime-web': `${CDN}onnxruntime-web@1.22.0/+esm` } });
26
+ document.head.prepend(im);
27
+ }
28
+ let pending = DEPS.length;
29
+ for (const { id, src } of DEPS) {
30
+ if (document.getElementById(id)) { if (--pending === 0) resolve(); continue; }
31
+ const s = document.createElement('script');
32
+ s.id = id;
33
+ s.src = src;
34
+ s.onload = () => { if (--pending === 0) resolve(); };
35
+ // A failed CDN load must not poison the memo: drop the rejected promise (and the dead <script>
36
+ // so the retry re-injects) so the next call can try again instead of failing forever.
37
+ s.onerror = () => { depsPromise = null; s.remove(); reject(new Error(`Failed to load ${src}`)); };
38
+ document.head.appendChild(s);
39
+ }
40
+ }));
41
+ }
42
+
43
+ // Pre-warm everything that gates 'listening' EXCEPT what needs the click gesture (mic + STT token):
44
+ // the CDN scripts AND the Silero v5 VAD ONNX session (WASM compile + model build — the biggest
45
+ // chunk of the cold-start path). Memoized and best-effort: safe to call on every mount; a failure
46
+ // just means the first start() pays the normal cold cost.
47
+ export function prewarmVoice() {
48
+ return loadVoiceDeps().then(() => warmVad()).catch(() => {});
49
+ }
package/src/index.js ADDED
@@ -0,0 +1,15 @@
1
+ export {
2
+ VoiceAgent, StreamingTTS, PiperTTS,
3
+ unlockAudio, warmVad,
4
+ isSelfEcho, novelChars, toolResultText,
5
+ VOICE_SYSMSG,
6
+ } from './voice-agent.js';
7
+ export {
8
+ STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
9
+ makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT,
10
+ warmDeepgramToken,
11
+ } from './stt.js';
12
+ export { ElevenLabsTTS } from './tts-elevenlabs.js';
13
+ export { loadVoiceDeps, prewarmVoice } from './deps.js';
14
+ export { makeOpenAILLM } from './llm-openai.js';
15
+ export { TUNING } from './tuning.js';
@@ -0,0 +1,122 @@
1
+ // ── Default LLM: any OpenAI-compatible /chat/completions endpoint ───────────────────────────────
2
+ // Normalizes the OpenAI streaming wire format to the VoiceAgent chunk contract:
3
+ // { text } — one per content delta (streamed straight into the TTS)
4
+ // { tool, args, id }— one per COMPLETED tool call (arguments accumulate across deltas; a call
5
+ // whose accumulated arguments never parse as JSON is DROPPED with a warning —
6
+ // firing a side-effecting tool with guessed/empty args would be worse)
7
+ //
8
+ // Works against OpenAI, Groq, Cerebras, Together, OpenRouter, Ollama, vLLM, LiteLLM… anything that
9
+ // speaks /chat/completions SSE. NEVER ship a provider secret key to a public page — in production
10
+ // point `llmUrl` at your own proxy route that injects the key server-side.
11
+ //
12
+ // Options:
13
+ // llmUrl — full /chat/completions URL (required unless you pass a custom `llm` to VoiceAgent)
14
+ // apiKey — sent as `Authorization: Bearer …` (omit when your proxy handles auth)
15
+ // model — model name; omitted from the body when empty (lets a proxy pick its default)
16
+ // maxTokens
17
+ // tools — VoiceAgent's tool map { name: { description, params, required? } }; converted to
18
+ // OpenAI tool specs (params = JSON-schema `properties`; `required` lists mandatory
19
+ // keys, defaulting to all of them)
20
+ // fetchFn — fetch override (tests, custom agents/headers); defaults to global fetch
21
+ // extraBody— merged into the request body (temperature, provider-specific knobs, …)
22
+ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 1024, tools = {}, fetchFn, extraBody = {} } = {}) {
23
+ const toolSpecs = Object.entries(tools).map(([name, t]) => ({
24
+ type: 'function',
25
+ function: {
26
+ name,
27
+ description: t.description ?? '',
28
+ parameters: { type: 'object', properties: t.params ?? {}, required: t.required ?? Object.keys(t.params ?? {}) },
29
+ },
30
+ }));
31
+
32
+ return async function* (history, system, signal) {
33
+ if (!llmUrl) throw new Error('VoiceAgent: no `llm` generator and no `llmUrl` configured — pass one of them');
34
+ const doFetch = fetchFn ?? fetch;
35
+ const r = await doFetch(llmUrl, {
36
+ method: 'POST', signal,
37
+ headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
38
+ body: JSON.stringify({
39
+ stream: true,
40
+ messages: [{ role: 'system', content: system }, ...history],
41
+ max_tokens: maxTokens,
42
+ ...(model ? { model } : {}),
43
+ ...(toolSpecs.length ? { tools: toolSpecs } : {}),
44
+ ...extraBody,
45
+ }),
46
+ });
47
+ if (!r.ok) {
48
+ const body = await r.text().catch(() => '');
49
+ let msg; try { msg = JSON.parse(body)?.error?.message ?? JSON.parse(body)?.error; } catch {}
50
+ throw new Error(msg || `LLM ${r.status} ${r.statusText}`);
51
+ }
52
+
53
+ // Tool-call deltas arrive fragmented ({index, id?, function:{name?, arguments?}} per chunk) —
54
+ // accumulate per index, emit when the provider signals the calls are complete (finish_reason
55
+ // 'tool_calls') or, defensively, at stream end.
56
+ const pending = new Map(); // index -> { id, name, args: '' }
57
+ const flush = function* () {
58
+ for (const c of pending.values()) {
59
+ if (!c.name) continue;
60
+ let args;
61
+ // Truncated/invalid argument JSON (aborted stream, provider bug) → DROP the call. The agent
62
+ // dispatches tools immediately, so guessing `{}` could fire a real side effect with the
63
+ // wrong arguments.
64
+ try { args = c.args ? JSON.parse(c.args) : {}; }
65
+ catch { console.warn(`makeOpenAILLM: dropping tool call "${c.name}" — arguments never parsed: ${c.args}`); continue; }
66
+ yield { tool: c.name, args, ...(c.id ? { id: c.id } : {}) };
67
+ }
68
+ pending.clear();
69
+ };
70
+
71
+ // One SSE data payload → zero or more chunks. Returns 'done' on [DONE].
72
+ const handle = function* (payload) {
73
+ if (payload === '[DONE]') return; // caller checks the sentinel itself
74
+ let m; try { m = JSON.parse(payload); } catch { return; } // tolerate keep-alive junk
75
+ if (m.error) throw new Error(m.error.message ?? String(m.error));
76
+ const choice = m.choices?.[0];
77
+ if (!choice) return;
78
+ const d = choice.delta ?? {};
79
+ if (d.content) yield { text: d.content };
80
+ for (const tc of d.tool_calls ?? []) {
81
+ const cur = pending.get(tc.index ?? 0) ?? { id: '', name: '', args: '' };
82
+ if (tc.id) cur.id = tc.id;
83
+ // Names are ASSIGNED, not concatenated: OpenAI sends the name once, but some compatible
84
+ // providers re-send the full name on later deltas — concatenating would corrupt it.
85
+ if (tc.function?.name) cur.name = tc.function.name;
86
+ if (tc.function?.arguments) cur.args += tc.function.arguments;
87
+ pending.set(tc.index ?? 0, cur);
88
+ }
89
+ if (choice.finish_reason === 'tool_calls') yield* flush();
90
+ };
91
+
92
+ const reader = r.body.getReader();
93
+ const dec = new TextDecoder();
94
+ let buf = '';
95
+ try {
96
+ while (true) {
97
+ const { value, done } = await reader.read();
98
+ buf += done ? dec.decode() : dec.decode(value, { stream: true }); // flush the decoder at EOF
99
+ let nl;
100
+ while ((nl = buf.indexOf('\n')) >= 0) {
101
+ const line = buf.slice(0, nl).trim();
102
+ buf = buf.slice(nl + 1);
103
+ if (!line.startsWith('data:')) continue;
104
+ const payload = line.slice(5).trim();
105
+ if (payload === '[DONE]') { yield* flush(); return; }
106
+ yield* handle(payload);
107
+ }
108
+ if (done) break;
109
+ }
110
+ // EOF: a final record without a trailing newline is still a record — parse the tail, then
111
+ // flush any buffered tool calls (defensive: stream ended without finish_reason/[DONE]).
112
+ const tail = buf.trim();
113
+ if (tail.startsWith('data:')) {
114
+ const payload = tail.slice(5).trim();
115
+ if (payload !== '[DONE]') yield* handle(payload);
116
+ }
117
+ yield* flush();
118
+ } finally {
119
+ reader.cancel().catch(() => {});
120
+ }
121
+ };
122
+ }