@todoforai/voiceloop 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -52,6 +52,26 @@ button.onclick = async () => {
52
52
 
53
53
  That's it. Speak, get spoken answers, interrupt at will.
54
54
 
55
+ ## Try it in 20 seconds
56
+
57
+ ```sh
58
+ npx serve . # from a clone
59
+ npx serve node_modules/@todoforai/voiceloop # from npm
60
+ ```
61
+
62
+ Open **`/examples/demo.html`** — the whole loop behind one orb, with the live
63
+ interim transcript, click-to-interrupt, and the two numbers that decide whether
64
+ it feels alive: **first token** and **first sound**. It starts in **echo mode**
65
+ — browser-native Web Speech STT, local Piper TTS and a stand-in "LLM" that
66
+ repeats you — so it runs with no keys, no backend and no cost. Open *settings*
67
+ to point it at any OpenAI-compatible endpoint (Ollama, your proxy) and a cloud
68
+ STT provider.
69
+
70
+ `/examples/simple-browser.html` is the same loop with no UI at all — 100 lines,
71
+ the place to start reading.
72
+
73
+ (`file://` won't work — mic and module imports need `http://localhost` or https.)
74
+
55
75
  ## How the latency adds up
56
76
 
57
77
  ```
@@ -0,0 +1,410 @@
1
+ <!doctype html>
2
+ <!-- voiceloop demo: the whole loop behind one button, plus the numbers that say
3
+ whether it feels seamless. No bundler, no framework, no dependencies.
4
+
5
+ from a clone: npx serve . → /examples/demo.html
6
+ from npm: npx serve node_modules/@todoforai/voiceloop
7
+
8
+ file:// won't work — mic + module imports need http://localhost or https.
9
+ Out of the box it runs in ECHO mode: Web Speech STT + local Piper TTS +
10
+ a fake "LLM" that repeats you. No keys, no backend, no cost. Open
11
+ settings to point it at a real LLM. -->
12
+ <html lang="en">
13
+ <head>
14
+ <meta charset="utf-8">
15
+ <meta name="viewport" content="width=device-width, initial-scale=1">
16
+ <title>voiceloop</title>
17
+ <style>
18
+ :root {
19
+ color-scheme: dark;
20
+ --bg:#0b0d10; --panel:#14171c; --line:#242932; --fg:#eceef1; --dim:#79818d;
21
+ --c:#4a5361; /* current state colour — everything reads from here */
22
+ }
23
+ * { box-sizing:border-box }
24
+ body { margin:0; background:var(--bg); color:var(--fg);
25
+ font:15px/1.55 ui-sans-serif,system-ui,-apple-system,sans-serif }
26
+ main { max-width:660px; margin:0 auto; padding:36px 18px 48px;
27
+ display:flex; flex-direction:column; align-items:center; gap:16px }
28
+
29
+ /* ── the orb: four radial blobs drifting behind a glass shell ──────────────
30
+ Plain CSS on purpose. A WebGPU version looks marginally better and is dark
31
+ for every Linux visitor (Chrome blocklists the adapter there), and an SVG
32
+ goo filter costs real CPU per frame for the same silhouette. This is ~25
33
+ lines, runs on the compositor, and degrades to a flat disc under
34
+ prefers-reduced-motion. */
35
+ #mic { position:relative; width:210px; height:210px; border-radius:50%; border:0; padding:0;
36
+ cursor:pointer; overflow:hidden; background:#0c0f14; display:grid; place-items:center;
37
+ --a:#2b3d5c; --b:#39506e; --cc:#2a3550; --d:#334463; --spin:30s; --glow:.14;
38
+ transition:transform .18s ease-out }
39
+ /* Each blob is a soft disc smaller than the orb, wandering on its own ellipse.
40
+ Blurred and screen-blended, four of them read as one body that swells and
41
+ pinches where they overlap — metaballs, minus the shader. */
42
+ #mic i { position:absolute; width:52%; height:52%; left:24%; top:24%; border-radius:50%;
43
+ filter:blur(13px); mix-blend-mode:screen; animation:wander var(--spin) ease-in-out infinite;
44
+ transition:background 1s } /* colours cross-fade; the speed change snaps (animation-* isn't animatable) */
45
+ #mic i:nth-child(1) { background:radial-gradient(circle, var(--a) 0 26%, transparent 66%) }
46
+ #mic i:nth-child(2) { background:radial-gradient(circle, var(--b) 0 24%, transparent 64%);
47
+ animation-duration:calc(var(--spin)*.72); animation-direction:reverse }
48
+ #mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0 28%, transparent 68%);
49
+ animation-duration:calc(var(--spin)*1.4); animation-delay:calc(var(--spin)*-.3) }
50
+ #mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0 22%, transparent 62%);
51
+ animation-duration:calc(var(--spin)*.9); animation-direction:reverse;
52
+ animation-delay:calc(var(--spin)*-.55) }
53
+ /* glass shell: specular cap + rim shadow, one pseudo each */
54
+ #mic::before { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
55
+ background:radial-gradient(120% 90% at 32% 20%, rgba(255,255,255,.30) 0 10%,
56
+ rgba(255,255,255,.05) 26%, transparent 52%);
57
+ box-shadow:inset 0 0 46px rgba(0,0,0,.75), inset 0 1px 1px rgba(255,255,255,.24) }
58
+ #mic::after { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
59
+ box-shadow:0 0 44px 8px var(--b); opacity:var(--glow); transition:opacity .8s }
60
+ @keyframes wander {
61
+ 0%,100% { transform:translate(-34%,-22%) scale(1) }
62
+ 25% { transform:translate( 30%,-34%) scale(1.18) }
63
+ 50% { transform:translate( 36%, 28%) scale(.86) }
64
+ 75% { transform:translate(-28%, 34%) scale(1.12) }
65
+ }
66
+ /* the glyph leads while the orb is dark; once the loop is live the orb says it better */
67
+ #glyph { position:relative; font-size:36px; line-height:1; transition:opacity .4s;
68
+ text-shadow:0 2px 12px #000 }
69
+ body:not([data-state="idle"]) #glyph { opacity:0 }
70
+
71
+ /* same object, more energy — the whole state display is these four lines */
72
+ body[data-state="listening"] #mic { --a:#12b87e; --b:#39d98a; --cc:#0e7fb4; --d:#5fd9b4; --spin:14s; --glow:.45 }
73
+ body[data-state="thinking"] #mic { --a:#e0a12e; --b:#ff9f43; --cc:#a63fc0; --d:#ffd166; --spin:6s; --glow:.65 }
74
+ body[data-state="speaking"] #mic { --a:#ff7a2d; --b:#ff5f6d; --cc:#ffa94d; --d:#b83fd0; --spin:4s; --glow:.9 }
75
+ /* VAD: the orb swells the instant the mic hears you — the "it's alive" tell */
76
+ body[data-vad="1"] #mic { transform:scale(1.035) }
77
+
78
+ body[data-state="listening"] { --c:#39d98a }
79
+ body[data-state="thinking"] { --c:#ffc453 }
80
+ body[data-state="speaking"] { --c:#ff8a3d }
81
+
82
+ @media (prefers-reduced-motion: reduce) {
83
+ #mic i { animation:none } #mic { transition:none }
84
+ }
85
+
86
+ #status { color:var(--c); font-size:14px; min-height:22px; text-align:center; transition:color .3s }
87
+ #lat { font:12px/1 ui-monospace,monospace; color:var(--dim); min-height:14px }
88
+
89
+ /* ── transcript ─────────────────────────────────────────────────────── */
90
+ #chat { width:100%; display:flex; flex-direction:column; gap:10px }
91
+ #chat:not(:empty) { margin-top:6px }
92
+ .msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
93
+ .msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
94
+ .msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
95
+ /* An echoed reply is not an answer — mark it so the transcript can't be read as a model talking. */
96
+ .msg.ai.echoed { border-left:2px solid #e0a94a }
97
+ .msg.ai.echoed::after { content:'echo — no model'; display:block; margin-top:5px;
98
+ font-size:10px; letter-spacing:.04em; color:#e0a94a; opacity:.85 }
99
+ .msg.interim { opacity:.5 }
100
+ .msg .tail { opacity:.35 } /* generated, not yet spoken */
101
+ .msg.tool { align-self:flex-start; background:#161d17; color:#8fc79a;
102
+ font:12px/1.4 ui-monospace,monospace }
103
+ .msg.note { align-self:center; background:none; color:var(--dim); font-size:12px; padding:0 }
104
+ .msg.err { align-self:center; background:#38141d; color:#ffb0bd; font-size:13px }
105
+
106
+ /* ── everything else, out of the way ────────────────────────────────── */
107
+ .bar { width:100%; display:flex; gap:8px }
108
+ input, select, button.btn { background:#0f1216; color:var(--fg); border:1px solid var(--line);
109
+ border-radius:9px; padding:9px 12px; font:inherit }
110
+ input:focus, select:focus { outline:none; border-color:var(--c) }
111
+ button.btn { cursor:pointer } button.btn:hover { border-color:#3a424e }
112
+ #typed { flex:1 }
113
+ details { width:100%; border-top:1px solid var(--line); padding-top:10px }
114
+ summary { cursor:pointer; color:var(--dim); font-size:13px; list-style:none }
115
+ summary::before { content:'▸ ' } details[open] summary::before { content:'▾ ' }
116
+ .grid { display:flex; flex-wrap:wrap; gap:10px; padding:12px 0 4px }
117
+ label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
118
+ #log { font:11px/1.5 ui-monospace,monospace; color:var(--dim);
119
+ max-height:220px; overflow:auto; white-space:pre-wrap; padding-top:8px }
120
+ /* Which code is actually running — the demo must never let you mistake the browser's own
121
+ recognizer for voiceloop's pipeline, or an echo stand-in for a real LLM. */
122
+ #pipeline { font-size:11.5px; color:var(--dim); text-align:center; line-height:1.7; max-width:46ch }
123
+ #pipeline b { color:#c8d0da; font-weight:600 }
124
+ #pipeline .warn { color:#e0a94a }
125
+ </style>
126
+ </head>
127
+ <body>
128
+ <main>
129
+ <button id="mic" title="click to start / stop · click while it talks to interrupt">
130
+ <i></i><i></i><i></i><i></i><span id="glyph">🎙</span>
131
+ </button>
132
+ <div id="status">click to start · then just talk</div>
133
+ <div id="lat"></div>
134
+ <div id="pipeline"></div>
135
+
136
+ <div id="chat"></div>
137
+
138
+ <div class="bar">
139
+ <input id="typed" placeholder="…or type a turn">
140
+ <button class="btn" id="send">Send</button>
141
+ </div>
142
+
143
+ <details>
144
+ <summary>settings</summary>
145
+ <div class="grid">
146
+ <label>mode<select id="mode">
147
+ <option value="echo">echo — no keys, no backend</option>
148
+ <option value="llm">full loop — your LLM</option>
149
+ </select></label>
150
+ <label>LLM url<input id="llmUrl" size="30" value="http://localhost:11434/v1/chat/completions"></label>
151
+ <label>model<input id="model" size="14" value="llama3.2"></label>
152
+ <label>STT<select id="sttProvider"></select></label>
153
+ <label>STT token url<input id="sttTokenUrl" size="30" placeholder="your backend route (cloud STT only)"></label>
154
+ <label>api key<input id="apiKey" size="18" placeholder="Bearer for LLM · X-API-Key for token url"></label>
155
+ <label>lang<select id="sttLang"></select></label>
156
+ <label>voice<select id="voice"></select></label>
157
+ <label>mic<select id="micDeviceId"><option value="">default</option></select></label>
158
+ </div>
159
+ <div class="bar">
160
+ <button class="btn" id="clear">Clear</button>
161
+ <button class="btn" id="muteMic">Mute mic</button>
162
+ <button class="btn" id="muteTts">Mute voice</button>
163
+ </div>
164
+ </details>
165
+
166
+ <details><summary>log</summary><div id="log"></div></details>
167
+ </main>
168
+
169
+ <script type="module">
170
+ import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice, STT_PROVIDERS, sttSelfCaptures, LANG_NAMES } from '../src/index.js';
171
+
172
+ // Load VAD/WASM + build the Silero session while the page sits idle — but ONLY for a provider that
173
+ // will actually use it. A selfCapture provider (webspeech) builds no pipeline, so prewarming there
174
+ // downloads an ONNX model to feed a detector that never runs.
175
+ const prewarmIfPipelined = () => { if (!sttSelfCaptures($('sttProvider').value)) prewarmVoice(); };
176
+
177
+ const $ = id => document.getElementById(id);
178
+ const fill = (id, entries) => $(id).append(...entries.map(([value, textContent]) =>
179
+ Object.assign(document.createElement('option'), { value, textContent })));
180
+
181
+ // Every choice below is READ from the library, never retyped here: a provider added to STT_PROVIDERS
182
+ // or a language added to LANG_NAMES shows up in this page by itself. A hand-typed copy would be a
183
+ // second source of truth that silently rots — and a demo that lies about what the library supports.
184
+ fill('sttProvider', Object.keys(STT_PROVIDERS).map(p => [p, p === 'webspeech' ? 'webspeech — free, in-browser' : p]));
185
+ fill('sttLang', Object.entries(LANG_NAMES).map(([code, name]) => [code, `${code} — ${name}`]));
186
+ const SETTINGS = ['mode','llmUrl','model','sttProvider','sttTokenUrl','apiKey','sttLang','voice','micDeviceId'];
187
+ for (const id of SETTINGS) {
188
+ const v = localStorage.getItem('vl.' + id); if (v) $(id).value = v;
189
+ $(id).addEventListener('change', () => {
190
+ localStorage.setItem('vl.' + id, $(id).value);
191
+ onSettingChanged(); // the banner must never describe the PREVIOUS choice
192
+ });
193
+ }
194
+ const log = (...a) => {
195
+ const line = `${new Date().toISOString().slice(11,23)} ${a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ')}`;
196
+ $('log').textContent = (line + '\n' + $('log').textContent).slice(0, 20000);
197
+ };
198
+
199
+ // ── transcript ──────────────────────────────────────────────────────────────
200
+ const chat = $('chat');
201
+ let interimEl = null, aiEl = null, vadOff = null;
202
+ const toolEls = new Map();
203
+ const bubble = (cls, text = '') => {
204
+ const el = document.createElement('div');
205
+ el.className = `msg ${cls}`; el.textContent = text;
206
+ chat.append(el); el.scrollIntoView({ block:'nearest' });
207
+ return el;
208
+ };
209
+
210
+ // ── latency: the two numbers that decide whether this feels alive ───────────
211
+ let t0 = 0, tText = 0, tAudio = 0;
212
+ const showLat = () => $('lat').textContent = [
213
+ tText && `first token ${tText - t0}ms`,
214
+ tAudio && `first sound ${tAudio - t0}ms`,
215
+ ].filter(Boolean).join(' · ');
216
+
217
+ // undefined (not '') when unset — so PiperTTS applies its OWN default voice instead of this page
218
+ // re-declaring one. The picker is populated async below and adopts whatever it resolved to.
219
+ const tts = new PiperTTS(localStorage.getItem('vl.voice') || undefined);
220
+ let agent = null;
221
+
222
+ // Echo mode: a stand-in "LLM" that repeats the turn VERBATIM — exercises the entire
223
+ // VAD→STT→…→TTS loop, barge-in included, with no key and no backend. Nothing is added to the
224
+ // text: a reply that is exactly the transcript is the whole point, since any prefix would be
225
+ // this page's words in the reply and would blur what the recognizer actually heard. That it's
226
+ // an echo and not a model is stated in the pipeline banner and on the bubble itself.
227
+ async function* echoLLM(history) {
228
+ const last = [...history].reverse().find(m => m.role === 'user')?.content ?? '';
229
+ for (const w of last.match(/\S+\s*/g) ?? []) {
230
+ yield { text: w }; await new Promise(r => setTimeout(r, 25));
231
+ }
232
+ }
233
+
234
+ // ── what is ACTUALLY running ────────────────────────────────────────────────
235
+ // A demo that hides which engine served a reply is worse than no demo: a browser-native
236
+ // recognizer looks identical to voiceloop's pipeline, and an echo stand-in looks identical
237
+ // to a model. Both get named on screen, always.
238
+ const describePipeline = (a) => {
239
+ const echo = $('mode').value === 'echo';
240
+ // Before the first click there is no session, so ask the provider statically — reading
241
+ // `a?.stt?.selfCapture` off a null agent silently yields false and the banner claims the
242
+ // pipeline is running when it is exactly what gets bypassed.
243
+ const self = a?.stt ? !!a.stt.selfCapture : sttSelfCaptures($('sttProvider').value);
244
+ const stt = self
245
+ ? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span>`
246
+ : `<b>${$('sttProvider').value}</b> via voiceloop's pipeline — Silero VAD → feed → commit`;
247
+ const llm = echo
248
+ ? '<span class="warn"><b>echo</b> — no model: replies repeat your words back</span>'
249
+ : `<b>${$('model').value || 'your model'}</b> at ${new URL($('llmUrl').value).host}`;
250
+ return `STT: ${stt}<br>LLM: ${llm}`;
251
+ };
252
+ const showPipeline = () => { $('pipeline').innerHTML = describePipeline(agent); };
253
+ // Hoisted (function declaration) so the settings listeners above can call it before this point.
254
+ function onSettingChanged() { showPipeline(); prewarmIfPipelined(); }
255
+
256
+ // A typed turn drives the same state machine without a mic — don't claim we're listening then.
257
+ let micOn = false;
258
+ const STATUS = {
259
+ idle: () => micOn ? 'listening — just talk' : 'click to start · then just talk',
260
+ listening: () => micOn ? 'listening — just talk' : 'ready · click to talk',
261
+ thinking: () => 'thinking…',
262
+ speaking: () => micOn ? 'speaking — talk over it to interrupt' : 'speaking — click to interrupt',
263
+ };
264
+ const setState = s => {
265
+ document.body.dataset.state = micOn || s !== 'listening' ? s : 'idle';
266
+ $('status').textContent = STATUS[s]?.() ?? s;
267
+ };
268
+
269
+ function onEvent(e) {
270
+ if (e.type !== 'stt' || e.turnComplete) log(e);
271
+ switch (e.type) {
272
+ case 'state':
273
+ // The agent can stop itself (mic denied, STT socket died) — the mic light
274
+ // follows the agent, never the other way round.
275
+ if (e.state === 'idle' && agent?.closed) { micOn = false; document.body.dataset.vad = '0'; }
276
+ setState(e.state);
277
+ break;
278
+ case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
279
+ case 'stt':
280
+ // A selfCapture provider runs no agent VAD, so no 'vad' event ever arrives and the orb would
281
+ // sit dead still while the browser hears you perfectly — looking exactly like a broken mic.
282
+ // Live partials are the only "I hear you" signal on that path, so drive the pulse from them.
283
+ if (agent?.stt?.selfCapture) {
284
+ document.body.dataset.vad = e.turnComplete ? '0' : '1';
285
+ clearTimeout(vadOff);
286
+ if (!e.turnComplete) vadOff = setTimeout(() => { document.body.dataset.vad = '0'; }, 700);
287
+ }
288
+ if (e.turnComplete) {
289
+ interimEl?.remove(); interimEl = null;
290
+ bubble('user', e.text);
291
+ aiEl = null; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
292
+ } else {
293
+ interimEl ??= bubble('user interim');
294
+ interimEl.textContent = `${e.committed ?? ''} ${e.text}`.trim();
295
+ interimEl.scrollIntoView({ block:'nearest' });
296
+ }
297
+ break;
298
+ case 'assistant': {
299
+ // Empty and non-final means "that draft never happened" (interrupted before
300
+ // a word was spoken) — drop the bubble rather than blank it.
301
+ if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; break; }
302
+ if (!tText && t0) { tText = Date.now(); showLat(); }
303
+ // `full` only exists once the TTS progress cursor is live — i.e. a clip is
304
+ // actually playing. The 'speaking' state is just the first token, not sound.
305
+ if (e.full != null && t0 && !tAudio) { tAudio = Date.now(); showLat(); }
306
+ // Tag the bubble (never the text) when no model produced it — the reply is a verbatim echo,
307
+ // and only the UI should say so.
308
+ aiEl ??= bubble($('mode').value === 'echo' ? 'ai echoed' : 'ai');
309
+ aiEl.textContent = e.text; // spoken so far
310
+ const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
311
+ if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
312
+ aiEl.scrollIntoView({ block:'nearest' });
313
+ break;
314
+ }
315
+ case 'tool': {
316
+ // running and result arrive as two events with one id: update, don't stack.
317
+ const el = toolEls.get(e.id) ?? bubble('tool');
318
+ if (e.id) toolEls.set(e.id, el);
319
+ el.textContent = `🔧 ${e.name}(${JSON.stringify(e.args)})${e.running ? ' …' : ` → ${JSON.stringify(e.result ?? '')}`.slice(0,200)}`;
320
+ break;
321
+ }
322
+ case 'echo': bubble('note', `self-echo dropped: ${e.text}`); break;
323
+ case 'error': bubble('err', e.error); break;
324
+ }
325
+ }
326
+
327
+ const muted = id => $(id).dataset.on === '1';
328
+ const makeAgent = () => {
329
+ const a = new VoiceAgent({
330
+ llmUrl: $('llmUrl').value.trim(),
331
+ model: $('model').value.trim(),
332
+ apiKey: $('apiKey').value.trim(),
333
+ sttProvider: $('sttProvider').value,
334
+ sttTokenUrl: $('sttTokenUrl').value.trim(),
335
+ sttLang: $('sttLang').value,
336
+ micDeviceId: $('micDeviceId').value,
337
+ persona: 'You are a concise, friendly voice assistant. Keep answers to a sentence or two.',
338
+ tts,
339
+ llm: $('mode').value === 'echo' ? echoLLM : undefined,
340
+ onEvent,
341
+ });
342
+ a.setMuted(muted('muteMic')); a.setTtsMuted(muted('muteTts')); // the toggles survive a rebuild
343
+ log('agent created; stt =', a.sttProvider); showPipeline(); // now reflects the LIVE session, not just the form
344
+ return a;
345
+ };
346
+
347
+ // ── one button does everything ──────────────────────────────────────────────
348
+ const stop = () => {
349
+ agent?.destroy(); agent = null; micOn = false;
350
+ document.body.dataset.vad = '0'; setState('idle');
351
+ onSettingChanged(); // describe the restored settings, and prewarm only if they'll use the VAD
352
+ };
353
+ $('mic').onclick = async () => {
354
+ unlockAudio(); // must be inside the gesture (autoplay policy)
355
+ // A reply in flight? then the click means "shut up", whether the mic is open
356
+ // or not — it cancels the turn and keeps the session.
357
+ if (agent && (agent.state === 'thinking' || agent.state === 'speaking')) return agent.interrupt();
358
+ if (micOn) return stop();
359
+ agent ??= makeAgent();
360
+ micOn = true; setState('listening');
361
+ try { await agent.start(); }
362
+ catch (e) { bubble('err', String(e?.message ?? e)); stop(); }
363
+ };
364
+
365
+ $('send').onclick = () => {
366
+ const text = $('typed').value.trim(); if (!text) return;
367
+ unlockAudio();
368
+ agent ??= makeAgent();
369
+ agent.sendUserText(text); // emits its own user bubble via the stt turnComplete event
370
+ $('typed').value = '';
371
+ };
372
+ $('typed').onkeydown = e => { if (e.key === 'Enter') $('send').click(); };
373
+
374
+ $('clear').onclick = () => { chat.textContent = ''; $('log').textContent = ''; $('lat').textContent = '';
375
+ interimEl = aiEl = null; toolEls.clear(); };
376
+ const toggle = (id, fn) => $(id).onclick = () => {
377
+ const on = $(id).dataset.on !== '1';
378
+ $(id).dataset.on = on ? '1' : '0';
379
+ $(id).style.borderColor = on ? 'var(--c)' : '';
380
+ fn(on);
381
+ };
382
+ toggle('muteMic', on => agent?.setMuted(on));
383
+ toggle('muteTts', on => agent?.setTtsMuted(on));
384
+
385
+ // Voice switches live. The rest are constructor options, so they can only apply
386
+ // to a NEW agent — drop the current one so the next turn builds it from these.
387
+ $('voice').addEventListener('change', () => { tts.setVoice($('voice').value); log('voice →', $('voice').value); });
388
+ for (const id of SETTINGS.filter(i => i !== 'voice'))
389
+ $(id).addEventListener('change', () => {
390
+ if (micOn) return bubble('note', `stop the mic to apply ${id}`);
391
+ if (agent) { agent.destroy(); agent = null; log(id, '→ agent rebuilt on next turn'); }
392
+ });
393
+
394
+ // ── device + voice lists ────────────────────────────────────────────────────
395
+ tts.voices().then(vs => {
396
+ fill('voice', vs.map(v => [v.id, v.id]));
397
+ $('voice').value = localStorage.getItem('vl.voice') || tts.voiceId; // the library's own default, not a copy of it
398
+ tts.setVoice($('voice').value);
399
+ }).catch(e => log('voice list failed', String(e)));
400
+ navigator.mediaDevices?.enumerateDevices().then(ds => {
401
+ for (const d of ds.filter(d => d.kind === 'audioinput'))
402
+ $('micDeviceId').append(Object.assign(document.createElement('option'),
403
+ { value: d.deviceId, textContent: d.label || d.deviceId.slice(0,8) }));
404
+ }).catch(() => {});
405
+
406
+ setState('idle');
407
+ window.agent = () => agent; // console poking: agent().history, …
408
+ </script>
409
+ </body>
410
+ </html>
package/package.json CHANGED
@@ -1,14 +1,18 @@
1
1
  {
2
2
  "name": "@todoforai/voiceloop",
3
- "version": "0.1.2",
3
+ "version": "0.1.4",
4
4
  "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
7
7
  "exports": {
8
- ".": "./src/index.js",
8
+ ".": {
9
+ "types": "./src/index.d.ts",
10
+ "default": "./src/index.js"
11
+ },
9
12
  "./stt": "./src/stt.js",
10
13
  "./tuning": "./src/tuning.js",
11
- "./llm-openai": "./src/llm-openai.js"
14
+ "./llm-openai": "./src/llm-openai.js",
15
+ "./package.json": "./package.json"
12
16
  },
13
17
  "files": [
14
18
  "src",
@@ -48,5 +52,6 @@
48
52
  },
49
53
  "publishConfig": {
50
54
  "access": "public"
51
- }
55
+ },
56
+ "types": "src/index.d.ts"
52
57
  }
package/src/index.d.ts ADDED
@@ -0,0 +1,320 @@
1
+ // Public types for voiceloop — VAD → STT → LLM → TTS with real barge-in.
2
+ // Hand-written (the source is plain JS); see README.md for the prose version of every option.
3
+
4
+ // Events the agent emits on `onEvent` — discriminated on `type` so a switch narrows the rest.
5
+ export type VoiceAgentEvent =
6
+ | { type: 'state'; state: 'idle' | 'listening' | 'thinking' | 'speaking' }
7
+ | { type: 'stt'; turnComplete: boolean; text: string; ms: number; committed?: string } // text=live tail; committed=locked segments (Speechmatics AddTranscript); turnComplete=user turn ended → send to LLM
8
+ | { type: 'assistant'; text: string; final: boolean; full?: string } // text=spoken-so-far (solid, advances live as TTS plays); full=complete answer → tail full.slice(text.length) not-yet-spoken (dim)
9
+ | { type: 'vad'; active: boolean } // VAD hears speech?
10
+ // running:true = loop call announced, executing now (display-only spinner; no result yet).
11
+ // The outcome event follows with the same `id` and replaces it in place.
12
+ | { type: 'tool'; name: string; args: Record<string, unknown>; result?: unknown; id?: string; running?: true }
13
+ | { type: 'echo'; text: string } // dropped self-echo (the agent's own TTS leaked into the mic) — diagnostics only
14
+ | { type: 'diag'; diag: 'aec-fallback'; message: string } // once/session: AEC loopback failed → TTS plays un-cancelled (self-echo only text-filtered)
15
+ | { type: 'diag'; diag: 'presynth'; text: string } // a reply's first clip was synthesized speculatively (latency diagnostics)
16
+ | { type: 'error'; error: string };
17
+
18
+ export interface VoiceTool {
19
+ description?: string;
20
+ params?: Record<string, unknown>; // JSON-schema `properties` for the arguments
21
+ /** Which `params` the model MUST supply. Defaults to ALL of them — set this to make some
22
+ * optional, otherwise the model is forced to invent a value for every field. */
23
+ required?: string[];
24
+ /** A tool with no `run` is dropped, never advertised: the model would call it, nothing would
25
+ * happen, and the transcript would still show it as having run — a silent lie. */
26
+ run?: (args: Record<string, unknown>) => unknown;
27
+ }
28
+
29
+ // One LLM delta: either spoken text, or a single tool call (args already parsed). Agent-loop tool
30
+ // chunks additionally carry the provider call `id` (two same-args calls, e.g. a retry, stay distinct
31
+ // acts) and the outcome (`result`/`error`) — the loop already executed the tool, so the agent
32
+ // reports it instead of running it again.
33
+ export type LLMChunk =
34
+ | { text: string }
35
+ | { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
36
+ // Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
37
+ // The default (agent-llm.js, vendored Pi loop) accepts an optional 4th arg: `toolGate` is awaited
38
+ // before ANY tool executes — speculative prefetch passes one that resolves on turn commit, so a
39
+ // speculation streams text early but can never fire a side effect from uncommitted speech.
40
+ // Custom LLMs are a low-level seam (tests, embedding hosts): their bare tool chunks are executed
41
+ // by the agent, but the results are NOT fed back into that LLM mid-turn — such calls are terminal
42
+ // for the turn. They may ignore toolGate.
43
+ export type LLM = (
44
+ history: Array<{ role: string; content: string }>,
45
+ system: string,
46
+ signal: AbortSignal,
47
+ options?: { toolGate?: Promise<unknown> },
48
+ ) => AsyncIterable<LLMChunk>;
49
+
50
+ // Pluggable TTS contract. `speak` resolves with the text actually HEARD (full on completion,
51
+ // a prefix on barge-in). The optional members back the voice picker / switching.
52
+ export interface VoiceTTS {
53
+ speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
54
+ stop?(): void;
55
+ /** Eagerly prepare the synth engine (load models, JIT) so the first utterance has no cold start. */
56
+ warm?(): Promise<void>;
57
+ voices?(): Promise<Array<{ id: string; name: string; language: string }>>;
58
+ setVoice?(voiceId: string): void;
59
+ setSpeed?(speed: number): void;
60
+ /** Register a callback fired while a clip plays with the chars spoken SO FAR across the whole
61
+ * reply — drives the host's live spoken cursor. Pass null to clear. */
62
+ setOnProgress?(fn: ((spokenText: string) => void) | null): void;
63
+ }
64
+
65
+ export interface VoiceAgentOptions {
66
+ sysmsg?: string;
67
+ /** Persona/system prompt prepended to the conversation context. Defaults to the built-in
68
+ * voice persona (VOICE_SYSMSG); pass your own agent's system message to override it. */
69
+ persona?: string;
70
+ /** LLM model id sent to the backend `/llm` route; empty → the backend's VOICE_MODEL default. */
71
+ model?: string;
72
+ llm?: LLM;
73
+ /** Test seam for the agent-loop LLM transport; defaults to global fetch. */
74
+ fetchFn?: typeof fetch;
75
+ apiKey?: string;
76
+ /** Full endpoint URLs, not a base: voiceloop composes no routes, so a host can point the LLM and
77
+ * the STT token at unrelated origins (a backend proxy for one, a direct provider for the other).
78
+ * Deriving `${baseUrl}/llm` + `${baseUrl}/stt/<provider>-token` is the HOST's job — and each STT
79
+ * provider mints its own token type, so one shared token URL cannot serve all of them. */
80
+ llmUrl?: string;
81
+ maxTokens?: number;
82
+ tts?: VoiceTTS;
83
+ /** Speaking speed for the default Piper TTS (1.2 → 20% faster). Ignored if `tts` is passed. */
84
+ speed?: number;
85
+ sttLang?: string;
86
+ micDeviceId?: string;
87
+ /** STT backend: 'webspeech' (default — the browser's own on-device SpeechRecognition; free, no
88
+ * token, no socket, captures its own mic) | 'speechmatics' | 'elevenlabs' (Scribe) | 'deepgram'
89
+ * (Flux — native model-based end-of-turn detection; the agent feeds it the full mic stream and
90
+ * lets it close turns itself). The cloud ones connect the browser DIRECTLY to the provider with a
91
+ * short-TTL token the backend mints (raw key stays server-side).
92
+ * 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
93
+ * the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
94
+ sttProvider?: SttProvider;
95
+ /** ONE endpoint that mints the token. When each provider needs a different route or credential,
96
+ * use `getSttToken` instead — it is handed the resolved provider, so the downgrade rule above
97
+ * stays in this library instead of being mirrored by every host. */
98
+ sttTokenUrl?: string;
99
+ /** Mint a short-TTL STT token yourself, for auth that doesn't fit one POST route. Called with the
100
+ * provider actually RUNNING (post-downgrade), never the requested one. Wins over sttTokenUrl. */
101
+ getSttToken?: (provider: SttProvider) => Promise<SttToken>;
102
+ sttUsageUrl?: string;
103
+ sttUrl?: string;
104
+ sttModel?: string;
105
+ /** Deepgram Flux end-of-turn confidence (0.5–0.9) — its model closes the turn itself once its
106
+ * confidence the user is done exceeds this threshold. Lower = snappier turn ends, higher = more
107
+ * patient across mid-thought pauses. Deepgram-only; default from tuning.js (0.7). */
108
+ sttEotThreshold?: number;
109
+ /** Every tool the model may call. voiceloop ships NO built-in tools — a voice library has no
110
+ * business knowing what a "todo" is — so product tools (and their `run`) are supplied here. */
111
+ tools?: Record<string, VoiceTool>;
112
+ keyterms?: string[];
113
+ preroll?: number;
114
+ vadOptions?: Record<string, unknown>;
115
+ /** Semantic end-of-turn hook run when VAD hears end-of-speech: return false to KEEP listening (the
116
+ * user only paused mid-thought), true (default, when omitted) to close the turn. Lets you swap the
117
+ * "is the turn over?" decision (silence-only | SLM | smart-turn) without touching the STT pipeline. */
118
+ turnDetector?: (text: string) => boolean | Promise<boolean>;
119
+ /** Hard cap (ms) a turnDetector may keep a turn open across a pause before it's force-committed, so a
120
+ * user who stops mid-sentence can't hang the turn forever. Default 4000. Ignored without a detector. */
121
+ maxPauseMs?: number;
122
+ /** Min transcribed chars of the user's interim (heard WHILE the agent is speaking) needed to count as a
123
+ * real barge-in and cut the reply. Higher = short backchannels ("mhm","yeah") don't interrupt; 0 =
124
+ * any transcribed char interrupts. Default 3 (see tuning.js TUNING.BARGE_IN_MIN_CHARS). */
125
+ bargeInMinChars?: number;
126
+ onEvent?: (event: VoiceAgentEvent) => void;
127
+ }
128
+
129
+ /** Call from a user-gesture handler (e.g. the mic button click) to bless the shared TTS audio
130
+ * element on mobile — without this, iOS rejects the async play() of synthesized speech. */
131
+ export function unlockAudio(): void;
132
+ /** Pre-build the Silero v5 VAD ONNX session (no mic) so the first start() reaches 'listening'
133
+ * much faster. Best-effort + memoized; requires the vad-web UMD bundle already loaded. */
134
+ export function warmVad(): Promise<void>;
135
+ /** Pre-mint + cache the short-TTL Deepgram STT token off the critical path (e.g. at mic-button
136
+ * mount), so a voice start only pays the WS handshake. Best-effort + deduped; no-op while a
137
+ * fresh cached token remains.
138
+ * `sttTokenUrl` is the EXACT endpoint that mints a Deepgram token (e.g. `/api/v1/stt/deepgram-token`),
139
+ * the same value passed as the `sttTokenUrl` option — NOT an API base. A host that mints tokens
140
+ * itself passes its `getSttToken` as `getToken` and can skip the URL entirely (warm for the
141
+ * provider it will actually run: `() => getSttToken('deepgram')`). Warming with the wrong URL
142
+ * silently no-ops the optimization: the real session just mints again on click. */
143
+ export function warmDeepgramToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
144
+ /** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
145
+ * host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
146
+ export const VOICE_SYSMSG: string;
147
+ /** Language codes the agent can instruct the LLM to reply in, code → English name. The `sttLang`
148
+ * values that actually steer the reply language — exported so a host builds its language picker
149
+ * from this instead of hand-listing a subset that silently rots as the map grows. */
150
+ export const LANG_NAMES: Record<string, string>;
151
+
152
+
153
+ export class VoiceAgent {
154
+ constructor(options?: VoiceAgentOptions);
155
+ /** The live conversation the LLM sees. A mutable reference the host may read and push into —
156
+ * carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
157
+ history: Array<{ role: string; content: string }>;
158
+ start(deviceId?: string): Promise<void>;
159
+ /** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
160
+ * start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
161
+ * fully release the mic when ending the session. */
162
+ stop(): void;
163
+ /** End for real: pause + release the mic, audio graph and VAD model (turns off the OS mic indicator).
164
+ * Call this when dropping the agent reference so a paused-but-discarded agent never leaks the mic. */
165
+ destroy(): void;
166
+ listVoices(): Promise<Array<{ id: string; name: string; language: string }>>;
167
+ setVoice(voiceId: string): void;
168
+ setSpeed(speed: number): void;
169
+ /** Update the host context (e.g. a re-opened chat); applies from the next turn — no restart. */
170
+ setSysmsg(sysmsg?: string): void;
171
+ /** Mute/unmute the mic (disables the stream's audio tracks → silence to VAD+STT); keeps the pipeline live. */
172
+ setMuted(muted: boolean): void;
173
+ /** Mute/unmute the AI's spoken output (TTS): the reply still streams as text but is never voiced. */
174
+ setTtsMuted(muted: boolean): void;
175
+ /** Move the playhead to a position in the current reply (non-whitespace char count) and play forward
176
+ * from there. Hits any already-streamed sentence (backward replay is cached, forward skips ahead);
177
+ * beyond streamed text is a no-op. Returns true on a hit, false otherwise. */
178
+ seek(nwIndex: number): boolean;
179
+ /** Re-speak a FINISHED reply from `fromNw` (a non-whitespace offset — the tapped word), interruptibly.
180
+ * Pure TTS (no LLM, no history change). `onProgress(spokenSoFar, done)` is driven by the host so it can
181
+ * update the exact transcript message it tapped: spokenSoFar is the spoken prefix, done flips true when
182
+ * the replay ends (played out, barged-in, or superseded) with spokenSoFar = the whole reply. */
183
+ replay(text: string, fromNw?: number, onProgress?: (spokenSoFar: string, done: boolean) => void): Promise<void>;
184
+ /** Cut off the current in-flight turn (abort the LLM, stop TTS) without stopping the agent —
185
+ * the mic keeps listening. No-op when not thinking/speaking. A host-callable barge-in. */
186
+ interrupt(): void;
187
+ /** Hold: keep listening/transcribing into history but DON'T reply. Entering hold silences the
188
+ * current reply (kept as the full streamed answer so far); releasing flushes all accumulated
189
+ * turns through the LLM in one go. */
190
+ setHeld(held: boolean): void;
191
+ /** Inject an out-of-band turn (e.g. a background TODO finished): speaks a reply. No-op once stopped. */
192
+ notify(text: string): Promise<void>;
193
+ /** Send a typed user turn (renders as a user bubble, then the agent replies in the background).
194
+ * Needs no mic — accepts even while paused/stopped (revives the reply loop; start() re-arms audio
195
+ * separately). Returns true when accepted (sent or queued while held), false for blank text or
196
+ * once destroy()ed (terminal). */
197
+ sendUserText(text: string): boolean;
198
+ /** Diagnostic tap: the last ~30s of the exact audio handed to STT as a 16kHz mono WAV, plus any
199
+ * detected delivery stalls (wall-clock gaps between capture chunks; offsets relative to the WAV).
200
+ * Play it back after a bad transcription run: gaps/corrupt audio → capture layer; clean audio →
201
+ * provider-side. */
202
+ dumpAudio(): { wav: Blob; gaps: Array<{ atSample: number; atSec: number; gapMs: number }>; seconds: number };
203
+ state: 'idle' | 'listening' | 'thinking' | 'speaking';
204
+ /** The STT provider actually in use — differs from the requested one when 'webspeech' was
205
+ * downgraded to a cloud provider (browser without SpeechRecognition). */
206
+ readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
207
+ /** True once the pipeline has been torn down (stop() or a fatal STT error self-stopped it). */
208
+ readonly closed: boolean;
209
+ }
210
+
211
+ export class PiperTTS implements VoiceTTS {
212
+ constructor(voiceId?: string, speed?: number);
213
+ voices(): Promise<Array<{ id: string; name: string; language: string }>>;
214
+ setVoice(voiceId: string): void;
215
+ setSpeed(speed: number): void;
216
+ speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
217
+ stop(): void;
218
+ /** Eagerly download+compile the ONNX model and JIT the WASM path so the first reply has no cold start. */
219
+ warm(): Promise<void>;
220
+ }
221
+
222
+ // ── STT providers ────────────────────────────────────────────────────────────────────────────
223
+ // A provider is a factory: it receives the callbacks below plus its own credentials/tuning, and
224
+ // returns a live session. VoiceAgent wires these itself — call them directly only when embedding
225
+ // STT without the agent.
226
+ export interface STTCallbacks {
227
+ onPartial?: (text: string, ms: number, committed?: string) => void;
228
+ onFinal?: (text: string, ms: number) => void;
229
+ onError?: (error: unknown) => void;
230
+ onFatal?: (error: unknown) => void;
231
+ onClose?: () => void;
232
+ isClosed?: () => boolean;
233
+ }
234
+ /** A live STT session. The capability flags tell the agent how much of the audio path the provider
235
+ * owns, so it builds only the missing half — a provider with its own mic and its own end-of-turn
236
+ * (WebSpeech) needs no VAD/capture pipeline at all. */
237
+ export interface STTSession {
238
+ /** Provider owns the mic → the agent builds no capture pipeline. Implies nativeEOT. */
239
+ selfCapture?: boolean;
240
+ /** Provider closes turns itself (unsolicited onFinal) → the agent runs no VAD-commit. */
241
+ nativeEOT?: boolean;
242
+ /** Provider wants the full mic stream, including silence (its EOT model needs real pauses). */
243
+ continuous?: boolean;
244
+ /** Pre-open the socket (token + handshake) so the first utterance isn't stalled. Idempotent. */
245
+ open?(): void;
246
+ /** Push PCM. Opens the socket lazily if `open()` was never called. */
247
+ feed?(pcm: Int16Array): void;
248
+ /** End of utterance: ask the provider to finalize. No-op providers with nativeEOT. */
249
+ commit?(): void;
250
+ /** Mute/unmute without tearing the session down (selfCapture providers). */
251
+ setEnabled?(enabled: boolean): void;
252
+ close(): void;
253
+ }
254
+ export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
255
+
256
+ /** The built-in STT provider ids — the keys of STT_PROVIDERS. */
257
+ export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
258
+ /** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
259
+ * provider reports one (used to cache and pre-mint just before expiry). */
260
+ export interface SttToken { token: string; expires_in?: number }
261
+
262
+ export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
263
+ /** Does this provider own the mic and its own end-of-turn? Then the agent builds NO capture pipeline
264
+ * and runs no VAD for it — so it emits no `vad` events, and prewarming the VAD/ONNX model is a
265
+ * wasted download. Answerable without constructing a session, so a host can decide both up front. */
266
+ export function sttSelfCaptures(provider: string): boolean;
267
+ export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
268
+ export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
269
+ export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
270
+ export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
271
+ /** Web Speech exists only in Chromium/Safari; elsewhere 'webspeech' resolves to this cloud provider. */
272
+ export const WEBSPEECH_FALLBACK: 'elevenlabs';
273
+ export function webSpeechSupported(): boolean;
274
+ /** The provider id actually used: 'webspeech' downgrades to WEBSPEECH_FALLBACK in a browser
275
+ * without SpeechRecognition. Call it to label the UI ("using cloud recognition") — but NOT to pick
276
+ * a token route: `getSttToken` is already handed the resolved id. */
277
+ export function resolveSttProvider(id: string): SttProvider;
278
+
279
+ // ── TTS ──────────────────────────────────────────────────────────────────────────────────────
280
+ /** Base class for streaming TTS engines: sentence-splits the incoming token stream, synthesizes
281
+ * the next clip while the current one plays, and reports the spoken cursor via setOnProgress.
282
+ * Subclass and implement `_synth(text)` to add an engine. */
283
+ export class StreamingTTS implements VoiceTTS {
284
+ constructor(voiceId?: string, speed?: number);
285
+ speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
286
+ stop(): void;
287
+ setOnProgress(fn: (spoken: string, scope: string) => void): void;
288
+ /** Jump playback to a word index within the current reply. Returns false when the seek can't be
289
+ * honoured (nothing playing, or past the text streamed so far) — check it before moving a cursor. */
290
+ seek(nwIndex: number): boolean;
291
+ }
292
+ /** Human-grade cloud voices. Pass `ttsUrl` (your backend proxy, key stays server-side); `apiKey`
293
+ * talks to ElevenLabs directly and is dev-only. */
294
+ export class ElevenLabsTTS extends StreamingTTS {
295
+ constructor(options?: { ttsUrl?: string; apiKey?: string; voiceId?: string; modelId?: string; format?: string });
296
+ warm(): Promise<void>;
297
+ }
298
+
299
+ // ── LLM ──────────────────────────────────────────────────────────────────────────────────────
300
+ /** Any OpenAI-compatible /chat/completions endpoint as an LLM adapter, with tool-calling wired
301
+ * to the `tools` map. `ttsUrl`-style proxying applies: prefer a backend route over apiKey. */
302
+ export function makeOpenAILLM(options?: {
303
+ llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
304
+ tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
305
+ }): LLM;
306
+
307
+ // ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
308
+ /** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
309
+ * session — call it while your UI is idle so the first start() is instant. */
310
+ export function loadVoiceDeps(): Promise<unknown>;
311
+ export function prewarmVoice(): void;
312
+ /** Every latency/sensitivity constant (VAD thresholds, barge-in chars, prefetch window, echo
313
+ * match threshold). Mutate before constructing an agent to retune globally. */
314
+ export const TUNING: Record<string, any>;
315
+ /** Echo filtering internals, exported for tests and custom mic gating: `isSelfEcho` decides
316
+ * whether heard text is the agent's own voice, `novelChars` counts what is genuinely new. */
317
+ export function isSelfEcho(heard: string, reply: string, options?: { threshold?: number; window?: number; minHits?: number }): boolean;
318
+ export function novelChars(heard: string, reply: string): number;
319
+ /** Render a tool result the way the agent reports it to the model (and to `tool` events). */
320
+ export function toolResultText(result: unknown): string;
package/src/index.js CHANGED
@@ -2,11 +2,11 @@ export {
2
2
  VoiceAgent, StreamingTTS, PiperTTS,
3
3
  unlockAudio, warmVad,
4
4
  isSelfEcho, novelChars, toolResultText,
5
- VOICE_SYSMSG,
5
+ VOICE_SYSMSG, LANG_NAMES,
6
6
  } from './voice-agent.js';
7
7
  export {
8
8
  STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
9
- makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT,
9
+ makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, sttSelfCaptures,
10
10
  warmDeepgramToken,
11
11
  } from './stt.js';
12
12
  export { ElevenLabsTTS } from './tts-elevenlabs.js';
package/src/stt.js CHANGED
@@ -39,8 +39,10 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
39
39
  // AUTH (cloud providers): the raw provider key must stay server-side. Either
40
40
  // • `sttTokenUrl` — your backend route that mints a short-TTL token: POST, optional X-API-Key
41
41
  // (`apiKey`), responds { token, expires_in? }; or
42
- // • `getToken` — an async () => ({ token, expires_in? }) callback, when your auth doesn't fit
43
- // that one route shape. Takes precedence over sttTokenUrl.
42
+ // • `getToken` — an async (provider) => ({ token, expires_in? }) callback, when your auth doesn't
43
+ // fit that one route shape. Takes precedence over sttTokenUrl. VoiceAgent calls it with the
44
+ // provider actually RUNNING (post-downgrade), so a host whose credential differs per provider
45
+ // writes one function instead of mirroring the downgrade rule.
44
46
 
45
47
  // ── Shared socket-lifecycle helpers ─────────────────────────────────────────
46
48
  // The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
@@ -57,7 +59,14 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
57
59
  async function mintSttToken(url, apiKey, onFatal, getToken) {
58
60
  if (getToken) {
59
61
  try {
60
- const body = await getToken();
62
+ // HOST code on the critical path: bound it exactly like the built-in fetch below, or a minter
63
+ // that never settles stalls voice start forever with no error. We can't cancel the host's
64
+ // promise — we just stop waiting on it (it may still resolve into the void, which is fine:
65
+ // a token is idempotent to mint).
66
+ const body = await Promise.race([
67
+ getToken(),
68
+ new Promise((_, rej) => setTimeout(() => rej(new Error('timed out')), TUNING.STT.tokenFetchTimeoutMs)),
69
+ ]);
61
70
  if (!body?.token) throw new Error('getToken returned no token');
62
71
  return body;
63
72
  } catch (e) { onFatal(`STT token: ${e.message}`); return null; }
@@ -71,7 +80,10 @@ async function mintSttToken(url, apiKey, onFatal, getToken) {
71
80
  return null;
72
81
  }
73
82
  const body = await res.json().catch(() => ({}));
74
- if (!res.ok || !body.token) { onFatal(`STT token: ${body.error || body.message || res.statusText}`); return null; }
83
+ // `message` first: a token route's error body usually puts the reason there and the bare status
84
+ // name in `error` ({ error: 'Unauthorized', message: 'API key not found' }) — the reason is what
85
+ // tells the developer their key is wrong rather than their URL.
86
+ if (!res.ok || !body.token) { onFatal(`STT token: ${body.message || body.error || res.statusText}`); return null; }
75
87
  return body;
76
88
  }
77
89
 
@@ -586,10 +598,12 @@ export function makeDeepgramSTT(opts) {
586
598
  // result window). Blindly appending would double words ("hello there" + "there how" → "there there").
587
599
  // Every final is de-overlapped against the turn-so-far before it's appended.
588
600
  export function makeWebSpeechSTT(opts) {
589
- const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onFatal, onClose, isClosed } = opts;
601
+ const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
590
602
  const Rec = typeof window !== 'undefined' && (window.SpeechRecognition || window.webkitSpeechRecognition);
591
603
  const EOT_MS = TUNING.WEBSPEECH_EOT_MS ?? 900;
592
604
  const TAIL_MAX_DEFERS = TUNING.WEBSPEECH_TAIL_MAX_DEFERS ?? 3;
605
+ const MAX_ERROR_RESTARTS = TUNING.WEBSPEECH_MAX_ERROR_RESTARTS ?? 5;
606
+ const ERROR_BACKOFF_MS = TUNING.WEBSPEECH_ERROR_BACKOFF_MS ?? 300;
593
607
  let rec = null; // the live SpeechRecognition instance (null between restarts / when stopped)
594
608
  let sttStart = 0; // performance.now() at the current turn's first activity (for onPartial/onFinal ms)
595
609
  let turn = ''; // committed (isFinal) text of the CURRENT turn, merged across recognizer restarts
@@ -600,6 +614,9 @@ export function makeWebSpeechSTT(opts) {
600
614
  let tailDefers = 0; // consecutive closes deferred waiting on an un-promoted tail (bounded, reset on new text)
601
615
  let want = false; // intent to listen — onend auto-restarts while true
602
616
  let muted = false; // mic off — stop the recognizer and don't restart
617
+ let lastError = ''; // error of the recognizer currently ending (drives the backoff/give-up path)
618
+ let errorRestarts = 0; // consecutive failed launches with no recognized text between them
619
+ let restartTimer = null; // pending backed-off relaunch
603
620
  // AEC'D CAPTURE (SpeechRecognition.start(audioTrack), spec'd + shipped in recent Chromium): by
604
621
  // default the engine opens its OWN mic, bypassing every protection the agent's pipeline has — the
605
622
  // browser AEC referenced against the WebRTC TTS loopback (voice-agent initAecLoopback) and noise
@@ -724,6 +741,7 @@ export function makeWebSpeechSTT(opts) {
724
741
  }
725
742
  final = final.trim(); interim = interim.trim();
726
743
  if (!turn && !interimTail && (final || interim)) sttStart = performance.now(); // first activity of a fresh turn
744
+ if (final || interim) errorRestarts = 0; // the engine is demonstrably working — forget past failures
727
745
  commitFinal(final); // de-overlaps the seam, then locks the newly-final words into the turn
728
746
  // Keep the live tail as its own state so a recognizer restart (which drops interim) can replay it
729
747
  // instead of blanking the draft. A frame that only finalized words (interim === '') clears it —
@@ -736,8 +754,11 @@ export function makeWebSpeechSTT(opts) {
736
754
  };
737
755
  r.onerror = (event) => {
738
756
  if (r !== rec) return;
739
- if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); }
740
- // 'no-speech' / 'aborted' / 'network' are recoverable — onend restarts; don't surface.
757
+ if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); return; }
758
+ // 'no-speech' / 'aborted' are the engine's normal idle churn — onend just relaunches, silently.
759
+ // Anything else ('network', 'audio-capture', …) is a FAILED launch: remembered here so onend
760
+ // backs off instead of hot-looping, and gives up out loud once the engine looks dead.
761
+ if (event.error !== 'no-speech' && event.error !== 'aborted') lastError = event.error;
741
762
  };
742
763
  r.onend = () => {
743
764
  if (r !== rec) return; // superseded instance's late onend — ignore
@@ -753,8 +774,19 @@ export function makeWebSpeechSTT(opts) {
753
774
  // untouched — if this really was end-of-turn, that timer fires on its own EOT_MS after the last
754
775
  // word (recognizer alive or not); if the user is still talking, the new recognizer's text
755
776
  // re-arms it. No flush decision belongs here — that's what makes the boundary restart-immune.
756
- if (want && !muted && !isClosed()) start();
757
- else onClose?.();
777
+ const err = lastError; lastError = '';
778
+ if (!(want && !muted && !isClosed())) { onClose?.(); return; }
779
+ if (!err) { errorRestarts = 0; start(); return; }
780
+ // A launch that only produced an error and no text: the engine may be permanently unable to run
781
+ // here (speech service unreachable, unreadable capture device). Retry with backoff, then stop —
782
+ // an endless silent restart loop is indistinguishable from "listening" and never recovers.
783
+ if (++errorRestarts > MAX_ERROR_RESTARTS) {
784
+ want = false;
785
+ onFatal?.(`Speech recognition: ${err} (${MAX_ERROR_RESTARTS} restarts failed — the browser's speech service is unreachable)`);
786
+ return;
787
+ }
788
+ onError?.(`Speech recognition: ${err} — retrying (${errorRestarts}/${MAX_ERROR_RESTARTS})`);
789
+ restartTimer = setTimeout(() => { restartTimer = null; if (want && !muted && !isClosed()) start(); }, Math.min(ERROR_BACKOFF_MS * 2 ** (errorRestarts - 1), 4000));
758
790
  };
759
791
  rec = r;
760
792
  let ok = false;
@@ -768,7 +800,7 @@ export function makeWebSpeechSTT(opts) {
768
800
 
769
801
  // Stop the current recognizer and drop all in-progress state (mute/close). The stopped recognizer's
770
802
  // late onend is stale (rec nulled → r !== rec), so it can't auto-restart.
771
- const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
803
+ const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; clearTimeout(restartTimer); restartTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; lastError = ''; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
772
804
 
773
805
  return {
774
806
  selfCapture: true, // owns its mic → the agent builds no capture pipeline
@@ -777,7 +809,7 @@ export function makeWebSpeechSTT(opts) {
777
809
  if (!Rec) { onFatal?.('Speech recognition not supported in this browser'); return; }
778
810
  // start() first, synchronously — Safari requires SpeechRecognition.start inside the click
779
811
  // gesture. The AEC'd track lands async right after and upgrades the recognizer at the seam.
780
- want = true; start();
812
+ want = true; errorRestarts = 0; start();
781
813
  ensureTrack();
782
814
  },
783
815
  // Mute: Web Speech captures its OWN mic (independent of the agent), so mute must actually stop it.
@@ -786,7 +818,7 @@ export function makeWebSpeechSTT(opts) {
786
818
  setEnabled(on) {
787
819
  muted = !on;
788
820
  if (muted) { teardown(); dropMic(); onPartial('', 0, ''); } // drop the draft AND the mic (privacy light off)
789
- else if (want && !rec) { start(); ensureTrack(); }
821
+ else if (want && !rec) { errorRestarts = 0; start(); ensureTrack(); }
790
822
  },
791
823
  close() { want = false; teardown(); dropMic(); },
792
824
  };
@@ -800,6 +832,14 @@ export const STT_PROVIDERS = {
800
832
  deepgram: makeDeepgramSTT,
801
833
  };
802
834
 
835
+ // Self-capturing providers own the mic AND their end-of-turn, so the agent builds no capture
836
+ // pipeline and runs no VAD for them. Readable WITHOUT constructing a session, so a host can decide
837
+ // up front whether prewarming the VAD/ONNX model is worth the download (it isn't, for these) and
838
+ // whether to expect 'vad' events at all. Kept on the factory so it can't drift from the session's
839
+ // own selfCapture flag.
840
+ makeWebSpeechSTT.selfCapture = true;
841
+ export const sttSelfCaptures = (provider) => !!STT_PROVIDERS[provider]?.selfCapture;
842
+
803
843
  // Web Speech only exists in Chromium/Safari — Firefox and Linux WebKitGTK webviews ship no
804
844
  // SpeechRecognition at all. There the default 'webspeech' provider can only fatal ("Speech
805
845
  // recognition not supported in this browser") and voice mode is simply dead, so resolve it to a
package/src/tuning.js CHANGED
@@ -71,6 +71,17 @@ export const TUNING = {
71
71
  // Raise if slow engines still split turns; lower to bound the wait harder.
72
72
  WEBSPEECH_TAIL_MAX_DEFERS: 3,
73
73
 
74
+ // Web Speech errors that aren't auth ('network', 'audio-capture', …) are retried by relaunching the
75
+ // recognizer — but if the engine is genuinely broken (browser's speech service unreachable, a fake
76
+ // audio device it can't read) EVERY relaunch fails instantly and the old code hot-looped forever,
77
+ // silently, looking like "listening but deaf". So consecutive failures with no recognized text in
78
+ // between are counted: each one waits longer (backoff below), and this many gives up with onFatal
79
+ // instead of spinning. Reset by any result. 'no-speech'/'aborted' are normal and don't count.
80
+ WEBSPEECH_MAX_ERROR_RESTARTS: 5,
81
+ // First retry delay after a failing recognizer; doubles per consecutive failure (capped at 4s), so
82
+ // a dead speech service costs a handful of tries over a few seconds instead of a spin loop.
83
+ WEBSPEECH_ERROR_BACKOFF_MS: 300,
84
+
74
85
  // Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
75
86
  // agent starts generating a reply for the probable final text — overlapping the LLM's first-token
76
87
  // latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
@@ -54,7 +54,7 @@ const CAPTURE_WORKLET_URL = typeof URL !== 'undefined' && URL.createObjectURL
54
54
  // + the host's conversation context (if any).
55
55
  // The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
56
56
  // English under the English persona — so state the selected STT language explicitly.
57
- const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
57
+ export const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
58
58
  const langRule = (lang) => {
59
59
  const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
60
60
  return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
@@ -267,7 +267,10 @@ export class VoiceAgent {
267
267
  const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
268
268
  this.stt = makeStt({
269
269
  apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
270
- sttTokenUrl, getToken: getSttToken, sttUsageUrl, eotThreshold: sttEotThreshold,
270
+ // `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
271
+ // whose credential differs per provider never has to re-derive the webspeech downgrade
272
+ // (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
273
+ sttTokenUrl, getToken: getSttToken && (() => getSttToken(this.sttProvider)), sttUsageUrl, eotThreshold: sttEotThreshold,
271
274
  onPartial: (text, ms, committed = '') => {
272
275
  const interim = `${committed} ${text}`.trim();
273
276
  this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector
@@ -1853,6 +1856,10 @@ const CDN_PIPER = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-tts-web.j
1853
1856
  // per-call module setup is exactly what we bypass (see PiperTTS._session).
1854
1857
  const CDN_PIPER_PHONEMIZE = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-o91UDS6e.js`;
1855
1858
  const CDN_PIPER_WASM = `${CDN}@diffusionstudio/piper-wasm@1.0.0/build/piper_phonemize`;
1859
+ // Resolved through the importmap at runtime. Kept in a variable, not a string literal, because a
1860
+ // literal specifier is resolved by Vite's import analysis BEFORE it checks /* @vite-ignore */ —
1861
+ // which fails since onnxruntime-web isn't installed as a dependency.
1862
+ const ORT_SPECIFIER = 'onnxruntime-web';
1856
1863
 
1857
1864
  // OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
1858
1865
  // models already downloaded through the lib are reused (and vice versa).
@@ -1921,7 +1928,7 @@ export class PiperTTS extends StreamingTTS {
1921
1928
  const path = lib.PATH_MAP[voiceId];
1922
1929
  if (!path) throw new Error(`unknown Piper voice: ${voiceId}`);
1923
1930
  const [ort, phonMod, cfgBlob, modelBlob] = await Promise.all([
1924
- import(/* webpackIgnore: true */ /* @vite-ignore */ 'onnxruntime-web'),
1931
+ import(/* webpackIgnore: true */ /* @vite-ignore */ ORT_SPECIFIER),
1925
1932
  import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
1926
1933
  opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
1927
1934
  opfsCachedFetch(`${lib.HF_BASE}/${path}`),