@todoforai/voiceloop 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -92,6 +92,10 @@
92
92
  .msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
93
93
  .msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
94
94
  .msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
95
+ /* An echoed reply is not an answer — mark it so the transcript can't be read as a model talking. */
96
+ .msg.ai.echoed { border-left:2px solid #e0a94a }
97
+ .msg.ai.echoed::after { content:'echo — no model'; display:block; margin-top:5px;
98
+ font-size:10px; letter-spacing:.04em; color:#e0a94a; opacity:.85 }
95
99
  .msg.interim { opacity:.5 }
96
100
  .msg .tail { opacity:.35 } /* generated, not yet spoken */
97
101
  .msg.tool { align-self:flex-start; background:#161d17; color:#8fc79a;
@@ -113,6 +117,11 @@
113
117
  label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
114
118
  #log { font:11px/1.5 ui-monospace,monospace; color:var(--dim);
115
119
  max-height:220px; overflow:auto; white-space:pre-wrap; padding-top:8px }
120
+ /* Which code is actually running — the demo must never let you mistake the browser's own
121
+ recognizer for voiceloop's pipeline, or an echo stand-in for a real LLM. */
122
+ #pipeline { font-size:11.5px; color:var(--dim); text-align:center; line-height:1.7; max-width:46ch }
123
+ #pipeline b { color:#c8d0da; font-weight:600 }
124
+ #pipeline .warn { color:#e0a94a }
116
125
  </style>
117
126
  </head>
118
127
  <body>
@@ -122,6 +131,7 @@
122
131
  </button>
123
132
  <div id="status">click to start · then just talk</div>
124
133
  <div id="lat"></div>
134
+ <div id="pipeline"></div>
125
135
 
126
136
  <div id="chat"></div>
127
137
 
@@ -139,18 +149,11 @@
139
149
  </select></label>
140
150
  <label>LLM url<input id="llmUrl" size="30" value="http://localhost:11434/v1/chat/completions"></label>
141
151
  <label>model<input id="model" size="14" value="llama3.2"></label>
142
- <label>STT<select id="sttProvider">
143
- <option value="webspeech">webspeech — free, in-browser</option>
144
- <option value="deepgram">deepgram</option>
145
- <option value="elevenlabs">elevenlabs</option>
146
- <option value="speechmatics">speechmatics</option>
147
- </select></label>
152
+ <label>STT<select id="sttProvider"></select></label>
148
153
  <label>STT token url<input id="sttTokenUrl" size="30" placeholder="your backend route (cloud STT only)"></label>
149
154
  <label>api key<input id="apiKey" size="18" placeholder="Bearer for LLM · X-API-Key for token url"></label>
150
- <label>lang<select id="sttLang">
151
- <option value="en">en</option><option value="hu">hu</option><option value="de">de</option>
152
- </select></label>
153
- <label>voice<select id="voice"><option value="en_US-joe-medium">en_US-joe-medium</option></select></label>
155
+ <label>lang<select id="sttLang"></select></label>
156
+ <label>voice<select id="voice"></select></label>
154
157
  <label>mic<select id="micDeviceId"><option value="">default</option></select></label>
155
158
  </div>
156
159
  <div class="bar">
@@ -164,15 +167,29 @@
164
167
  </main>
165
168
 
166
169
  <script type="module">
167
- import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice } from '../src/index.js';
170
+ import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice, STT_PROVIDERS, sttSelfCaptures, LANG_NAMES } from '../src/index.js';
168
171
 
169
- prewarmVoice(); // load VAD/WASM + build the Silero session while the page sits idle
172
+ // Load VAD/WASM + build the Silero session while the page sits idle — but ONLY for a provider that
173
+ // will actually use it. A selfCapture provider (webspeech) builds no pipeline, so prewarming there
174
+ // downloads an ONNX model to feed a detector that never runs.
175
+ const prewarmIfPipelined = () => { if (!sttSelfCaptures($('sttProvider').value)) prewarmVoice(); };
170
176
 
171
177
  const $ = id => document.getElementById(id);
178
+ const fill = (id, entries) => $(id).append(...entries.map(([value, textContent]) =>
179
+ Object.assign(document.createElement('option'), { value, textContent })));
180
+
181
+ // Every choice below is READ from the library, never retyped here: a provider added to STT_PROVIDERS
182
+ // or a language added to LANG_NAMES shows up in this page by itself. A hand-typed copy would be a
183
+ // second source of truth that silently rots — and a demo that lies about what the library supports.
184
+ fill('sttProvider', Object.keys(STT_PROVIDERS).map(p => [p, p === 'webspeech' ? 'webspeech — free, in-browser' : p]));
185
+ fill('sttLang', Object.entries(LANG_NAMES).map(([code, name]) => [code, `${code} — ${name}`]));
172
186
  const SETTINGS = ['mode','llmUrl','model','sttProvider','sttTokenUrl','apiKey','sttLang','voice','micDeviceId'];
173
187
  for (const id of SETTINGS) {
174
188
  const v = localStorage.getItem('vl.' + id); if (v) $(id).value = v;
175
- $(id).addEventListener('change', () => localStorage.setItem('vl.' + id, $(id).value));
189
+ $(id).addEventListener('change', () => {
190
+ localStorage.setItem('vl.' + id, $(id).value);
191
+ onSettingChanged(); // the banner must never describe the PREVIOUS choice
192
+ });
176
193
  }
177
194
  const log = (...a) => {
178
195
  const line = `${new Date().toISOString().slice(11,23)} ${a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ')}`;
@@ -181,7 +198,7 @@ const log = (...a) => {
181
198
 
182
199
  // ── transcript ──────────────────────────────────────────────────────────────
183
200
  const chat = $('chat');
184
- let interimEl = null, aiEl = null;
201
+ let interimEl = null, aiEl = null, vadOff = null;
185
202
  const toolEls = new Map();
186
203
  const bubble = (cls, text = '') => {
187
204
  const el = document.createElement('div');
@@ -197,18 +214,45 @@ const showLat = () => $('lat').textContent = [
197
214
  tAudio && `first sound ${tAudio - t0}ms`,
198
215
  ].filter(Boolean).join(' · ');
199
216
 
200
- const tts = new PiperTTS($('voice').value);
217
+ // undefined (not '') when unset — so PiperTTS applies its OWN default voice instead of this page
218
+ // re-declaring one. The picker is populated async below and adopts whatever it resolved to.
219
+ const tts = new PiperTTS(localStorage.getItem('vl.voice') || undefined);
201
220
  let agent = null;
202
221
 
203
- // Echo mode: a stand-in "LLM" that repeats the turn — exercises the entire
204
- // VAD→STT→…→TTS loop, barge-in included, with no key and no backend.
222
+ // Echo mode: a stand-in "LLM" that repeats the turn VERBATIM — exercises the entire
223
+ // VAD→STT→…→TTS loop, barge-in included, with no key and no backend. Nothing is added to the
224
+ // text: a reply that is exactly the transcript is the whole point, since any prefix would be
225
+ // this page's words in the reply and would blur what the recognizer actually heard. That it's
226
+ // an echo and not a model is stated in the pipeline banner and on the bubble itself.
205
227
  async function* echoLLM(history) {
206
228
  const last = [...history].reverse().find(m => m.role === 'user')?.content ?? '';
207
- for (const w of `You said: ${last}`.match(/\S+\s*/g) ?? []) {
229
+ for (const w of last.match(/\S+\s*/g) ?? []) {
208
230
  yield { text: w }; await new Promise(r => setTimeout(r, 25));
209
231
  }
210
232
  }
211
233
 
234
+ // ── what is ACTUALLY running ────────────────────────────────────────────────
235
+ // A demo that hides which engine served a reply is worse than no demo: a browser-native
236
+ // recognizer looks identical to voiceloop's pipeline, and an echo stand-in looks identical
237
+ // to a model. Both get named on screen, always.
238
+ const describePipeline = (a) => {
239
+ const echo = $('mode').value === 'echo';
240
+ // Before the first click there is no session, so ask the provider statically — reading
241
+ // `a?.stt?.selfCapture` off a null agent silently yields false and the banner claims the
242
+ // pipeline is running when it is exactly what gets bypassed.
243
+ const self = a?.stt ? !!a.stt.selfCapture : sttSelfCaptures($('sttProvider').value);
244
+ const stt = self
245
+ ? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span>`
246
+ : `<b>${$('sttProvider').value}</b> via voiceloop's pipeline — Silero VAD → feed → commit`;
247
+ const llm = echo
248
+ ? '<span class="warn"><b>echo</b> — no model: replies repeat your words back</span>'
249
+ : `<b>${$('model').value || 'your model'}</b> at ${new URL($('llmUrl').value).host}`;
250
+ return `STT: ${stt}<br>LLM: ${llm}`;
251
+ };
252
+ const showPipeline = () => { $('pipeline').innerHTML = describePipeline(agent); };
253
+ // Hoisted (function declaration) so the settings listeners above can call it before this point.
254
+ function onSettingChanged() { showPipeline(); prewarmIfPipelined(); }
255
+
212
256
  // A typed turn drives the same state machine without a mic — don't claim we're listening then.
213
257
  let micOn = false;
214
258
  const STATUS = {
@@ -233,6 +277,14 @@ function onEvent(e) {
233
277
  break;
234
278
  case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
235
279
  case 'stt':
280
+ // A selfCapture provider runs no agent VAD, so no 'vad' event ever arrives and the orb would
281
+ // sit dead still while the browser hears you perfectly — looking exactly like a broken mic.
282
+ // Live partials are the only "I hear you" signal on that path, so drive the pulse from them.
283
+ if (agent?.stt?.selfCapture) {
284
+ document.body.dataset.vad = e.turnComplete ? '0' : '1';
285
+ clearTimeout(vadOff);
286
+ if (!e.turnComplete) vadOff = setTimeout(() => { document.body.dataset.vad = '0'; }, 700);
287
+ }
236
288
  if (e.turnComplete) {
237
289
  interimEl?.remove(); interimEl = null;
238
290
  bubble('user', e.text);
@@ -251,7 +303,9 @@ function onEvent(e) {
251
303
  // `full` only exists once the TTS progress cursor is live — i.e. a clip is
252
304
  // actually playing. The 'speaking' state is just the first token, not sound.
253
305
  if (e.full != null && t0 && !tAudio) { tAudio = Date.now(); showLat(); }
254
- aiEl ??= bubble('ai');
306
+ // Tag the bubble (never the text) when no model produced it — the reply is a verbatim echo,
307
+ // and only the UI should say so.
308
+ aiEl ??= bubble($('mode').value === 'echo' ? 'ai echoed' : 'ai');
255
309
  aiEl.textContent = e.text; // spoken so far
256
310
  const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
257
311
  if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
@@ -286,7 +340,7 @@ const makeAgent = () => {
286
340
  onEvent,
287
341
  });
288
342
  a.setMuted(muted('muteMic')); a.setTtsMuted(muted('muteTts')); // the toggles survive a rebuild
289
- log('agent created; stt =', a.sttProvider);
343
+ log('agent created; stt =', a.sttProvider); showPipeline(); // now reflects the LIVE session, not just the form
290
344
  return a;
291
345
  };
292
346
 
@@ -294,6 +348,7 @@ const makeAgent = () => {
294
348
  const stop = () => {
295
349
  agent?.destroy(); agent = null; micOn = false;
296
350
  document.body.dataset.vad = '0'; setState('idle');
351
+ onSettingChanged(); // describe the restored settings, and prewarm only if they'll use the VAD
297
352
  };
298
353
  $('mic').onclick = async () => {
299
354
  unlockAudio(); // must be inside the gesture (autoplay policy)
@@ -338,8 +393,8 @@ for (const id of SETTINGS.filter(i => i !== 'voice'))
338
393
 
339
394
  // ── device + voice lists ────────────────────────────────────────────────────
340
395
  tts.voices().then(vs => {
341
- $('voice').innerHTML = vs.map(v => `<option value="${v.id}">${v.id}</option>`).join('');
342
- $('voice').value = localStorage.getItem('vl.voice') || 'en_US-joe-medium';
396
+ fill('voice', vs.map(v => [v.id, v.id]));
397
+ $('voice').value = localStorage.getItem('vl.voice') || tts.voiceId; // the library's own default, not a copy of it
343
398
  tts.setVoice($('voice').value);
344
399
  }).catch(e => log('voice list failed', String(e)));
345
400
  navigator.mediaDevices?.enumerateDevices().then(ds => {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@todoforai/voiceloop",
3
- "version": "0.1.3",
3
+ "version": "0.1.4",
4
4
  "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
package/src/index.d.ts CHANGED
@@ -91,8 +91,14 @@ export interface VoiceAgentOptions {
91
91
  * short-TTL token the backend mints (raw key stays server-side).
92
92
  * 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
93
93
  * the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
94
- sttProvider?: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
94
+ sttProvider?: SttProvider;
95
+ /** ONE endpoint that mints the token. When each provider needs a different route or credential,
96
+ * use `getSttToken` instead — it is handed the resolved provider, so the downgrade rule above
97
+ * stays in this library instead of being mirrored by every host. */
95
98
  sttTokenUrl?: string;
99
+ /** Mint a short-TTL STT token yourself, for auth that doesn't fit one POST route. Called with the
100
+ * provider actually RUNNING (post-downgrade), never the requested one. Wins over sttTokenUrl. */
101
+ getSttToken?: (provider: SttProvider) => Promise<SttToken>;
96
102
  sttUsageUrl?: string;
97
103
  sttUrl?: string;
98
104
  sttModel?: string;
@@ -130,13 +136,18 @@ export function warmVad(): Promise<void>;
130
136
  * mount), so a voice start only pays the WS handshake. Best-effort + deduped; no-op while a
131
137
  * fresh cached token remains.
132
138
  * `sttTokenUrl` is the EXACT endpoint that mints a Deepgram token (e.g. `/api/v1/stt/deepgram-token`),
133
- * the same value passed as the `sttTokenUrl` option — NOT an API base. Pass `getSttToken` here as
134
- * `getToken` if the host mints tokens itself. Warming with the wrong URL silently no-ops the
135
- * optimization: the real session just mints again on click. */
136
- export function warmDeepgramToken(sttTokenUrl: string, apiKey?: string, getToken?: () => Promise<string>): Promise<void>;
139
+ * the same value passed as the `sttTokenUrl` option — NOT an API base. A host that mints tokens
140
+ * itself passes its `getSttToken` as `getToken` and can skip the URL entirely (warm for the
141
+ * provider it will actually run: `() => getSttToken('deepgram')`). Warming with the wrong URL
142
+ * silently no-ops the optimization: the real session just mints again on click. */
143
+ export function warmDeepgramToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
137
144
  /** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
138
145
  * host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
139
146
  export const VOICE_SYSMSG: string;
147
+ /** Language codes the agent can instruct the LLM to reply in, code → English name. The `sttLang`
148
+ * values that actually steer the reply language — exported so a host builds its language picker
149
+ * from this instead of hand-listing a subset that silently rots as the map grows. */
150
+ export const LANG_NAMES: Record<string, string>;
140
151
 
141
152
 
142
153
  export class VoiceAgent {
@@ -242,7 +253,17 @@ export interface STTSession {
242
253
  }
243
254
  export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
244
255
 
245
- export const STT_PROVIDERS: Record<'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram', STTFactory>;
256
+ /** The built-in STT provider ids — the keys of STT_PROVIDERS. */
257
+ export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
258
+ /** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
259
+ * provider reports one (used to cache and pre-mint just before expiry). */
260
+ export interface SttToken { token: string; expires_in?: number }
261
+
262
+ export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
263
+ /** Does this provider own the mic and its own end-of-turn? Then the agent builds NO capture pipeline
264
+ * and runs no VAD for it — so it emits no `vad` events, and prewarming the VAD/ONNX model is a
265
+ * wasted download. Answerable without constructing a session, so a host can decide both up front. */
266
+ export function sttSelfCaptures(provider: string): boolean;
246
267
  export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
247
268
  export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
248
269
  export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
@@ -251,8 +272,9 @@ export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>):
251
272
  export const WEBSPEECH_FALLBACK: 'elevenlabs';
252
273
  export function webSpeechSupported(): boolean;
253
274
  /** The provider id actually used: 'webspeech' downgrades to WEBSPEECH_FALLBACK in a browser
254
- * without SpeechRecognition. Call it to label the UI ("using cloud recognition"). */
255
- export function resolveSttProvider(id: string): string;
275
+ * without SpeechRecognition. Call it to label the UI ("using cloud recognition") — but NOT to pick
276
+ * a token route: `getSttToken` is already handed the resolved id. */
277
+ export function resolveSttProvider(id: string): SttProvider;
256
278
 
257
279
  // ── TTS ──────────────────────────────────────────────────────────────────────────────────────
258
280
  /** Base class for streaming TTS engines: sentence-splits the incoming token stream, synthesizes
package/src/index.js CHANGED
@@ -2,11 +2,11 @@ export {
2
2
  VoiceAgent, StreamingTTS, PiperTTS,
3
3
  unlockAudio, warmVad,
4
4
  isSelfEcho, novelChars, toolResultText,
5
- VOICE_SYSMSG,
5
+ VOICE_SYSMSG, LANG_NAMES,
6
6
  } from './voice-agent.js';
7
7
  export {
8
8
  STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
9
- makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT,
9
+ makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, sttSelfCaptures,
10
10
  warmDeepgramToken,
11
11
  } from './stt.js';
12
12
  export { ElevenLabsTTS } from './tts-elevenlabs.js';
package/src/stt.js CHANGED
@@ -39,8 +39,10 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
39
39
  // AUTH (cloud providers): the raw provider key must stay server-side. Either
40
40
  // • `sttTokenUrl` — your backend route that mints a short-TTL token: POST, optional X-API-Key
41
41
  // (`apiKey`), responds { token, expires_in? }; or
42
- // • `getToken` — an async () => ({ token, expires_in? }) callback, when your auth doesn't fit
43
- // that one route shape. Takes precedence over sttTokenUrl.
42
+ // • `getToken` — an async (provider) => ({ token, expires_in? }) callback, when your auth doesn't
43
+ // fit that one route shape. Takes precedence over sttTokenUrl. VoiceAgent calls it with the
44
+ // provider actually RUNNING (post-downgrade), so a host whose credential differs per provider
45
+ // writes one function instead of mirroring the downgrade rule.
44
46
 
45
47
  // ── Shared socket-lifecycle helpers ─────────────────────────────────────────
46
48
  // The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
@@ -57,7 +59,14 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
57
59
  async function mintSttToken(url, apiKey, onFatal, getToken) {
58
60
  if (getToken) {
59
61
  try {
60
- const body = await getToken();
62
+ // HOST code on the critical path: bound it exactly like the built-in fetch below, or a minter
63
+ // that never settles stalls voice start forever with no error. We can't cancel the host's
64
+ // promise — we just stop waiting on it (it may still resolve into the void, which is fine:
65
+ // a token is idempotent to mint).
66
+ const body = await Promise.race([
67
+ getToken(),
68
+ new Promise((_, rej) => setTimeout(() => rej(new Error('timed out')), TUNING.STT.tokenFetchTimeoutMs)),
69
+ ]);
61
70
  if (!body?.token) throw new Error('getToken returned no token');
62
71
  return body;
63
72
  } catch (e) { onFatal(`STT token: ${e.message}`); return null; }
@@ -589,10 +598,12 @@ export function makeDeepgramSTT(opts) {
589
598
  // result window). Blindly appending would double words ("hello there" + "there how" → "there there").
590
599
  // Every final is de-overlapped against the turn-so-far before it's appended.
591
600
  export function makeWebSpeechSTT(opts) {
592
- const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onFatal, onClose, isClosed } = opts;
601
+ const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
593
602
  const Rec = typeof window !== 'undefined' && (window.SpeechRecognition || window.webkitSpeechRecognition);
594
603
  const EOT_MS = TUNING.WEBSPEECH_EOT_MS ?? 900;
595
604
  const TAIL_MAX_DEFERS = TUNING.WEBSPEECH_TAIL_MAX_DEFERS ?? 3;
605
+ const MAX_ERROR_RESTARTS = TUNING.WEBSPEECH_MAX_ERROR_RESTARTS ?? 5;
606
+ const ERROR_BACKOFF_MS = TUNING.WEBSPEECH_ERROR_BACKOFF_MS ?? 300;
596
607
  let rec = null; // the live SpeechRecognition instance (null between restarts / when stopped)
597
608
  let sttStart = 0; // performance.now() at the current turn's first activity (for onPartial/onFinal ms)
598
609
  let turn = ''; // committed (isFinal) text of the CURRENT turn, merged across recognizer restarts
@@ -603,6 +614,9 @@ export function makeWebSpeechSTT(opts) {
603
614
  let tailDefers = 0; // consecutive closes deferred waiting on an un-promoted tail (bounded, reset on new text)
604
615
  let want = false; // intent to listen — onend auto-restarts while true
605
616
  let muted = false; // mic off — stop the recognizer and don't restart
617
+ let lastError = ''; // error of the recognizer currently ending (drives the backoff/give-up path)
618
+ let errorRestarts = 0; // consecutive failed launches with no recognized text between them
619
+ let restartTimer = null; // pending backed-off relaunch
606
620
  // AEC'D CAPTURE (SpeechRecognition.start(audioTrack), spec'd + shipped in recent Chromium): by
607
621
  // default the engine opens its OWN mic, bypassing every protection the agent's pipeline has — the
608
622
  // browser AEC referenced against the WebRTC TTS loopback (voice-agent initAecLoopback) and noise
@@ -727,6 +741,7 @@ export function makeWebSpeechSTT(opts) {
727
741
  }
728
742
  final = final.trim(); interim = interim.trim();
729
743
  if (!turn && !interimTail && (final || interim)) sttStart = performance.now(); // first activity of a fresh turn
744
+ if (final || interim) errorRestarts = 0; // the engine is demonstrably working — forget past failures
730
745
  commitFinal(final); // de-overlaps the seam, then locks the newly-final words into the turn
731
746
  // Keep the live tail as its own state so a recognizer restart (which drops interim) can replay it
732
747
  // instead of blanking the draft. A frame that only finalized words (interim === '') clears it —
@@ -739,8 +754,11 @@ export function makeWebSpeechSTT(opts) {
739
754
  };
740
755
  r.onerror = (event) => {
741
756
  if (r !== rec) return;
742
- if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); }
743
- // 'no-speech' / 'aborted' / 'network' are recoverable — onend restarts; don't surface.
757
+ if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); return; }
758
+ // 'no-speech' / 'aborted' are the engine's normal idle churn — onend just relaunches, silently.
759
+ // Anything else ('network', 'audio-capture', …) is a FAILED launch: remembered here so onend
760
+ // backs off instead of hot-looping, and gives up out loud once the engine looks dead.
761
+ if (event.error !== 'no-speech' && event.error !== 'aborted') lastError = event.error;
744
762
  };
745
763
  r.onend = () => {
746
764
  if (r !== rec) return; // superseded instance's late onend — ignore
@@ -756,8 +774,19 @@ export function makeWebSpeechSTT(opts) {
756
774
  // untouched — if this really was end-of-turn, that timer fires on its own EOT_MS after the last
757
775
  // word (recognizer alive or not); if the user is still talking, the new recognizer's text
758
776
  // re-arms it. No flush decision belongs here — that's what makes the boundary restart-immune.
759
- if (want && !muted && !isClosed()) start();
760
- else onClose?.();
777
+ const err = lastError; lastError = '';
778
+ if (!(want && !muted && !isClosed())) { onClose?.(); return; }
779
+ if (!err) { errorRestarts = 0; start(); return; }
780
+ // A launch that only produced an error and no text: the engine may be permanently unable to run
781
+ // here (speech service unreachable, unreadable capture device). Retry with backoff, then stop —
782
+ // an endless silent restart loop is indistinguishable from "listening" and never recovers.
783
+ if (++errorRestarts > MAX_ERROR_RESTARTS) {
784
+ want = false;
785
+ onFatal?.(`Speech recognition: ${err} (${MAX_ERROR_RESTARTS} restarts failed — the browser's speech service is unreachable)`);
786
+ return;
787
+ }
788
+ onError?.(`Speech recognition: ${err} — retrying (${errorRestarts}/${MAX_ERROR_RESTARTS})`);
789
+ restartTimer = setTimeout(() => { restartTimer = null; if (want && !muted && !isClosed()) start(); }, Math.min(ERROR_BACKOFF_MS * 2 ** (errorRestarts - 1), 4000));
761
790
  };
762
791
  rec = r;
763
792
  let ok = false;
@@ -771,7 +800,7 @@ export function makeWebSpeechSTT(opts) {
771
800
 
772
801
  // Stop the current recognizer and drop all in-progress state (mute/close). The stopped recognizer's
773
802
  // late onend is stale (rec nulled → r !== rec), so it can't auto-restart.
774
- const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
803
+ const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; clearTimeout(restartTimer); restartTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; lastError = ''; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
775
804
 
776
805
  return {
777
806
  selfCapture: true, // owns its mic → the agent builds no capture pipeline
@@ -780,7 +809,7 @@ export function makeWebSpeechSTT(opts) {
780
809
  if (!Rec) { onFatal?.('Speech recognition not supported in this browser'); return; }
781
810
  // start() first, synchronously — Safari requires SpeechRecognition.start inside the click
782
811
  // gesture. The AEC'd track lands async right after and upgrades the recognizer at the seam.
783
- want = true; start();
812
+ want = true; errorRestarts = 0; start();
784
813
  ensureTrack();
785
814
  },
786
815
  // Mute: Web Speech captures its OWN mic (independent of the agent), so mute must actually stop it.
@@ -789,7 +818,7 @@ export function makeWebSpeechSTT(opts) {
789
818
  setEnabled(on) {
790
819
  muted = !on;
791
820
  if (muted) { teardown(); dropMic(); onPartial('', 0, ''); } // drop the draft AND the mic (privacy light off)
792
- else if (want && !rec) { start(); ensureTrack(); }
821
+ else if (want && !rec) { errorRestarts = 0; start(); ensureTrack(); }
793
822
  },
794
823
  close() { want = false; teardown(); dropMic(); },
795
824
  };
@@ -803,6 +832,14 @@ export const STT_PROVIDERS = {
803
832
  deepgram: makeDeepgramSTT,
804
833
  };
805
834
 
835
+ // Self-capturing providers own the mic AND their end-of-turn, so the agent builds no capture
836
+ // pipeline and runs no VAD for them. Readable WITHOUT constructing a session, so a host can decide
837
+ // up front whether prewarming the VAD/ONNX model is worth the download (it isn't, for these) and
838
+ // whether to expect 'vad' events at all. Kept on the factory so it can't drift from the session's
839
+ // own selfCapture flag.
840
+ makeWebSpeechSTT.selfCapture = true;
841
+ export const sttSelfCaptures = (provider) => !!STT_PROVIDERS[provider]?.selfCapture;
842
+
806
843
  // Web Speech only exists in Chromium/Safari — Firefox and Linux WebKitGTK webviews ship no
807
844
  // SpeechRecognition at all. There the default 'webspeech' provider can only fatal ("Speech
808
845
  // recognition not supported in this browser") and voice mode is simply dead, so resolve it to a
package/src/tuning.js CHANGED
@@ -71,6 +71,17 @@ export const TUNING = {
71
71
  // Raise if slow engines still split turns; lower to bound the wait harder.
72
72
  WEBSPEECH_TAIL_MAX_DEFERS: 3,
73
73
 
74
+ // Web Speech errors that aren't auth ('network', 'audio-capture', …) are retried by relaunching the
75
+ // recognizer — but if the engine is genuinely broken (browser's speech service unreachable, a fake
76
+ // audio device it can't read) EVERY relaunch fails instantly and the old code hot-looped forever,
77
+ // silently, looking like "listening but deaf". So consecutive failures with no recognized text in
78
+ // between are counted: each one waits longer (backoff below), and this many gives up with onFatal
79
+ // instead of spinning. Reset by any result. 'no-speech'/'aborted' are normal and don't count.
80
+ WEBSPEECH_MAX_ERROR_RESTARTS: 5,
81
+ // First retry delay after a failing recognizer; doubles per consecutive failure (capped at 4s), so
82
+ // a dead speech service costs a handful of tries over a few seconds instead of a spin loop.
83
+ WEBSPEECH_ERROR_BACKOFF_MS: 300,
84
+
74
85
  // Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
75
86
  // agent starts generating a reply for the probable final text — overlapping the LLM's first-token
76
87
  // latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
@@ -54,7 +54,7 @@ const CAPTURE_WORKLET_URL = typeof URL !== 'undefined' && URL.createObjectURL
54
54
  // + the host's conversation context (if any).
55
55
  // The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
56
56
  // English under the English persona — so state the selected STT language explicitly.
57
- const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
57
+ export const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
58
58
  const langRule = (lang) => {
59
59
  const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
60
60
  return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
@@ -267,7 +267,10 @@ export class VoiceAgent {
267
267
  const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
268
268
  this.stt = makeStt({
269
269
  apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
270
- sttTokenUrl, getToken: getSttToken, sttUsageUrl, eotThreshold: sttEotThreshold,
270
+ // `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
271
+ // whose credential differs per provider never has to re-derive the webspeech downgrade
272
+ // (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
273
+ sttTokenUrl, getToken: getSttToken && (() => getSttToken(this.sttProvider)), sttUsageUrl, eotThreshold: sttEotThreshold,
271
274
  onPartial: (text, ms, committed = '') => {
272
275
  const interim = `${committed} ${text}`.trim();
273
276
  this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector