@todoforai/voiceloop 0.1.26 → 0.1.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -163,7 +163,7 @@ iOS ducks the speakers while the recognizer holds the mic — which is exactly w
163
163
  | `elevenlabs` | VAD-gated | Scribe v2 realtime, great accuracy |
164
164
  | `deepgram` | **Native** (Flux) | Provider's own turn model hears the full stream |
165
165
  | `speechmatics` | VAD-gated | Cheapest per second, locked-words-only finals |
166
- | `soniox` | **Native** (stt-rt-v5) | Provider's own endpointing, 60+ languages incl. Hungarian, $0.12/hr |
166
+ | `soniox` | **Native** (stt-rt-v6) | Provider's own endpointing, 60+ languages incl. Hungarian, $0.12/hr |
167
167
 
168
168
  ## Bring your own LLM
169
169
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@todoforai/voiceloop",
3
- "version": "0.1.26",
3
+ "version": "0.1.29",
4
4
  "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
package/src/index.d.ts CHANGED
@@ -20,7 +20,8 @@ export type VoiceAgentEvent =
20
20
  | { type: 'tool'; name: string; args: Record<string, unknown>; result?: unknown; id?: string; running?: true }
21
21
  | { type: 'echo'; text: string } // dropped self-echo (the agent's own TTS leaked into the mic) — diagnostics only
22
22
  | { type: 'diag'; diag: 'aec-fallback'; message: string } // once/session: AEC loopback failed → TTS plays un-cancelled (self-echo only text-filtered)
23
- | { type: 'diag'; diag: 'presynth'; text: string } // a reply's first clip was synthesized speculatively (latency diagnostics)
23
+ | { type: 'diag'; diag: 'presynth'; text: string }
24
+ | { type: 'diag'; diag: 'tts_ws_fallback' | 'tts_ws_error'; error: string } // ElevenLabsTTS socket path: a clip fell back to HTTP / the provider sent an error frame // a reply's first clip was synthesized speculatively (latency diagnostics)
24
25
  // Did that speculation pay off? hit = the real reply opened with the presynthesized text (clip
25
26
  // reused, latency saved); miss = it didn't (clip discarded). `pre` is what we had guessed.
26
27
  | { type: 'diag'; diag: 'presynth-hit' | 'presynth-miss'; text: string; pre: string }
@@ -99,6 +100,8 @@ export interface VoiceTTS {
99
100
  * `scope` is that prefix PLUS the whole clip now playing: the cursor lags real audio, so echo
100
101
  * suppression judges against this upper bound rather than the spoken text. */
101
102
  setOnProgress?(fn: TtsProgress | null): void;
103
+ /** Release held resources (sockets); VoiceAgent.destroy calls it. */
104
+ close?(): void;
102
105
  }
103
106
 
104
107
  /** Host-supplied mic capture (React Native, Electron, tests) — replaces getUserMedia + AudioWorklet +
@@ -149,7 +152,7 @@ export interface VoiceAgentOptions {
149
152
  /** STT backend: 'webspeech' (default — the browser's own on-device SpeechRecognition; free, no
150
153
  * token, no socket, captures its own mic) | 'speechmatics' | 'elevenlabs' (Scribe) | 'deepgram'
151
154
  * (Flux — native model-based end-of-turn detection; the agent feeds it the full mic stream and
152
- * lets it close turns itself) | 'soniox' (stt-rt-v5 — same native-endpointing shape as Flux). The cloud ones connect the browser DIRECTLY to the provider with a
155
+ * lets it close turns itself) | 'soniox' (stt-rt-v6 — same native-endpointing shape as Flux). The cloud ones connect the browser DIRECTLY to the provider with a
153
156
  * short-TTL token the backend mints (raw key stays server-side).
154
157
  * 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
155
158
  * the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
@@ -341,8 +344,10 @@ export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSe
341
344
  /** The built-in STT provider ids — the keys of STT_PROVIDERS. */
342
345
  export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram' | 'soniox';
343
346
  /** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
344
- * provider reports one (used to cache and pre-mint just before expiry). */
345
- export interface SttToken { token: string; expires_in?: number }
347
+ * provider reports one (used to cache and pre-mint just before expiry). `url`: the provider WS
348
+ * endpoint matching the token's region (Soniox keys are region-bound, US/EU); overrides the
349
+ * built-in default, an explicit `sttUrl` still wins. */
350
+ export interface SttToken { token: string; expires_in?: number; url?: string }
346
351
 
347
352
  export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
348
353
  /** Does this provider own the mic and its own end-of-turn? Then the agent builds NO capture pipeline
@@ -383,8 +388,15 @@ export class StreamingTTS implements VoiceTTS {
383
388
  * talks to ElevenLabs directly and is dev-only. `headers` authenticate proxy requests
384
389
  * (e.g. { 'X-API-Key': ... }) and are never sent on a direct ElevenLabs call. */
385
390
  export class ElevenLabsTTS extends StreamingTTS {
386
- constructor(options?: { ttsUrl?: string; apiKey?: string; voiceId?: string; modelId?: string; format?: string; headers?: Record<string, string>; languageCode?: string });
391
+ /** `getWsToken` enables the persistent browser↔ElevenLabs WebSocket path (mint a `tts_websocket`
392
+ * single-use token server-side); `usageUrl` receives POST { chars, voice_id, model_id } per
393
+ * socket clip so the backend can meter it. Without `getWsToken` every clip goes over HTTP via
394
+ * `ttsUrl`; with it, HTTP stays the fallback for any socket failure. */
395
+ constructor(options?: { ttsUrl?: string; apiKey?: string; voiceId?: string; modelId?: string; format?: string; headers?: Record<string, string>; languageCode?: string;
396
+ getWsToken?: () => Promise<{ token?: string }>; usageUrl?: string });
387
397
  warm(): Promise<void>;
398
+ /** Close the socket (VoiceAgent.destroy calls it). */
399
+ close(): void;
388
400
  }
389
401
 
390
402
  // ── LLM ──────────────────────────────────────────────────────────────────────────────────────
package/src/stt.js CHANGED
@@ -59,6 +59,20 @@ const newSocket = (url, protocols) => {
59
59
  // provider actually RUNNING (post-downgrade), so a host whose credential differs per provider
60
60
  // writes one function instead of mirroring the downgrade rule.
61
61
 
62
+ // A TRANSIENT close (going away / abnormal / server restart) on a socket the server already
63
+ // answered on — network handoff, phone screen-off / frozen tab, proxy idle cut, all seen as 1006 —
64
+ // is nothing the user can act on: the continuous providers reopen on the very next mic frame, so
65
+ // it's a console warning, not a chat error row. Anything else (a socket cut before any answer, a
66
+ // policy/auth/protocol close) is still reported, so a broken setup can't go silently deaf.
67
+ // onerror alone is never reported: the browser always follows it with onclose (which has the code)
68
+ // — reporting both doubled every drop into "STT socket error" + "STT closed 1006".
69
+ const TRANSIENT_CLOSE = new Set([1001, 1005, 1006, 1011, 1012, 1013, 1014]);
70
+ function reportClose(ev, live, onError) {
71
+ if (ev.code === 1000) return;
72
+ const msg = `STT closed ${ev.code}: ${ev.reason || 'no reason'}`;
73
+ if (live && TRANSIENT_CLOSE.has(ev.code)) console.warn(`${msg} — reconnecting`); else onError(msg);
74
+ }
75
+
62
76
  // ── Shared socket-lifecycle helpers ─────────────────────────────────────────
63
77
  // The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
64
78
  // Deepgram's cumulative-turn + token cache, ElevenLabs' simple commit) — those stay separate on
@@ -162,8 +176,8 @@ export function makeElevenLabsSTT(opts) {
162
176
  sttStart = performance.now();
163
177
  ws = newSocket(`${sttUrl}?${p}`);
164
178
  ws.onopen = () => { const q = outbox; outbox = []; for (const it of q) it === 'commit' ? sendCommit() : sendAudio(it); };
165
- ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('STT ws error', ev); } };
166
- ws.onclose = ev => { usage.flush(); onClose?.(); if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); console.error('STT ws closed', ev.code, ev.reason); } };
179
+ ws.onerror = ev => { if (!isClosed()) console.warn('STT ws error', ev); };
180
+ ws.onclose = ev => { usage.flush(); onClose?.(); if (!isClosed() && ev.code !== 1000) onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); };
167
181
  ws.onmessage = ev => {
168
182
  const m = JSON.parse(ev.data);
169
183
  const ms = Math.round(performance.now() - sttStart);
@@ -307,6 +321,7 @@ export function makeSpeechmaticsSTT(opts) {
307
321
  sttStart = performance.now();
308
322
  ws = newSocket(`${SM_URL}?jwt=${encodeURIComponent(body.token)}`);
309
323
  ws.binaryType = 'arraybuffer';
324
+ let opened = false; // server has answered → session is live (see reportClose)
310
325
  ws.onopen = () => {
311
326
  // Declare the audio format + transcription config, then flush buffered audio once started.
312
327
  ws.send(JSON.stringify({
@@ -328,12 +343,13 @@ export function makeSpeechmaticsSTT(opts) {
328
343
  }));
329
344
  };
330
345
  const sock = ws; // this handler's own socket — feed() may reopen while it's still CLOSING
331
- ws.onerror = ev => { if (sock === ws && !isClosed()) { onError('STT socket error'); console.error('SM ws error', ev); } };
346
+ ws.onerror = ev => { if (sock === ws && !isClosed()) console.warn('SM ws error', ev); };
332
347
  // Guard on sock === ws: a superseded socket's late onclose must not kill the NEW socket's
333
348
  // keepalive (feed() reopens on readyState>=2, i.e. while the old one is still CLOSING).
334
- ws.onclose = ev => { if (sock === ws) { clearInterval(keepalive); keepalive = null; } usage.flush(); onClose?.(); if (!isClosed() && sock === ws && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); } };
349
+ ws.onclose = ev => { if (sock === ws) { clearInterval(keepalive); keepalive = null; } usage.flush(); onClose?.(); if (!isClosed() && sock === ws) reportClose(ev, opened, onError); };
335
350
  ws.onmessage = ev => {
336
351
  if (sock !== ws) return; // stale message from a superseded socket (its clock is dead)
352
+ opened = true;
337
353
  const m = JSON.parse(ev.data);
338
354
  const ms = Math.round(performance.now() - sttStart);
339
355
  switch (m.message) {
@@ -535,16 +551,18 @@ export function makeDeepgramSTT(opts) {
535
551
  // /v1/auth/grant). The token only needs to be valid at handshake time; the socket persists.
536
552
  ws = newSocket(`${DG_URL}?${p}`, ['bearer', token]);
537
553
  ws.binaryType = 'arraybuffer';
554
+ let opened = false; // server has answered → session is live (see reportClose)
538
555
  ws.onopen = () => { const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c); };
539
- ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('DG ws error', ev); } };
556
+ ws.onerror = ev => { if (!isClosed()) console.warn('DG ws error', ev); };
540
557
  ws.onclose = ev => {
541
558
  usage.flush(); onClose?.();
542
559
  // Handshake rejected (auth) — likely a stale cached token (clock drift / revocation): drop the
543
560
  // cache so the next open() re-mints instead of failing forever within the cached window.
544
561
  if (ev.code === 1008 || ev.code === 4001 || ev.code === 1002) { dgToken = null; dgTokenExp = 0; }
545
- if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); }
562
+ if (!isClosed()) reportClose(ev, opened, onError);
546
563
  };
547
564
  ws.onmessage = ev => {
565
+ opened = true;
548
566
  if (typeof ev.data !== 'string') return;
549
567
  const m = JSON.parse(ev.data);
550
568
  // Continuous stream: the socket lives for the whole session, so the ms clock re-anchors at
@@ -607,10 +625,12 @@ export function makeDeepgramSTT(opts) {
607
625
  };
608
626
  }
609
627
 
610
- // ── Soniox realtime (stt-rt-v5) ──────────────────────────────────────────────
628
+ // ── Soniox realtime (stt-rt-v6) ──────────────────────────────────────────────
611
629
  // Browser↔Soniox direct WS. Auth: the backend mints a temporary API key (POST /v1/auth/temporary-api-key,
612
- // usage_type transcribe_websocket) and the browser sends it in the FIRST JSON config message — no
613
- // URL/subprotocol auth. Audio is raw binary pcm_s16le @16k after the config frame.
630
+ // usage_type transcribe_websocket) and the browser sends it WITH THE CONNECTION, as the WebSocket
631
+ // subprotocol pair ['soniox-api-key', <key>] (a browser can't set an Authorization header). Soniox
632
+ // refuses a key sent only in the config message from 2027-01-15. Audio is raw binary pcm_s16le @16k
633
+ // after the config frame.
614
634
  //
615
635
  // Turn model — TOKENS, not transcripts: every message carries a `tokens` array; `is_final: true`
616
636
  // tokens arrive exactly once and are appended, `is_final: false` tokens are the whole current
@@ -623,8 +643,11 @@ const SONIOX_URL = 'wss://stt-rt.soniox.com/transcribe-websocket';
623
643
  const SONIOX_KEEPALIVE_MS = 10000; // server closes after >20s with no audio/keepalive
624
644
  // Soniox errors arrive as { error_code: <http status>, error_type, error_message }. 4xx won't fix
625
645
  // itself with a reconnect (bad key/config, balance exhausted — seen live as 402): fatal, so the host
626
- // stops instead of the continuous feed re-minting + reconnecting forever. 5xx/429 stay recoverable.
627
- const sonioxFatal = (code) => code >= 400 && code < 500 && code !== 429;
646
+ // stops instead of the continuous feed re-minting + reconnecting forever. 5xx/429 stay recoverable,
647
+ // and so do the per-STREAM 4xx a fresh socket fixes: 408 request_timeout (audio feed stalled, e.g.
648
+ // iOS backgrounding — seen live after ~8min), 413 max_duration_reached, 403 temp_api_key_session_expired.
649
+ const SONIOX_STREAM_ERRORS = new Set(['request_timeout', 'max_duration_reached', 'temp_api_key_session_expired']);
650
+ const sonioxFatal = (code, type) => code >= 400 && code < 500 && code !== 429 && code !== 408 && !SONIOX_STREAM_ERRORS.has(type);
628
651
  // Soniox's `<end>` is ACOUSTIC-LEANING: it fires ~250ms into any pause, including mid-sentence
629
652
  // ones — measured on the bench hesitation audio it closed "Hi, I need some help with—" / "Um." /
630
653
  // "with a dinner reservation." as three turns (Flux keeps that as one). Its own signal for "not
@@ -634,14 +657,16 @@ const sonioxFatal = (code) => code >= 400 && code < 500 && code !== 429;
634
657
  const SONIOX_HOLD_MS = 1500;
635
658
  // Same token cache/prewarm as Deepgram (see dgToken): the temporary-key mint + WS open cost ~2s on
636
659
  // the first utterance when paid cold on the mic click.
637
- let sxToken = null, sxTokenExp = 0, sxMintPromise = null;
660
+ // The token body may carry `url`: a Soniox key is bound to its region (US/EU/…), so the backend that
661
+ // minted it names the matching WS host and the client never hardcodes a region.
662
+ let sxToken = null, sxTokenExp = 0, sxUrl = '', sxMintPromise = null;
638
663
  // One acquisition path for prewarm AND open(): a click landing mid-prewarm joins that mint
639
664
  // instead of starting a second one. `slackMs` = how much life the cached key must have left.
640
665
  function acquireSonioxToken(sttTokenUrl, apiKey, onFatal, getToken, slackMs) {
641
666
  if (sxToken && performance.now() < sxTokenExp - slackMs) return Promise.resolve(sxToken);
642
667
  return (sxMintPromise ??= (async () => {
643
668
  const body = await mintSttToken(sttTokenUrl, apiKey, onFatal, getToken);
644
- if (body) { sxToken = body.token; sxTokenExp = performance.now() + (body.expires_in ?? 300) * 1000; }
669
+ if (body) { sxToken = body.token; sxUrl = body.url || ''; sxTokenExp = performance.now() + (body.expires_in ?? 300) * 1000; }
645
670
  return body?.token ?? null;
646
671
  })().finally(() => { sxMintPromise = null; }));
647
672
  }
@@ -652,8 +677,7 @@ const sonioxContinues = (t) => /[—–-]$/.test(t) || /^(um+|uh+|hmm+|er+|ah+)[
652
677
  export function makeSonioxSTT(opts) {
653
678
  const { apiKey, sttModel, sttLang = 'en', keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
654
679
  onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
655
- const sttUrl = opts.sttUrl || SONIOX_URL;
656
- const model = sttModel || 'stt-rt-v5';
680
+ const model = sttModel || 'stt-rt-v6';
657
681
  let ws = null, opening = false, outbox = [], outboxSamples = 0, keepalive = null;
658
682
  // Turn state. Soniox finals are APPEND-ONLY (no cumulative restatement like Flux), so a socket
659
683
  // boundary must wipe them: a stale prefix can't be repaired by the next stream.
@@ -712,12 +736,13 @@ export function makeSonioxSTT(opts) {
712
736
  // so the host's closing latch always resolves.
713
737
  resetText(); dropHold(); // a held cut-off phrase belongs to the dead stream too
714
738
  clearInterval(keepalive); keepalive = null; // the superseded socket's timer must not outlive it
715
- const sock = ws = newSocket(sttUrl);
739
+ const sock = ws = newSocket(opts.sttUrl || sxUrl || SONIOX_URL, ['soniox-api-key', token]);
716
740
  sock.binaryType = 'arraybuffer';
741
+ let opened = false; // server has answered → session is live (see reportClose)
717
742
  sock.onopen = () => {
718
743
  if (sock !== ws) return;
719
744
  const config = {
720
- api_key: token, model, audio_format: 'pcm_s16le', sample_rate: 16000, num_channels: 1,
745
+ model, audio_format: 'pcm_s16le', sample_rate: 16000, num_channels: 1,
721
746
  enable_endpoint_detection: true,
722
747
  // Level 2 measured (bench smalltalk, 5 runs): 1042 → 768 ms voice→voice, 0 mid-sentence
723
748
  // splits; level 3 was faster still (726) but split "Hey there. | Can you hear me?" every run.
@@ -731,21 +756,25 @@ export function makeSonioxSTT(opts) {
731
756
  if (awaitingFinal) sock.send(JSON.stringify({ type: 'finalize' })); // a commit() that landed while connecting
732
757
  keepalive = setInterval(() => { if (sock.readyState === 1) sock.send(JSON.stringify({ type: 'keepalive' })); }, SONIOX_KEEPALIVE_MS);
733
758
  };
734
- sock.onerror = ev => { if (sock === ws && !isClosed()) { onError('STT socket error'); console.error('Soniox ws error', ev); } };
759
+ sock.onerror = ev => { if (sock === ws && !isClosed()) console.warn('Soniox ws error', ev); };
735
760
  sock.onclose = ev => {
736
761
  usage.flush(); onClose?.();
737
762
  if (sock !== ws) return; // superseded socket's late close: the new one's state is untouched
738
763
  clearInterval(keepalive); keepalive = null;
739
- if (!isClosed() && ev.code !== 1000) onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`);
764
+ if (!isClosed()) reportClose(ev, opened, onError);
740
765
  };
741
766
  sock.onmessage = ev => {
742
767
  if (sock !== ws || isClosed() || typeof ev.data !== 'string') return;
768
+ opened = true;
743
769
  const m = JSON.parse(ev.data);
744
770
  if (m.error_code || m.error_message) {
745
771
  const msg = `${m.error_type || m.error_code || 'error'}: ${m.error_message || ''}`;
746
- console.error('Soniox error msg', m);
747
- if (+m.error_code === 401) { sxToken = null; sxTokenExp = 0; } // a cached key the server rejects is dead
748
- if (sonioxFatal(+m.error_code)) onFatal(`STT ${msg}`); else onError(msg);
772
+ const code = +m.error_code, fatal = sonioxFatal(code, m.error_type);
773
+ (fatal ? console.error : console.warn)('Soniox error msg', m);
774
+ if (code === 401 || code === 403) { sxToken = null; sxTokenExp = 0; } // a cached key the server rejects is dead
775
+ // Per-stream errors (feed stalled while the phone slept, max duration): the server closes
776
+ // and the next feed() reopens — nothing for the user to see. 429/5xx are still surfaced.
777
+ if (fatal) onFatal(`STT ${msg}`); else if (!SONIOX_STREAM_ERRORS.has(m.error_type) && code !== 408) onError(msg);
749
778
  return;
750
779
  }
751
780
  if (!Array.isArray(m.tokens)) return;
@@ -10,16 +10,47 @@
10
10
  // every proxy request (synth + voices), never on a direct ElevenLabs call.
11
11
  // • apiKey — direct browser→ElevenLabs (dev/bench only: the key is visible in the page).
12
12
  //
13
+ // Transport, two paths (both end in the same Blob, so playback can't tell them apart):
14
+ // • WebSocket (fast path) — one persistent browser↔ElevenLabs multi-context socket, one context
15
+ // per sentence. Measured from Budapest: ~195 ms to a full clip vs ~330 ms through an HTTP proxy
16
+ // hop (every sentence pays TLS+proxy RTT over HTTP; the socket pays it once). Opt in with
17
+ // `getWsToken: () => Promise<{ token }>` — a backend minter of ElevenLabs' `tts_websocket`
18
+ // single-use token (sutkn_…, POST /v1/single-use-token/tts_websocket with the xi-api-key), the
19
+ // same shape as the STT token flow. The key never reaches the page. Direct `apiKey` mode uses
20
+ // the socket too (xi_api_key in the first frame).
21
+ // • HTTP (fallback) — the proxy/direct POST above. Used when no WS auth is configured, when the
22
+ // socket can't be opened/minted, and for ANY clip whose socket turn fails or stalls
23
+ // (TUNING-free constant below): the clip falls back transparently and the socket is reopened
24
+ // on the next sentence. Speed never costs reliability.
25
+ //
26
+ // Billing on the socket path: the backend can't meter a direct browser↔ElevenLabs socket, so the
27
+ // client reports what it synthesized — `usageUrl` gets POST { chars, voice_id, model_id } per clip
28
+ // (trust-but-verify, same stance as STT seconds). HTTP-proxy clips are metered by the proxy itself.
29
+ //
13
30
  // Latency: eleven_flash_v2_5 is EL's low-latency model (~75ms model time); the rest of a clip's
14
31
  // cost is network TTFB + transfer, which presynth (during the end-of-turn debounce) and the base
15
32
  // class's ahead-of-playback synth queue hide for all but the first clip of a cold turn.
16
33
  import { StreamingTTS } from './voice-agent.js';
17
34
 
18
35
  const EL_API = 'https://api.elevenlabs.io/v1/text-to-speech';
36
+ const EL_WS = 'wss://api.elevenlabs.io/v1/text-to-speech';
37
+ // A socket turn that hasn't produced its final frame by then is abandoned for the HTTP path — a
38
+ // stalled socket must not cost more than one HTTP round-trip's worth of silence.
39
+ const WS_CLIP_TIMEOUT_MS = 2500;
40
+ // Server-side inactivity limit is 20 s by default, 180 s max: ask for the max and ping under it so
41
+ // a socket opened at session start is still warm when the user finally speaks.
42
+ const WS_INACTIVITY_S = 180;
43
+ const WS_PING_MS = 60_000;
44
+ // After a socket failure (open, auth, stall) HTTP serves every clip for this long before the
45
+ // socket is tried again — one failure must not turn into a reconnect per sentence.
46
+ const WS_BACKOFF_MS = 30_000;
47
+ // Token mint + handshake together; past this the clip (and warm()) gives up on the socket.
48
+ const WS_OPEN_TIMEOUT_MS = 4000;
19
49
 
20
50
  export class ElevenLabsTTS extends StreamingTTS {
21
51
  constructor({ ttsUrl, apiKey, voiceId = 'JBFqnCBsd6RMkjVDRZzb' /* George */,
22
- modelId = 'eleven_flash_v2_5', format = 'mp3_44100_64', headers = {}, languageCode = '' } = {}) {
52
+ modelId = 'eleven_flash_v2_5', format = 'mp3_44100_64', headers = {}, languageCode = '',
53
+ getWsToken, usageUrl = '' } = {}) {
23
54
  super(voiceId);
24
55
  if (!ttsUrl && !apiKey) throw new Error('ElevenLabsTTS needs ttsUrl (backend proxy) or apiKey (direct, dev only)');
25
56
  this.ttsUrl = ttsUrl; this.apiKey = apiKey; this.modelId = modelId; this.format = format;
@@ -27,8 +58,14 @@ export class ElevenLabsTTS extends StreamingTTS {
27
58
  // English and spoken with English phonetics. Base code only; '' = let the model guess.
28
59
  this.languageCode = String(languageCode || '').split(/[-_]/)[0].toLowerCase();
29
60
  this.headers = headers; // proxy-path auth (X-API-Key etc.) — never sent to ElevenLabs directly
61
+ this.getWsToken = getWsToken; this.usageUrl = usageUrl;
62
+ this._conn = null; this._opening = null; this._ctxN = 0; this._gen = 0; this._closed = false;
63
+ this._wsDisabledUntil = 0; // after a socket failure HTTP serves until then (see WS_BACKOFF_MS)
30
64
  }
31
65
 
66
+ get _wsEnabled() { return !!(this.getWsToken || (this.apiKey && !this.ttsUrl)); }
67
+ get _wsReady() { return this._wsEnabled && !this._closed && Date.now() >= this._wsDisabledUntil; }
68
+
32
69
  // Listing voices needs the key, so on the proxy path it has to come from YOUR backend: a GET on
33
70
  // the same ttsUrl returns [{ id, name }] (see examples/token-worker). Direct mode asks ElevenLabs
34
71
  // with the key it already has. Either way a failure is non-fatal — an empty list just means the
@@ -45,9 +82,22 @@ export class ElevenLabsTTS extends StreamingTTS {
45
82
  } catch { return []; }
46
83
  }
47
84
 
85
+ // The socket is bound to voice + model + language: changing the voice retires it (an in-flight
86
+ // open too, via the generation bump) and the next sentence opens a fresh one.
87
+ setVoice(voiceId) { if (voiceId !== this.voiceId) { this._gen++; this._retire(this._conn, false); } super.setVoice(voiceId); }
88
+
48
89
  // Pre-establish the connection (DNS + TLS + any proxy hop) so the first real clip only pays TTFB.
49
- // A HEAD to the API root is enough to open the socket; failures are non-fatal.
90
+ // Socket path: open (and keep) the socket now. HTTP path: a HEAD is enough to open the socket;
91
+ // failures are non-fatal.
50
92
  async warm() {
93
+ if (this._wsEnabled) {
94
+ if (!this._wsReady) return;
95
+ // Open the socket AND run a throwaway clip through it: a fresh multi-stream socket's first
96
+ // clip costs ~450 ms (model/voice warm-up per connection), every later one ~130 ms. Paying
97
+ // that during the warm (user not yet speaking) keeps the first real sentence fast.
98
+ await this._synthWs('Ok.', undefined, true).catch(() => {});
99
+ return;
100
+ }
51
101
  const url = this.ttsUrl ?? EL_API;
52
102
  try { await fetch(url, { method: 'HEAD', signal: AbortSignal.timeout(3000) }); } catch {}
53
103
  }
@@ -55,6 +105,15 @@ export class ElevenLabsTTS extends StreamingTTS {
55
105
  async _synth(text, signal) {
56
106
  text = text?.trim();
57
107
  if (signal?.aborted || !text) return null;
108
+ if (this._wsReady) {
109
+ const blob = await this._synthWs(text, signal).catch((e) => { this.onEvent?.({ type: 'diag', diag: 'tts_ws_fallback', error: String(e?.message ?? e) }); return undefined; });
110
+ if (blob !== undefined) return blob; // null = aborted, Blob = done; undefined = fall back
111
+ if (signal?.aborted || this._closed) return null;
112
+ }
113
+ return this._synthHttp(text, signal);
114
+ }
115
+
116
+ async _synthHttp(text, signal) {
58
117
  const direct = !this.ttsUrl;
59
118
  const url = direct ? `${EL_API}/${this.voiceId}?output_format=${this.format}` : this.ttsUrl;
60
119
  const res = await fetch(url, {
@@ -72,4 +131,136 @@ export class ElevenLabsTTS extends StreamingTTS {
72
131
  const blob = await res.blob();
73
132
  return signal?.aborted ? null : blob;
74
133
  }
134
+
135
+ // ── WebSocket path ─────────────────────────────────────────────────────────────────────────
136
+ // One multi-context socket; each sentence is its own context (text+flush, then close_context),
137
+ // audio frames arrive base64 in JSON and are concatenated into one mp3 Blob on is_final. All
138
+ // per-socket state (pending contexts, ping timer) lives on a `conn` record so a stale socket's
139
+ // late close/error can only ever touch ITS OWN contexts, never a replacement's.
140
+ async _synthWs(text, signal, isWarm = false) {
141
+ const conn = await this._connect(signal);
142
+ if (signal?.aborted) return null;
143
+ const { ws } = conn;
144
+ const id = `c${++this._ctxN}`;
145
+ const chunks = [];
146
+ const blob = await new Promise((resolve, reject) => {
147
+ let settled = false;
148
+ const settle = (fn, v) => { if (settled) return; settled = true; clearTimeout(timer); conn.pending.delete(id); signal?.removeEventListener('abort', onAbort); fn(v); };
149
+ const timer = setTimeout(() => fail(new Error('ws clip timeout')), WS_CLIP_TIMEOUT_MS);
150
+ // A failed clip retires the socket with backoff (HTTP serves meanwhile); an abort just drops
151
+ // the context — the socket is healthy.
152
+ const fail = (e) => { settle(reject, e); this._retire(conn, true); };
153
+ const onAbort = () => { try { ws.send(JSON.stringify({ context_id: id, close_context: true })); } catch {} settle(resolve, null); };
154
+ signal?.addEventListener('abort', onAbort, { once: true });
155
+ conn.pending.set(id, {
156
+ audio: (b64) => chunks.push(Uint8Array.from(atob(b64), (c) => c.charCodeAt(0))),
157
+ // A final with no audio for real text is a provider failure, not an abort: fall back.
158
+ final: () => chunks.length ? settle(resolve, new Blob(chunks, { type: 'audio/mpeg' })) : fail(new Error('ws clip empty')),
159
+ error: fail,
160
+ });
161
+ try {
162
+ ws.send(JSON.stringify({ context_id: id, text: text + ' ', flush: true }));
163
+ ws.send(JSON.stringify({ context_id: id, close_context: true }));
164
+ } catch (e) { fail(e); }
165
+ });
166
+ if (blob && this.usageUrl && !isWarm) this._reportUsage(text.length);
167
+ return blob;
168
+ }
169
+
170
+ // The live connection, or a shared in-flight open. The whole open (token mint + handshake) is
171
+ // bounded by WS_OPEN_TIMEOUT_MS; a caller's own abort releases just that caller.
172
+ _connect(signal) {
173
+ if (this._conn?.ws.readyState === 1) return Promise.resolve(this._conn);
174
+ if (!this._opening) {
175
+ this._opening = this._open().finally(() => { this._opening = null; });
176
+ this._opening.catch(() => {});
177
+ }
178
+ if (!signal) return this._opening;
179
+ return new Promise((resolve, reject) => {
180
+ const onAbort = () => reject(new Error('aborted'));
181
+ signal.addEventListener('abort', onAbort, { once: true });
182
+ this._opening.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort));
183
+ });
184
+ }
185
+
186
+ async _open() {
187
+ if (this._closed) throw new Error('closed');
188
+ const gen = ++this._gen;
189
+ const WS = globalThis.WebSocket;
190
+ const conn = { ws: null, pending: new Map(), pingTimer: null, retired: false };
191
+ let deadlineTimer;
192
+ const deadline = new Promise((_, rej) => { deadlineTimer = setTimeout(() => rej(new Error('ws open timeout')), WS_OPEN_TIMEOUT_MS); });
193
+ try {
194
+ if (!WS) throw new Error('WebSocket unavailable');
195
+ const q = new URLSearchParams({ model_id: this.modelId, output_format: this.format, inactivity_timeout: String(WS_INACTIVITY_S) });
196
+ if (this.languageCode) q.set('language_code', this.languageCode);
197
+ if (this.getWsToken) {
198
+ const { token } = await Promise.race([this.getWsToken(), deadline]);
199
+ if (!token) throw new Error('no tts ws token');
200
+ q.set('single_use_token', token);
201
+ }
202
+ if (this._closed || gen !== this._gen) throw new Error('superseded');
203
+ const ws = conn.ws = new WS(`${EL_WS}/${this.voiceId}/multi-stream-input?${q}`);
204
+ await Promise.race([deadline, new Promise((resolve, reject) => {
205
+ ws.onopen = resolve;
206
+ ws.onerror = () => reject(new Error('ws open failed'));
207
+ ws.onclose = (e) => reject(new Error(`ws closed ${e.code}`));
208
+ })]);
209
+ if (this._closed || gen !== this._gen) throw new Error('superseded');
210
+ if (this.apiKey && !this.getWsToken) ws.send(JSON.stringify({ context_id: 'auth', text: ' ', xi_api_key: this.apiKey }));
211
+ } catch (e) {
212
+ // Every open failure (mint, handshake, timeout, closed meanwhile) backs off the socket path.
213
+ this._retire(conn, true);
214
+ throw e;
215
+ } finally { clearTimeout(deadlineTimer); }
216
+ const { ws } = conn;
217
+ ws.onmessage = (e) => {
218
+ let m; try { m = JSON.parse(e.data); } catch { return; }
219
+ const ctx = m.contextId ?? m.context_id;
220
+ const p = conn.pending.get(ctx);
221
+ if (m.error) {
222
+ this.onEvent?.({ type: 'diag', diag: 'tts_ws_error', error: m.message ?? m.error });
223
+ // Context-scoped error → that clip only (an already-aborted context's error is noise);
224
+ // connection-scoped (no context) → every pending clip, and the socket is retired.
225
+ if (ctx) p?.error(new Error(m.message ?? m.error)); else this._retire(conn, true, new Error(m.message ?? m.error));
226
+ return;
227
+ }
228
+ if (!p) return;
229
+ if (m.audio) p.audio(m.audio);
230
+ if (m.isFinal || m.is_final) p.final();
231
+ };
232
+ ws.onclose = () => this._retire(conn, true, new Error('ws closed'));
233
+ ws.onerror = () => {};
234
+ // keep_context_alive resets the server's inactivity timer without synthesizing anything.
235
+ conn.pingTimer = setInterval(() => { try { ws.send(JSON.stringify({ context_id: 'ka', keep_context_alive: true })); } catch {} }, WS_PING_MS);
236
+ conn.pingTimer.unref?.(); // never keeps a Node process alive (tests, SSR)
237
+ this._conn = conn;
238
+ return conn;
239
+ }
240
+
241
+ // The one teardown path. `failed` = back off (HTTP serves WS_BACKOFF_MS); intentional closes
242
+ // (setVoice, close()) don't. Pending contexts of THIS conn are failed with `err`, which sends
243
+ // them to the HTTP fallback via _synth's catch. Idempotent; a stale conn can't touch a newer one.
244
+ _retire(conn, failed, err = new Error('ws retired')) {
245
+ if (!conn || conn.retired) return;
246
+ conn.retired = true;
247
+ clearInterval(conn.pingTimer);
248
+ if (this._conn === conn) this._conn = null;
249
+ if (failed) this._wsDisabledUntil = Date.now() + WS_BACKOFF_MS;
250
+ const { ws } = conn;
251
+ if (ws && ws.readyState <= 1) { try { ws.send(JSON.stringify({ close_socket: true })); } catch {} try { ws.close(); } catch {} }
252
+ for (const p of [...conn.pending.values()]) p.error(err);
253
+ }
254
+
255
+ _reportUsage(chars) {
256
+ fetch(this.usageUrl, {
257
+ method: 'POST', keepalive: true,
258
+ headers: { 'Content-Type': 'application/json', ...this.headers },
259
+ body: JSON.stringify({ chars, voice_id: this.voiceId, model_id: this.modelId }),
260
+ }).catch(() => {});
261
+ }
262
+
263
+ // Release the socket; the host calls this when the agent is destroyed (VoiceAgent.destroy →
264
+ // tts.close?.()). An in-flight open is superseded by the generation bump and retires itself.
265
+ close() { this._closed = true; this._gen++; this._retire(this._conn, false); }
75
266
  }
@@ -32,16 +32,22 @@ export const VOICE_SYSMSG =
32
32
 
33
33
  const CDN = 'https://cdn.jsdelivr.net/npm/';
34
34
 
35
- // AudioWorklet processor: accumulate the audio-thread's 128-sample render quanta into ~256ms chunks
36
- // (4096 samples @16kHz — the cadence the pipeline's bootstrap-VAD/preroll math assumes) and post each
37
- // to the main thread. Kept as a source string → Blob URL so no separate asset/bundler config is needed.
35
+ // AudioWorklet processor: downsample the context's rate to 16kHz (box-average per output sample;
36
+ // passthrough when the context already runs at 16kHz) and accumulate ~256ms chunks (4096 samples
37
+ // @16kHz — the cadence the pipeline's bootstrap-VAD/preroll math assumes), posting each to the main
38
+ // thread. The context runs at the device's native rate: Safari/iOS may ignore or mishandle a 16kHz
39
+ // context fed by a 48kHz mic, so the resample happens here where we control it.
40
+ // Kept as a source string → Blob URL so no separate asset/bundler config is needed.
38
41
  const CAPTURE_WORKLET_SRC = `
39
42
  registerProcessor('capture-processor', class extends AudioWorkletProcessor {
40
- constructor() { super(); this._buf = new Float32Array(4096); this._n = 0; }
43
+ constructor() { super(); this._buf = new Float32Array(4096); this._n = 0; this._step = sampleRate / 16000; this._acc = 0; this._sum = 0; this._cnt = 0; }
41
44
  process(inputs) {
42
45
  const ch = inputs[0]?.[0];
43
46
  if (ch) for (let i = 0; i < ch.length; i++) {
44
- this._buf[this._n++] = ch[i];
47
+ this._sum += ch[i]; this._cnt++;
48
+ if (++this._acc < this._step) continue;
49
+ this._acc -= this._step;
50
+ this._buf[this._n++] = this._sum / this._cnt; this._sum = 0; this._cnt = 0;
45
51
  if (this._n === this._buf.length) { this.port.postMessage(this._buf.slice()); this._n = 0; }
46
52
  }
47
53
  return true;
@@ -55,11 +61,16 @@ const CAPTURE_WORKLET_URL_GET = () => captureWorkletUrl ??= URL.createObjectURL(
55
61
  // Compose the full system prompt: a persona (base voice persona by default, or a host-supplied one)
56
62
  // + the host's conversation context (if any).
57
63
  // The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
58
- // English under the English persona — so state the selected STT language explicitly.
64
+ // English under the English persona — so state the selected STT language as the DEFAULT. Only a
65
+ // default: the setting is often stale (web defaults to 'en', mobile to the device locale) while
66
+ // multilingual STT transcribes whatever is spoken, and a hard "ALWAYS reply in English" made the
67
+ // agent refuse a Hungarian speaker's explicit "answer in Hungarian".
59
68
  export const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
60
69
  const langRule = (lang) => {
61
70
  const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
62
- return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
71
+ return name
72
+ ? `Reply in the language the user speaks — including tool-call acknowledgements. Default to ${name} (their selected language); switch when they clearly speak another language or ask for one, and stay in it.`
73
+ : 'Reply in the language the user speaks — including tool-call acknowledgements.';
63
74
  };
64
75
  // A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
65
76
  // what order a provider streamed the keys in (used for dedup).
@@ -443,7 +454,11 @@ export class VoiceAgent {
443
454
  // stale continuation must clean up its own objects only.
444
455
  let ctx;
445
456
  try {
446
- ctx = new AudioContext({ sampleRate: 16000 });
457
+ ctx = new (window.AudioContext || window.webkitAudioContext)(); // native rate — the worklet resamples to 16kHz
458
+ // Created after awaits (outside the click gesture): iOS starts it 'suspended' and suspends/'interrupts'
459
+ // it again on backgrounding/calls. An active mic capture lets WebKit run it without a gesture.
460
+ const wake = () => { if (ctx.state !== 'running' && ctx.state !== 'closed') ctx.resume().catch(() => {}); };
461
+ ctx.onstatechange = wake; wake();
447
462
  const src = ctx.createMediaStreamSource(stream);
448
463
  await ctx.audioWorklet.addModule(CAPTURE_WORKLET_URL_GET());
449
464
  if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); ctx.close(); return; } // superseded/stopped while loading
@@ -537,7 +552,7 @@ export class VoiceAgent {
537
552
  // End the session for real: pause logic + release the mic/VAD (turns off the OS mic indicator). The
538
553
  // host calls this when it drops the agent (end/close/reset/new-chat) so a paused-but-discarded agent
539
554
  // never leaks the microphone.
540
- destroy() { this._destroyed = true; this.stop(); this.stt.close(); this._teardownPipeline(); }
555
+ destroy() { this._destroyed = true; this.stop(); this.stt.close(); this.tts.close?.(); this._teardownPipeline(); }
541
556
 
542
557
  // Host-triggered barge-in: cut off the current in-flight turn (abort the LLM, stop TTS) WITHOUT
543
558
  // tearing down the pipeline — the mic keeps listening. Mirrors `_onBargeIn` but callable from a