@todoforai/voiceloop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/stt.js ADDED
@@ -0,0 +1,815 @@
1
+ // stt.js — pluggable STT providers for the VoiceAgent. One small contract, many backends.
2
+ //
3
+ // A provider is a factory `(opts) => STT` where STT is:
4
+ // open() [optional] pre-open the socket (token fetch + WS handshake) before any audio, so the
5
+ // first utterance isn't stalled by a cold connect; idempotent, and feed() still opens lazily
6
+ // feed(i16) push a 16kHz mono Int16Array chunk (provider opens its socket lazily on first feed)
7
+ // commit() VAD speech-end → finalize the current utterance (no-op if nothing was fed since)
8
+ // close() teardown the socket and drop buffered audio
9
+ // It reports back through callbacks passed in `opts`:
10
+ // onPartial(text, ms, committed='') live interim TAIL (called many times/utterance); `committed`
11
+ // is the word-level locked prefix to render solid (Speechmatics)
12
+ // onFinal(text) one finalized utterance → becomes a user turn
13
+ // onError(msg) recoverable error, surfaced to the host (session stays alive)
14
+ // onFatal(msg) unrecoverable (auth/config) — host should stop() the agent
15
+ // onClose() socket closed (used for usage flushing)
16
+ // isClosed() agent stopped — don't reopen / don't emit
17
+ // A provider may also expose capability flags the agent adapts to:
18
+ // continuous: true feed the FULL mic stream (no VAD gating/preroll) — the provider's own model
19
+ // needs to hear the silences (e.g. Deepgram Flux end-of-turn detection)
20
+ // nativeEOT: true the provider closes turns itself (unsolicited onFinal) — the agent skips its
21
+ // VAD-driven commit() and just handles onFinal when it arrives
22
+ // selfCapture: true the provider OWNS the whole audio path: its own mic + its own end-of-turn
23
+ // (e.g. the browser Web Speech API). The agent builds NO mic/AudioWorklet/VAD
24
+ // pipeline for it — start() just opens the provider. Implies nativeEOT (the
25
+ // provider closes its own turns via unsolicited onFinal). It exposes open()/
26
+ // close() + setEnabled() (mute), and drives barge-in the normal way via onPartial.
27
+ //
28
+ // Adding a provider = write a factory + register it in STT_PROVIDERS below. The VoiceAgent owns
29
+ // the mic + VAD and just forwards feed/commit/close, so providers never touch audio capture.
30
+
31
+ import { TUNING } from './tuning.js';
32
+
33
+ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at least this often
34
+
35
+ // Common opts shape (provided by the VoiceAgent):
36
+ // { apiKey, sttUrl, sttModel, sttLang, keyterms, sttTokenUrl, getToken, sttUsageUrl,
37
+ // onPartial, onFinal, onError, onClose, isClosed }
38
+ //
39
+ // AUTH (cloud providers): the raw provider key must stay server-side. Either
40
+ // • `sttTokenUrl` — your backend route that mints a short-TTL token: POST, optional X-API-Key
41
+ // (`apiKey`), responds { token, expires_in? }; or
42
+ // • `getToken` — an async () => ({ token, expires_in? }) callback, when your auth doesn't fit
43
+ // that one route shape. Takes precedence over sttTokenUrl.
44
+
45
+ // ── Shared socket-lifecycle helpers ─────────────────────────────────────────
46
+ // The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
47
+ // Deepgram's cumulative-turn + token cache, ElevenLabs' simple commit) — those stay separate on
48
+ // purpose. But token-fetch-with-timeout and usage-reporting are byte-for-byte the same shape in all
49
+ // three; duplicating THOSE is a pure bug magnet (a timeout fix would need applying 3x), so they're
50
+ // factored out here.
51
+
52
+ // Mint a short-TTL token — via the host's `getToken` callback when given, else a POST to
53
+ // `sttTokenUrl`. Bounded by tokenFetchTimeoutMs so a hung/unreachable backend can't stall
54
+ // voice-mode startup forever. Calls onFatal itself on any failure (network, timeout, non-2xx,
55
+ // missing token) and returns null — the caller just checks for null and bails (what buffered
56
+ // state to clear on failure differs per provider, so that stays with the caller).
57
+ async function mintSttToken(url, apiKey, onFatal, getToken) {
58
+ if (getToken) {
59
+ try {
60
+ const body = await getToken();
61
+ if (!body?.token) throw new Error('getToken returned no token');
62
+ return body;
63
+ } catch (e) { onFatal(`STT token: ${e.message}`); return null; }
64
+ }
65
+ if (!url) { onFatal('STT token: no sttTokenUrl/getToken configured — cloud STT needs a token route (see README)'); return null; }
66
+ let res;
67
+ try {
68
+ res = await fetch(url, { method: 'POST', headers: apiKey ? { 'X-API-Key': apiKey } : {}, signal: AbortSignal.timeout(TUNING.STT.tokenFetchTimeoutMs) });
69
+ } catch (e) {
70
+ onFatal(`STT token: ${e.name === 'TimeoutError' ? 'timed out' : e.message}`);
71
+ return null;
72
+ }
73
+ const body = await res.json().catch(() => ({}));
74
+ if (!res.ok || !body.token) { onFatal(`STT token: ${body.error || body.message || res.statusText}`); return null; }
75
+ return body;
76
+ }
77
+
78
+ // Accumulate streamed samples and flush them to /stt/usage — inline once USAGE_FLUSH_SECONDS is
79
+ // crossed, and again whenever the caller flushes on close (billing is trust-but-verify: the client
80
+ // reports what it streamed since the backend can't meter a direct browser↔provider socket).
81
+ function makeUsageReporter(sttUsageUrl, apiKey, provider, extra = {}) {
82
+ let samples = 0;
83
+ const flush = () => {
84
+ const seconds = samples / 16000; samples = 0;
85
+ if (seconds <= 0 || !sttUsageUrl) return;
86
+ fetch(sttUsageUrl, {
87
+ method: 'POST', keepalive: true,
88
+ headers: { 'Content-Type': 'application/json', 'X-API-Key': apiKey },
89
+ body: JSON.stringify({ seconds, provider, ...extra }),
90
+ }).catch(() => {});
91
+ };
92
+ return { add: (n) => { samples += n; if (samples >= 16000 * USAGE_FLUSH_SECONDS) flush(); }, flush };
93
+ }
94
+
95
+ // ── ElevenLabs Scribe realtime ──────────────────────────────────────────────
96
+ // Browser↔ElevenLabs direct WS. Auth via a single-use token minted by the backend (/stt/token);
97
+ // the key stays server-side. Billing is trust-but-verify: the client reports streamed seconds to
98
+ // /stt/usage (the backend can't meter a direct browser↔ElevenLabs socket).
99
+ export function makeElevenLabsSTT(opts) {
100
+ // `||` fallbacks, not destructuring defaults: VoiceAgent passes '' for unset knobs, which a
101
+ // `= default` would NOT replace — an empty model_id makes ElevenLabs hard-close the socket (1006).
102
+ const { apiKey, sttLang, keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
103
+ onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
104
+ const sttUrl = opts.sttUrl || 'wss://api.elevenlabs.io/v1/speech-to-text/realtime';
105
+ const sttModel = opts.sttModel || 'scribe_v2_realtime';
106
+ let ws = null, opening = false, outbox = [], sentSinceCommit = false, sttStart = 0;
107
+ const usage = makeUsageReporter(sttUsageUrl, apiKey, 'elevenlabs');
108
+
109
+ const sendAudio = (i16) => {
110
+ if (!ws || ws.readyState !== 1) { outbox.push(i16); return; }
111
+ let bin = ''; const b = new Uint8Array(i16.buffer); for (let i = 0; i < b.length; i++) bin += String.fromCharCode(b[i]);
112
+ ws.send(JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: btoa(bin) }));
113
+ usage.add(i16.length);
114
+ };
115
+ const sendCommit = () => ws.send(JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: '', commit: true }));
116
+
117
+ const open = async () => {
118
+ if (opening) return; // re-entry guard: the async token fetch could otherwise open two sockets
119
+ opening = true;
120
+ try {
121
+ const body = await mintSttToken(sttTokenUrl, apiKey, onFatal, getToken);
122
+ if (!body) {
123
+ // Auth/config failure won't fix itself — drop buffered audio (don't replay stale speech on
124
+ // a later recovery); mintSttToken already called onFatal so the host stops instead of
125
+ // re-firing on every utterance.
126
+ outbox = []; sentSinceCommit = false;
127
+ return;
128
+ }
129
+ if (isClosed()) return;
130
+ const p = new URLSearchParams({ model_id: sttModel, commit_strategy: 'manual', audio_format: 'pcm_16000', token: body.token });
131
+ if (sttLang) p.set('language_code', sttLang);
132
+ for (const k of keyterms) p.append('keyterms', k);
133
+ sttStart = performance.now();
134
+ ws = new WebSocket(`${sttUrl}?${p}`);
135
+ ws.onopen = () => { const q = outbox; outbox = []; for (const it of q) it === 'commit' ? sendCommit() : sendAudio(it); };
136
+ ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('STT ws error', ev); } };
137
+ ws.onclose = ev => { usage.flush(); onClose?.(); if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); console.error('STT ws closed', ev.code, ev.reason); } };
138
+ ws.onmessage = ev => {
139
+ const m = JSON.parse(ev.data);
140
+ const ms = Math.round(performance.now() - sttStart);
141
+ if (m.message_type?.endsWith('_error')) { onError(`${m.message_type}: ${m.error || m.reason || m.message || ''}`); console.error('STT error msg', m); return; }
142
+ if (m.message_type === 'partial_transcript') onPartial(m.text || '', ms);
143
+ else if (m.message_type?.startsWith('committed') && m.text) onFinal(m.text, ms);
144
+ };
145
+ } finally {
146
+ opening = false;
147
+ }
148
+ };
149
+
150
+ return {
151
+ // Pre-open the socket (token fetch + WS handshake) before any audio so the first utterance isn't
152
+ // stalled by cold connect. Idempotent (open() has a re-entry guard); feed() still opens lazily.
153
+ open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
154
+ feed(i16) {
155
+ if (!isClosed() && (!ws || ws.readyState >= 2)) open();
156
+ sendAudio(i16); sentSinceCommit = true;
157
+ },
158
+ commit() {
159
+ // No audio since the last commit (VAD can fire several times per utterance, or only the AI's own
160
+ // echo was heard): don't ask ElevenLabs to finalize (that would dup/empty-turn), but still emit an
161
+ // empty close. The agent sets _turnClosing before calling commit() and clears it on onFinal — so
162
+ // without this an empty VAD end would leave _turnClosing stuck (mirrors the Speechmatics contract).
163
+ if (!sentSinceCommit) { onFinal('', Math.round(performance.now() - sttStart)); return; }
164
+ sentSinceCommit = false;
165
+ if (ws?.readyState === 1) sendCommit(); else outbox.push('commit');
166
+ },
167
+ close() { outbox = []; sentSinceCommit = false; usage.flush(); ws?.close(); ws = null; },
168
+ };
169
+ }
170
+
171
+ // ── Speechmatics realtime ────────────────────────────────────────────────────
172
+ // Browser↔Speechmatics direct WS. Like ElevenLabs: the backend mints a short-TTL JWT (the raw key
173
+ // stays server-side), the browser connects with ?jwt=. Differs in wire shape — a JSON config
174
+ // message (StartRecognition) then BINARY audio frames (not base64), and JSON transcript messages
175
+ // (AddPartialTranscript / AddTranscript / EndOfUtterance). Billing is trust-but-verify: we report
176
+ // streamed seconds to /stt/usage tagged provider:'speechmatics'.
177
+ //
178
+ // TURN MODEL — LOCKED WORDS ARE THE ONLY SOURCE OF TRUTH:
179
+ // AddTranscript = permanently locked words → append to `committed`.
180
+ // AddPartialTranscript = the flickering not-yet-locked tail → shown live, NEVER written to history.
181
+ // EndOfUtterance = the turn boundary (follows our ForceEndOfUtterance, or the silence trigger).
182
+ // It always trails the flushing AddTranscript, so on arrival `committed` holds
183
+ // every locked word → onFinal(committed), no tail folding (the old mergeTail
184
+ // path was the source of the "four, five, four five" duplication). Only the
185
+ // SAFETY-TIMER close (EndOfUtterance never arrived) folds the interim tail in
186
+ // — see flushFinal's `withTail`.
187
+ // commit() = ForceEndOfUtterance; we wait for EndOfUtterance (bounded safety fallback if the socket
188
+ // never answers). NB: requires conversation_config.end_of_utterance_silence_trigger in StartRecognition.
189
+ //
190
+ // ORDERING: end_times are absolute seconds from StartRecognition (never reset per turn). `cursor` =
191
+ // end_time of the last accepted word; anything ending at/before it is an out-of-order straggler from
192
+ // a closed turn → dropped. The cursor only advances and survives reset() (close() zeroes it).
193
+ const SM_URL = 'wss://eu.rt.speechmatics.com/v2';
194
+ export function makeSpeechmaticsSTT(opts) {
195
+ const { apiKey, sttModel, sttLang = 'en', sttTokenUrl, getToken, sttUsageUrl,
196
+ onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
197
+ const tokenUrl = sttTokenUrl;
198
+ let ws = null, opening = false, started = false, outbox = [], sentSinceCommit = false, sttStart = 0;
199
+ // committed = locked words (the turn's text; the partial tail is display-only). cursor = monotonic
200
+ // end_time (s) of the last accepted word — the straggler gate (see header). turnOpen = words landed
201
+ // since the last close. awaitingFinal = ForceEndOfUtterance in flight, waiting on EndOfUtterance.
202
+ // sentSamples = stream position (total samples fed this socket, outbox included); commit() snapshots
203
+ // it as commitBoundary — if the safety timer closes a turn WITHOUT the EndOfUtterance handshake, the
204
+ // cursor advances to that boundary, gating the old turn's late flush (audio fed before the commit)
205
+ // while the next turn's words (fed after) flow immediately.
206
+ let committed = '', cursor = 0, turnOpen = false, awaitingFinal = false, safetyTimer = null, pendingForceEnd = false;
207
+ let sentSamples = 0, commitBoundary = 0;
208
+ // lastTail = the most recent interim tail (AddPartialTranscript). Normally display-only, but on a
209
+ // SAFETY-TIMER close (the flushing AddTranscript never arrived) it's the only copy of the user's
210
+ // last words — fold it into that final instead of stripping it. Cleared whenever words lock.
211
+ let lastTail = '';
212
+ // keepalive: audio is VAD-gated, so quiet listening sends NOTHING — Speechmatics closes an idle
213
+ // socket after ~3min of no audio. Trickle 0.1s of silence when idle so the session survives long
214
+ // silences instead of dying + reconnecting (which loses the first words to the token/handshake).
215
+ let keepalive = null, lastSendMs = 0;
216
+ // Send ForceEndOfUtterance now if the socket is live, else defer it until RecognitionStarted (a VAD
217
+ // commit can land before the socket finishes opening; without this the force-end is silently dropped).
218
+ const sendForceEnd = () => {
219
+ if (ws?.readyState === 1 && started) { ws.send(JSON.stringify({ message: 'ForceEndOfUtterance' })); pendingForceEnd = false; }
220
+ else pendingForceEnd = true;
221
+ };
222
+ // Close the turn: emit the LOCKED words (committed) as the final — never the tail. By the time
223
+ // EndOfUtterance lands, Speechmatics has already flushed the turn's words as AddTranscript, so
224
+ // `committed` is complete. Called on EndOfUtterance, or by the safety timer if it never arrives.
225
+ // `withTail`: closing WITHOUT the EndOfUtterance handshake (safety timer) — the flushing
226
+ // AddTranscript never landed, so the last interim tail is the only copy of the user's final words.
227
+ // Fold it in rather than strip it; the cursor advance (armSafety) gates the late flush so the same
228
+ // words can't ALSO land as a locked duplicate afterwards. The normal EOU path stays locked-words-only.
229
+ const flushFinal = (withTail = false) => {
230
+ clearTimeout(safetyTimer); safetyTimer = null;
231
+ awaitingFinal = false; turnOpen = false; sentSinceCommit = false; // turn closed → nothing left to finalize
232
+ const t = (withTail && lastTail ? `${committed} ${lastTail}` : committed).trim();
233
+ committed = ''; lastTail = ''; // keep cursor: monotonic gate survives the turn
234
+ // Always notify — even empty — so the agent clears its "closing" state and never wedges waiting.
235
+ onFinal(t, Math.round(performance.now() - sttStart));
236
+ };
237
+ const armSafety = (ms) => { clearTimeout(safetyTimer); safetyTimer = setTimeout(() => {
238
+ // Closing WITHOUT the EndOfUtterance handshake: the old turn's late flush may hold words the
239
+ // cursor never saw locked. Advance it to the commit-time stream boundary — the old turn's words
240
+ // (fed before the commit) end at/before it and are gated; the next turn's flow immediately.
241
+ cursor = Math.max(cursor, commitBoundary);
242
+ flushFinal(true);
243
+ }, ms); };
244
+
245
+ const usage = makeUsageReporter(sttUsageUrl, apiKey, 'speechmatics', { model: sttModel || 'standard' });
246
+
247
+ const sendAudio = (i16) => {
248
+ if (!ws || ws.readyState !== 1 || !started) { outbox.push(i16); return; }
249
+ ws.send(i16.buffer); // raw binary AddAudio (pcm_s16le @16k, declared in StartRecognition)
250
+ lastSendMs = performance.now();
251
+ usage.add(i16.length);
252
+ };
253
+
254
+ const open = async () => {
255
+ if (opening) return;
256
+ opening = true;
257
+ // NEW SOCKET = NEW STREAM CLOCK: end_times restart at 0 (they're relative to the new socket's
258
+ // StartRecognition), so per-socket state must reset. Without this, a silent reconnect (e.g.
259
+ // Speechmatics' idle timeout closed the old socket, feed() reopened) leaves the old
260
+ // cursor/sentSamples in place — every word of the new stream ends "behind" the stale cursor and
261
+ // is dropped as a straggler → the session goes PERMANENTLY DEAF. Stale `started` similarly made
262
+ // feed() send audio before the new RecognitionStarted handshake. Reset SYNCHRONOUSLY, before the
263
+ // awaited token fetch: feed() calls open() first and increments sentSamples after, so audio
264
+ // queued while the socket opens is counted into THIS socket's clock (the commitBoundary safety
265
+ // gate stays correct). `committed` survives: words locked before a mid-turn drop still belong to
266
+ // the open turn's final.
267
+ cursor = 0; sentSamples = 0; commitBoundary = 0; started = false;
268
+ try {
269
+ const body = await mintSttToken(tokenUrl, apiKey, onFatal, getToken);
270
+ if (!body) { outbox = []; sentSinceCommit = false; return; }
271
+ if (isClosed()) return;
272
+ sttStart = performance.now();
273
+ ws = new WebSocket(`${SM_URL}?jwt=${encodeURIComponent(body.token)}`);
274
+ ws.binaryType = 'arraybuffer';
275
+ ws.onopen = () => {
276
+ // Declare the audio format + transcription config, then flush buffered audio once started.
277
+ ws.send(JSON.stringify({
278
+ message: 'StartRecognition',
279
+ audio_format: { type: 'raw', encoding: 'pcm_s16le', sample_rate: 16000 },
280
+ // max_delay = seconds Speechmatics waits before LOCKING words into a final (AddTranscript); the
281
+ // 4s default leaves a long flickering PARTIAL tail (6-7 words) before words move to `committed`.
282
+ // We tie it to the model's accuracy headroom: 'enhanced' (their demo's combo) tolerates max_delay:1
283
+ // for a snappy 1-4 word tail; 'standard' lacks that headroom, so 2s (Speechmatics' documented
284
+ // "optimal balance", ~1% loss vs ~<5% at 1s) buys back the context it needs. max_delay_mode stays
285
+ // default ('flexible') so entities (numbers/dates) still finalize whole. (Range 0.7–4.)
286
+ // conversation_config.end_of_utterance_silence_trigger is REQUIRED for ForceEndOfUtterance /
287
+ // EndOfUtterance to work at all — without it commit() is silently ignored and the turn only
288
+ // ever closes via the safety timer. Keep it < max_delay. The AGENT owns the boundary (VAD +
289
+ // turnDetector → commit → ForceEndOfUtterance); a silence-triggered EndOfUtterance can also
290
+ // close a turn, but since the agent feeds VAD-GATED audio (little post-speech silence reaches
291
+ // Speechmatics), commit() is the boundary that matters in practice.
292
+ transcription_config: { language: (sttLang || 'en').slice(0, 2), operating_point: sttModel || 'standard', enable_partials: true, max_delay: sttModel === 'enhanced' ? TUNING.STT.maxDelayEnhanced : TUNING.STT.maxDelayStandard, conversation_config: { end_of_utterance_silence_trigger: TUNING.STT.eouSilenceTrigger } },
293
+ }));
294
+ };
295
+ const sock = ws; // this handler's own socket — feed() may reopen while it's still CLOSING
296
+ ws.onerror = ev => { if (sock === ws && !isClosed()) { onError('STT socket error'); console.error('SM ws error', ev); } };
297
+ // Guard on sock === ws: a superseded socket's late onclose must not kill the NEW socket's
298
+ // keepalive (feed() reopens on readyState>=2, i.e. while the old one is still CLOSING).
299
+ ws.onclose = ev => { if (sock === ws) { clearInterval(keepalive); keepalive = null; } usage.flush(); onClose?.(); if (!isClosed() && sock === ws && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); } };
300
+ ws.onmessage = ev => {
301
+ if (sock !== ws) return; // stale message from a superseded socket (its clock is dead)
302
+ const m = JSON.parse(ev.data);
303
+ const ms = Math.round(performance.now() - sttStart);
304
+ switch (m.message) {
305
+ case 'RecognitionStarted': {
306
+ started = true; const q = outbox; outbox = []; for (const c of q) sendAudio(c); if (pendingForceEnd) sendForceEnd();
307
+ // Idle keepalive (see declaration): 0.1s of silence whenever nothing was sent for 20s
308
+ // AND no turn is open/closing — injected silence must never trip the provider's
309
+ // end-of-utterance silence trigger on a live turn. Cost is negligible (~0.5% stream time).
310
+ // Counts into `sentSamples` like any other sent audio: it genuinely advances Speechmatics'
311
+ // stream clock (end_times are relative to ALL audio sent), so the commitBoundary safety
312
+ // gate must stay in sync or a late straggler could slip past cursor after an idle period.
313
+ clearInterval(keepalive);
314
+ keepalive = setInterval(() => {
315
+ if (ws?.readyState === 1 && started && !turnOpen && !awaitingFinal && !sentSinceCommit
316
+ && performance.now() - lastSendMs > 20000) {
317
+ const silence = new Int16Array(1600);
318
+ sentSamples += silence.length;
319
+ sendAudio(silence);
320
+ }
321
+ }, 5000);
322
+ keepalive.unref?.(); // Node (tests): don't hold the event loop open; no-op in browsers
323
+ break;
324
+ }
325
+ // Live tail — display-only replacement, never committed to history. During an OPEN turn
326
+ // Speechmatics restates the tail around the just-locked boundary, so a partial's end_time can
327
+ // legitimately land at/behind the last locked word's cursor — gating those there silently
328
+ // drops live updates and the visible tail blanks/freezes for ~max_delay after each lock
329
+ // ("cuts every few seconds" during continuous speech). Only drop behind-cursor partials once
330
+ // the turn is CLOSED/closing (!turnOpen || awaitingFinal), so a stale old-turn partial can't
331
+ // leak a ghost transcript / false barge-in (onPartial also feeds the barge-in check).
332
+ case 'AddPartialTranscript': {
333
+ const end = m.metadata?.end_time ?? 0;
334
+ if (cursor > 0 && end > 0 && end <= cursor && (!turnOpen || awaitingFinal)) break;
335
+ turnOpen = true;
336
+ lastTail = (m.metadata?.transcript || '').trim();
337
+ onPartial(lastTail, ms, committed);
338
+ break;
339
+ }
340
+ // Words permanently locked → append to `committed`, advance the cursor. Never closes the
341
+ // turn — EndOfUtterance does (below).
342
+ case 'AddTranscript': {
343
+ const end = m.metadata?.end_time ?? 0;
344
+ if (cursor > 0 && end > 0 && end <= cursor) break; // straggler for a closed turn → drop
345
+ const frag = (m.metadata?.transcript || '').trim();
346
+ if (frag) { committed = committed ? `${committed} ${frag}` : frag; turnOpen = true; }
347
+ cursor = Math.max(cursor, end);
348
+ // The lock usually consumes the whole tail; a PREFIX lock (frag = the tail's first words)
349
+ // leaves the remainder as the only copy of the newest words — keep it for a safety close.
350
+ // A diverging lock (revised words) clears the tail so the fold can't duplicate.
351
+ lastTail = frag && lastTail.startsWith(frag) ? lastTail.slice(frag.length).trim() : '';
352
+ onPartial('', ms, committed); // tail consumed, committed grew → re-render solid
353
+ break;
354
+ }
355
+ // Speechmatics' turn boundary (after our ForceEndOfUtterance, or its own silence trigger).
356
+ // The flushing AddTranscript has already landed, so `committed` is complete → close on it.
357
+ case 'EndOfUtterance': {
358
+ if (!awaitingFinal && !turnOpen) break; // nothing open to close (stray/duplicate)
359
+ flushFinal();
360
+ break;
361
+ }
362
+ case 'Error': onError(`${m.type || 'error'}: ${m.reason || ''}`); break;
363
+ }
364
+ };
365
+ } finally {
366
+ opening = false;
367
+ }
368
+ };
369
+
370
+ return {
371
+ // Pre-open the socket (token fetch + WS handshake) before any audio so the first utterance isn't
372
+ // stalled by cold connect. Idempotent (open() has a re-entry guard); feed() still opens lazily.
373
+ open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
374
+ feed(i16) {
375
+ if (!isClosed() && (!ws || ws.readyState >= 2)) open();
376
+ sentSamples += i16.length; // stream position (counts outbox-queued audio too — sent in order)
377
+ sendAudio(i16); sentSinceCommit = true;
378
+ },
379
+ // Start a fresh utterance: drop leftover committed words so a prior turn can't bleed into this
380
+ // one. Called on a fresh VAD speech-start when the previous turn actually CLOSED — a turn the
381
+ // detector kept open is intentionally NOT reset. `cursor` survives (the straggler gate; end_times
382
+ // don't reset per turn); pending force-end state is cleared.
383
+ reset() { clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false; pendingForceEnd = false; committed = ''; lastTail = ''; turnOpen = false; },
384
+ // VAD speech-end = the utterance boundary. Ask Speechmatics to finalize NOW (ForceEndOfUtterance →
385
+ // it flushes the turn's words as AddTranscript, then emits EndOfUtterance); awaitingFinal makes that
386
+ // EndOfUtterance close the turn. A short safety timer closes it anyway if the socket never answers.
387
+ commit() {
388
+ if (awaitingFinal) return; // already closing — its flushFinal will notify
389
+ // Nothing to finalize (stray VAD end, e.g. only the AI's own echo): the agent waits on onFinal
390
+ // to clear its closing latch, so emit an empty close.
391
+ if (!turnOpen && !sentSinceCommit) { onFinal('', Math.round(performance.now() - sttStart)); return; }
392
+ if (!sentSinceCommit) { flushFinal(); return; } // no new audio this turn: close on the words we have
393
+ commitBoundary = sentSamples / 16000; // this turn's audio all ends at/before here (safety gate)
394
+ sentSinceCommit = false;
395
+ awaitingFinal = true;
396
+ sendForceEnd(); // sends now, or defers to RecognitionStarted if still opening
397
+ armSafety(TUNING.STT.forceEndSafetyMs); // fallback if EndOfUtterance never arrives (tuning.js)
398
+ },
399
+ close() { clearTimeout(safetyTimer); safetyTimer = null; clearInterval(keepalive); keepalive = null; outbox = []; committed = ''; lastTail = ''; cursor = 0; sentSamples = 0; commitBoundary = 0; turnOpen = false; awaitingFinal = false; sentSinceCommit = false; started = false; pendingForceEnd = false; usage.flush(); ws?.close(); ws = null; },
400
+ };
401
+ }
402
+
403
+ // ── Deepgram Flux realtime ───────────────────────────────────────────────────
404
+ // Browser↔Deepgram direct WS, same token pattern as the others: the backend mints a short-TTL
405
+ // access token (POST /v1/auth/grant, key stays server-side) and the browser authenticates via the
406
+ // WebSocket subprotocol ['bearer', token] (browsers can't set an Authorization header). Audio is
407
+ // raw binary pcm16 @16k like Speechmatics.
408
+ //
409
+ // Flux differs from Speechmatics/ElevenLabs in WHO owns the turn boundary: its model does native
410
+ // end-of-turn detection (semantic + acoustic) and emits TurnInfo events — `Update` while the turn
411
+ // is in progress (transcript = the WHOLE turn so far, not a fragment) and `EndOfTurn` when it
412
+ // decides the user is done. For that to work it must hear the real silences, so this provider sets
413
+ // continuous: true → the agent feeds the FULL mic stream (no VAD gating / preroll)
414
+ // nativeEOT: true → the agent does NOT commit() on VAD speech-end; onFinal arrives unsolicited
415
+ // from EndOfTurn and the agent's onFinal path closes the turn.
416
+ // commit()/reset() are still implemented (VAD-gated fallback contract) so a host that ignores the
417
+ // flags gets working — if slower — turn handling: commit() waits briefly for a native EndOfTurn and
418
+ // otherwise flushes the latest turn transcript.
419
+ const DG_URL = 'wss://api.deepgram.com/v2/listen';
420
+ // Deepgram's auth endpoints are US-hosted — a token mint costs ~0.6-1s from EU, and it sits on the
421
+ // CRITICAL PATH of every socket open (the token rides in the WS handshake, so nothing can connect
422
+ // until it arrives). The backend mints with a 300s TTL; cache it module-wide (shared across agent
423
+ // instances/reopens) and reuse until shortly before expiry, so a stop()/start() resume — which
424
+ // closes and reopens the socket — only pays the ~0.4s handshake instead of the full mint chain.
425
+ let dgToken = null, dgTokenExp = 0;
426
+ // Pre-mint + cache the Deepgram token OFF the critical path (call at prewarm time, e.g. mic-button
427
+ // mount): during a cold voice start the mint races Piper/ONNX downloads for bandwidth and main
428
+ // thread (measured 2.7s in-session vs 0.6s standalone), so paying it early means the click only owes
429
+ // the ~0.4s WS handshake. Best-effort + deduped; makeDeepgramSTT reads the same cache.
430
+ let dgMintPromise = null;
431
+ export function warmDeepgramToken(sttTokenUrl, apiKey, getToken) {
432
+ if (dgToken && performance.now() < dgTokenExp - 30000) return Promise.resolve();
433
+ return (dgMintPromise ??= (async () => {
434
+ const body = await mintSttToken(sttTokenUrl, apiKey, () => {}, getToken); // best-effort: failure just means open() re-mints
435
+ if (body) { dgToken = body.token; dgTokenExp = performance.now() + (body.expires_in ?? 300) * 1000; }
436
+ })().finally(() => { dgMintPromise = null; }));
437
+ }
438
+ export function makeDeepgramSTT(opts) {
439
+ const { apiKey, sttModel, sttLang = 'en', keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
440
+ eotThreshold = TUNING.STT.fluxEotThreshold, onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
441
+ const tokenUrl = sttTokenUrl;
442
+ const model = sttModel || 'flux-general-en';
443
+ let ws = null, opening = false, outbox = [], sttStart = 0;
444
+ // TurnInfo.transcript is cumulative for the CURRENT turn (Flux restarts it on each new turn_index),
445
+ // so `turnText` is simply the latest transcript — no committed/tail split needed.
446
+ let turnText = '', audioSinceFinal = false, awaitingFinal = false, safetyTimer = null;
447
+ const usage = makeUsageReporter(sttUsageUrl, apiKey, 'deepgram', { model });
448
+
449
+ // Pre-open buffer is a bounded PREROLL (~1s), not a backlog: Flux is a real-time model — burst-
450
+ // sending seconds of buffered audio after a slow token fetch/handshake makes its whole timeline lag
451
+ // real time (garbled starts, late EndOfTurns). Keep only the newest ~1s so the user's first words
452
+ // survive the connect without shifting Flux's clock.
453
+ const OUTBOX_MAX_SAMPLES = 32000; // ~2s: enough to cover the WS handshake so opening words survive
454
+ let outboxSamples = 0;
455
+ const sendAudio = (i16) => {
456
+ if (!ws || ws.readyState !== 1) {
457
+ outbox.push(i16); outboxSamples += i16.length;
458
+ while (outboxSamples > OUTBOX_MAX_SAMPLES && outbox.length > 1) outboxSamples -= outbox.shift().length;
459
+ return;
460
+ }
461
+ ws.send(i16.buffer); // raw binary linear16 @16k (declared in the URL query)
462
+ usage.add(i16.length);
463
+ };
464
+
465
+ // Close the turn from OUR side (host-called commit() fallback only — the agent never calls this in
466
+ // native mode). NB: because Flux transcripts are cumulative per turn, flushing a still-open turn
467
+ // emits a prefix Flux will restate — only useful for hosts that ignore the nativeEOT flag.
468
+ const flushFinal = () => {
469
+ clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false;
470
+ const t = turnText; turnText = ''; audioSinceFinal = false;
471
+ onFinal(t, Math.round(performance.now() - sttStart));
472
+ };
473
+ const armSafety = () => { clearTimeout(safetyTimer); safetyTimer = setTimeout(flushFinal, TUNING.STT.forceEndSafetyMs); };
474
+
475
+ const open = async () => {
476
+ if (opening) return;
477
+ opening = true;
478
+ try {
479
+ // Cached token still fresh? Skip the mint round trip entirely (see dgToken above). 15s slack:
480
+ // the token only needs to be valid at handshake time, but leave margin for a slow connect.
481
+ let token = (dgToken && performance.now() < dgTokenExp - 15000) ? dgToken : null;
482
+ if (!token) {
483
+ const body = await mintSttToken(tokenUrl, apiKey, onFatal, getToken);
484
+ if (!body) { outbox = []; outboxSamples = 0; return; }
485
+ token = dgToken = body.token;
486
+ dgTokenExp = performance.now() + (body.expires_in ?? 300) * 1000;
487
+ }
488
+ if (isClosed()) return;
489
+ const p = new URLSearchParams({ model, encoding: 'linear16', sample_rate: '16000', eot_threshold: String(eotThreshold) });
490
+ if (model.endsWith('multi') && sttLang) p.append('language_hint', sttLang.slice(0, 2));
491
+ for (const k of keyterms) p.append('keyterm', k);
492
+ sttStart = performance.now();
493
+ // Browser WS auth: Sec-WebSocket-Protocol "bearer, <token>" (short-TTL access token from
494
+ // /v1/auth/grant). The token only needs to be valid at handshake time; the socket persists.
495
+ ws = new WebSocket(`${DG_URL}?${p}`, ['bearer', token]);
496
+ ws.binaryType = 'arraybuffer';
497
+ ws.onopen = () => { const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c); };
498
+ ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('DG ws error', ev); } };
499
+ ws.onclose = ev => {
500
+ usage.flush(); onClose?.();
501
+ // Handshake rejected (auth) — likely a stale cached token (clock drift / revocation): drop the
502
+ // cache so the next open() re-mints instead of failing forever within the cached window.
503
+ if (ev.code === 1008 || ev.code === 4001 || ev.code === 1002) { dgToken = null; dgTokenExp = 0; }
504
+ if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); }
505
+ };
506
+ ws.onmessage = ev => {
507
+ if (typeof ev.data !== 'string') return;
508
+ const m = JSON.parse(ev.data);
509
+ const ms = Math.round(performance.now() - sttStart);
510
+ if (m.type === 'TurnInfo') {
511
+ const t = (m.transcript || '').trim();
512
+ if (m.event === 'EndOfTurn') {
513
+ // Native turn boundary: Flux decided the user is done. Emit the whole turn as final —
514
+ // unsolicited from the agent's point of view (its onFinal path closes the turn cleanly).
515
+ clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false;
516
+ turnText = ''; audioSinceFinal = false;
517
+ onFinal(t, ms);
518
+ } else { // Update / StartOfTurn (EagerEndOfTurn/TurnResumed only occur with eager_eot_threshold set)
519
+ turnText = t;
520
+ onPartial(t, ms);
521
+ }
522
+ } else if (m.type === 'Error') { onError(`${m.code || 'error'}: ${m.description || ''}`); console.error('DG error msg', m); }
523
+ };
524
+ } finally {
525
+ opening = false;
526
+ }
527
+ };
528
+
529
+ return {
530
+ continuous: true, // agent feeds the full mic stream (Flux needs real silences for its EOT model)
531
+ nativeEOT: true, // agent skips VAD-commit; EndOfTurn → unsolicited onFinal closes the turn
532
+ open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
533
+ feed(i16) {
534
+ if (!isClosed() && (!ws || ws.readyState >= 2)) open();
535
+ sendAudio(i16); audioSinceFinal = true;
536
+ },
537
+ reset() { turnText = ''; clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false; },
538
+ // VAD-gated fallback: give the native EndOfTurn a short window to land (it usually beats this —
539
+ // the trailing VAD-redemption silence is in the stream), then flush what we have. If no audio was
540
+ // fed since the last final there's no turn — emit an empty close so the host's closing latch clears.
541
+ commit() {
542
+ if (awaitingFinal) return;
543
+ if (!audioSinceFinal) { onFinal('', Math.round(performance.now() - sttStart)); return; }
544
+ awaitingFinal = true;
545
+ armSafety();
546
+ },
547
+ close() {
548
+ clearTimeout(safetyTimer); safetyTimer = null; outbox = []; outboxSamples = 0; turnText = ''; audioSinceFinal = false; awaitingFinal = false;
549
+ usage.flush();
550
+ if (ws?.readyState === 1) { try { ws.send(JSON.stringify({ type: 'CloseStream' })); } catch { /* closing anyway */ } }
551
+ ws?.close(); ws = null;
552
+ },
553
+ };
554
+ }
555
+
556
+ // ── Browser Web Speech API (local, no cloud, no token) ───────────────────────
557
+ // The browser's own SpeechRecognition engine (Chrome/Safari `webkitSpeechRecognition`). It OWNS the
558
+ // whole audio path — its own mic capture AND its own silence-based end-of-turn — so the agent builds
559
+ // NO getUserMedia / AudioWorklet / Silero-VAD pipeline for it (`selfCapture: true`, which the agent
560
+ // reads as "just open() me"). It's `nativeEOT: true`: the provider closes turns itself, so there's no
561
+ // VAD-commit — a short silence-debounce after the engine's own isFinal fires onFinal as ONE turn.
562
+ // interim results → onPartial(tail) live draft + word-based barge-in while the agent is speaking
563
+ // final results → accumulate + (re)arm the end-of-turn timer; if the user keeps talking within
564
+ // the debounce window the fragments merge, else the timer flushes the turn
565
+ // No backend token, no /stt/usage reporting (free + on-device), no socket. Only where the API exists
566
+ // (else open() reports a fatal so the host can fall back / surface it).
567
+ // Browser Web Speech API STT. The agent hands it nothing (selfCapture) — it owns its mic and its own
568
+ // end-of-turn (nativeEOT). It reports transcripts via onPartial (live draft, drives the user bubble +
569
+ // word-based barge-in) and onFinal (one committed user turn).
570
+ //
571
+ // THE HARD PART — Chrome fragments ONE logical listening session into MANY short-lived recognizer
572
+ // instances. A `SpeechRecognition` auto-`onend`s after a brief internal silence (even mid-sentence);
573
+ // we recreate it in onend to keep listening. A single user utterance therefore spans several
574
+ // recognizers, and TWO artefacts of that churn must be neutralised so the agent sees one clean turn:
575
+ //
576
+ // 1. LIVENESS RACE — the restart gap (onend → new onstart) is NOT a user pause, just a technicality.
577
+ // The turn boundary is decided SOLELY by a silence debounce measured from the last time real TEXT
578
+ // arrived (`lastActivity`), completely independent of recognizer lifecycle. A restart never arms,
579
+ // fires, or cancels that clock. So whether or not a recognizer happens to be alive when the timer
580
+ // elapses is irrelevant — no race. (The old bug: the debounce fired DURING a restart, emitting the
581
+ // opening as its own turn; the fresh recognizer then re-finalized the same buffered audio → the
582
+ // opening was duplicated into a second turn, which barged in and killed the reply.)
583
+ //
584
+ // 2. SEAM DUPLICATION — when Chrome auto-ends mid-word it re-delivers the tail it was still buffering
585
+ // as the NEXT recognizer's first final (and can even restate a tail within one recognizer's own
586
+ // result window). Blindly appending would double words ("hello there" + "there how" → "there there").
587
+ // Every final is de-overlapped against the turn-so-far before it's appended.
588
+ export function makeWebSpeechSTT(opts) {
589
+ const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onFatal, onClose, isClosed } = opts;
590
+ const Rec = typeof window !== 'undefined' && (window.SpeechRecognition || window.webkitSpeechRecognition);
591
+ const EOT_MS = TUNING.WEBSPEECH_EOT_MS ?? 900;
592
+ const TAIL_MAX_DEFERS = TUNING.WEBSPEECH_TAIL_MAX_DEFERS ?? 3;
593
+ let rec = null; // the live SpeechRecognition instance (null between restarts / when stopped)
594
+ let sttStart = 0; // performance.now() at the current turn's first activity (for onPartial/onFinal ms)
595
+ let turn = ''; // committed (isFinal) text of the CURRENT turn, merged across recognizer restarts
596
+ let interimTail = ''; // last live (not-yet-final) tail — KEPT across recognizer restarts so the draft never blanks
597
+ let eotTimer = null; // silence debounce: closes the turn EOT_MS after the last real text activity
598
+ let justClosed = ''; // tail of the turn we JUST flushed — a late re-finalized seam is deduped against it
599
+ let justClosedTimer = null; // clears justClosed after a short guard window (a genuine next turn isn't a seam)
600
+ let tailDefers = 0; // consecutive closes deferred waiting on an un-promoted tail (bounded, reset on new text)
601
+ let want = false; // intent to listen — onend auto-restarts while true
602
+ let muted = false; // mic off — stop the recognizer and don't restart
603
+ // AEC'D CAPTURE (SpeechRecognition.start(audioTrack), spec'd + shipped in recent Chromium): by
604
+ // default the engine opens its OWN mic, bypassing every protection the agent's pipeline has — the
605
+ // browser AEC referenced against the WebRTC TTS loopback (voice-agent initAecLoopback) and noise
606
+ // suppression. So on speakers the recognizer hears the agent's own reply. Handing it OUR
607
+ // getUserMedia track (echoCancellation+noiseSuppression on) routes the SAME protected audio the
608
+ // cloud providers get. Older engines throw on start(track) — one throw disables it for the
609
+ // session and we fall back to the engine's own capture (old behavior, text-filter guarded).
610
+ let micStream = null; // AEC'd capture the recognizer runs on — null → engine's own mic
611
+ let micTrack = null;
612
+ let trackArg = true; // start(track) supported? optimistic; first throw turns it off
613
+ let recHasTrack = false; // whether the CURRENT recognizer was started on micTrack (for the upgrade restart)
614
+ let micGen = 0; // bumped by dropMic — a getUserMedia resolving from an older generation is stopped, not published (mute/close during acquisition can't leak an OS mic)
615
+ let lastLoggedMode = null; // capture-mode console.info dedup — log only on change, not per restart
616
+ let micPending = null; // the single in-flight acquisition — concurrent ensureTrack calls share it
617
+ const acquireMic = () => {
618
+ if (micPending) return micPending;
619
+ if (micTrack?.readyState === 'live' || !trackArg || !navigator.mediaDevices?.getUserMedia) return Promise.resolve();
620
+ const gen = micGen;
621
+ micPending = navigator.mediaDevices.getUserMedia({ audio: {
622
+ echoCancellation: true, noiseSuppression: true, autoGainControl: true,
623
+ ...(micDeviceId ? { deviceId: { exact: micDeviceId } } : {}),
624
+ } }).then((s) => {
625
+ // Stale (muted/closed/dropped while pending) → stop immediately; publishing it would leak the mic.
626
+ if (gen !== micGen || muted || !want || isClosed()) { s.getTracks().forEach((t) => t.stop()); return; }
627
+ micStream?.getTracks().forEach((t) => t.stop()); // never hold two captures
628
+ micStream = s; micTrack = s.getAudioTracks()[0] ?? null;
629
+ // Device unplugged mid-session: forget the dead track so the next boundary re-acquires (or the
630
+ // recognizer falls back to its own capture) instead of silently keeping a corpse around.
631
+ if (micTrack) micTrack.addEventListener('ended', () => { if (micStream === s) { s.getTracks().forEach((t) => t.stop()); micStream = null; micTrack = null; } });
632
+ }).catch(() => { /* denied/no device → engine's own capture */ })
633
+ .finally(() => { micPending = null; });
634
+ return micPending;
635
+ };
636
+ const dropMic = () => { micGen++; micStream?.getTracks().forEach((t) => t.stop()); micStream = null; micTrack = null; };
637
+ // Acquire (or re-acquire) the track, then upgrade a recognizer that started track-less: abort it
638
+ // and let onend relaunch via start(), which now has micTrack. Only at a turn boundary — an abort
639
+ // mid-utterance would drop buffered speech. If acquisition lands mid-utterance, flush() re-runs
640
+ // this at the turn boundary, so the upgrade isn't lost.
641
+ const ensureTrack = () => acquireMic().then(() => {
642
+ if (trackArg && micTrack?.readyState === 'live' && rec && !recHasTrack && !turn && !interimTail) { try { rec.abort(); } catch { /* onend relaunches anyway */ } }
643
+ });
644
+ // SpeechRecognition wants a BCP-47 tag (en-US, hu-HU); our sttLang is often a bare code (en, hu).
645
+ const langTag = (l) => ({ en: 'en-US', hu: 'hu-HU', de: 'de-DE' }[String(l || '').slice(0, 2).toLowerCase()] || l || 'en-US');
646
+
647
+ // Chars of `next` to drop so a re-finalized tail isn't appended twice: the longest suffix of `prev`
648
+ // that is also a prefix of `next`, aligned to word boundaries (so "there"⊄"therefore"). Returns the
649
+ // overlap length in `next`. Case-insensitive compare; the caller slices the ORIGINAL-cased `next`.
650
+ const seamOverlap = (prev, next) => {
651
+ const a = prev.toLowerCase(), b = next.toLowerCase();
652
+ const max = Math.min(a.length, b.length);
653
+ for (let n = max; n > 0; n--) {
654
+ const boundary = b.length === n || b[n] === ' '; // the overlap ends on a word edge in `next`
655
+ const aligned = a.length === n || a[a.length - n - 1] === ' '; // ...and on one in `prev`
656
+ if (boundary && aligned && a.slice(a.length - n) === b.slice(0, n)) return n;
657
+ }
658
+ return 0;
659
+ };
660
+
661
+ // Append a newly-finalized fragment to the turn, de-overlapping the restart/restate seam first.
662
+ // seamOverlap is word-boundary aligned, so it drops a re-finalized tail (whole or partial) without
663
+ // eating a legitimate suffix that merely shares letters ("goodbye" then "bye" keeps "bye").
664
+ const commitFinal = (frag) => {
665
+ frag = frag.trim();
666
+ if (!frag) return;
667
+ // If a turn just flushed (Chrome auto-restarted right as the silence clock closed the turn), the
668
+ // fresh recognizer often re-finalizes that turn's tail as this turn's FIRST final — a duplicate.
669
+ // De-overlap it against the just-closed tail so the repeat is dropped instead of becoming a bogus
670
+ // new turn (the barge-in-killing dup). Only applies while `turn` is still empty (start of a turn).
671
+ if (!turn && justClosed) { const ov = seamOverlap(justClosed, frag); frag = ov ? frag.slice(ov).trimStart() : frag; if (!frag) return; }
672
+ if (turn) { const ov = seamOverlap(turn, frag); frag = ov ? frag.slice(ov).trimStart() : frag; }
673
+ if (frag) turn = turn ? `${turn} ${frag}` : frag;
674
+ };
675
+
676
+ const flush = () => { // the silence debounce elapsed: the turn-so-far is one complete user turn
677
+ clearTimeout(eotTimer); eotTimer = null;
678
+ // NOT QUIESCENT YET: a live recognizer still holding an un-promoted tail means the utterance is
679
+ // by definition unfinished — Chrome's endpointer can take longer than EOT_MS to promote trailing
680
+ // words to isFinal. Closing here would emit a TRUNCATED turn and let the late final open a bogus
681
+ // second one — and since the two texts are DISJOINT ("book a meeting" / "for tomorrow at five"),
682
+ // the justClosed seam-guard has no overlap to dedup and can't catch it. So wait for the engine
683
+ // instead of guessing with a clock. The wait is bounded twice over: onend clears the tail (the
684
+ // engine gave up on it), and TAIL_MAX_DEFERS caps a tail that never resolves. The cap only bounds
685
+ // the WAIT — it does not make that path safe: closing there emits the same truncated turn, just
686
+ // ≥TAIL_MAX_DEFERS windows later. It's a liveness backstop for an engine that never resolves nor
687
+ // ends, not a second correctness guarantee (the real fix is waiting for quiescence at all).
688
+ if (rec && interimTail && tailDefers++ < TAIL_MAX_DEFERS) { armEot(); return; }
689
+ interimTail = ''; tailDefers = 0;
690
+ // Only FINALIZED words become a turn. A tail that survived to end-of-turn WITHOUT ever finalizing
691
+ // is noise (throat-clears, background speech) — Chrome finalizes real speech, so a never-final
692
+ // interim is dropped, not committed. `turn` already holds every finalized word (incl. tails folded
693
+ // in on restart by onend, which the fresh recognizer's re-finalization would have confirmed).
694
+ const text = turn.trim(); turn = '';
695
+ if (!text) return; // blank silence → not a turn
696
+ // Guard the seam: if Chrome auto-restarted exactly as this turn closed, the fresh recognizer may
697
+ // re-finalize this turn's tail as the NEXT turn's opening. Remember it briefly so commitFinal drops
698
+ // that late duplicate instead of emitting a bogus turn (the barge-in-killer). One-debounce window —
699
+ // a genuine next turn arriving later is NOT a seam.
700
+ justClosed = text; clearTimeout(justClosedTimer); justClosedTimer = setTimeout(() => { justClosed = ''; }, EOT_MS);
701
+ onPartial('', 0, ''); // clear the live draft the just-committed turn was still showing
702
+ onFinal(text, Math.round(performance.now() - sttStart));
703
+ // Turn boundary = the safe seam for the AEC'd-track upgrade (and for re-acquiring a track that
704
+ // died mid-turn) — if the track landed while the user was speaking, this is where it takes effect.
705
+ if (want && !muted && !isClosed()) ensureTrack();
706
+ };
707
+ // A SILENCE clock, not a recognizer clock: (re)armed on ANY real text (final OR live interim) so a
708
+ // mid-thought pause keeps one turn together, and closed only after EOT_MS with no new text at all —
709
+ // recognizer restarts in between neither arm nor fire it. That's what makes the turn boundary immune
710
+ // to Chrome's internal churn.
711
+ const armEot = () => { clearTimeout(eotTimer); eotTimer = setTimeout(flush, EOT_MS); };
712
+
713
+ const start = () => {
714
+ if (!Rec || rec || muted || isClosed()) return;
715
+ const r = new Rec();
716
+ r.continuous = true; r.interimResults = true; r.lang = langTag(sttLang);
717
+ r.onstart = () => { if (!turn && !interimTail) sttStart = performance.now(); }; // anchor ms at the turn's first activity (final OR live interim), not each restart
718
+ r.onresult = (event) => {
719
+ if (r !== rec) return; // stale recognizer (superseded by a restart) — ignore its late events
720
+ let final = '', interim = '';
721
+ for (let i = event.resultIndex; i < event.results.length; i++) {
722
+ const t = event.results[i][0].transcript;
723
+ if (event.results[i].isFinal) final += t; else interim += t;
724
+ }
725
+ final = final.trim(); interim = interim.trim();
726
+ if (!turn && !interimTail && (final || interim)) sttStart = performance.now(); // first activity of a fresh turn
727
+ commitFinal(final); // de-overlaps the seam, then locks the newly-final words into the turn
728
+ // Keep the live tail as its own state so a recognizer restart (which drops interim) can replay it
729
+ // instead of blanking the draft. A frame that only finalized words (interim === '') clears it —
730
+ // those words moved into `turn`, so the tail is genuinely empty now, not lost.
731
+ interimTail = interim;
732
+ if (final || interim) { tailDefers = 0; armEot(); } // any real text (re)arms the silence debounce (and is fresh progress, so the tail-deferral budget resets) — see armEot
733
+ // Live draft = committed turn-so-far + the still-forming interim tail. Drives the user bubble and
734
+ // word-based barge-in (the agent cuts its reply the instant real words land).
735
+ onPartial(interimTail, Math.round(performance.now() - sttStart), turn);
736
+ };
737
+ r.onerror = (event) => {
738
+ if (r !== rec) return;
739
+ if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); }
740
+ // 'no-speech' / 'aborted' / 'network' are recoverable — onend restarts; don't surface.
741
+ };
742
+ r.onend = () => {
743
+ if (r !== rec) return; // superseded instance's late onend — ignore
744
+ rec = null;
745
+ // The engine dropped its buffered tail, so it no longer represents PENDING words — a close
746
+ // deferred on it (see flush) must not keep waiting. The draft still shows the committed `turn`,
747
+ // and the fresh recognizer re-finalizes real speech, so clearing it here costs no flicker.
748
+ interimTail = '';
749
+ // Chrome drops any not-yet-final interim on auto-end. DON'T commit it here — only finalized words
750
+ // count (a never-final tail is noise). If the tail was real speech, the fresh recognizer will
751
+ // re-finalize it (commitFinal then locks it in, deduping the seam).
752
+ // Keep the SAME logical session alive by relaunching. The turn and its silence timer are
753
+ // untouched — if this really was end-of-turn, that timer fires on its own EOT_MS after the last
754
+ // word (recognizer alive or not); if the user is still talking, the new recognizer's text
755
+ // re-arms it. No flush decision belongs here — that's what makes the boundary restart-immune.
756
+ if (want && !muted && !isClosed()) start();
757
+ else onClose?.();
758
+ };
759
+ rec = r;
760
+ let ok = false;
761
+ if (trackArg && micTrack?.readyState === 'live') {
762
+ try { r.start(micTrack); ok = true; recHasTrack = true; }
763
+ catch (e) { trackArg = false; dropMic(); console.info('[webspeech] start(track) rejected — engine\'s own mic (no AEC):', e?.message || e); } // engine predates start(track) — release the now-useless capture (don't hold two mics)
764
+ }
765
+ if (!ok) { recHasTrack = false; try { r.start(); } catch { rec = null; } } // start() throws if already running — treat as no-op
766
+ if (rec && recHasTrack !== lastLoggedMode) { lastLoggedMode = recHasTrack; console.info(`[webspeech] capture: ${recHasTrack ? "AEC'd track (start(track))" : "engine's own mic (unprotected)"}`); }
767
+ };
768
+
769
+ // Stop the current recognizer and drop all in-progress state (mute/close). The stopped recognizer's
770
+ // late onend is stale (rec nulled → r !== rec), so it can't auto-restart.
771
+ const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
772
+
773
+ return {
774
+ selfCapture: true, // owns its mic → the agent builds no capture pipeline
775
+ nativeEOT: true, // owns its end-of-turn → the agent runs no VAD-commit
776
+ open() {
777
+ if (!Rec) { onFatal?.('Speech recognition not supported in this browser'); return; }
778
+ // start() first, synchronously — Safari requires SpeechRecognition.start inside the click
779
+ // gesture. The AEC'd track lands async right after and upgrades the recognizer at the seam.
780
+ want = true; start();
781
+ ensureTrack();
782
+ },
783
+ // Mute: Web Speech captures its OWN mic (independent of the agent), so mute must actually stop it.
784
+ // `rec` is nulled by teardown() so the stopped recognizer's late onend is stale (r !== rec) and
785
+ // can't auto-restart; unmute relaunches.
786
+ setEnabled(on) {
787
+ muted = !on;
788
+ if (muted) { teardown(); dropMic(); onPartial('', 0, ''); } // drop the draft AND the mic (privacy light off)
789
+ else if (want && !rec) { start(); ensureTrack(); }
790
+ },
791
+ close() { want = false; teardown(); dropMic(); },
792
+ };
793
+ }
794
+
795
+ // Registry: provider id → factory. Add a backend here and it's selectable everywhere.
796
+ export const STT_PROVIDERS = {
797
+ webspeech: makeWebSpeechSTT,
798
+ speechmatics: makeSpeechmaticsSTT,
799
+ elevenlabs: makeElevenLabsSTT,
800
+ deepgram: makeDeepgramSTT,
801
+ };
802
+
803
+ // Web Speech only exists in Chromium/Safari — Firefox and Linux WebKitGTK webviews ship no
804
+ // SpeechRecognition at all. There the default 'webspeech' provider can only fatal ("Speech
805
+ // recognition not supported in this browser") and voice mode is simply dead, so resolve it to a
806
+ // cloud provider instead (needs a token route configured — see the auth notes at the top).
807
+ export const WEBSPEECH_FALLBACK = 'elevenlabs';
808
+ export const webSpeechSupported = () =>
809
+ typeof window !== 'undefined' && !!(window.SpeechRecognition || window.webkitSpeechRecognition);
810
+ /** Provider id actually used for `id` here — 'webspeech' downgrades to the cloud fallback in a real
811
+ * browser that lacks SpeechRecognition. Hosts can call it to label/inform ("using cloud recognition").
812
+ * Outside a browser (SSR/tests) nothing is downgraded: there's no mic there either, so the honest
813
+ * "not supported" fatal is the right answer. */
814
+ export const resolveSttProvider = (id) =>
815
+ (id === 'webspeech' && typeof window !== 'undefined' && !webSpeechSupported() ? WEBSPEECH_FALLBACK : id);