@todoforai/voiceloop 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +239 -0
- package/examples/simple-browser.html +100 -0
- package/package.json +48 -0
- package/src/deps.js +49 -0
- package/src/index.js +15 -0
- package/src/llm-openai.js +122 -0
- package/src/stt.js +815 -0
- package/src/tts-elevenlabs.js +50 -0
- package/src/tuning.js +182 -0
- package/src/voice-agent.js +1984 -0
package/src/stt.js
ADDED
|
@@ -0,0 +1,815 @@
|
|
|
1
|
+
// stt.js — pluggable STT providers for the VoiceAgent. One small contract, many backends.
|
|
2
|
+
//
|
|
3
|
+
// A provider is a factory `(opts) => STT` where STT is:
|
|
4
|
+
// open() [optional] pre-open the socket (token fetch + WS handshake) before any audio, so the
|
|
5
|
+
// first utterance isn't stalled by a cold connect; idempotent, and feed() still opens lazily
|
|
6
|
+
// feed(i16) push a 16kHz mono Int16Array chunk (provider opens its socket lazily on first feed)
|
|
7
|
+
// commit() VAD speech-end → finalize the current utterance (no-op if nothing was fed since)
|
|
8
|
+
// close() teardown the socket and drop buffered audio
|
|
9
|
+
// It reports back through callbacks passed in `opts`:
|
|
10
|
+
// onPartial(text, ms, committed='') live interim TAIL (called many times/utterance); `committed`
|
|
11
|
+
// is the word-level locked prefix to render solid (Speechmatics)
|
|
12
|
+
// onFinal(text) one finalized utterance → becomes a user turn
|
|
13
|
+
// onError(msg) recoverable error, surfaced to the host (session stays alive)
|
|
14
|
+
// onFatal(msg) unrecoverable (auth/config) — host should stop() the agent
|
|
15
|
+
// onClose() socket closed (used for usage flushing)
|
|
16
|
+
// isClosed() agent stopped — don't reopen / don't emit
|
|
17
|
+
// A provider may also expose capability flags the agent adapts to:
|
|
18
|
+
// continuous: true feed the FULL mic stream (no VAD gating/preroll) — the provider's own model
|
|
19
|
+
// needs to hear the silences (e.g. Deepgram Flux end-of-turn detection)
|
|
20
|
+
// nativeEOT: true the provider closes turns itself (unsolicited onFinal) — the agent skips its
|
|
21
|
+
// VAD-driven commit() and just handles onFinal when it arrives
|
|
22
|
+
// selfCapture: true the provider OWNS the whole audio path: its own mic + its own end-of-turn
|
|
23
|
+
// (e.g. the browser Web Speech API). The agent builds NO mic/AudioWorklet/VAD
|
|
24
|
+
// pipeline for it — start() just opens the provider. Implies nativeEOT (the
|
|
25
|
+
// provider closes its own turns via unsolicited onFinal). It exposes open()/
|
|
26
|
+
// close() + setEnabled() (mute), and drives barge-in the normal way via onPartial.
|
|
27
|
+
//
|
|
28
|
+
// Adding a provider = write a factory + register it in STT_PROVIDERS below. The VoiceAgent owns
|
|
29
|
+
// the mic + VAD and just forwards feed/commit/close, so providers never touch audio capture.
|
|
30
|
+
|
|
31
|
+
import { TUNING } from './tuning.js';
|
|
32
|
+
|
|
33
|
+
const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at least this often
|
|
34
|
+
|
|
35
|
+
// Common opts shape (provided by the VoiceAgent):
|
|
36
|
+
// { apiKey, sttUrl, sttModel, sttLang, keyterms, sttTokenUrl, getToken, sttUsageUrl,
|
|
37
|
+
// onPartial, onFinal, onError, onClose, isClosed }
|
|
38
|
+
//
|
|
39
|
+
// AUTH (cloud providers): the raw provider key must stay server-side. Either
|
|
40
|
+
// • `sttTokenUrl` — your backend route that mints a short-TTL token: POST, optional X-API-Key
|
|
41
|
+
// (`apiKey`), responds { token, expires_in? }; or
|
|
42
|
+
// • `getToken` — an async () => ({ token, expires_in? }) callback, when your auth doesn't fit
|
|
43
|
+
// that one route shape. Takes precedence over sttTokenUrl.
|
|
44
|
+
|
|
45
|
+
// ── Shared socket-lifecycle helpers ─────────────────────────────────────────
|
|
46
|
+
// The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
|
|
47
|
+
// Deepgram's cumulative-turn + token cache, ElevenLabs' simple commit) — those stay separate on
|
|
48
|
+
// purpose. But token-fetch-with-timeout and usage-reporting are byte-for-byte the same shape in all
|
|
49
|
+
// three; duplicating THOSE is a pure bug magnet (a timeout fix would need applying 3x), so they're
|
|
50
|
+
// factored out here.
|
|
51
|
+
|
|
52
|
+
// Mint a short-TTL token — via the host's `getToken` callback when given, else a POST to
|
|
53
|
+
// `sttTokenUrl`. Bounded by tokenFetchTimeoutMs so a hung/unreachable backend can't stall
|
|
54
|
+
// voice-mode startup forever. Calls onFatal itself on any failure (network, timeout, non-2xx,
|
|
55
|
+
// missing token) and returns null — the caller just checks for null and bails (what buffered
|
|
56
|
+
// state to clear on failure differs per provider, so that stays with the caller).
|
|
57
|
+
async function mintSttToken(url, apiKey, onFatal, getToken) {
|
|
58
|
+
if (getToken) {
|
|
59
|
+
try {
|
|
60
|
+
const body = await getToken();
|
|
61
|
+
if (!body?.token) throw new Error('getToken returned no token');
|
|
62
|
+
return body;
|
|
63
|
+
} catch (e) { onFatal(`STT token: ${e.message}`); return null; }
|
|
64
|
+
}
|
|
65
|
+
if (!url) { onFatal('STT token: no sttTokenUrl/getToken configured — cloud STT needs a token route (see README)'); return null; }
|
|
66
|
+
let res;
|
|
67
|
+
try {
|
|
68
|
+
res = await fetch(url, { method: 'POST', headers: apiKey ? { 'X-API-Key': apiKey } : {}, signal: AbortSignal.timeout(TUNING.STT.tokenFetchTimeoutMs) });
|
|
69
|
+
} catch (e) {
|
|
70
|
+
onFatal(`STT token: ${e.name === 'TimeoutError' ? 'timed out' : e.message}`);
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const body = await res.json().catch(() => ({}));
|
|
74
|
+
if (!res.ok || !body.token) { onFatal(`STT token: ${body.error || body.message || res.statusText}`); return null; }
|
|
75
|
+
return body;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Accumulate streamed samples and flush them to /stt/usage — inline once USAGE_FLUSH_SECONDS is
|
|
79
|
+
// crossed, and again whenever the caller flushes on close (billing is trust-but-verify: the client
|
|
80
|
+
// reports what it streamed since the backend can't meter a direct browser↔provider socket).
|
|
81
|
+
function makeUsageReporter(sttUsageUrl, apiKey, provider, extra = {}) {
|
|
82
|
+
let samples = 0;
|
|
83
|
+
const flush = () => {
|
|
84
|
+
const seconds = samples / 16000; samples = 0;
|
|
85
|
+
if (seconds <= 0 || !sttUsageUrl) return;
|
|
86
|
+
fetch(sttUsageUrl, {
|
|
87
|
+
method: 'POST', keepalive: true,
|
|
88
|
+
headers: { 'Content-Type': 'application/json', 'X-API-Key': apiKey },
|
|
89
|
+
body: JSON.stringify({ seconds, provider, ...extra }),
|
|
90
|
+
}).catch(() => {});
|
|
91
|
+
};
|
|
92
|
+
return { add: (n) => { samples += n; if (samples >= 16000 * USAGE_FLUSH_SECONDS) flush(); }, flush };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ── ElevenLabs Scribe realtime ──────────────────────────────────────────────
|
|
96
|
+
// Browser↔ElevenLabs direct WS. Auth via a single-use token minted by the backend (/stt/token);
|
|
97
|
+
// the key stays server-side. Billing is trust-but-verify: the client reports streamed seconds to
|
|
98
|
+
// /stt/usage (the backend can't meter a direct browser↔ElevenLabs socket).
|
|
99
|
+
export function makeElevenLabsSTT(opts) {
|
|
100
|
+
// `||` fallbacks, not destructuring defaults: VoiceAgent passes '' for unset knobs, which a
|
|
101
|
+
// `= default` would NOT replace — an empty model_id makes ElevenLabs hard-close the socket (1006).
|
|
102
|
+
const { apiKey, sttLang, keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
|
|
103
|
+
onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
104
|
+
const sttUrl = opts.sttUrl || 'wss://api.elevenlabs.io/v1/speech-to-text/realtime';
|
|
105
|
+
const sttModel = opts.sttModel || 'scribe_v2_realtime';
|
|
106
|
+
let ws = null, opening = false, outbox = [], sentSinceCommit = false, sttStart = 0;
|
|
107
|
+
const usage = makeUsageReporter(sttUsageUrl, apiKey, 'elevenlabs');
|
|
108
|
+
|
|
109
|
+
const sendAudio = (i16) => {
|
|
110
|
+
if (!ws || ws.readyState !== 1) { outbox.push(i16); return; }
|
|
111
|
+
let bin = ''; const b = new Uint8Array(i16.buffer); for (let i = 0; i < b.length; i++) bin += String.fromCharCode(b[i]);
|
|
112
|
+
ws.send(JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: btoa(bin) }));
|
|
113
|
+
usage.add(i16.length);
|
|
114
|
+
};
|
|
115
|
+
const sendCommit = () => ws.send(JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: '', commit: true }));
|
|
116
|
+
|
|
117
|
+
const open = async () => {
|
|
118
|
+
if (opening) return; // re-entry guard: the async token fetch could otherwise open two sockets
|
|
119
|
+
opening = true;
|
|
120
|
+
try {
|
|
121
|
+
const body = await mintSttToken(sttTokenUrl, apiKey, onFatal, getToken);
|
|
122
|
+
if (!body) {
|
|
123
|
+
// Auth/config failure won't fix itself — drop buffered audio (don't replay stale speech on
|
|
124
|
+
// a later recovery); mintSttToken already called onFatal so the host stops instead of
|
|
125
|
+
// re-firing on every utterance.
|
|
126
|
+
outbox = []; sentSinceCommit = false;
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (isClosed()) return;
|
|
130
|
+
const p = new URLSearchParams({ model_id: sttModel, commit_strategy: 'manual', audio_format: 'pcm_16000', token: body.token });
|
|
131
|
+
if (sttLang) p.set('language_code', sttLang);
|
|
132
|
+
for (const k of keyterms) p.append('keyterms', k);
|
|
133
|
+
sttStart = performance.now();
|
|
134
|
+
ws = new WebSocket(`${sttUrl}?${p}`);
|
|
135
|
+
ws.onopen = () => { const q = outbox; outbox = []; for (const it of q) it === 'commit' ? sendCommit() : sendAudio(it); };
|
|
136
|
+
ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('STT ws error', ev); } };
|
|
137
|
+
ws.onclose = ev => { usage.flush(); onClose?.(); if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); console.error('STT ws closed', ev.code, ev.reason); } };
|
|
138
|
+
ws.onmessage = ev => {
|
|
139
|
+
const m = JSON.parse(ev.data);
|
|
140
|
+
const ms = Math.round(performance.now() - sttStart);
|
|
141
|
+
if (m.message_type?.endsWith('_error')) { onError(`${m.message_type}: ${m.error || m.reason || m.message || ''}`); console.error('STT error msg', m); return; }
|
|
142
|
+
if (m.message_type === 'partial_transcript') onPartial(m.text || '', ms);
|
|
143
|
+
else if (m.message_type?.startsWith('committed') && m.text) onFinal(m.text, ms);
|
|
144
|
+
};
|
|
145
|
+
} finally {
|
|
146
|
+
opening = false;
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
// Pre-open the socket (token fetch + WS handshake) before any audio so the first utterance isn't
|
|
152
|
+
// stalled by cold connect. Idempotent (open() has a re-entry guard); feed() still opens lazily.
|
|
153
|
+
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
154
|
+
feed(i16) {
|
|
155
|
+
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
156
|
+
sendAudio(i16); sentSinceCommit = true;
|
|
157
|
+
},
|
|
158
|
+
commit() {
|
|
159
|
+
// No audio since the last commit (VAD can fire several times per utterance, or only the AI's own
|
|
160
|
+
// echo was heard): don't ask ElevenLabs to finalize (that would dup/empty-turn), but still emit an
|
|
161
|
+
// empty close. The agent sets _turnClosing before calling commit() and clears it on onFinal — so
|
|
162
|
+
// without this an empty VAD end would leave _turnClosing stuck (mirrors the Speechmatics contract).
|
|
163
|
+
if (!sentSinceCommit) { onFinal('', Math.round(performance.now() - sttStart)); return; }
|
|
164
|
+
sentSinceCommit = false;
|
|
165
|
+
if (ws?.readyState === 1) sendCommit(); else outbox.push('commit');
|
|
166
|
+
},
|
|
167
|
+
close() { outbox = []; sentSinceCommit = false; usage.flush(); ws?.close(); ws = null; },
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ── Speechmatics realtime ────────────────────────────────────────────────────
|
|
172
|
+
// Browser↔Speechmatics direct WS. Like ElevenLabs: the backend mints a short-TTL JWT (the raw key
|
|
173
|
+
// stays server-side), the browser connects with ?jwt=. Differs in wire shape — a JSON config
|
|
174
|
+
// message (StartRecognition) then BINARY audio frames (not base64), and JSON transcript messages
|
|
175
|
+
// (AddPartialTranscript / AddTranscript / EndOfUtterance). Billing is trust-but-verify: we report
|
|
176
|
+
// streamed seconds to /stt/usage tagged provider:'speechmatics'.
|
|
177
|
+
//
|
|
178
|
+
// TURN MODEL — LOCKED WORDS ARE THE ONLY SOURCE OF TRUTH:
|
|
179
|
+
// AddTranscript = permanently locked words → append to `committed`.
|
|
180
|
+
// AddPartialTranscript = the flickering not-yet-locked tail → shown live, NEVER written to history.
|
|
181
|
+
// EndOfUtterance = the turn boundary (follows our ForceEndOfUtterance, or the silence trigger).
|
|
182
|
+
// It always trails the flushing AddTranscript, so on arrival `committed` holds
|
|
183
|
+
// every locked word → onFinal(committed), no tail folding (the old mergeTail
|
|
184
|
+
// path was the source of the "four, five, four five" duplication). Only the
|
|
185
|
+
// SAFETY-TIMER close (EndOfUtterance never arrived) folds the interim tail in
|
|
186
|
+
// — see flushFinal's `withTail`.
|
|
187
|
+
// commit() = ForceEndOfUtterance; we wait for EndOfUtterance (bounded safety fallback if the socket
|
|
188
|
+
// never answers). NB: requires conversation_config.end_of_utterance_silence_trigger in StartRecognition.
|
|
189
|
+
//
|
|
190
|
+
// ORDERING: end_times are absolute seconds from StartRecognition (never reset per turn). `cursor` =
|
|
191
|
+
// end_time of the last accepted word; anything ending at/before it is an out-of-order straggler from
|
|
192
|
+
// a closed turn → dropped. The cursor only advances and survives reset() (close() zeroes it).
|
|
193
|
+
const SM_URL = 'wss://eu.rt.speechmatics.com/v2';
|
|
194
|
+
export function makeSpeechmaticsSTT(opts) {
|
|
195
|
+
const { apiKey, sttModel, sttLang = 'en', sttTokenUrl, getToken, sttUsageUrl,
|
|
196
|
+
onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
197
|
+
const tokenUrl = sttTokenUrl;
|
|
198
|
+
let ws = null, opening = false, started = false, outbox = [], sentSinceCommit = false, sttStart = 0;
|
|
199
|
+
// committed = locked words (the turn's text; the partial tail is display-only). cursor = monotonic
|
|
200
|
+
// end_time (s) of the last accepted word — the straggler gate (see header). turnOpen = words landed
|
|
201
|
+
// since the last close. awaitingFinal = ForceEndOfUtterance in flight, waiting on EndOfUtterance.
|
|
202
|
+
// sentSamples = stream position (total samples fed this socket, outbox included); commit() snapshots
|
|
203
|
+
// it as commitBoundary — if the safety timer closes a turn WITHOUT the EndOfUtterance handshake, the
|
|
204
|
+
// cursor advances to that boundary, gating the old turn's late flush (audio fed before the commit)
|
|
205
|
+
// while the next turn's words (fed after) flow immediately.
|
|
206
|
+
let committed = '', cursor = 0, turnOpen = false, awaitingFinal = false, safetyTimer = null, pendingForceEnd = false;
|
|
207
|
+
let sentSamples = 0, commitBoundary = 0;
|
|
208
|
+
// lastTail = the most recent interim tail (AddPartialTranscript). Normally display-only, but on a
|
|
209
|
+
// SAFETY-TIMER close (the flushing AddTranscript never arrived) it's the only copy of the user's
|
|
210
|
+
// last words — fold it into that final instead of stripping it. Cleared whenever words lock.
|
|
211
|
+
let lastTail = '';
|
|
212
|
+
// keepalive: audio is VAD-gated, so quiet listening sends NOTHING — Speechmatics closes an idle
|
|
213
|
+
// socket after ~3min of no audio. Trickle 0.1s of silence when idle so the session survives long
|
|
214
|
+
// silences instead of dying + reconnecting (which loses the first words to the token/handshake).
|
|
215
|
+
let keepalive = null, lastSendMs = 0;
|
|
216
|
+
// Send ForceEndOfUtterance now if the socket is live, else defer it until RecognitionStarted (a VAD
|
|
217
|
+
// commit can land before the socket finishes opening; without this the force-end is silently dropped).
|
|
218
|
+
const sendForceEnd = () => {
|
|
219
|
+
if (ws?.readyState === 1 && started) { ws.send(JSON.stringify({ message: 'ForceEndOfUtterance' })); pendingForceEnd = false; }
|
|
220
|
+
else pendingForceEnd = true;
|
|
221
|
+
};
|
|
222
|
+
// Close the turn: emit the LOCKED words (committed) as the final — never the tail. By the time
|
|
223
|
+
// EndOfUtterance lands, Speechmatics has already flushed the turn's words as AddTranscript, so
|
|
224
|
+
// `committed` is complete. Called on EndOfUtterance, or by the safety timer if it never arrives.
|
|
225
|
+
// `withTail`: closing WITHOUT the EndOfUtterance handshake (safety timer) — the flushing
|
|
226
|
+
// AddTranscript never landed, so the last interim tail is the only copy of the user's final words.
|
|
227
|
+
// Fold it in rather than strip it; the cursor advance (armSafety) gates the late flush so the same
|
|
228
|
+
// words can't ALSO land as a locked duplicate afterwards. The normal EOU path stays locked-words-only.
|
|
229
|
+
const flushFinal = (withTail = false) => {
|
|
230
|
+
clearTimeout(safetyTimer); safetyTimer = null;
|
|
231
|
+
awaitingFinal = false; turnOpen = false; sentSinceCommit = false; // turn closed → nothing left to finalize
|
|
232
|
+
const t = (withTail && lastTail ? `${committed} ${lastTail}` : committed).trim();
|
|
233
|
+
committed = ''; lastTail = ''; // keep cursor: monotonic gate survives the turn
|
|
234
|
+
// Always notify — even empty — so the agent clears its "closing" state and never wedges waiting.
|
|
235
|
+
onFinal(t, Math.round(performance.now() - sttStart));
|
|
236
|
+
};
|
|
237
|
+
const armSafety = (ms) => { clearTimeout(safetyTimer); safetyTimer = setTimeout(() => {
|
|
238
|
+
// Closing WITHOUT the EndOfUtterance handshake: the old turn's late flush may hold words the
|
|
239
|
+
// cursor never saw locked. Advance it to the commit-time stream boundary — the old turn's words
|
|
240
|
+
// (fed before the commit) end at/before it and are gated; the next turn's flow immediately.
|
|
241
|
+
cursor = Math.max(cursor, commitBoundary);
|
|
242
|
+
flushFinal(true);
|
|
243
|
+
}, ms); };
|
|
244
|
+
|
|
245
|
+
const usage = makeUsageReporter(sttUsageUrl, apiKey, 'speechmatics', { model: sttModel || 'standard' });
|
|
246
|
+
|
|
247
|
+
const sendAudio = (i16) => {
|
|
248
|
+
if (!ws || ws.readyState !== 1 || !started) { outbox.push(i16); return; }
|
|
249
|
+
ws.send(i16.buffer); // raw binary AddAudio (pcm_s16le @16k, declared in StartRecognition)
|
|
250
|
+
lastSendMs = performance.now();
|
|
251
|
+
usage.add(i16.length);
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
const open = async () => {
|
|
255
|
+
if (opening) return;
|
|
256
|
+
opening = true;
|
|
257
|
+
// NEW SOCKET = NEW STREAM CLOCK: end_times restart at 0 (they're relative to the new socket's
|
|
258
|
+
// StartRecognition), so per-socket state must reset. Without this, a silent reconnect (e.g.
|
|
259
|
+
// Speechmatics' idle timeout closed the old socket, feed() reopened) leaves the old
|
|
260
|
+
// cursor/sentSamples in place — every word of the new stream ends "behind" the stale cursor and
|
|
261
|
+
// is dropped as a straggler → the session goes PERMANENTLY DEAF. Stale `started` similarly made
|
|
262
|
+
// feed() send audio before the new RecognitionStarted handshake. Reset SYNCHRONOUSLY, before the
|
|
263
|
+
// awaited token fetch: feed() calls open() first and increments sentSamples after, so audio
|
|
264
|
+
// queued while the socket opens is counted into THIS socket's clock (the commitBoundary safety
|
|
265
|
+
// gate stays correct). `committed` survives: words locked before a mid-turn drop still belong to
|
|
266
|
+
// the open turn's final.
|
|
267
|
+
cursor = 0; sentSamples = 0; commitBoundary = 0; started = false;
|
|
268
|
+
try {
|
|
269
|
+
const body = await mintSttToken(tokenUrl, apiKey, onFatal, getToken);
|
|
270
|
+
if (!body) { outbox = []; sentSinceCommit = false; return; }
|
|
271
|
+
if (isClosed()) return;
|
|
272
|
+
sttStart = performance.now();
|
|
273
|
+
ws = new WebSocket(`${SM_URL}?jwt=${encodeURIComponent(body.token)}`);
|
|
274
|
+
ws.binaryType = 'arraybuffer';
|
|
275
|
+
ws.onopen = () => {
|
|
276
|
+
// Declare the audio format + transcription config, then flush buffered audio once started.
|
|
277
|
+
ws.send(JSON.stringify({
|
|
278
|
+
message: 'StartRecognition',
|
|
279
|
+
audio_format: { type: 'raw', encoding: 'pcm_s16le', sample_rate: 16000 },
|
|
280
|
+
// max_delay = seconds Speechmatics waits before LOCKING words into a final (AddTranscript); the
|
|
281
|
+
// 4s default leaves a long flickering PARTIAL tail (6-7 words) before words move to `committed`.
|
|
282
|
+
// We tie it to the model's accuracy headroom: 'enhanced' (their demo's combo) tolerates max_delay:1
|
|
283
|
+
// for a snappy 1-4 word tail; 'standard' lacks that headroom, so 2s (Speechmatics' documented
|
|
284
|
+
// "optimal balance", ~1% loss vs ~<5% at 1s) buys back the context it needs. max_delay_mode stays
|
|
285
|
+
// default ('flexible') so entities (numbers/dates) still finalize whole. (Range 0.7–4.)
|
|
286
|
+
// conversation_config.end_of_utterance_silence_trigger is REQUIRED for ForceEndOfUtterance /
|
|
287
|
+
// EndOfUtterance to work at all — without it commit() is silently ignored and the turn only
|
|
288
|
+
// ever closes via the safety timer. Keep it < max_delay. The AGENT owns the boundary (VAD +
|
|
289
|
+
// turnDetector → commit → ForceEndOfUtterance); a silence-triggered EndOfUtterance can also
|
|
290
|
+
// close a turn, but since the agent feeds VAD-GATED audio (little post-speech silence reaches
|
|
291
|
+
// Speechmatics), commit() is the boundary that matters in practice.
|
|
292
|
+
transcription_config: { language: (sttLang || 'en').slice(0, 2), operating_point: sttModel || 'standard', enable_partials: true, max_delay: sttModel === 'enhanced' ? TUNING.STT.maxDelayEnhanced : TUNING.STT.maxDelayStandard, conversation_config: { end_of_utterance_silence_trigger: TUNING.STT.eouSilenceTrigger } },
|
|
293
|
+
}));
|
|
294
|
+
};
|
|
295
|
+
const sock = ws; // this handler's own socket — feed() may reopen while it's still CLOSING
|
|
296
|
+
ws.onerror = ev => { if (sock === ws && !isClosed()) { onError('STT socket error'); console.error('SM ws error', ev); } };
|
|
297
|
+
// Guard on sock === ws: a superseded socket's late onclose must not kill the NEW socket's
|
|
298
|
+
// keepalive (feed() reopens on readyState>=2, i.e. while the old one is still CLOSING).
|
|
299
|
+
ws.onclose = ev => { if (sock === ws) { clearInterval(keepalive); keepalive = null; } usage.flush(); onClose?.(); if (!isClosed() && sock === ws && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); } };
|
|
300
|
+
ws.onmessage = ev => {
|
|
301
|
+
if (sock !== ws) return; // stale message from a superseded socket (its clock is dead)
|
|
302
|
+
const m = JSON.parse(ev.data);
|
|
303
|
+
const ms = Math.round(performance.now() - sttStart);
|
|
304
|
+
switch (m.message) {
|
|
305
|
+
case 'RecognitionStarted': {
|
|
306
|
+
started = true; const q = outbox; outbox = []; for (const c of q) sendAudio(c); if (pendingForceEnd) sendForceEnd();
|
|
307
|
+
// Idle keepalive (see declaration): 0.1s of silence whenever nothing was sent for 20s
|
|
308
|
+
// AND no turn is open/closing — injected silence must never trip the provider's
|
|
309
|
+
// end-of-utterance silence trigger on a live turn. Cost is negligible (~0.5% stream time).
|
|
310
|
+
// Counts into `sentSamples` like any other sent audio: it genuinely advances Speechmatics'
|
|
311
|
+
// stream clock (end_times are relative to ALL audio sent), so the commitBoundary safety
|
|
312
|
+
// gate must stay in sync or a late straggler could slip past cursor after an idle period.
|
|
313
|
+
clearInterval(keepalive);
|
|
314
|
+
keepalive = setInterval(() => {
|
|
315
|
+
if (ws?.readyState === 1 && started && !turnOpen && !awaitingFinal && !sentSinceCommit
|
|
316
|
+
&& performance.now() - lastSendMs > 20000) {
|
|
317
|
+
const silence = new Int16Array(1600);
|
|
318
|
+
sentSamples += silence.length;
|
|
319
|
+
sendAudio(silence);
|
|
320
|
+
}
|
|
321
|
+
}, 5000);
|
|
322
|
+
keepalive.unref?.(); // Node (tests): don't hold the event loop open; no-op in browsers
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
// Live tail — display-only replacement, never committed to history. During an OPEN turn
|
|
326
|
+
// Speechmatics restates the tail around the just-locked boundary, so a partial's end_time can
|
|
327
|
+
// legitimately land at/behind the last locked word's cursor — gating those there silently
|
|
328
|
+
// drops live updates and the visible tail blanks/freezes for ~max_delay after each lock
|
|
329
|
+
// ("cuts every few seconds" during continuous speech). Only drop behind-cursor partials once
|
|
330
|
+
// the turn is CLOSED/closing (!turnOpen || awaitingFinal), so a stale old-turn partial can't
|
|
331
|
+
// leak a ghost transcript / false barge-in (onPartial also feeds the barge-in check).
|
|
332
|
+
case 'AddPartialTranscript': {
|
|
333
|
+
const end = m.metadata?.end_time ?? 0;
|
|
334
|
+
if (cursor > 0 && end > 0 && end <= cursor && (!turnOpen || awaitingFinal)) break;
|
|
335
|
+
turnOpen = true;
|
|
336
|
+
lastTail = (m.metadata?.transcript || '').trim();
|
|
337
|
+
onPartial(lastTail, ms, committed);
|
|
338
|
+
break;
|
|
339
|
+
}
|
|
340
|
+
// Words permanently locked → append to `committed`, advance the cursor. Never closes the
|
|
341
|
+
// turn — EndOfUtterance does (below).
|
|
342
|
+
case 'AddTranscript': {
|
|
343
|
+
const end = m.metadata?.end_time ?? 0;
|
|
344
|
+
if (cursor > 0 && end > 0 && end <= cursor) break; // straggler for a closed turn → drop
|
|
345
|
+
const frag = (m.metadata?.transcript || '').trim();
|
|
346
|
+
if (frag) { committed = committed ? `${committed} ${frag}` : frag; turnOpen = true; }
|
|
347
|
+
cursor = Math.max(cursor, end);
|
|
348
|
+
// The lock usually consumes the whole tail; a PREFIX lock (frag = the tail's first words)
|
|
349
|
+
// leaves the remainder as the only copy of the newest words — keep it for a safety close.
|
|
350
|
+
// A diverging lock (revised words) clears the tail so the fold can't duplicate.
|
|
351
|
+
lastTail = frag && lastTail.startsWith(frag) ? lastTail.slice(frag.length).trim() : '';
|
|
352
|
+
onPartial('', ms, committed); // tail consumed, committed grew → re-render solid
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
// Speechmatics' turn boundary (after our ForceEndOfUtterance, or its own silence trigger).
|
|
356
|
+
// The flushing AddTranscript has already landed, so `committed` is complete → close on it.
|
|
357
|
+
case 'EndOfUtterance': {
|
|
358
|
+
if (!awaitingFinal && !turnOpen) break; // nothing open to close (stray/duplicate)
|
|
359
|
+
flushFinal();
|
|
360
|
+
break;
|
|
361
|
+
}
|
|
362
|
+
case 'Error': onError(`${m.type || 'error'}: ${m.reason || ''}`); break;
|
|
363
|
+
}
|
|
364
|
+
};
|
|
365
|
+
} finally {
|
|
366
|
+
opening = false;
|
|
367
|
+
}
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
return {
|
|
371
|
+
// Pre-open the socket (token fetch + WS handshake) before any audio so the first utterance isn't
|
|
372
|
+
// stalled by cold connect. Idempotent (open() has a re-entry guard); feed() still opens lazily.
|
|
373
|
+
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
374
|
+
feed(i16) {
|
|
375
|
+
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
376
|
+
sentSamples += i16.length; // stream position (counts outbox-queued audio too — sent in order)
|
|
377
|
+
sendAudio(i16); sentSinceCommit = true;
|
|
378
|
+
},
|
|
379
|
+
// Start a fresh utterance: drop leftover committed words so a prior turn can't bleed into this
|
|
380
|
+
// one. Called on a fresh VAD speech-start when the previous turn actually CLOSED — a turn the
|
|
381
|
+
// detector kept open is intentionally NOT reset. `cursor` survives (the straggler gate; end_times
|
|
382
|
+
// don't reset per turn); pending force-end state is cleared.
|
|
383
|
+
reset() { clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false; pendingForceEnd = false; committed = ''; lastTail = ''; turnOpen = false; },
|
|
384
|
+
// VAD speech-end = the utterance boundary. Ask Speechmatics to finalize NOW (ForceEndOfUtterance →
|
|
385
|
+
// it flushes the turn's words as AddTranscript, then emits EndOfUtterance); awaitingFinal makes that
|
|
386
|
+
// EndOfUtterance close the turn. A short safety timer closes it anyway if the socket never answers.
|
|
387
|
+
commit() {
|
|
388
|
+
if (awaitingFinal) return; // already closing — its flushFinal will notify
|
|
389
|
+
// Nothing to finalize (stray VAD end, e.g. only the AI's own echo): the agent waits on onFinal
|
|
390
|
+
// to clear its closing latch, so emit an empty close.
|
|
391
|
+
if (!turnOpen && !sentSinceCommit) { onFinal('', Math.round(performance.now() - sttStart)); return; }
|
|
392
|
+
if (!sentSinceCommit) { flushFinal(); return; } // no new audio this turn: close on the words we have
|
|
393
|
+
commitBoundary = sentSamples / 16000; // this turn's audio all ends at/before here (safety gate)
|
|
394
|
+
sentSinceCommit = false;
|
|
395
|
+
awaitingFinal = true;
|
|
396
|
+
sendForceEnd(); // sends now, or defers to RecognitionStarted if still opening
|
|
397
|
+
armSafety(TUNING.STT.forceEndSafetyMs); // fallback if EndOfUtterance never arrives (tuning.js)
|
|
398
|
+
},
|
|
399
|
+
close() { clearTimeout(safetyTimer); safetyTimer = null; clearInterval(keepalive); keepalive = null; outbox = []; committed = ''; lastTail = ''; cursor = 0; sentSamples = 0; commitBoundary = 0; turnOpen = false; awaitingFinal = false; sentSinceCommit = false; started = false; pendingForceEnd = false; usage.flush(); ws?.close(); ws = null; },
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// ── Deepgram Flux realtime ───────────────────────────────────────────────────
|
|
404
|
+
// Browser↔Deepgram direct WS, same token pattern as the others: the backend mints a short-TTL
|
|
405
|
+
// access token (POST /v1/auth/grant, key stays server-side) and the browser authenticates via the
|
|
406
|
+
// WebSocket subprotocol ['bearer', token] (browsers can't set an Authorization header). Audio is
|
|
407
|
+
// raw binary pcm16 @16k like Speechmatics.
|
|
408
|
+
//
|
|
409
|
+
// Flux differs from Speechmatics/ElevenLabs in WHO owns the turn boundary: its model does native
|
|
410
|
+
// end-of-turn detection (semantic + acoustic) and emits TurnInfo events — `Update` while the turn
|
|
411
|
+
// is in progress (transcript = the WHOLE turn so far, not a fragment) and `EndOfTurn` when it
|
|
412
|
+
// decides the user is done. For that to work it must hear the real silences, so this provider sets
|
|
413
|
+
// continuous: true → the agent feeds the FULL mic stream (no VAD gating / preroll)
|
|
414
|
+
// nativeEOT: true → the agent does NOT commit() on VAD speech-end; onFinal arrives unsolicited
|
|
415
|
+
// from EndOfTurn and the agent's onFinal path closes the turn.
|
|
416
|
+
// commit()/reset() are still implemented (VAD-gated fallback contract) so a host that ignores the
|
|
417
|
+
// flags gets working — if slower — turn handling: commit() waits briefly for a native EndOfTurn and
|
|
418
|
+
// otherwise flushes the latest turn transcript.
|
|
419
|
+
const DG_URL = 'wss://api.deepgram.com/v2/listen';
|
|
420
|
+
// Deepgram's auth endpoints are US-hosted — a token mint costs ~0.6-1s from EU, and it sits on the
|
|
421
|
+
// CRITICAL PATH of every socket open (the token rides in the WS handshake, so nothing can connect
|
|
422
|
+
// until it arrives). The backend mints with a 300s TTL; cache it module-wide (shared across agent
|
|
423
|
+
// instances/reopens) and reuse until shortly before expiry, so a stop()/start() resume — which
|
|
424
|
+
// closes and reopens the socket — only pays the ~0.4s handshake instead of the full mint chain.
|
|
425
|
+
let dgToken = null, dgTokenExp = 0;
|
|
426
|
+
// Pre-mint + cache the Deepgram token OFF the critical path (call at prewarm time, e.g. mic-button
|
|
427
|
+
// mount): during a cold voice start the mint races Piper/ONNX downloads for bandwidth and main
|
|
428
|
+
// thread (measured 2.7s in-session vs 0.6s standalone), so paying it early means the click only owes
|
|
429
|
+
// the ~0.4s WS handshake. Best-effort + deduped; makeDeepgramSTT reads the same cache.
|
|
430
|
+
let dgMintPromise = null;
|
|
431
|
+
export function warmDeepgramToken(sttTokenUrl, apiKey, getToken) {
|
|
432
|
+
if (dgToken && performance.now() < dgTokenExp - 30000) return Promise.resolve();
|
|
433
|
+
return (dgMintPromise ??= (async () => {
|
|
434
|
+
const body = await mintSttToken(sttTokenUrl, apiKey, () => {}, getToken); // best-effort: failure just means open() re-mints
|
|
435
|
+
if (body) { dgToken = body.token; dgTokenExp = performance.now() + (body.expires_in ?? 300) * 1000; }
|
|
436
|
+
})().finally(() => { dgMintPromise = null; }));
|
|
437
|
+
}
|
|
438
|
+
export function makeDeepgramSTT(opts) {
|
|
439
|
+
const { apiKey, sttModel, sttLang = 'en', keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
|
|
440
|
+
eotThreshold = TUNING.STT.fluxEotThreshold, onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
441
|
+
const tokenUrl = sttTokenUrl;
|
|
442
|
+
const model = sttModel || 'flux-general-en';
|
|
443
|
+
let ws = null, opening = false, outbox = [], sttStart = 0;
|
|
444
|
+
// TurnInfo.transcript is cumulative for the CURRENT turn (Flux restarts it on each new turn_index),
|
|
445
|
+
// so `turnText` is simply the latest transcript — no committed/tail split needed.
|
|
446
|
+
let turnText = '', audioSinceFinal = false, awaitingFinal = false, safetyTimer = null;
|
|
447
|
+
const usage = makeUsageReporter(sttUsageUrl, apiKey, 'deepgram', { model });
|
|
448
|
+
|
|
449
|
+
// Pre-open buffer is a bounded PREROLL (~1s), not a backlog: Flux is a real-time model — burst-
|
|
450
|
+
// sending seconds of buffered audio after a slow token fetch/handshake makes its whole timeline lag
|
|
451
|
+
// real time (garbled starts, late EndOfTurns). Keep only the newest ~1s so the user's first words
|
|
452
|
+
// survive the connect without shifting Flux's clock.
|
|
453
|
+
const OUTBOX_MAX_SAMPLES = 32000; // ~2s: enough to cover the WS handshake so opening words survive
|
|
454
|
+
let outboxSamples = 0;
|
|
455
|
+
const sendAudio = (i16) => {
|
|
456
|
+
if (!ws || ws.readyState !== 1) {
|
|
457
|
+
outbox.push(i16); outboxSamples += i16.length;
|
|
458
|
+
while (outboxSamples > OUTBOX_MAX_SAMPLES && outbox.length > 1) outboxSamples -= outbox.shift().length;
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
ws.send(i16.buffer); // raw binary linear16 @16k (declared in the URL query)
|
|
462
|
+
usage.add(i16.length);
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
// Close the turn from OUR side (host-called commit() fallback only — the agent never calls this in
|
|
466
|
+
// native mode). NB: because Flux transcripts are cumulative per turn, flushing a still-open turn
|
|
467
|
+
// emits a prefix Flux will restate — only useful for hosts that ignore the nativeEOT flag.
|
|
468
|
+
const flushFinal = () => {
|
|
469
|
+
clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false;
|
|
470
|
+
const t = turnText; turnText = ''; audioSinceFinal = false;
|
|
471
|
+
onFinal(t, Math.round(performance.now() - sttStart));
|
|
472
|
+
};
|
|
473
|
+
const armSafety = () => { clearTimeout(safetyTimer); safetyTimer = setTimeout(flushFinal, TUNING.STT.forceEndSafetyMs); };
|
|
474
|
+
|
|
475
|
+
const open = async () => {
|
|
476
|
+
if (opening) return;
|
|
477
|
+
opening = true;
|
|
478
|
+
try {
|
|
479
|
+
// Cached token still fresh? Skip the mint round trip entirely (see dgToken above). 15s slack:
|
|
480
|
+
// the token only needs to be valid at handshake time, but leave margin for a slow connect.
|
|
481
|
+
let token = (dgToken && performance.now() < dgTokenExp - 15000) ? dgToken : null;
|
|
482
|
+
if (!token) {
|
|
483
|
+
const body = await mintSttToken(tokenUrl, apiKey, onFatal, getToken);
|
|
484
|
+
if (!body) { outbox = []; outboxSamples = 0; return; }
|
|
485
|
+
token = dgToken = body.token;
|
|
486
|
+
dgTokenExp = performance.now() + (body.expires_in ?? 300) * 1000;
|
|
487
|
+
}
|
|
488
|
+
if (isClosed()) return;
|
|
489
|
+
const p = new URLSearchParams({ model, encoding: 'linear16', sample_rate: '16000', eot_threshold: String(eotThreshold) });
|
|
490
|
+
if (model.endsWith('multi') && sttLang) p.append('language_hint', sttLang.slice(0, 2));
|
|
491
|
+
for (const k of keyterms) p.append('keyterm', k);
|
|
492
|
+
sttStart = performance.now();
|
|
493
|
+
// Browser WS auth: Sec-WebSocket-Protocol "bearer, <token>" (short-TTL access token from
|
|
494
|
+
// /v1/auth/grant). The token only needs to be valid at handshake time; the socket persists.
|
|
495
|
+
ws = new WebSocket(`${DG_URL}?${p}`, ['bearer', token]);
|
|
496
|
+
ws.binaryType = 'arraybuffer';
|
|
497
|
+
ws.onopen = () => { const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c); };
|
|
498
|
+
ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('DG ws error', ev); } };
|
|
499
|
+
ws.onclose = ev => {
|
|
500
|
+
usage.flush(); onClose?.();
|
|
501
|
+
// Handshake rejected (auth) — likely a stale cached token (clock drift / revocation): drop the
|
|
502
|
+
// cache so the next open() re-mints instead of failing forever within the cached window.
|
|
503
|
+
if (ev.code === 1008 || ev.code === 4001 || ev.code === 1002) { dgToken = null; dgTokenExp = 0; }
|
|
504
|
+
if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); }
|
|
505
|
+
};
|
|
506
|
+
ws.onmessage = ev => {
|
|
507
|
+
if (typeof ev.data !== 'string') return;
|
|
508
|
+
const m = JSON.parse(ev.data);
|
|
509
|
+
const ms = Math.round(performance.now() - sttStart);
|
|
510
|
+
if (m.type === 'TurnInfo') {
|
|
511
|
+
const t = (m.transcript || '').trim();
|
|
512
|
+
if (m.event === 'EndOfTurn') {
|
|
513
|
+
// Native turn boundary: Flux decided the user is done. Emit the whole turn as final —
|
|
514
|
+
// unsolicited from the agent's point of view (its onFinal path closes the turn cleanly).
|
|
515
|
+
clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false;
|
|
516
|
+
turnText = ''; audioSinceFinal = false;
|
|
517
|
+
onFinal(t, ms);
|
|
518
|
+
} else { // Update / StartOfTurn (EagerEndOfTurn/TurnResumed only occur with eager_eot_threshold set)
|
|
519
|
+
turnText = t;
|
|
520
|
+
onPartial(t, ms);
|
|
521
|
+
}
|
|
522
|
+
} else if (m.type === 'Error') { onError(`${m.code || 'error'}: ${m.description || ''}`); console.error('DG error msg', m); }
|
|
523
|
+
};
|
|
524
|
+
} finally {
|
|
525
|
+
opening = false;
|
|
526
|
+
}
|
|
527
|
+
};
|
|
528
|
+
|
|
529
|
+
return {
|
|
530
|
+
continuous: true, // agent feeds the full mic stream (Flux needs real silences for its EOT model)
|
|
531
|
+
nativeEOT: true, // agent skips VAD-commit; EndOfTurn → unsolicited onFinal closes the turn
|
|
532
|
+
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
533
|
+
feed(i16) {
|
|
534
|
+
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
535
|
+
sendAudio(i16); audioSinceFinal = true;
|
|
536
|
+
},
|
|
537
|
+
reset() { turnText = ''; clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false; },
|
|
538
|
+
// VAD-gated fallback: give the native EndOfTurn a short window to land (it usually beats this —
|
|
539
|
+
// the trailing VAD-redemption silence is in the stream), then flush what we have. If no audio was
|
|
540
|
+
// fed since the last final there's no turn — emit an empty close so the host's closing latch clears.
|
|
541
|
+
commit() {
|
|
542
|
+
if (awaitingFinal) return;
|
|
543
|
+
if (!audioSinceFinal) { onFinal('', Math.round(performance.now() - sttStart)); return; }
|
|
544
|
+
awaitingFinal = true;
|
|
545
|
+
armSafety();
|
|
546
|
+
},
|
|
547
|
+
close() {
|
|
548
|
+
clearTimeout(safetyTimer); safetyTimer = null; outbox = []; outboxSamples = 0; turnText = ''; audioSinceFinal = false; awaitingFinal = false;
|
|
549
|
+
usage.flush();
|
|
550
|
+
if (ws?.readyState === 1) { try { ws.send(JSON.stringify({ type: 'CloseStream' })); } catch { /* closing anyway */ } }
|
|
551
|
+
ws?.close(); ws = null;
|
|
552
|
+
},
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// ── Browser Web Speech API (local, no cloud, no token) ───────────────────────
|
|
557
|
+
// The browser's own SpeechRecognition engine (Chrome/Safari `webkitSpeechRecognition`). It OWNS the
|
|
558
|
+
// whole audio path — its own mic capture AND its own silence-based end-of-turn — so the agent builds
|
|
559
|
+
// NO getUserMedia / AudioWorklet / Silero-VAD pipeline for it (`selfCapture: true`, which the agent
|
|
560
|
+
// reads as "just open() me"). It's `nativeEOT: true`: the provider closes turns itself, so there's no
|
|
561
|
+
// VAD-commit — a short silence-debounce after the engine's own isFinal fires onFinal as ONE turn.
|
|
562
|
+
// interim results → onPartial(tail) live draft + word-based barge-in while the agent is speaking
|
|
563
|
+
// final results → accumulate + (re)arm the end-of-turn timer; if the user keeps talking within
|
|
564
|
+
// the debounce window the fragments merge, else the timer flushes the turn
|
|
565
|
+
// No backend token, no /stt/usage reporting (free + on-device), no socket. Only where the API exists
|
|
566
|
+
// (else open() reports a fatal so the host can fall back / surface it).
|
|
567
|
+
// Browser Web Speech API STT. The agent hands it nothing (selfCapture) — it owns its mic and its own
|
|
568
|
+
// end-of-turn (nativeEOT). It reports transcripts via onPartial (live draft, drives the user bubble +
|
|
569
|
+
// word-based barge-in) and onFinal (one committed user turn).
|
|
570
|
+
//
|
|
571
|
+
// THE HARD PART — Chrome fragments ONE logical listening session into MANY short-lived recognizer
|
|
572
|
+
// instances. A `SpeechRecognition` auto-`onend`s after a brief internal silence (even mid-sentence);
|
|
573
|
+
// we recreate it in onend to keep listening. A single user utterance therefore spans several
|
|
574
|
+
// recognizers, and TWO artefacts of that churn must be neutralised so the agent sees one clean turn:
|
|
575
|
+
//
|
|
576
|
+
// 1. LIVENESS RACE — the restart gap (onend → new onstart) is NOT a user pause, just a technicality.
|
|
577
|
+
// The turn boundary is decided SOLELY by a silence debounce measured from the last time real TEXT
|
|
578
|
+
// arrived (`lastActivity`), completely independent of recognizer lifecycle. A restart never arms,
|
|
579
|
+
// fires, or cancels that clock. So whether or not a recognizer happens to be alive when the timer
|
|
580
|
+
// elapses is irrelevant — no race. (The old bug: the debounce fired DURING a restart, emitting the
|
|
581
|
+
// opening as its own turn; the fresh recognizer then re-finalized the same buffered audio → the
|
|
582
|
+
// opening was duplicated into a second turn, which barged in and killed the reply.)
|
|
583
|
+
//
|
|
584
|
+
// 2. SEAM DUPLICATION — when Chrome auto-ends mid-word it re-delivers the tail it was still buffering
|
|
585
|
+
// as the NEXT recognizer's first final (and can even restate a tail within one recognizer's own
|
|
586
|
+
// result window). Blindly appending would double words ("hello there" + "there how" → "there there").
|
|
587
|
+
// Every final is de-overlapped against the turn-so-far before it's appended.
|
|
588
|
+
export function makeWebSpeechSTT(opts) {
|
|
589
|
+
const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onFatal, onClose, isClosed } = opts;
|
|
590
|
+
const Rec = typeof window !== 'undefined' && (window.SpeechRecognition || window.webkitSpeechRecognition);
|
|
591
|
+
const EOT_MS = TUNING.WEBSPEECH_EOT_MS ?? 900;
|
|
592
|
+
const TAIL_MAX_DEFERS = TUNING.WEBSPEECH_TAIL_MAX_DEFERS ?? 3;
|
|
593
|
+
let rec = null; // the live SpeechRecognition instance (null between restarts / when stopped)
|
|
594
|
+
let sttStart = 0; // performance.now() at the current turn's first activity (for onPartial/onFinal ms)
|
|
595
|
+
let turn = ''; // committed (isFinal) text of the CURRENT turn, merged across recognizer restarts
|
|
596
|
+
let interimTail = ''; // last live (not-yet-final) tail — KEPT across recognizer restarts so the draft never blanks
|
|
597
|
+
let eotTimer = null; // silence debounce: closes the turn EOT_MS after the last real text activity
|
|
598
|
+
let justClosed = ''; // tail of the turn we JUST flushed — a late re-finalized seam is deduped against it
|
|
599
|
+
let justClosedTimer = null; // clears justClosed after a short guard window (a genuine next turn isn't a seam)
|
|
600
|
+
let tailDefers = 0; // consecutive closes deferred waiting on an un-promoted tail (bounded, reset on new text)
|
|
601
|
+
let want = false; // intent to listen — onend auto-restarts while true
|
|
602
|
+
let muted = false; // mic off — stop the recognizer and don't restart
|
|
603
|
+
// AEC'D CAPTURE (SpeechRecognition.start(audioTrack), spec'd + shipped in recent Chromium): by
|
|
604
|
+
// default the engine opens its OWN mic, bypassing every protection the agent's pipeline has — the
|
|
605
|
+
// browser AEC referenced against the WebRTC TTS loopback (voice-agent initAecLoopback) and noise
|
|
606
|
+
// suppression. So on speakers the recognizer hears the agent's own reply. Handing it OUR
|
|
607
|
+
// getUserMedia track (echoCancellation+noiseSuppression on) routes the SAME protected audio the
|
|
608
|
+
// cloud providers get. Older engines throw on start(track) — one throw disables it for the
|
|
609
|
+
// session and we fall back to the engine's own capture (old behavior, text-filter guarded).
|
|
610
|
+
let micStream = null; // AEC'd capture the recognizer runs on — null → engine's own mic
|
|
611
|
+
let micTrack = null;
|
|
612
|
+
let trackArg = true; // start(track) supported? optimistic; first throw turns it off
|
|
613
|
+
let recHasTrack = false; // whether the CURRENT recognizer was started on micTrack (for the upgrade restart)
|
|
614
|
+
let micGen = 0; // bumped by dropMic — a getUserMedia resolving from an older generation is stopped, not published (mute/close during acquisition can't leak an OS mic)
|
|
615
|
+
let lastLoggedMode = null; // capture-mode console.info dedup — log only on change, not per restart
|
|
616
|
+
let micPending = null; // the single in-flight acquisition — concurrent ensureTrack calls share it
|
|
617
|
+
const acquireMic = () => {
|
|
618
|
+
if (micPending) return micPending;
|
|
619
|
+
if (micTrack?.readyState === 'live' || !trackArg || !navigator.mediaDevices?.getUserMedia) return Promise.resolve();
|
|
620
|
+
const gen = micGen;
|
|
621
|
+
micPending = navigator.mediaDevices.getUserMedia({ audio: {
|
|
622
|
+
echoCancellation: true, noiseSuppression: true, autoGainControl: true,
|
|
623
|
+
...(micDeviceId ? { deviceId: { exact: micDeviceId } } : {}),
|
|
624
|
+
} }).then((s) => {
|
|
625
|
+
// Stale (muted/closed/dropped while pending) → stop immediately; publishing it would leak the mic.
|
|
626
|
+
if (gen !== micGen || muted || !want || isClosed()) { s.getTracks().forEach((t) => t.stop()); return; }
|
|
627
|
+
micStream?.getTracks().forEach((t) => t.stop()); // never hold two captures
|
|
628
|
+
micStream = s; micTrack = s.getAudioTracks()[0] ?? null;
|
|
629
|
+
// Device unplugged mid-session: forget the dead track so the next boundary re-acquires (or the
|
|
630
|
+
// recognizer falls back to its own capture) instead of silently keeping a corpse around.
|
|
631
|
+
if (micTrack) micTrack.addEventListener('ended', () => { if (micStream === s) { s.getTracks().forEach((t) => t.stop()); micStream = null; micTrack = null; } });
|
|
632
|
+
}).catch(() => { /* denied/no device → engine's own capture */ })
|
|
633
|
+
.finally(() => { micPending = null; });
|
|
634
|
+
return micPending;
|
|
635
|
+
};
|
|
636
|
+
const dropMic = () => { micGen++; micStream?.getTracks().forEach((t) => t.stop()); micStream = null; micTrack = null; };
|
|
637
|
+
// Acquire (or re-acquire) the track, then upgrade a recognizer that started track-less: abort it
|
|
638
|
+
// and let onend relaunch via start(), which now has micTrack. Only at a turn boundary — an abort
|
|
639
|
+
// mid-utterance would drop buffered speech. If acquisition lands mid-utterance, flush() re-runs
|
|
640
|
+
// this at the turn boundary, so the upgrade isn't lost.
|
|
641
|
+
const ensureTrack = () => acquireMic().then(() => {
|
|
642
|
+
if (trackArg && micTrack?.readyState === 'live' && rec && !recHasTrack && !turn && !interimTail) { try { rec.abort(); } catch { /* onend relaunches anyway */ } }
|
|
643
|
+
});
|
|
644
|
+
// SpeechRecognition wants a BCP-47 tag (en-US, hu-HU); our sttLang is often a bare code (en, hu).
|
|
645
|
+
const langTag = (l) => ({ en: 'en-US', hu: 'hu-HU', de: 'de-DE' }[String(l || '').slice(0, 2).toLowerCase()] || l || 'en-US');
|
|
646
|
+
|
|
647
|
+
// Chars of `next` to drop so a re-finalized tail isn't appended twice: the longest suffix of `prev`
|
|
648
|
+
// that is also a prefix of `next`, aligned to word boundaries (so "there"⊄"therefore"). Returns the
|
|
649
|
+
// overlap length in `next`. Case-insensitive compare; the caller slices the ORIGINAL-cased `next`.
|
|
650
|
+
const seamOverlap = (prev, next) => {
|
|
651
|
+
const a = prev.toLowerCase(), b = next.toLowerCase();
|
|
652
|
+
const max = Math.min(a.length, b.length);
|
|
653
|
+
for (let n = max; n > 0; n--) {
|
|
654
|
+
const boundary = b.length === n || b[n] === ' '; // the overlap ends on a word edge in `next`
|
|
655
|
+
const aligned = a.length === n || a[a.length - n - 1] === ' '; // ...and on one in `prev`
|
|
656
|
+
if (boundary && aligned && a.slice(a.length - n) === b.slice(0, n)) return n;
|
|
657
|
+
}
|
|
658
|
+
return 0;
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
// Append a newly-finalized fragment to the turn, de-overlapping the restart/restate seam first.
|
|
662
|
+
// seamOverlap is word-boundary aligned, so it drops a re-finalized tail (whole or partial) without
|
|
663
|
+
// eating a legitimate suffix that merely shares letters ("goodbye" then "bye" keeps "bye").
|
|
664
|
+
const commitFinal = (frag) => {
|
|
665
|
+
frag = frag.trim();
|
|
666
|
+
if (!frag) return;
|
|
667
|
+
// If a turn just flushed (Chrome auto-restarted right as the silence clock closed the turn), the
|
|
668
|
+
// fresh recognizer often re-finalizes that turn's tail as this turn's FIRST final — a duplicate.
|
|
669
|
+
// De-overlap it against the just-closed tail so the repeat is dropped instead of becoming a bogus
|
|
670
|
+
// new turn (the barge-in-killing dup). Only applies while `turn` is still empty (start of a turn).
|
|
671
|
+
if (!turn && justClosed) { const ov = seamOverlap(justClosed, frag); frag = ov ? frag.slice(ov).trimStart() : frag; if (!frag) return; }
|
|
672
|
+
if (turn) { const ov = seamOverlap(turn, frag); frag = ov ? frag.slice(ov).trimStart() : frag; }
|
|
673
|
+
if (frag) turn = turn ? `${turn} ${frag}` : frag;
|
|
674
|
+
};
|
|
675
|
+
|
|
676
|
+
const flush = () => { // the silence debounce elapsed: the turn-so-far is one complete user turn
|
|
677
|
+
clearTimeout(eotTimer); eotTimer = null;
|
|
678
|
+
// NOT QUIESCENT YET: a live recognizer still holding an un-promoted tail means the utterance is
|
|
679
|
+
// by definition unfinished — Chrome's endpointer can take longer than EOT_MS to promote trailing
|
|
680
|
+
// words to isFinal. Closing here would emit a TRUNCATED turn and let the late final open a bogus
|
|
681
|
+
// second one — and since the two texts are DISJOINT ("book a meeting" / "for tomorrow at five"),
|
|
682
|
+
// the justClosed seam-guard has no overlap to dedup and can't catch it. So wait for the engine
|
|
683
|
+
// instead of guessing with a clock. The wait is bounded twice over: onend clears the tail (the
|
|
684
|
+
// engine gave up on it), and TAIL_MAX_DEFERS caps a tail that never resolves. The cap only bounds
|
|
685
|
+
// the WAIT — it does not make that path safe: closing there emits the same truncated turn, just
|
|
686
|
+
// ≥TAIL_MAX_DEFERS windows later. It's a liveness backstop for an engine that never resolves nor
|
|
687
|
+
// ends, not a second correctness guarantee (the real fix is waiting for quiescence at all).
|
|
688
|
+
if (rec && interimTail && tailDefers++ < TAIL_MAX_DEFERS) { armEot(); return; }
|
|
689
|
+
interimTail = ''; tailDefers = 0;
|
|
690
|
+
// Only FINALIZED words become a turn. A tail that survived to end-of-turn WITHOUT ever finalizing
|
|
691
|
+
// is noise (throat-clears, background speech) — Chrome finalizes real speech, so a never-final
|
|
692
|
+
// interim is dropped, not committed. `turn` already holds every finalized word (incl. tails folded
|
|
693
|
+
// in on restart by onend, which the fresh recognizer's re-finalization would have confirmed).
|
|
694
|
+
const text = turn.trim(); turn = '';
|
|
695
|
+
if (!text) return; // blank silence → not a turn
|
|
696
|
+
// Guard the seam: if Chrome auto-restarted exactly as this turn closed, the fresh recognizer may
|
|
697
|
+
// re-finalize this turn's tail as the NEXT turn's opening. Remember it briefly so commitFinal drops
|
|
698
|
+
// that late duplicate instead of emitting a bogus turn (the barge-in-killer). One-debounce window —
|
|
699
|
+
// a genuine next turn arriving later is NOT a seam.
|
|
700
|
+
justClosed = text; clearTimeout(justClosedTimer); justClosedTimer = setTimeout(() => { justClosed = ''; }, EOT_MS);
|
|
701
|
+
onPartial('', 0, ''); // clear the live draft the just-committed turn was still showing
|
|
702
|
+
onFinal(text, Math.round(performance.now() - sttStart));
|
|
703
|
+
// Turn boundary = the safe seam for the AEC'd-track upgrade (and for re-acquiring a track that
|
|
704
|
+
// died mid-turn) — if the track landed while the user was speaking, this is where it takes effect.
|
|
705
|
+
if (want && !muted && !isClosed()) ensureTrack();
|
|
706
|
+
};
|
|
707
|
+
// A SILENCE clock, not a recognizer clock: (re)armed on ANY real text (final OR live interim) so a
|
|
708
|
+
// mid-thought pause keeps one turn together, and closed only after EOT_MS with no new text at all —
|
|
709
|
+
// recognizer restarts in between neither arm nor fire it. That's what makes the turn boundary immune
|
|
710
|
+
// to Chrome's internal churn.
|
|
711
|
+
const armEot = () => { clearTimeout(eotTimer); eotTimer = setTimeout(flush, EOT_MS); };
|
|
712
|
+
|
|
713
|
+
const start = () => {
|
|
714
|
+
if (!Rec || rec || muted || isClosed()) return;
|
|
715
|
+
const r = new Rec();
|
|
716
|
+
r.continuous = true; r.interimResults = true; r.lang = langTag(sttLang);
|
|
717
|
+
r.onstart = () => { if (!turn && !interimTail) sttStart = performance.now(); }; // anchor ms at the turn's first activity (final OR live interim), not each restart
|
|
718
|
+
r.onresult = (event) => {
|
|
719
|
+
if (r !== rec) return; // stale recognizer (superseded by a restart) — ignore its late events
|
|
720
|
+
let final = '', interim = '';
|
|
721
|
+
for (let i = event.resultIndex; i < event.results.length; i++) {
|
|
722
|
+
const t = event.results[i][0].transcript;
|
|
723
|
+
if (event.results[i].isFinal) final += t; else interim += t;
|
|
724
|
+
}
|
|
725
|
+
final = final.trim(); interim = interim.trim();
|
|
726
|
+
if (!turn && !interimTail && (final || interim)) sttStart = performance.now(); // first activity of a fresh turn
|
|
727
|
+
commitFinal(final); // de-overlaps the seam, then locks the newly-final words into the turn
|
|
728
|
+
// Keep the live tail as its own state so a recognizer restart (which drops interim) can replay it
|
|
729
|
+
// instead of blanking the draft. A frame that only finalized words (interim === '') clears it —
|
|
730
|
+
// those words moved into `turn`, so the tail is genuinely empty now, not lost.
|
|
731
|
+
interimTail = interim;
|
|
732
|
+
if (final || interim) { tailDefers = 0; armEot(); } // any real text (re)arms the silence debounce (and is fresh progress, so the tail-deferral budget resets) — see armEot
|
|
733
|
+
// Live draft = committed turn-so-far + the still-forming interim tail. Drives the user bubble and
|
|
734
|
+
// word-based barge-in (the agent cuts its reply the instant real words land).
|
|
735
|
+
onPartial(interimTail, Math.round(performance.now() - sttStart), turn);
|
|
736
|
+
};
|
|
737
|
+
r.onerror = (event) => {
|
|
738
|
+
if (r !== rec) return;
|
|
739
|
+
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); }
|
|
740
|
+
// 'no-speech' / 'aborted' / 'network' are recoverable — onend restarts; don't surface.
|
|
741
|
+
};
|
|
742
|
+
r.onend = () => {
|
|
743
|
+
if (r !== rec) return; // superseded instance's late onend — ignore
|
|
744
|
+
rec = null;
|
|
745
|
+
// The engine dropped its buffered tail, so it no longer represents PENDING words — a close
|
|
746
|
+
// deferred on it (see flush) must not keep waiting. The draft still shows the committed `turn`,
|
|
747
|
+
// and the fresh recognizer re-finalizes real speech, so clearing it here costs no flicker.
|
|
748
|
+
interimTail = '';
|
|
749
|
+
// Chrome drops any not-yet-final interim on auto-end. DON'T commit it here — only finalized words
|
|
750
|
+
// count (a never-final tail is noise). If the tail was real speech, the fresh recognizer will
|
|
751
|
+
// re-finalize it (commitFinal then locks it in, deduping the seam).
|
|
752
|
+
// Keep the SAME logical session alive by relaunching. The turn and its silence timer are
|
|
753
|
+
// untouched — if this really was end-of-turn, that timer fires on its own EOT_MS after the last
|
|
754
|
+
// word (recognizer alive or not); if the user is still talking, the new recognizer's text
|
|
755
|
+
// re-arms it. No flush decision belongs here — that's what makes the boundary restart-immune.
|
|
756
|
+
if (want && !muted && !isClosed()) start();
|
|
757
|
+
else onClose?.();
|
|
758
|
+
};
|
|
759
|
+
rec = r;
|
|
760
|
+
let ok = false;
|
|
761
|
+
if (trackArg && micTrack?.readyState === 'live') {
|
|
762
|
+
try { r.start(micTrack); ok = true; recHasTrack = true; }
|
|
763
|
+
catch (e) { trackArg = false; dropMic(); console.info('[webspeech] start(track) rejected — engine\'s own mic (no AEC):', e?.message || e); } // engine predates start(track) — release the now-useless capture (don't hold two mics)
|
|
764
|
+
}
|
|
765
|
+
if (!ok) { recHasTrack = false; try { r.start(); } catch { rec = null; } } // start() throws if already running — treat as no-op
|
|
766
|
+
if (rec && recHasTrack !== lastLoggedMode) { lastLoggedMode = recHasTrack; console.info(`[webspeech] capture: ${recHasTrack ? "AEC'd track (start(track))" : "engine's own mic (unprotected)"}`); }
|
|
767
|
+
};
|
|
768
|
+
|
|
769
|
+
// Stop the current recognizer and drop all in-progress state (mute/close). The stopped recognizer's
|
|
770
|
+
// late onend is stale (rec nulled → r !== rec), so it can't auto-restart.
|
|
771
|
+
const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
|
|
772
|
+
|
|
773
|
+
return {
|
|
774
|
+
selfCapture: true, // owns its mic → the agent builds no capture pipeline
|
|
775
|
+
nativeEOT: true, // owns its end-of-turn → the agent runs no VAD-commit
|
|
776
|
+
open() {
|
|
777
|
+
if (!Rec) { onFatal?.('Speech recognition not supported in this browser'); return; }
|
|
778
|
+
// start() first, synchronously — Safari requires SpeechRecognition.start inside the click
|
|
779
|
+
// gesture. The AEC'd track lands async right after and upgrades the recognizer at the seam.
|
|
780
|
+
want = true; start();
|
|
781
|
+
ensureTrack();
|
|
782
|
+
},
|
|
783
|
+
// Mute: Web Speech captures its OWN mic (independent of the agent), so mute must actually stop it.
|
|
784
|
+
// `rec` is nulled by teardown() so the stopped recognizer's late onend is stale (r !== rec) and
|
|
785
|
+
// can't auto-restart; unmute relaunches.
|
|
786
|
+
setEnabled(on) {
|
|
787
|
+
muted = !on;
|
|
788
|
+
if (muted) { teardown(); dropMic(); onPartial('', 0, ''); } // drop the draft AND the mic (privacy light off)
|
|
789
|
+
else if (want && !rec) { start(); ensureTrack(); }
|
|
790
|
+
},
|
|
791
|
+
close() { want = false; teardown(); dropMic(); },
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
// Registry: provider id → factory. Add a backend here and it's selectable everywhere.
|
|
796
|
+
export const STT_PROVIDERS = {
|
|
797
|
+
webspeech: makeWebSpeechSTT,
|
|
798
|
+
speechmatics: makeSpeechmaticsSTT,
|
|
799
|
+
elevenlabs: makeElevenLabsSTT,
|
|
800
|
+
deepgram: makeDeepgramSTT,
|
|
801
|
+
};
|
|
802
|
+
|
|
803
|
+
// Web Speech only exists in Chromium/Safari — Firefox and Linux WebKitGTK webviews ship no
|
|
804
|
+
// SpeechRecognition at all. There the default 'webspeech' provider can only fatal ("Speech
|
|
805
|
+
// recognition not supported in this browser") and voice mode is simply dead, so resolve it to a
|
|
806
|
+
// cloud provider instead (needs a token route configured — see the auth notes at the top).
|
|
807
|
+
export const WEBSPEECH_FALLBACK = 'elevenlabs';
|
|
808
|
+
export const webSpeechSupported = () =>
|
|
809
|
+
typeof window !== 'undefined' && !!(window.SpeechRecognition || window.webkitSpeechRecognition);
|
|
810
|
+
/** Provider id actually used for `id` here — 'webspeech' downgrades to the cloud fallback in a real
|
|
811
|
+
* browser that lacks SpeechRecognition. Hosts can call it to label/inform ("using cloud recognition").
|
|
812
|
+
* Outside a browser (SSR/tests) nothing is downgraded: there's no mic there either, so the honest
|
|
813
|
+
* "not supported" fatal is the right answer. */
|
|
814
|
+
export const resolveSttProvider = (id) =>
|
|
815
|
+
(id === 'webspeech' && typeof window !== 'undefined' && !webSpeechSupported() ? WEBSPEECH_FALLBACK : id);
|