@todoforai/voiceloop 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/examples/demo.html +77 -22
- package/package.json +1 -1
- package/src/index.d.ts +30 -8
- package/src/index.js +2 -2
- package/src/stt.js +50 -11
- package/src/tuning.js +11 -0
- package/src/voice-agent.js +5 -2
package/README.md
CHANGED
|
@@ -123,7 +123,12 @@ new VoiceAgent({ sttProvider: 'speechmatics', sttTokenUrl: '/api/stt/token' })
|
|
|
123
123
|
Cloud providers authenticate with a **short-TTL token minted by your backend** so the raw API key never reaches the page. Either:
|
|
124
124
|
|
|
125
125
|
- `sttTokenUrl` — a POST route on your server that returns `{ token }` (mint it against the provider's temp-token API with your secret key), or
|
|
126
|
-
- `getSttToken` — an async callback `() => ({ token })` when your auth doesn't fit a bare POST.
|
|
126
|
+
- `getSttToken` — an async callback `(provider) => ({ token })` when your auth doesn't fit a bare POST, or when each provider mints a different credential. It is handed the provider actually running (after the fallback below), so you pick the route without re-deriving that rule:
|
|
127
|
+
|
|
128
|
+
```js
|
|
129
|
+
const ROUTE = { elevenlabs: '/api/stt/token', deepgram: '/api/stt/deepgram-token' };
|
|
130
|
+
new VoiceAgent({ sttProvider, getSttToken: p => fetch(ROUTE[p], { method: 'POST' }).then(r => r.json()) })
|
|
131
|
+
```
|
|
127
132
|
|
|
128
133
|
On browsers without SpeechRecognition (Firefox, WebKitGTK) `webspeech` falls back to `elevenlabs` — which needs a token route configured, so pass `sttTokenUrl`/`getSttToken` if you want the fallback to work.
|
|
129
134
|
|
package/examples/demo.html
CHANGED
|
@@ -92,6 +92,10 @@
|
|
|
92
92
|
.msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
|
|
93
93
|
.msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
|
|
94
94
|
.msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
|
|
95
|
+
/* An echoed reply is not an answer — mark it so the transcript can't be read as a model talking. */
|
|
96
|
+
.msg.ai.echoed { border-left:2px solid #e0a94a }
|
|
97
|
+
.msg.ai.echoed::after { content:'echo — no model'; display:block; margin-top:5px;
|
|
98
|
+
font-size:10px; letter-spacing:.04em; color:#e0a94a; opacity:.85 }
|
|
95
99
|
.msg.interim { opacity:.5 }
|
|
96
100
|
.msg .tail { opacity:.35 } /* generated, not yet spoken */
|
|
97
101
|
.msg.tool { align-self:flex-start; background:#161d17; color:#8fc79a;
|
|
@@ -113,6 +117,11 @@
|
|
|
113
117
|
label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
|
|
114
118
|
#log { font:11px/1.5 ui-monospace,monospace; color:var(--dim);
|
|
115
119
|
max-height:220px; overflow:auto; white-space:pre-wrap; padding-top:8px }
|
|
120
|
+
/* Which code is actually running — the demo must never let you mistake the browser's own
|
|
121
|
+
recognizer for voiceloop's pipeline, or an echo stand-in for a real LLM. */
|
|
122
|
+
#pipeline { font-size:11.5px; color:var(--dim); text-align:center; line-height:1.7; max-width:46ch }
|
|
123
|
+
#pipeline b { color:#c8d0da; font-weight:600 }
|
|
124
|
+
#pipeline .warn { color:#e0a94a }
|
|
116
125
|
</style>
|
|
117
126
|
</head>
|
|
118
127
|
<body>
|
|
@@ -122,6 +131,7 @@
|
|
|
122
131
|
</button>
|
|
123
132
|
<div id="status">click to start · then just talk</div>
|
|
124
133
|
<div id="lat"></div>
|
|
134
|
+
<div id="pipeline"></div>
|
|
125
135
|
|
|
126
136
|
<div id="chat"></div>
|
|
127
137
|
|
|
@@ -139,18 +149,11 @@
|
|
|
139
149
|
</select></label>
|
|
140
150
|
<label>LLM url<input id="llmUrl" size="30" value="http://localhost:11434/v1/chat/completions"></label>
|
|
141
151
|
<label>model<input id="model" size="14" value="llama3.2"></label>
|
|
142
|
-
<label>STT<select id="sttProvider">
|
|
143
|
-
<option value="webspeech">webspeech — free, in-browser</option>
|
|
144
|
-
<option value="deepgram">deepgram</option>
|
|
145
|
-
<option value="elevenlabs">elevenlabs</option>
|
|
146
|
-
<option value="speechmatics">speechmatics</option>
|
|
147
|
-
</select></label>
|
|
152
|
+
<label>STT<select id="sttProvider"></select></label>
|
|
148
153
|
<label>STT token url<input id="sttTokenUrl" size="30" placeholder="your backend route (cloud STT only)"></label>
|
|
149
154
|
<label>api key<input id="apiKey" size="18" placeholder="Bearer for LLM · X-API-Key for token url"></label>
|
|
150
|
-
<label>lang<select id="sttLang">
|
|
151
|
-
|
|
152
|
-
</select></label>
|
|
153
|
-
<label>voice<select id="voice"><option value="en_US-joe-medium">en_US-joe-medium</option></select></label>
|
|
155
|
+
<label>lang<select id="sttLang"></select></label>
|
|
156
|
+
<label>voice<select id="voice"></select></label>
|
|
154
157
|
<label>mic<select id="micDeviceId"><option value="">default</option></select></label>
|
|
155
158
|
</div>
|
|
156
159
|
<div class="bar">
|
|
@@ -164,15 +167,29 @@
|
|
|
164
167
|
</main>
|
|
165
168
|
|
|
166
169
|
<script type="module">
|
|
167
|
-
import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice } from '../src/index.js';
|
|
170
|
+
import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice, STT_PROVIDERS, sttSelfCaptures, LANG_NAMES } from '../src/index.js';
|
|
168
171
|
|
|
169
|
-
|
|
172
|
+
// Load VAD/WASM + build the Silero session while the page sits idle — but ONLY for a provider that
|
|
173
|
+
// will actually use it. A selfCapture provider (webspeech) builds no pipeline, so prewarming there
|
|
174
|
+
// downloads an ONNX model to feed a detector that never runs.
|
|
175
|
+
const prewarmIfPipelined = () => { if (!sttSelfCaptures($('sttProvider').value)) prewarmVoice(); };
|
|
170
176
|
|
|
171
177
|
const $ = id => document.getElementById(id);
|
|
178
|
+
const fill = (id, entries) => $(id).append(...entries.map(([value, textContent]) =>
|
|
179
|
+
Object.assign(document.createElement('option'), { value, textContent })));
|
|
180
|
+
|
|
181
|
+
// Every choice below is READ from the library, never retyped here: a provider added to STT_PROVIDERS
|
|
182
|
+
// or a language added to LANG_NAMES shows up in this page by itself. A hand-typed copy would be a
|
|
183
|
+
// second source of truth that silently rots — and a demo that lies about what the library supports.
|
|
184
|
+
fill('sttProvider', Object.keys(STT_PROVIDERS).map(p => [p, p === 'webspeech' ? 'webspeech — free, in-browser' : p]));
|
|
185
|
+
fill('sttLang', Object.entries(LANG_NAMES).map(([code, name]) => [code, `${code} — ${name}`]));
|
|
172
186
|
const SETTINGS = ['mode','llmUrl','model','sttProvider','sttTokenUrl','apiKey','sttLang','voice','micDeviceId'];
|
|
173
187
|
for (const id of SETTINGS) {
|
|
174
188
|
const v = localStorage.getItem('vl.' + id); if (v) $(id).value = v;
|
|
175
|
-
$(id).addEventListener('change', () =>
|
|
189
|
+
$(id).addEventListener('change', () => {
|
|
190
|
+
localStorage.setItem('vl.' + id, $(id).value);
|
|
191
|
+
onSettingChanged(); // the banner must never describe the PREVIOUS choice
|
|
192
|
+
});
|
|
176
193
|
}
|
|
177
194
|
const log = (...a) => {
|
|
178
195
|
const line = `${new Date().toISOString().slice(11,23)} ${a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ')}`;
|
|
@@ -181,7 +198,7 @@ const log = (...a) => {
|
|
|
181
198
|
|
|
182
199
|
// ── transcript ──────────────────────────────────────────────────────────────
|
|
183
200
|
const chat = $('chat');
|
|
184
|
-
let interimEl = null, aiEl = null;
|
|
201
|
+
let interimEl = null, aiEl = null, vadOff = null;
|
|
185
202
|
const toolEls = new Map();
|
|
186
203
|
const bubble = (cls, text = '') => {
|
|
187
204
|
const el = document.createElement('div');
|
|
@@ -197,18 +214,45 @@ const showLat = () => $('lat').textContent = [
|
|
|
197
214
|
tAudio && `first sound ${tAudio - t0}ms`,
|
|
198
215
|
].filter(Boolean).join(' · ');
|
|
199
216
|
|
|
200
|
-
|
|
217
|
+
// undefined (not '') when unset — so PiperTTS applies its OWN default voice instead of this page
|
|
218
|
+
// re-declaring one. The picker is populated async below and adopts whatever it resolved to.
|
|
219
|
+
const tts = new PiperTTS(localStorage.getItem('vl.voice') || undefined);
|
|
201
220
|
let agent = null;
|
|
202
221
|
|
|
203
|
-
// Echo mode: a stand-in "LLM" that repeats the turn — exercises the entire
|
|
204
|
-
// VAD→STT→…→TTS loop, barge-in included, with no key and no backend.
|
|
222
|
+
// Echo mode: a stand-in "LLM" that repeats the turn VERBATIM — exercises the entire
|
|
223
|
+
// VAD→STT→…→TTS loop, barge-in included, with no key and no backend. Nothing is added to the
|
|
224
|
+
// text: a reply that is exactly the transcript is the whole point, since any prefix would be
|
|
225
|
+
// this page's words in the reply and would blur what the recognizer actually heard. That it's
|
|
226
|
+
// an echo and not a model is stated in the pipeline banner and on the bubble itself.
|
|
205
227
|
async function* echoLLM(history) {
|
|
206
228
|
const last = [...history].reverse().find(m => m.role === 'user')?.content ?? '';
|
|
207
|
-
for (const w of
|
|
229
|
+
for (const w of last.match(/\S+\s*/g) ?? []) {
|
|
208
230
|
yield { text: w }; await new Promise(r => setTimeout(r, 25));
|
|
209
231
|
}
|
|
210
232
|
}
|
|
211
233
|
|
|
234
|
+
// ── what is ACTUALLY running ────────────────────────────────────────────────
|
|
235
|
+
// A demo that hides which engine served a reply is worse than no demo: a browser-native
|
|
236
|
+
// recognizer looks identical to voiceloop's pipeline, and an echo stand-in looks identical
|
|
237
|
+
// to a model. Both get named on screen, always.
|
|
238
|
+
const describePipeline = (a) => {
|
|
239
|
+
const echo = $('mode').value === 'echo';
|
|
240
|
+
// Before the first click there is no session, so ask the provider statically — reading
|
|
241
|
+
// `a?.stt?.selfCapture` off a null agent silently yields false and the banner claims the
|
|
242
|
+
// pipeline is running when it is exactly what gets bypassed.
|
|
243
|
+
const self = a?.stt ? !!a.stt.selfCapture : sttSelfCaptures($('sttProvider').value);
|
|
244
|
+
const stt = self
|
|
245
|
+
? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span>`
|
|
246
|
+
: `<b>${$('sttProvider').value}</b> via voiceloop's pipeline — Silero VAD → feed → commit`;
|
|
247
|
+
const llm = echo
|
|
248
|
+
? '<span class="warn"><b>echo</b> — no model: replies repeat your words back</span>'
|
|
249
|
+
: `<b>${$('model').value || 'your model'}</b> at ${new URL($('llmUrl').value).host}`;
|
|
250
|
+
return `STT: ${stt}<br>LLM: ${llm}`;
|
|
251
|
+
};
|
|
252
|
+
const showPipeline = () => { $('pipeline').innerHTML = describePipeline(agent); };
|
|
253
|
+
// Hoisted (function declaration) so the settings listeners above can call it before this point.
|
|
254
|
+
function onSettingChanged() { showPipeline(); prewarmIfPipelined(); }
|
|
255
|
+
|
|
212
256
|
// A typed turn drives the same state machine without a mic — don't claim we're listening then.
|
|
213
257
|
let micOn = false;
|
|
214
258
|
const STATUS = {
|
|
@@ -233,6 +277,14 @@ function onEvent(e) {
|
|
|
233
277
|
break;
|
|
234
278
|
case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
|
|
235
279
|
case 'stt':
|
|
280
|
+
// A selfCapture provider runs no agent VAD, so no 'vad' event ever arrives and the orb would
|
|
281
|
+
// sit dead still while the browser hears you perfectly — looking exactly like a broken mic.
|
|
282
|
+
// Live partials are the only "I hear you" signal on that path, so drive the pulse from them.
|
|
283
|
+
if (agent?.stt?.selfCapture) {
|
|
284
|
+
document.body.dataset.vad = e.turnComplete ? '0' : '1';
|
|
285
|
+
clearTimeout(vadOff);
|
|
286
|
+
if (!e.turnComplete) vadOff = setTimeout(() => { document.body.dataset.vad = '0'; }, 700);
|
|
287
|
+
}
|
|
236
288
|
if (e.turnComplete) {
|
|
237
289
|
interimEl?.remove(); interimEl = null;
|
|
238
290
|
bubble('user', e.text);
|
|
@@ -251,7 +303,9 @@ function onEvent(e) {
|
|
|
251
303
|
// `full` only exists once the TTS progress cursor is live — i.e. a clip is
|
|
252
304
|
// actually playing. The 'speaking' state is just the first token, not sound.
|
|
253
305
|
if (e.full != null && t0 && !tAudio) { tAudio = Date.now(); showLat(); }
|
|
254
|
-
|
|
306
|
+
// Tag the bubble (never the text) when no model produced it — the reply is a verbatim echo,
|
|
307
|
+
// and only the UI should say so.
|
|
308
|
+
aiEl ??= bubble($('mode').value === 'echo' ? 'ai echoed' : 'ai');
|
|
255
309
|
aiEl.textContent = e.text; // spoken so far
|
|
256
310
|
const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
|
|
257
311
|
if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
|
|
@@ -286,7 +340,7 @@ const makeAgent = () => {
|
|
|
286
340
|
onEvent,
|
|
287
341
|
});
|
|
288
342
|
a.setMuted(muted('muteMic')); a.setTtsMuted(muted('muteTts')); // the toggles survive a rebuild
|
|
289
|
-
log('agent created; stt =', a.sttProvider);
|
|
343
|
+
log('agent created; stt =', a.sttProvider); showPipeline(); // now reflects the LIVE session, not just the form
|
|
290
344
|
return a;
|
|
291
345
|
};
|
|
292
346
|
|
|
@@ -294,6 +348,7 @@ const makeAgent = () => {
|
|
|
294
348
|
const stop = () => {
|
|
295
349
|
agent?.destroy(); agent = null; micOn = false;
|
|
296
350
|
document.body.dataset.vad = '0'; setState('idle');
|
|
351
|
+
onSettingChanged(); // describe the restored settings, and prewarm only if they'll use the VAD
|
|
297
352
|
};
|
|
298
353
|
$('mic').onclick = async () => {
|
|
299
354
|
unlockAudio(); // must be inside the gesture (autoplay policy)
|
|
@@ -338,8 +393,8 @@ for (const id of SETTINGS.filter(i => i !== 'voice'))
|
|
|
338
393
|
|
|
339
394
|
// ── device + voice lists ────────────────────────────────────────────────────
|
|
340
395
|
tts.voices().then(vs => {
|
|
341
|
-
|
|
342
|
-
$('voice').value = localStorage.getItem('vl.voice') || '
|
|
396
|
+
fill('voice', vs.map(v => [v.id, v.id]));
|
|
397
|
+
$('voice').value = localStorage.getItem('vl.voice') || tts.voiceId; // the library's own default, not a copy of it
|
|
343
398
|
tts.setVoice($('voice').value);
|
|
344
399
|
}).catch(e => log('voice list failed', String(e)));
|
|
345
400
|
navigator.mediaDevices?.enumerateDevices().then(ds => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -91,8 +91,14 @@ export interface VoiceAgentOptions {
|
|
|
91
91
|
* short-TTL token the backend mints (raw key stays server-side).
|
|
92
92
|
* 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
|
|
93
93
|
* the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
|
|
94
|
-
sttProvider?:
|
|
94
|
+
sttProvider?: SttProvider;
|
|
95
|
+
/** ONE endpoint that mints the token. When each provider needs a different route or credential,
|
|
96
|
+
* use `getSttToken` instead — it is handed the resolved provider, so the downgrade rule above
|
|
97
|
+
* stays in this library instead of being mirrored by every host. */
|
|
95
98
|
sttTokenUrl?: string;
|
|
99
|
+
/** Mint a short-TTL STT token yourself, for auth that doesn't fit one POST route. Called with the
|
|
100
|
+
* provider actually RUNNING (post-downgrade), never the requested one. Wins over sttTokenUrl. */
|
|
101
|
+
getSttToken?: (provider: SttProvider) => Promise<SttToken>;
|
|
96
102
|
sttUsageUrl?: string;
|
|
97
103
|
sttUrl?: string;
|
|
98
104
|
sttModel?: string;
|
|
@@ -130,13 +136,18 @@ export function warmVad(): Promise<void>;
|
|
|
130
136
|
* mount), so a voice start only pays the WS handshake. Best-effort + deduped; no-op while a
|
|
131
137
|
* fresh cached token remains.
|
|
132
138
|
* `sttTokenUrl` is the EXACT endpoint that mints a Deepgram token (e.g. `/api/v1/stt/deepgram-token`),
|
|
133
|
-
* the same value passed as the `sttTokenUrl` option — NOT an API base.
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
|
|
139
|
+
* the same value passed as the `sttTokenUrl` option — NOT an API base. A host that mints tokens
|
|
140
|
+
* itself passes its `getSttToken` as `getToken` and can skip the URL entirely (warm for the
|
|
141
|
+
* provider it will actually run: `() => getSttToken('deepgram')`). Warming with the wrong URL
|
|
142
|
+
* silently no-ops the optimization: the real session just mints again on click. */
|
|
143
|
+
export function warmDeepgramToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
|
|
137
144
|
/** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
|
|
138
145
|
* host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
|
|
139
146
|
export const VOICE_SYSMSG: string;
|
|
147
|
+
/** Language codes the agent can instruct the LLM to reply in, code → English name. The `sttLang`
|
|
148
|
+
* values that actually steer the reply language — exported so a host builds its language picker
|
|
149
|
+
* from this instead of hand-listing a subset that silently rots as the map grows. */
|
|
150
|
+
export const LANG_NAMES: Record<string, string>;
|
|
140
151
|
|
|
141
152
|
|
|
142
153
|
export class VoiceAgent {
|
|
@@ -242,7 +253,17 @@ export interface STTSession {
|
|
|
242
253
|
}
|
|
243
254
|
export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
|
|
244
255
|
|
|
245
|
-
|
|
256
|
+
/** The built-in STT provider ids — the keys of STT_PROVIDERS. */
|
|
257
|
+
export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
258
|
+
/** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
|
|
259
|
+
* provider reports one (used to cache and pre-mint just before expiry). */
|
|
260
|
+
export interface SttToken { token: string; expires_in?: number }
|
|
261
|
+
|
|
262
|
+
export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
|
|
263
|
+
/** Does this provider own the mic and its own end-of-turn? Then the agent builds NO capture pipeline
|
|
264
|
+
* and runs no VAD for it — so it emits no `vad` events, and prewarming the VAD/ONNX model is a
|
|
265
|
+
* wasted download. Answerable without constructing a session, so a host can decide both up front. */
|
|
266
|
+
export function sttSelfCaptures(provider: string): boolean;
|
|
246
267
|
export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
247
268
|
export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
248
269
|
export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
@@ -251,8 +272,9 @@ export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>):
|
|
|
251
272
|
export const WEBSPEECH_FALLBACK: 'elevenlabs';
|
|
252
273
|
export function webSpeechSupported(): boolean;
|
|
253
274
|
/** The provider id actually used: 'webspeech' downgrades to WEBSPEECH_FALLBACK in a browser
|
|
254
|
-
* without SpeechRecognition. Call it to label the UI ("using cloud recognition")
|
|
255
|
-
|
|
275
|
+
* without SpeechRecognition. Call it to label the UI ("using cloud recognition") — but NOT to pick
|
|
276
|
+
* a token route: `getSttToken` is already handed the resolved id. */
|
|
277
|
+
export function resolveSttProvider(id: string): SttProvider;
|
|
256
278
|
|
|
257
279
|
// ── TTS ──────────────────────────────────────────────────────────────────────────────────────
|
|
258
280
|
/** Base class for streaming TTS engines: sentence-splits the incoming token stream, synthesizes
|
package/src/index.js
CHANGED
|
@@ -2,11 +2,11 @@ export {
|
|
|
2
2
|
VoiceAgent, StreamingTTS, PiperTTS,
|
|
3
3
|
unlockAudio, warmVad,
|
|
4
4
|
isSelfEcho, novelChars, toolResultText,
|
|
5
|
-
VOICE_SYSMSG,
|
|
5
|
+
VOICE_SYSMSG, LANG_NAMES,
|
|
6
6
|
} from './voice-agent.js';
|
|
7
7
|
export {
|
|
8
8
|
STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
|
|
9
|
-
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT,
|
|
9
|
+
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, sttSelfCaptures,
|
|
10
10
|
warmDeepgramToken,
|
|
11
11
|
} from './stt.js';
|
|
12
12
|
export { ElevenLabsTTS } from './tts-elevenlabs.js';
|
package/src/stt.js
CHANGED
|
@@ -39,8 +39,10 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
|
|
|
39
39
|
// AUTH (cloud providers): the raw provider key must stay server-side. Either
|
|
40
40
|
// • `sttTokenUrl` — your backend route that mints a short-TTL token: POST, optional X-API-Key
|
|
41
41
|
// (`apiKey`), responds { token, expires_in? }; or
|
|
42
|
-
// • `getToken` — an async () => ({ token, expires_in? }) callback, when your auth doesn't
|
|
43
|
-
// that one route shape. Takes precedence over sttTokenUrl.
|
|
42
|
+
// • `getToken` — an async (provider) => ({ token, expires_in? }) callback, when your auth doesn't
|
|
43
|
+
// fit that one route shape. Takes precedence over sttTokenUrl. VoiceAgent calls it with the
|
|
44
|
+
// provider actually RUNNING (post-downgrade), so a host whose credential differs per provider
|
|
45
|
+
// writes one function instead of mirroring the downgrade rule.
|
|
44
46
|
|
|
45
47
|
// ── Shared socket-lifecycle helpers ─────────────────────────────────────────
|
|
46
48
|
// The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
|
|
@@ -56,11 +58,20 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
|
|
|
56
58
|
// state to clear on failure differs per provider, so that stays with the caller).
|
|
57
59
|
async function mintSttToken(url, apiKey, onFatal, getToken) {
|
|
58
60
|
if (getToken) {
|
|
61
|
+
// HOST code on the critical path: bound it exactly like the built-in fetch below, or a minter
|
|
62
|
+
// that never settles stalls voice start forever with no error. We can't cancel the host's
|
|
63
|
+
// promise — we just stop waiting on it (it may still resolve into the void, which is fine:
|
|
64
|
+
// a token is idempotent to mint).
|
|
65
|
+
let timer;
|
|
59
66
|
try {
|
|
60
|
-
const body = await
|
|
67
|
+
const body = await Promise.race([
|
|
68
|
+
getToken(),
|
|
69
|
+
new Promise((_, rej) => { timer = setTimeout(() => rej(new Error('timed out')), TUNING.STT.tokenFetchTimeoutMs); }),
|
|
70
|
+
]);
|
|
61
71
|
if (!body?.token) throw new Error('getToken returned no token');
|
|
62
72
|
return body;
|
|
63
73
|
} catch (e) { onFatal(`STT token: ${e.message}`); return null; }
|
|
74
|
+
finally { clearTimeout(timer); } // a fast mint must not hold the timer (and Node's loop) open
|
|
64
75
|
}
|
|
65
76
|
if (!url) { onFatal('STT token: no sttTokenUrl/getToken configured — cloud STT needs a token route (see README)'); return null; }
|
|
66
77
|
let res;
|
|
@@ -589,10 +600,12 @@ export function makeDeepgramSTT(opts) {
|
|
|
589
600
|
// result window). Blindly appending would double words ("hello there" + "there how" → "there there").
|
|
590
601
|
// Every final is de-overlapped against the turn-so-far before it's appended.
|
|
591
602
|
export function makeWebSpeechSTT(opts) {
|
|
592
|
-
const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onFatal, onClose, isClosed } = opts;
|
|
603
|
+
const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
593
604
|
const Rec = typeof window !== 'undefined' && (window.SpeechRecognition || window.webkitSpeechRecognition);
|
|
594
605
|
const EOT_MS = TUNING.WEBSPEECH_EOT_MS ?? 900;
|
|
595
606
|
const TAIL_MAX_DEFERS = TUNING.WEBSPEECH_TAIL_MAX_DEFERS ?? 3;
|
|
607
|
+
const MAX_ERROR_RESTARTS = TUNING.WEBSPEECH_MAX_ERROR_RESTARTS ?? 5;
|
|
608
|
+
const ERROR_BACKOFF_MS = TUNING.WEBSPEECH_ERROR_BACKOFF_MS ?? 300;
|
|
596
609
|
let rec = null; // the live SpeechRecognition instance (null between restarts / when stopped)
|
|
597
610
|
let sttStart = 0; // performance.now() at the current turn's first activity (for onPartial/onFinal ms)
|
|
598
611
|
let turn = ''; // committed (isFinal) text of the CURRENT turn, merged across recognizer restarts
|
|
@@ -603,6 +616,9 @@ export function makeWebSpeechSTT(opts) {
|
|
|
603
616
|
let tailDefers = 0; // consecutive closes deferred waiting on an un-promoted tail (bounded, reset on new text)
|
|
604
617
|
let want = false; // intent to listen — onend auto-restarts while true
|
|
605
618
|
let muted = false; // mic off — stop the recognizer and don't restart
|
|
619
|
+
let lastError = ''; // error of the recognizer currently ending (drives the backoff/give-up path)
|
|
620
|
+
let errorRestarts = 0; // consecutive failed launches with no recognized text between them
|
|
621
|
+
let restartTimer = null; // pending backed-off relaunch
|
|
606
622
|
// AEC'D CAPTURE (SpeechRecognition.start(audioTrack), spec'd + shipped in recent Chromium): by
|
|
607
623
|
// default the engine opens its OWN mic, bypassing every protection the agent's pipeline has — the
|
|
608
624
|
// browser AEC referenced against the WebRTC TTS loopback (voice-agent initAecLoopback) and noise
|
|
@@ -727,6 +743,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
727
743
|
}
|
|
728
744
|
final = final.trim(); interim = interim.trim();
|
|
729
745
|
if (!turn && !interimTail && (final || interim)) sttStart = performance.now(); // first activity of a fresh turn
|
|
746
|
+
if (final || interim) errorRestarts = 0; // the engine is demonstrably working — forget past failures
|
|
730
747
|
commitFinal(final); // de-overlaps the seam, then locks the newly-final words into the turn
|
|
731
748
|
// Keep the live tail as its own state so a recognizer restart (which drops interim) can replay it
|
|
732
749
|
// instead of blanking the draft. A frame that only finalized words (interim === '') clears it —
|
|
@@ -739,8 +756,11 @@ export function makeWebSpeechSTT(opts) {
|
|
|
739
756
|
};
|
|
740
757
|
r.onerror = (event) => {
|
|
741
758
|
if (r !== rec) return;
|
|
742
|
-
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); }
|
|
743
|
-
// 'no-speech' / 'aborted'
|
|
759
|
+
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); return; }
|
|
760
|
+
// 'no-speech' / 'aborted' are the engine's normal idle churn — onend just relaunches, silently.
|
|
761
|
+
// Anything else ('network', 'audio-capture', …) is a FAILED launch: remembered here so onend
|
|
762
|
+
// backs off instead of hot-looping, and gives up out loud once the engine looks dead.
|
|
763
|
+
if (event.error !== 'no-speech' && event.error !== 'aborted') lastError = event.error;
|
|
744
764
|
};
|
|
745
765
|
r.onend = () => {
|
|
746
766
|
if (r !== rec) return; // superseded instance's late onend — ignore
|
|
@@ -756,8 +776,19 @@ export function makeWebSpeechSTT(opts) {
|
|
|
756
776
|
// untouched — if this really was end-of-turn, that timer fires on its own EOT_MS after the last
|
|
757
777
|
// word (recognizer alive or not); if the user is still talking, the new recognizer's text
|
|
758
778
|
// re-arms it. No flush decision belongs here — that's what makes the boundary restart-immune.
|
|
759
|
-
|
|
760
|
-
|
|
779
|
+
const err = lastError; lastError = '';
|
|
780
|
+
if (!(want && !muted && !isClosed())) { onClose?.(); return; }
|
|
781
|
+
if (!err) { errorRestarts = 0; start(); return; }
|
|
782
|
+
// A launch that only produced an error and no text: the engine may be permanently unable to run
|
|
783
|
+
// here (speech service unreachable, unreadable capture device). Retry with backoff, then stop —
|
|
784
|
+
// an endless silent restart loop is indistinguishable from "listening" and never recovers.
|
|
785
|
+
if (++errorRestarts > MAX_ERROR_RESTARTS) {
|
|
786
|
+
want = false;
|
|
787
|
+
onFatal?.(`Speech recognition: ${err} (${MAX_ERROR_RESTARTS} restarts failed — the browser's speech service is unreachable)`);
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
onError?.(`Speech recognition: ${err} — retrying (${errorRestarts}/${MAX_ERROR_RESTARTS})`);
|
|
791
|
+
restartTimer = setTimeout(() => { restartTimer = null; if (want && !muted && !isClosed()) start(); }, Math.min(ERROR_BACKOFF_MS * 2 ** (errorRestarts - 1), 4000));
|
|
761
792
|
};
|
|
762
793
|
rec = r;
|
|
763
794
|
let ok = false;
|
|
@@ -771,7 +802,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
771
802
|
|
|
772
803
|
// Stop the current recognizer and drop all in-progress state (mute/close). The stopped recognizer's
|
|
773
804
|
// late onend is stale (rec nulled → r !== rec), so it can't auto-restart.
|
|
774
|
-
const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
|
|
805
|
+
const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; clearTimeout(restartTimer); restartTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; lastError = ''; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
|
|
775
806
|
|
|
776
807
|
return {
|
|
777
808
|
selfCapture: true, // owns its mic → the agent builds no capture pipeline
|
|
@@ -780,7 +811,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
780
811
|
if (!Rec) { onFatal?.('Speech recognition not supported in this browser'); return; }
|
|
781
812
|
// start() first, synchronously — Safari requires SpeechRecognition.start inside the click
|
|
782
813
|
// gesture. The AEC'd track lands async right after and upgrades the recognizer at the seam.
|
|
783
|
-
want = true; start();
|
|
814
|
+
want = true; errorRestarts = 0; start();
|
|
784
815
|
ensureTrack();
|
|
785
816
|
},
|
|
786
817
|
// Mute: Web Speech captures its OWN mic (independent of the agent), so mute must actually stop it.
|
|
@@ -789,7 +820,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
789
820
|
setEnabled(on) {
|
|
790
821
|
muted = !on;
|
|
791
822
|
if (muted) { teardown(); dropMic(); onPartial('', 0, ''); } // drop the draft AND the mic (privacy light off)
|
|
792
|
-
else if (want && !rec) { start(); ensureTrack(); }
|
|
823
|
+
else if (want && !rec) { errorRestarts = 0; start(); ensureTrack(); }
|
|
793
824
|
},
|
|
794
825
|
close() { want = false; teardown(); dropMic(); },
|
|
795
826
|
};
|
|
@@ -803,6 +834,14 @@ export const STT_PROVIDERS = {
|
|
|
803
834
|
deepgram: makeDeepgramSTT,
|
|
804
835
|
};
|
|
805
836
|
|
|
837
|
+
// Self-capturing providers own the mic AND their end-of-turn, so the agent builds no capture
|
|
838
|
+
// pipeline and runs no VAD for them. Readable WITHOUT constructing a session, so a host can decide
|
|
839
|
+
// up front whether prewarming the VAD/ONNX model is worth the download (it isn't, for these) and
|
|
840
|
+
// whether to expect 'vad' events at all. Kept on the factory so it can't drift from the session's
|
|
841
|
+
// own selfCapture flag.
|
|
842
|
+
makeWebSpeechSTT.selfCapture = true;
|
|
843
|
+
export const sttSelfCaptures = (provider) => !!STT_PROVIDERS[provider]?.selfCapture;
|
|
844
|
+
|
|
806
845
|
// Web Speech only exists in Chromium/Safari — Firefox and Linux WebKitGTK webviews ship no
|
|
807
846
|
// SpeechRecognition at all. There the default 'webspeech' provider can only fatal ("Speech
|
|
808
847
|
// recognition not supported in this browser") and voice mode is simply dead, so resolve it to a
|
package/src/tuning.js
CHANGED
|
@@ -71,6 +71,17 @@ export const TUNING = {
|
|
|
71
71
|
// Raise if slow engines still split turns; lower to bound the wait harder.
|
|
72
72
|
WEBSPEECH_TAIL_MAX_DEFERS: 3,
|
|
73
73
|
|
|
74
|
+
// Web Speech errors that aren't auth ('network', 'audio-capture', …) are retried by relaunching the
|
|
75
|
+
// recognizer — but if the engine is genuinely broken (browser's speech service unreachable, a fake
|
|
76
|
+
// audio device it can't read) EVERY relaunch fails instantly and the old code hot-looped forever,
|
|
77
|
+
// silently, looking like "listening but deaf". So consecutive failures with no recognized text in
|
|
78
|
+
// between are counted: each one waits longer (backoff below), and this many gives up with onFatal
|
|
79
|
+
// instead of spinning. Reset by any result. 'no-speech'/'aborted' are normal and don't count.
|
|
80
|
+
WEBSPEECH_MAX_ERROR_RESTARTS: 5,
|
|
81
|
+
// First retry delay after a failing recognizer; doubles per consecutive failure (capped at 4s), so
|
|
82
|
+
// a dead speech service costs a handful of tries over a few seconds instead of a spin loop.
|
|
83
|
+
WEBSPEECH_ERROR_BACKOFF_MS: 300,
|
|
84
|
+
|
|
74
85
|
// Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
|
|
75
86
|
// agent starts generating a reply for the probable final text — overlapping the LLM's first-token
|
|
76
87
|
// latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
|
package/src/voice-agent.js
CHANGED
|
@@ -54,7 +54,7 @@ const CAPTURE_WORKLET_URL = typeof URL !== 'undefined' && URL.createObjectURL
|
|
|
54
54
|
// + the host's conversation context (if any).
|
|
55
55
|
// The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
|
|
56
56
|
// English under the English persona — so state the selected STT language explicitly.
|
|
57
|
-
const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
|
|
57
|
+
export const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
|
|
58
58
|
const langRule = (lang) => {
|
|
59
59
|
const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
|
|
60
60
|
return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
|
|
@@ -267,7 +267,10 @@ export class VoiceAgent {
|
|
|
267
267
|
const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
|
|
268
268
|
this.stt = makeStt({
|
|
269
269
|
apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
270
|
-
|
|
270
|
+
// `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
|
|
271
|
+
// whose credential differs per provider never has to re-derive the webspeech downgrade
|
|
272
|
+
// (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
|
|
273
|
+
sttTokenUrl, getToken: getSttToken && (() => getSttToken(this.sttProvider)), sttUsageUrl, eotThreshold: sttEotThreshold,
|
|
271
274
|
onPartial: (text, ms, committed = '') => {
|
|
272
275
|
const interim = `${committed} ${text}`.trim();
|
|
273
276
|
this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector
|