@todoforai/voiceloop 0.1.20 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/package.json +1 -1
- package/src/index.d.ts +8 -3
- package/src/index.js +2 -1
- package/src/stt.js +190 -0
- package/src/voice-agent.js +3 -2
package/README.md
CHANGED
|
@@ -108,12 +108,19 @@ into the mic (no AEC):
|
|
|
108
108
|
| system | latency (clean) | talked through user | cut itself under echo |
|
|
109
109
|
|---|---|---|---|
|
|
110
110
|
| OpenAI Realtime (own LLM)* | 870ms | **0/30** | **17/30** |
|
|
111
|
+
| **voiceloop** (soniox + ElevenLabs flash)† | **770ms** | 7/30 | 0/30 |
|
|
111
112
|
| **voiceloop** (deepgram + ElevenLabs flash) | **980ms** | **0/30** | **0/30** |
|
|
112
113
|
| **voiceloop** (deepgram + Piper, free local TTS) | **970ms** | **0/30** | — |
|
|
113
114
|
| **voiceloop** (webspeech + Piper, zero-key default) | 2110ms | — | — |
|
|
114
115
|
| Pipecat 1.8.1 (same providers) | 1050ms | 2/30 | **20/30** |
|
|
115
116
|
| ElevenLabs ConvAI | 1450ms | **0/30** | 0/30 |
|
|
116
117
|
|
|
118
|
+
† Soniox row measured 2026-09-16 (5 runs × 6 turns each, scoring v2; the other rows are the
|
|
119
|
+
2026-08-31 v1 numbers). Soniox is the faster and cheaper STT ($0.12/hr vs $0.39/hr, 60+ languages)
|
|
120
|
+
but its endpointing is acoustic-leaning: it enters a mid-sentence pause on 15/30 hesitation turns and
|
|
121
|
+
talks through 7 of them (Flux: 2 → 0). Pick Flux when never-interrupt matters more than 100–400ms.
|
|
122
|
+
Full tables: [bench/results/SONIOX-VS-DEEPGRAM.md](bench/results/SONIOX-VS-DEEPGRAM.md).
|
|
123
|
+
|
|
117
124
|
The separating failure is echo: with no AEC, the other fast stacks hear their own voice as the
|
|
118
125
|
user and cut their own replies. voiceloop's word-level echo filter runs echo-coupled turns at
|
|
119
126
|
full speed (930ms — parity with clean).
|
|
@@ -156,6 +163,7 @@ iOS ducks the speakers while the recognizer holds the mic — which is exactly w
|
|
|
156
163
|
| `elevenlabs` | VAD-gated | Scribe v2 realtime, great accuracy |
|
|
157
164
|
| `deepgram` | **Native** (Flux) | Provider's own turn model hears the full stream |
|
|
158
165
|
| `speechmatics` | VAD-gated | Cheapest per second, locked-words-only finals |
|
|
166
|
+
| `soniox` | **Native** (stt-rt-v5) | Provider's own endpointing, 60+ languages incl. Hungarian, $0.12/hr |
|
|
159
167
|
|
|
160
168
|
## Bring your own LLM
|
|
161
169
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.22",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ export interface VoiceAgentOptions {
|
|
|
149
149
|
/** STT backend: 'webspeech' (default — the browser's own on-device SpeechRecognition; free, no
|
|
150
150
|
* token, no socket, captures its own mic) | 'speechmatics' | 'elevenlabs' (Scribe) | 'deepgram'
|
|
151
151
|
* (Flux — native model-based end-of-turn detection; the agent feeds it the full mic stream and
|
|
152
|
-
* lets it close turns itself). The cloud ones connect the browser DIRECTLY to the provider with a
|
|
152
|
+
* lets it close turns itself) | 'soniox' (stt-rt-v5 — same native-endpointing shape as Flux). The cloud ones connect the browser DIRECTLY to the provider with a
|
|
153
153
|
* short-TTL token the backend mints (raw key stays server-side).
|
|
154
154
|
* 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
|
|
155
155
|
* the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
|
|
@@ -175,6 +175,8 @@ export interface VoiceAgentOptions {
|
|
|
175
175
|
* confidence the user is done exceeds this threshold. Lower = snappier turn ends, higher = more
|
|
176
176
|
* patient across mid-thought pauses. Deepgram-only; default from tuning.js (0.7). */
|
|
177
177
|
sttEotThreshold?: number;
|
|
178
|
+
/** Provider-specific extras forwarded to the STT factory (Soniox: endpointLatencyLevel 0–3, endpointSensitivity −1..1). */
|
|
179
|
+
sttOptions?: Record<string, unknown>;
|
|
178
180
|
/** Every tool the model may call. voiceloop ships NO built-in tools — a voice library has no
|
|
179
181
|
* business knowing what a "todo" is — so product tools (and their `run`) are supplied here. */
|
|
180
182
|
tools?: Record<string, VoiceTool>;
|
|
@@ -215,6 +217,8 @@ export function warmVad(): Promise<void>;
|
|
|
215
217
|
* provider it will actually run: `() => getSttToken('deepgram')`). Warming with the wrong URL
|
|
216
218
|
* silently no-ops the optimization: the real session just mints again on click. */
|
|
217
219
|
export function warmDeepgramToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
|
|
220
|
+
/** Soniox counterpart of warmDeepgramToken: pre-mints and caches the temporary key. */
|
|
221
|
+
export function warmSonioxToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
|
|
218
222
|
/** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
|
|
219
223
|
* host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
|
|
220
224
|
export const VOICE_SYSMSG: string;
|
|
@@ -281,7 +285,7 @@ export class VoiceAgent {
|
|
|
281
285
|
state: 'idle' | 'listening' | 'thinking' | 'speaking';
|
|
282
286
|
/** The STT provider actually in use — differs from the requested one when 'webspeech' was
|
|
283
287
|
* downgraded to a cloud provider (browser without SpeechRecognition). */
|
|
284
|
-
readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
288
|
+
readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram' | 'soniox';
|
|
285
289
|
/** True once the pipeline has been torn down (stop() or a fatal STT error self-stopped it). */
|
|
286
290
|
readonly closed: boolean;
|
|
287
291
|
}
|
|
@@ -332,7 +336,7 @@ export interface STTSession {
|
|
|
332
336
|
export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
|
|
333
337
|
|
|
334
338
|
/** The built-in STT provider ids — the keys of STT_PROVIDERS. */
|
|
335
|
-
export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
339
|
+
export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram' | 'soniox';
|
|
336
340
|
/** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
|
|
337
341
|
* provider reports one (used to cache and pre-mint just before expiry). */
|
|
338
342
|
export interface SttToken { token: string; expires_in?: number }
|
|
@@ -343,6 +347,7 @@ export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
|
|
|
343
347
|
* wasted download. Answerable without constructing a session, so a host can decide both up front. */
|
|
344
348
|
export function sttSelfCaptures(provider: string): boolean;
|
|
345
349
|
export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
350
|
+
export function makeSonioxSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
346
351
|
export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
347
352
|
export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
348
353
|
export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
package/src/index.js
CHANGED
|
@@ -6,8 +6,9 @@ export {
|
|
|
6
6
|
} from './voice-agent.js';
|
|
7
7
|
export {
|
|
8
8
|
STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
|
|
9
|
-
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, sttSelfCaptures,
|
|
9
|
+
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, makeSonioxSTT, sttSelfCaptures,
|
|
10
10
|
warmDeepgramToken,
|
|
11
|
+
warmSonioxToken,
|
|
11
12
|
} from './stt.js';
|
|
12
13
|
export { ElevenLabsTTS } from './tts-elevenlabs.js';
|
|
13
14
|
export { loadVoiceDeps, prewarmVoice } from './deps.js';
|
package/src/stt.js
CHANGED
|
@@ -607,6 +607,195 @@ export function makeDeepgramSTT(opts) {
|
|
|
607
607
|
};
|
|
608
608
|
}
|
|
609
609
|
|
|
610
|
+
// ── Soniox realtime (stt-rt-v5) ──────────────────────────────────────────────
|
|
611
|
+
// Browser↔Soniox direct WS. Auth: the backend mints a temporary API key (POST /v1/auth/temporary-api-key,
|
|
612
|
+
// usage_type transcribe_websocket) and the browser sends it in the FIRST JSON config message — no
|
|
613
|
+
// URL/subprotocol auth. Audio is raw binary pcm_s16le @16k after the config frame.
|
|
614
|
+
//
|
|
615
|
+
// Turn model — TOKENS, not transcripts: every message carries a `tokens` array; `is_final: true`
|
|
616
|
+
// tokens arrive exactly once and are appended, `is_final: false` tokens are the whole current
|
|
617
|
+
// provisional tail and REPLACE the previous tail. Token text carries its own leading space (subword
|
|
618
|
+
// units), so the transcript is a plain concatenation. With enable_endpoint_detection the model does
|
|
619
|
+
// native end-of-turn (semantic + acoustic) and emits a final `<end>` token at the boundary — so like
|
|
620
|
+
// Flux this provider is `continuous` (must hear the silences) + `nativeEOT` (turn closes itself).
|
|
621
|
+
// commit() is the VAD-gated fallback ({"type":"finalize"} → tokens flush final + a `<fin>` marker).
|
|
622
|
+
const SONIOX_URL = 'wss://stt-rt.soniox.com/transcribe-websocket';
|
|
623
|
+
const SONIOX_KEEPALIVE_MS = 10000; // server closes after >20s with no audio/keepalive
|
|
624
|
+
// Soniox errors arrive as { error_code: <http status>, error_type, error_message }. 4xx won't fix
|
|
625
|
+
// itself with a reconnect (bad key/config, balance exhausted — seen live as 402): fatal, so the host
|
|
626
|
+
// stops instead of the continuous feed re-minting + reconnecting forever. 5xx/429 stay recoverable.
|
|
627
|
+
const sonioxFatal = (code) => code >= 400 && code < 500 && code !== 429;
|
|
628
|
+
// Soniox's `<end>` is ACOUSTIC-LEANING: it fires ~250ms into any pause, including mid-sentence
|
|
629
|
+
// ones — measured on the bench hesitation audio it closed "Hi, I need some help with—" / "Um." /
|
|
630
|
+
// "with a dinner reservation." as three turns (Flux keeps that as one). Its own signal for "not
|
|
631
|
+
// done" is the em-dash it appends to a cut-off phrase, so: a turn ending in one is held open and
|
|
632
|
+
// merged with what follows; anything else is trusted unless it's a bare filler ("Um."), which is
|
|
633
|
+
// also a continuation. The hold has a cap so a user who genuinely trails off still gets an answer.
|
|
634
|
+
const SONIOX_HOLD_MS = 1500;
|
|
635
|
+
// Same token cache/prewarm as Deepgram (see dgToken): the temporary-key mint + WS open cost ~2s on
|
|
636
|
+
// the first utterance when paid cold on the mic click.
|
|
637
|
+
let sxToken = null, sxTokenExp = 0, sxMintPromise = null;
|
|
638
|
+
// One acquisition path for prewarm AND open(): a click landing mid-prewarm joins that mint
|
|
639
|
+
// instead of starting a second one. `slackMs` = how much life the cached key must have left.
|
|
640
|
+
function acquireSonioxToken(sttTokenUrl, apiKey, onFatal, getToken, slackMs) {
|
|
641
|
+
if (sxToken && performance.now() < sxTokenExp - slackMs) return Promise.resolve(sxToken);
|
|
642
|
+
return (sxMintPromise ??= (async () => {
|
|
643
|
+
const body = await mintSttToken(sttTokenUrl, apiKey, onFatal, getToken);
|
|
644
|
+
if (body) { sxToken = body.token; sxTokenExp = performance.now() + (body.expires_in ?? 300) * 1000; }
|
|
645
|
+
return body?.token ?? null;
|
|
646
|
+
})().finally(() => { sxMintPromise = null; }));
|
|
647
|
+
}
|
|
648
|
+
export function warmSonioxToken(sttTokenUrl, apiKey, getToken) {
|
|
649
|
+
return acquireSonioxToken(sttTokenUrl, apiKey, () => {}, getToken, 30000).then(() => {}); // best-effort: open() re-mints
|
|
650
|
+
}
|
|
651
|
+
const sonioxContinues = (t) => /[—–-]$/.test(t) || /^(um+|uh+|hmm+|er+|ah+)[.,]?$/i.test(t);
|
|
652
|
+
export function makeSonioxSTT(opts) {
|
|
653
|
+
const { apiKey, sttModel, sttLang = 'en', keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
|
|
654
|
+
onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
655
|
+
const sttUrl = opts.sttUrl || SONIOX_URL;
|
|
656
|
+
const model = sttModel || 'stt-rt-v5';
|
|
657
|
+
let ws = null, opening = false, outbox = [], outboxSamples = 0, keepalive = null;
|
|
658
|
+
// Turn state. Soniox finals are APPEND-ONLY (no cumulative restatement like Flux), so a socket
|
|
659
|
+
// boundary must wipe them: a stale prefix can't be repaired by the next stream.
|
|
660
|
+
let finalText = '', tail = '', lastInterim = '', turnStart = 0;
|
|
661
|
+
let audioSinceFinal = false, awaitingFinal = false, safetyTimer = null;
|
|
662
|
+
let held = '', holdTimer = null; // turn closed on a continuation cue, waiting for the rest
|
|
663
|
+
const usage = makeUsageReporter(sttUsageUrl, apiKey, 'soniox', { model }, opts.fetchFn);
|
|
664
|
+
const OUTBOX_MAX_SAMPLES = 32000; // ~2s preroll across the token mint + handshake (see Deepgram)
|
|
665
|
+
|
|
666
|
+
const sendAudio = (i16) => {
|
|
667
|
+
if (!ws || ws.readyState !== 1) {
|
|
668
|
+
outbox.push(i16); outboxSamples += i16.length;
|
|
669
|
+
while (outboxSamples > OUTBOX_MAX_SAMPLES && outbox.length > 1) outboxSamples -= outbox.shift().length;
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
ws.send(i16.buffer.slice(i16.byteOffset, i16.byteOffset + i16.byteLength));
|
|
673
|
+
usage.add(i16.length);
|
|
674
|
+
};
|
|
675
|
+
const resetText = () => { finalText = ''; tail = ''; lastInterim = ''; turnStart = 0; };
|
|
676
|
+
const dropHold = () => { held = ''; clearTimeout(holdTimer); holdTimer = null; };
|
|
677
|
+
const resetTurn = () => { resetText(); dropHold(); clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false; };
|
|
678
|
+
const ms = () => Math.round(performance.now() - (turnStart || performance.now()));
|
|
679
|
+
const withHeld = (t) => (held ? `${held} ${t.trim()}` : t).trim();
|
|
680
|
+
const flushHeld = () => { // hold cap: nothing followed → the cut-off phrase was the whole turn
|
|
681
|
+
if ((finalText + tail).trim()) { holdTimer = setTimeout(flushHeld, SONIOX_HOLD_MS); return; } // continuation in progress: its <end> merges
|
|
682
|
+
const t = held, at = ms(); resetTurn(); audioSinceFinal = false; onFinal(t, at);
|
|
683
|
+
};
|
|
684
|
+
const closeTurn = (force = false) => {
|
|
685
|
+
const seg = (finalText + tail).trim(), t = withHeld(seg), at = ms();
|
|
686
|
+
if (!force && sonioxContinues(seg)) { // cut mid-thought: keep it as the prefix of the next segment
|
|
687
|
+
resetText(); held = t;
|
|
688
|
+
clearTimeout(holdTimer); holdTimer = setTimeout(flushHeld, SONIOX_HOLD_MS);
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
resetTurn(); audioSinceFinal = false;
|
|
692
|
+
onFinal(t, at);
|
|
693
|
+
};
|
|
694
|
+
// A partial that retracts to EMPTY must still reach the host (it drops a speculation built on
|
|
695
|
+
// withdrawn words) — but idle responses with nothing to say shouldn't spam empty callbacks.
|
|
696
|
+
const emitPartial = () => {
|
|
697
|
+
const interim = withHeld(finalText + tail);
|
|
698
|
+
if (interim === lastInterim) return;
|
|
699
|
+
lastInterim = interim;
|
|
700
|
+
onPartial(interim, ms());
|
|
701
|
+
};
|
|
702
|
+
|
|
703
|
+
const open = async () => {
|
|
704
|
+
if (opening) return;
|
|
705
|
+
opening = true;
|
|
706
|
+
try {
|
|
707
|
+
const token = await acquireSonioxToken(sttTokenUrl, apiKey, onFatal, getToken, 15000);
|
|
708
|
+
if (!token) { outbox = []; outboxSamples = 0; return; }
|
|
709
|
+
if (isClosed()) return;
|
|
710
|
+
// NEW SOCKET = NEW STREAM: whatever the old one left half-finalized is gone with it. A pending
|
|
711
|
+
// commit() survives (its finalize is re-sent on open, its safety timer still closes the turn)
|
|
712
|
+
// so the host's closing latch always resolves.
|
|
713
|
+
resetText(); dropHold(); // a held cut-off phrase belongs to the dead stream too
|
|
714
|
+
clearInterval(keepalive); keepalive = null; // the superseded socket's timer must not outlive it
|
|
715
|
+
const sock = ws = newSocket(sttUrl);
|
|
716
|
+
sock.binaryType = 'arraybuffer';
|
|
717
|
+
sock.onopen = () => {
|
|
718
|
+
if (sock !== ws) return;
|
|
719
|
+
const config = {
|
|
720
|
+
api_key: token, model, audio_format: 'pcm_s16le', sample_rate: 16000, num_channels: 1,
|
|
721
|
+
enable_endpoint_detection: true,
|
|
722
|
+
// Level 2 measured (bench smalltalk, 5 runs): 1042 → 768 ms voice→voice, 0 mid-sentence
|
|
723
|
+
// splits; level 3 was faster still (726) but split "Hey there. | Can you hear me?" every run.
|
|
724
|
+
endpoint_latency_adjustment_level: opts.endpointLatencyLevel ?? 2,
|
|
725
|
+
...(opts.endpointSensitivity != null && { endpoint_sensitivity: opts.endpointSensitivity }),
|
|
726
|
+
};
|
|
727
|
+
if (sttLang) config.language_hints = [sttLang.slice(0, 2)];
|
|
728
|
+
if (keyterms.length) config.context = { terms: keyterms };
|
|
729
|
+
sock.send(JSON.stringify(config));
|
|
730
|
+
const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c);
|
|
731
|
+
if (awaitingFinal) sock.send(JSON.stringify({ type: 'finalize' })); // a commit() that landed while connecting
|
|
732
|
+
keepalive = setInterval(() => { if (sock.readyState === 1) sock.send(JSON.stringify({ type: 'keepalive' })); }, SONIOX_KEEPALIVE_MS);
|
|
733
|
+
};
|
|
734
|
+
sock.onerror = ev => { if (sock === ws && !isClosed()) { onError('STT socket error'); console.error('Soniox ws error', ev); } };
|
|
735
|
+
sock.onclose = ev => {
|
|
736
|
+
usage.flush(); onClose?.();
|
|
737
|
+
if (sock !== ws) return; // superseded socket's late close: the new one's state is untouched
|
|
738
|
+
clearInterval(keepalive); keepalive = null;
|
|
739
|
+
if (!isClosed() && ev.code !== 1000) onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`);
|
|
740
|
+
};
|
|
741
|
+
sock.onmessage = ev => {
|
|
742
|
+
if (sock !== ws || isClosed() || typeof ev.data !== 'string') return;
|
|
743
|
+
const m = JSON.parse(ev.data);
|
|
744
|
+
if (m.error_code || m.error_message) {
|
|
745
|
+
const msg = `${m.error_type || m.error_code || 'error'}: ${m.error_message || ''}`;
|
|
746
|
+
console.error('Soniox error msg', m);
|
|
747
|
+
if (+m.error_code === 401) { sxToken = null; sxTokenExp = 0; } // a cached key the server rejects is dead
|
|
748
|
+
if (sonioxFatal(+m.error_code)) onFatal(`STT ${msg}`); else onError(msg);
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
if (!Array.isArray(m.tokens)) return;
|
|
752
|
+
// Tokens are chronological and a response may straddle a boundary (…, <end>, next-turn
|
|
753
|
+
// tokens): handle the markers at their POSITION, so text after one belongs to the next turn.
|
|
754
|
+
let newTail = '';
|
|
755
|
+
for (const t of m.tokens) {
|
|
756
|
+
if (t.text === '<end>' || t.text === '<fin>') { tail = newTail; newTail = ''; closeTurn(t.text === '<fin>'); continue; }
|
|
757
|
+
if (!turnStart) turnStart = performance.now(); // turn's ms clock anchors at its first token
|
|
758
|
+
if (t.is_final) finalText += t.text; else newTail += t.text;
|
|
759
|
+
}
|
|
760
|
+
tail = newTail;
|
|
761
|
+
emitPartial();
|
|
762
|
+
};
|
|
763
|
+
} catch (e) {
|
|
764
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
765
|
+
} finally {
|
|
766
|
+
opening = false;
|
|
767
|
+
}
|
|
768
|
+
};
|
|
769
|
+
|
|
770
|
+
return {
|
|
771
|
+
continuous: true, // native endpointing needs the real silences
|
|
772
|
+
nativeEOT: true, // `<end>` token → unsolicited onFinal closes the turn
|
|
773
|
+
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
774
|
+
feed(i16) {
|
|
775
|
+
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
776
|
+
sendAudio(i16); audioSinceFinal = true;
|
|
777
|
+
},
|
|
778
|
+
reset() { resetTurn(); },
|
|
779
|
+
// VAD-gated fallback (the agent never calls this in native mode): ask for a manual finalize and
|
|
780
|
+
// close on the `<fin>` marker, or on the safety timer if it never comes.
|
|
781
|
+
commit() {
|
|
782
|
+
if (awaitingFinal) return;
|
|
783
|
+
if (!audioSinceFinal) { onFinal('', ms()); return; }
|
|
784
|
+
awaitingFinal = true;
|
|
785
|
+
if (ws?.readyState === 1) ws.send(JSON.stringify({ type: 'finalize' })); // else onopen sends it
|
|
786
|
+
clearTimeout(safetyTimer); safetyTimer = setTimeout(() => closeTurn(true), TUNING.STT.forceEndSafetyMs);
|
|
787
|
+
},
|
|
788
|
+
close() {
|
|
789
|
+
resetTurn(); audioSinceFinal = false;
|
|
790
|
+
clearInterval(keepalive); keepalive = null; outbox = []; outboxSamples = 0;
|
|
791
|
+
usage.flush();
|
|
792
|
+
const sock = ws; ws = null;
|
|
793
|
+
if (sock?.readyState === 1) { try { sock.send(''); } catch { /* closing anyway */ } } // empty frame = end of stream
|
|
794
|
+
sock?.close();
|
|
795
|
+
},
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
|
|
610
799
|
// ── Browser Web Speech API (browser-managed, no token) ───────────────────────
|
|
611
800
|
// "Browser-native" is not "on-device": Chrome may route recognition through a vendor speech
|
|
612
801
|
// service. The win here is zero setup (no key, no token endpoint), not guaranteed privacy.
|
|
@@ -877,6 +1066,7 @@ export const STT_PROVIDERS = {
|
|
|
877
1066
|
speechmatics: makeSpeechmaticsSTT,
|
|
878
1067
|
elevenlabs: makeElevenLabsSTT,
|
|
879
1068
|
deepgram: makeDeepgramSTT,
|
|
1069
|
+
soniox: makeSonioxSTT,
|
|
880
1070
|
};
|
|
881
1071
|
|
|
882
1072
|
// Self-capturing providers own the mic AND their end-of-turn, so the agent builds no capture
|
package/src/voice-agent.js
CHANGED
|
@@ -161,6 +161,7 @@ export class VoiceAgent {
|
|
|
161
161
|
// `sttLang` — STT language code ('en'); also pins the reply language.
|
|
162
162
|
// `keyterms` — domain words to bias STT towards (cloud providers; max 50, ≤20 chars each).
|
|
163
163
|
// `sttUrl`/`sttModel`/`sttTokenUrl`/`sttUsageUrl`/`sttEotThreshold` — STT provider knobs (stt.js).
|
|
164
|
+
// `sttOptions` — provider-specific extras passed through as-is (e.g. Soniox endpointLatencyLevel/endpointSensitivity).
|
|
164
165
|
// `maxTokens`, `preroll` (chunks kept before VAD fires), `vadOptions` (Silero overrides).
|
|
165
166
|
// `turnDetector(text)` — async/sync predicate run when VAD hears end-of-speech: return false to
|
|
166
167
|
// KEEP listening (the user only paused mid-thought), true (default) to close the turn.
|
|
@@ -170,7 +171,7 @@ export class VoiceAgent {
|
|
|
170
171
|
// barge-in; higher ignores short backchannels ("mhm","yeah"). Defaults live in tuning.js.
|
|
171
172
|
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'level'|'echo'|'error'|'diag', … }.
|
|
172
173
|
constructor({ sysmsg = '', persona, model = '', llm, fetchFn, apiKey = '', llmUrl = '', maxTokens = TUNING.MAX_TOKENS,
|
|
173
|
-
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
174
|
+
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold, sttOptions = {},
|
|
174
175
|
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
175
176
|
tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
|
|
176
177
|
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, audioIO = null, onEvent = () => {} } = {}) {
|
|
@@ -199,7 +200,7 @@ export class VoiceAgent {
|
|
|
199
200
|
this.sttProvider = resolveSttProvider(sttProvider);
|
|
200
201
|
const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
|
|
201
202
|
this.stt = makeStt({
|
|
202
|
-
apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
203
|
+
...sttOptions, apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
203
204
|
// `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
|
|
204
205
|
// whose credential differs per provider never has to re-derive the webspeech downgrade
|
|
205
206
|
// (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
|