@todoforai/voiceloop 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/package.json +1 -1
- package/src/index.d.ts +12 -3
- package/src/index.js +2 -1
- package/src/stt.js +188 -0
- package/src/voice-agent.js +16 -2
package/README.md
CHANGED
|
@@ -156,6 +156,7 @@ iOS ducks the speakers while the recognizer holds the mic — which is exactly w
|
|
|
156
156
|
| `elevenlabs` | VAD-gated | Scribe v2 realtime, great accuracy |
|
|
157
157
|
| `deepgram` | **Native** (Flux) | Provider's own turn model hears the full stream |
|
|
158
158
|
| `speechmatics` | VAD-gated | Cheapest per second, locked-words-only finals |
|
|
159
|
+
| `soniox` | **Native** (stt-rt-v5) | Provider's own endpointing, 60+ languages incl. Hungarian, $0.12/hr |
|
|
159
160
|
|
|
160
161
|
## Bring your own LLM
|
|
161
162
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.21",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ export interface VoiceAgentOptions {
|
|
|
149
149
|
/** STT backend: 'webspeech' (default — the browser's own on-device SpeechRecognition; free, no
|
|
150
150
|
* token, no socket, captures its own mic) | 'speechmatics' | 'elevenlabs' (Scribe) | 'deepgram'
|
|
151
151
|
* (Flux — native model-based end-of-turn detection; the agent feeds it the full mic stream and
|
|
152
|
-
* lets it close turns itself). The cloud ones connect the browser DIRECTLY to the provider with a
|
|
152
|
+
* lets it close turns itself) | 'soniox' (stt-rt-v5 — same native-endpointing shape as Flux). The cloud ones connect the browser DIRECTLY to the provider with a
|
|
153
153
|
* short-TTL token the backend mints (raw key stays server-side).
|
|
154
154
|
* 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
|
|
155
155
|
* the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
|
|
@@ -175,6 +175,8 @@ export interface VoiceAgentOptions {
|
|
|
175
175
|
* confidence the user is done exceeds this threshold. Lower = snappier turn ends, higher = more
|
|
176
176
|
* patient across mid-thought pauses. Deepgram-only; default from tuning.js (0.7). */
|
|
177
177
|
sttEotThreshold?: number;
|
|
178
|
+
/** Provider-specific extras forwarded to the STT factory (Soniox: endpointLatencyLevel 0–3, endpointSensitivity −1..1). */
|
|
179
|
+
sttOptions?: Record<string, unknown>;
|
|
178
180
|
/** Every tool the model may call. voiceloop ships NO built-in tools — a voice library has no
|
|
179
181
|
* business knowing what a "todo" is — so product tools (and their `run`) are supplied here. */
|
|
180
182
|
tools?: Record<string, VoiceTool>;
|
|
@@ -215,6 +217,8 @@ export function warmVad(): Promise<void>;
|
|
|
215
217
|
* provider it will actually run: `() => getSttToken('deepgram')`). Warming with the wrong URL
|
|
216
218
|
* silently no-ops the optimization: the real session just mints again on click. */
|
|
217
219
|
export function warmDeepgramToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
|
|
220
|
+
/** Soniox counterpart of warmDeepgramToken: pre-mints and caches the temporary key. */
|
|
221
|
+
export function warmSonioxToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
|
|
218
222
|
/** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
|
|
219
223
|
* host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
|
|
220
224
|
export const VOICE_SYSMSG: string;
|
|
@@ -242,6 +246,10 @@ export class VoiceAgent {
|
|
|
242
246
|
setSpeed(speed: number): void;
|
|
243
247
|
/** Update the host context (e.g. a re-opened chat); applies from the next turn — no restart. */
|
|
244
248
|
setSysmsg(sysmsg?: string): void;
|
|
249
|
+
/** Append a synthetic tool_use/tool_result pair to history — live host context as an observation
|
|
250
|
+
* the agent "made", keeping the system prompt (and its cache prefix) untouched. `name` should be a
|
|
251
|
+
* tool the agent really has. Result text is bounded like real tool results. */
|
|
252
|
+
pushToolRecord(name: string, args?: Record<string, unknown>, result?: unknown): void;
|
|
245
253
|
/** Mute/unmute the mic (disables the stream's audio tracks → silence to VAD+STT); keeps the pipeline live. */
|
|
246
254
|
setMuted(muted: boolean): void;
|
|
247
255
|
/** Mute/unmute the AI's spoken output (TTS): the reply still streams as text but is never voiced. */
|
|
@@ -277,7 +285,7 @@ export class VoiceAgent {
|
|
|
277
285
|
state: 'idle' | 'listening' | 'thinking' | 'speaking';
|
|
278
286
|
/** The STT provider actually in use — differs from the requested one when 'webspeech' was
|
|
279
287
|
* downgraded to a cloud provider (browser without SpeechRecognition). */
|
|
280
|
-
readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
288
|
+
readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram' | 'soniox';
|
|
281
289
|
/** True once the pipeline has been torn down (stop() or a fatal STT error self-stopped it). */
|
|
282
290
|
readonly closed: boolean;
|
|
283
291
|
}
|
|
@@ -328,7 +336,7 @@ export interface STTSession {
|
|
|
328
336
|
export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
|
|
329
337
|
|
|
330
338
|
/** The built-in STT provider ids — the keys of STT_PROVIDERS. */
|
|
331
|
-
export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
339
|
+
export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram' | 'soniox';
|
|
332
340
|
/** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
|
|
333
341
|
* provider reports one (used to cache and pre-mint just before expiry). */
|
|
334
342
|
export interface SttToken { token: string; expires_in?: number }
|
|
@@ -339,6 +347,7 @@ export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
|
|
|
339
347
|
* wasted download. Answerable without constructing a session, so a host can decide both up front. */
|
|
340
348
|
export function sttSelfCaptures(provider: string): boolean;
|
|
341
349
|
export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
350
|
+
export function makeSonioxSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
342
351
|
export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
343
352
|
export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
344
353
|
export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
package/src/index.js
CHANGED
|
@@ -6,8 +6,9 @@ export {
|
|
|
6
6
|
} from './voice-agent.js';
|
|
7
7
|
export {
|
|
8
8
|
STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
|
|
9
|
-
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, sttSelfCaptures,
|
|
9
|
+
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, makeSonioxSTT, sttSelfCaptures,
|
|
10
10
|
warmDeepgramToken,
|
|
11
|
+
warmSonioxToken,
|
|
11
12
|
} from './stt.js';
|
|
12
13
|
export { ElevenLabsTTS } from './tts-elevenlabs.js';
|
|
13
14
|
export { loadVoiceDeps, prewarmVoice } from './deps.js';
|
package/src/stt.js
CHANGED
|
@@ -607,6 +607,193 @@ export function makeDeepgramSTT(opts) {
|
|
|
607
607
|
};
|
|
608
608
|
}
|
|
609
609
|
|
|
610
|
+
// ── Soniox realtime (stt-rt-v5) ──────────────────────────────────────────────
|
|
611
|
+
// Browser↔Soniox direct WS. Auth: the backend mints a temporary API key (POST /v1/auth/temporary-api-key,
|
|
612
|
+
// usage_type transcribe_websocket) and the browser sends it in the FIRST JSON config message — no
|
|
613
|
+
// URL/subprotocol auth. Audio is raw binary pcm_s16le @16k after the config frame.
|
|
614
|
+
//
|
|
615
|
+
// Turn model — TOKENS, not transcripts: every message carries a `tokens` array; `is_final: true`
|
|
616
|
+
// tokens arrive exactly once and are appended, `is_final: false` tokens are the whole current
|
|
617
|
+
// provisional tail and REPLACE the previous tail. Token text carries its own leading space (subword
|
|
618
|
+
// units), so the transcript is a plain concatenation. With enable_endpoint_detection the model does
|
|
619
|
+
// native end-of-turn (semantic + acoustic) and emits a final `<end>` token at the boundary — so like
|
|
620
|
+
// Flux this provider is `continuous` (must hear the silences) + `nativeEOT` (turn closes itself).
|
|
621
|
+
// commit() is the VAD-gated fallback ({"type":"finalize"} → tokens flush final + a `<fin>` marker).
|
|
622
|
+
const SONIOX_URL = 'wss://stt-rt.soniox.com/transcribe-websocket';
|
|
623
|
+
const SONIOX_KEEPALIVE_MS = 10000; // server closes after >20s with no audio/keepalive
|
|
624
|
+
// Soniox errors arrive as { error_code: <http status>, error_type, error_message }. 4xx won't fix
|
|
625
|
+
// itself with a reconnect (bad key/config, balance exhausted — seen live as 402): fatal, so the host
|
|
626
|
+
// stops instead of the continuous feed re-minting + reconnecting forever. 5xx/429 stay recoverable.
|
|
627
|
+
const sonioxFatal = (code) => code >= 400 && code < 500 && code !== 429;
|
|
628
|
+
// Soniox's `<end>` is ACOUSTIC-LEANING: it fires ~250ms into any pause, including mid-sentence
|
|
629
|
+
// ones — measured on the bench hesitation audio it closed "Hi, I need some help with—" / "Um." /
|
|
630
|
+
// "with a dinner reservation." as three turns (Flux keeps that as one). Its own signal for "not
|
|
631
|
+
// done" is the em-dash it appends to a cut-off phrase, so: a turn ending in one is held open and
|
|
632
|
+
// merged with what follows; anything else is trusted unless it's a bare filler ("Um."), which is
|
|
633
|
+
// also a continuation. The hold has a cap so a user who genuinely trails off still gets an answer.
|
|
634
|
+
const SONIOX_HOLD_MS = 1500;
|
|
635
|
+
// Same token cache/prewarm as Deepgram (see dgToken): the temporary-key mint + WS open cost ~2s on
|
|
636
|
+
// the first utterance when paid cold on the mic click.
|
|
637
|
+
let sxToken = null, sxTokenExp = 0, sxMintPromise = null;
|
|
638
|
+
export function warmSonioxToken(sttTokenUrl, apiKey, getToken) {
|
|
639
|
+
if (sxToken && performance.now() < sxTokenExp - 30000) return Promise.resolve();
|
|
640
|
+
return (sxMintPromise ??= (async () => {
|
|
641
|
+
const body = await mintSttToken(sttTokenUrl, apiKey, () => {}, getToken);
|
|
642
|
+
if (body) { sxToken = body.token; sxTokenExp = performance.now() + (body.expires_in ?? 300) * 1000; }
|
|
643
|
+
})().finally(() => { sxMintPromise = null; }));
|
|
644
|
+
}
|
|
645
|
+
const sonioxContinues = (t) => /[—–-]$/.test(t) || /^(um+|uh+|hmm+|er+|ah+)[.,]?$/i.test(t);
|
|
646
|
+
export function makeSonioxSTT(opts) {
|
|
647
|
+
const { apiKey, sttModel, sttLang = 'en', keyterms = [], sttTokenUrl, getToken, sttUsageUrl,
|
|
648
|
+
onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
649
|
+
const sttUrl = opts.sttUrl || SONIOX_URL;
|
|
650
|
+
const model = sttModel || 'stt-rt-v5';
|
|
651
|
+
let ws = null, opening = false, outbox = [], outboxSamples = 0, keepalive = null;
|
|
652
|
+
// Turn state. Soniox finals are APPEND-ONLY (no cumulative restatement like Flux), so a socket
|
|
653
|
+
// boundary must wipe them: a stale prefix can't be repaired by the next stream.
|
|
654
|
+
let finalText = '', tail = '', lastInterim = '', turnStart = 0;
|
|
655
|
+
let audioSinceFinal = false, awaitingFinal = false, safetyTimer = null;
|
|
656
|
+
let held = '', holdTimer = null; // turn closed on a continuation cue, waiting for the rest
|
|
657
|
+
const usage = makeUsageReporter(sttUsageUrl, apiKey, 'soniox', { model }, opts.fetchFn);
|
|
658
|
+
const OUTBOX_MAX_SAMPLES = 32000; // ~2s preroll across the token mint + handshake (see Deepgram)
|
|
659
|
+
|
|
660
|
+
const sendAudio = (i16) => {
|
|
661
|
+
if (!ws || ws.readyState !== 1) {
|
|
662
|
+
outbox.push(i16); outboxSamples += i16.length;
|
|
663
|
+
while (outboxSamples > OUTBOX_MAX_SAMPLES && outbox.length > 1) outboxSamples -= outbox.shift().length;
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
ws.send(i16.buffer.slice(i16.byteOffset, i16.byteOffset + i16.byteLength));
|
|
667
|
+
usage.add(i16.length);
|
|
668
|
+
};
|
|
669
|
+
const resetText = () => { finalText = ''; tail = ''; lastInterim = ''; turnStart = 0; };
|
|
670
|
+
const dropHold = () => { held = ''; clearTimeout(holdTimer); holdTimer = null; };
|
|
671
|
+
const resetTurn = () => { resetText(); dropHold(); clearTimeout(safetyTimer); safetyTimer = null; awaitingFinal = false; };
|
|
672
|
+
const ms = () => Math.round(performance.now() - (turnStart || performance.now()));
|
|
673
|
+
const withHeld = (t) => (held ? `${held} ${t.trim()}` : t).trim();
|
|
674
|
+
const flushHeld = () => { // hold cap: nothing followed → the cut-off phrase was the whole turn
|
|
675
|
+
if ((finalText + tail).trim()) { holdTimer = setTimeout(flushHeld, SONIOX_HOLD_MS); return; } // continuation in progress: its <end> merges
|
|
676
|
+
const t = held, at = ms(); resetTurn(); audioSinceFinal = false; onFinal(t, at);
|
|
677
|
+
};
|
|
678
|
+
const closeTurn = (force = false) => {
|
|
679
|
+
const seg = (finalText + tail).trim(), t = withHeld(seg), at = ms();
|
|
680
|
+
if (!force && sonioxContinues(seg)) { // cut mid-thought: keep it as the prefix of the next segment
|
|
681
|
+
resetText(); held = t;
|
|
682
|
+
clearTimeout(holdTimer); holdTimer = setTimeout(flushHeld, SONIOX_HOLD_MS);
|
|
683
|
+
return;
|
|
684
|
+
}
|
|
685
|
+
resetTurn(); audioSinceFinal = false;
|
|
686
|
+
onFinal(t, at);
|
|
687
|
+
};
|
|
688
|
+
// A partial that retracts to EMPTY must still reach the host (it drops a speculation built on
|
|
689
|
+
// withdrawn words) — but idle responses with nothing to say shouldn't spam empty callbacks.
|
|
690
|
+
const emitPartial = () => {
|
|
691
|
+
const interim = withHeld(finalText + tail);
|
|
692
|
+
if (interim === lastInterim) return;
|
|
693
|
+
lastInterim = interim;
|
|
694
|
+
onPartial(interim, ms());
|
|
695
|
+
};
|
|
696
|
+
|
|
697
|
+
const open = async () => {
|
|
698
|
+
if (opening) return;
|
|
699
|
+
opening = true;
|
|
700
|
+
try {
|
|
701
|
+
let token = (sxToken && performance.now() < sxTokenExp - 15000) ? sxToken : null;
|
|
702
|
+
if (!token) {
|
|
703
|
+
const body = await mintSttToken(sttTokenUrl, apiKey, onFatal, getToken);
|
|
704
|
+
if (!body) { outbox = []; outboxSamples = 0; return; }
|
|
705
|
+
token = sxToken = body.token; sxTokenExp = performance.now() + (body.expires_in ?? 300) * 1000;
|
|
706
|
+
}
|
|
707
|
+
if (isClosed()) return;
|
|
708
|
+
// NEW SOCKET = NEW STREAM: whatever the old one left half-finalized is gone with it. A pending
|
|
709
|
+
// commit() survives (its finalize is re-sent on open, its safety timer still closes the turn)
|
|
710
|
+
// so the host's closing latch always resolves.
|
|
711
|
+
resetText();
|
|
712
|
+
clearInterval(keepalive); keepalive = null; // the superseded socket's timer must not outlive it
|
|
713
|
+
const sock = ws = newSocket(sttUrl);
|
|
714
|
+
sock.binaryType = 'arraybuffer';
|
|
715
|
+
sock.onopen = () => {
|
|
716
|
+
if (sock !== ws) return;
|
|
717
|
+
const config = {
|
|
718
|
+
api_key: token, model, audio_format: 'pcm_s16le', sample_rate: 16000, num_channels: 1,
|
|
719
|
+
enable_endpoint_detection: true,
|
|
720
|
+
// Level 2 measured (bench smalltalk, 5 runs): 1042 → 768 ms voice→voice, 0 mid-sentence
|
|
721
|
+
// splits; level 3 was faster still (726) but split "Hey there. | Can you hear me?" every run.
|
|
722
|
+
endpoint_latency_adjustment_level: opts.endpointLatencyLevel ?? 2,
|
|
723
|
+
...(opts.endpointSensitivity != null && { endpoint_sensitivity: opts.endpointSensitivity }),
|
|
724
|
+
};
|
|
725
|
+
if (sttLang) config.language_hints = [sttLang.slice(0, 2)];
|
|
726
|
+
if (keyterms.length) config.context = { terms: keyterms };
|
|
727
|
+
sock.send(JSON.stringify(config));
|
|
728
|
+
const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c);
|
|
729
|
+
if (awaitingFinal) sock.send(JSON.stringify({ type: 'finalize' })); // a commit() that landed while connecting
|
|
730
|
+
keepalive = setInterval(() => { if (sock.readyState === 1) sock.send(JSON.stringify({ type: 'keepalive' })); }, SONIOX_KEEPALIVE_MS);
|
|
731
|
+
};
|
|
732
|
+
sock.onerror = ev => { if (sock === ws && !isClosed()) { onError('STT socket error'); console.error('Soniox ws error', ev); } };
|
|
733
|
+
sock.onclose = ev => {
|
|
734
|
+
usage.flush(); onClose?.();
|
|
735
|
+
if (sock !== ws) return; // superseded socket's late close: the new one's state is untouched
|
|
736
|
+
clearInterval(keepalive); keepalive = null;
|
|
737
|
+
if (!isClosed() && ev.code !== 1000) onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`);
|
|
738
|
+
};
|
|
739
|
+
sock.onmessage = ev => {
|
|
740
|
+
if (sock !== ws || isClosed() || typeof ev.data !== 'string') return;
|
|
741
|
+
const m = JSON.parse(ev.data);
|
|
742
|
+
if (m.error_code || m.error_message) {
|
|
743
|
+
const msg = `${m.error_type || m.error_code || 'error'}: ${m.error_message || ''}`;
|
|
744
|
+
console.error('Soniox error msg', m);
|
|
745
|
+
if (+m.error_code === 401) { sxToken = null; sxTokenExp = 0; } // a cached key the server rejects is dead
|
|
746
|
+
if (sonioxFatal(+m.error_code)) onFatal(`STT ${msg}`); else onError(msg);
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
if (!Array.isArray(m.tokens)) return;
|
|
750
|
+
// Tokens are chronological and a response may straddle a boundary (…, <end>, next-turn
|
|
751
|
+
// tokens): handle the markers at their POSITION, so text after one belongs to the next turn.
|
|
752
|
+
let newTail = '';
|
|
753
|
+
for (const t of m.tokens) {
|
|
754
|
+
if (t.text === '<end>' || t.text === '<fin>') { tail = newTail; newTail = ''; closeTurn(t.text === '<fin>'); continue; }
|
|
755
|
+
if (!turnStart) turnStart = performance.now(); // turn's ms clock anchors at its first token
|
|
756
|
+
if (t.is_final) finalText += t.text; else newTail += t.text;
|
|
757
|
+
}
|
|
758
|
+
tail = newTail;
|
|
759
|
+
emitPartial();
|
|
760
|
+
};
|
|
761
|
+
} catch (e) {
|
|
762
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
763
|
+
} finally {
|
|
764
|
+
opening = false;
|
|
765
|
+
}
|
|
766
|
+
};
|
|
767
|
+
|
|
768
|
+
return {
|
|
769
|
+
continuous: true, // native endpointing needs the real silences
|
|
770
|
+
nativeEOT: true, // `<end>` token → unsolicited onFinal closes the turn
|
|
771
|
+
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
772
|
+
feed(i16) {
|
|
773
|
+
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
774
|
+
sendAudio(i16); audioSinceFinal = true;
|
|
775
|
+
},
|
|
776
|
+
reset() { resetTurn(); },
|
|
777
|
+
// VAD-gated fallback (the agent never calls this in native mode): ask for a manual finalize and
|
|
778
|
+
// close on the `<fin>` marker, or on the safety timer if it never comes.
|
|
779
|
+
commit() {
|
|
780
|
+
if (awaitingFinal) return;
|
|
781
|
+
if (!audioSinceFinal) { onFinal('', ms()); return; }
|
|
782
|
+
awaitingFinal = true;
|
|
783
|
+
if (ws?.readyState === 1) ws.send(JSON.stringify({ type: 'finalize' })); // else onopen sends it
|
|
784
|
+
clearTimeout(safetyTimer); safetyTimer = setTimeout(() => closeTurn(true), TUNING.STT.forceEndSafetyMs);
|
|
785
|
+
},
|
|
786
|
+
close() {
|
|
787
|
+
resetTurn(); audioSinceFinal = false;
|
|
788
|
+
clearInterval(keepalive); keepalive = null; outbox = []; outboxSamples = 0;
|
|
789
|
+
usage.flush();
|
|
790
|
+
const sock = ws; ws = null;
|
|
791
|
+
if (sock?.readyState === 1) { try { sock.send(''); } catch { /* closing anyway */ } } // empty frame = end of stream
|
|
792
|
+
sock?.close();
|
|
793
|
+
},
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
|
|
610
797
|
// ── Browser Web Speech API (browser-managed, no token) ───────────────────────
|
|
611
798
|
// "Browser-native" is not "on-device": Chrome may route recognition through a vendor speech
|
|
612
799
|
// service. The win here is zero setup (no key, no token endpoint), not guaranteed privacy.
|
|
@@ -877,6 +1064,7 @@ export const STT_PROVIDERS = {
|
|
|
877
1064
|
speechmatics: makeSpeechmaticsSTT,
|
|
878
1065
|
elevenlabs: makeElevenLabsSTT,
|
|
879
1066
|
deepgram: makeDeepgramSTT,
|
|
1067
|
+
soniox: makeSonioxSTT,
|
|
880
1068
|
};
|
|
881
1069
|
|
|
882
1070
|
// Self-capturing providers own the mic AND their end-of-turn, so the agent builds no capture
|
package/src/voice-agent.js
CHANGED
|
@@ -161,6 +161,7 @@ export class VoiceAgent {
|
|
|
161
161
|
// `sttLang` — STT language code ('en'); also pins the reply language.
|
|
162
162
|
// `keyterms` — domain words to bias STT towards (cloud providers; max 50, ≤20 chars each).
|
|
163
163
|
// `sttUrl`/`sttModel`/`sttTokenUrl`/`sttUsageUrl`/`sttEotThreshold` — STT provider knobs (stt.js).
|
|
164
|
+
// `sttOptions` — provider-specific extras passed through as-is (e.g. Soniox endpointLatencyLevel/endpointSensitivity).
|
|
164
165
|
// `maxTokens`, `preroll` (chunks kept before VAD fires), `vadOptions` (Silero overrides).
|
|
165
166
|
// `turnDetector(text)` — async/sync predicate run when VAD hears end-of-speech: return false to
|
|
166
167
|
// KEEP listening (the user only paused mid-thought), true (default) to close the turn.
|
|
@@ -170,7 +171,7 @@ export class VoiceAgent {
|
|
|
170
171
|
// barge-in; higher ignores short backchannels ("mhm","yeah"). Defaults live in tuning.js.
|
|
171
172
|
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'level'|'echo'|'error'|'diag', … }.
|
|
172
173
|
constructor({ sysmsg = '', persona, model = '', llm, fetchFn, apiKey = '', llmUrl = '', maxTokens = TUNING.MAX_TOKENS,
|
|
173
|
-
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
174
|
+
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold, sttOptions = {},
|
|
174
175
|
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
175
176
|
tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
|
|
176
177
|
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, audioIO = null, onEvent = () => {} } = {}) {
|
|
@@ -199,7 +200,7 @@ export class VoiceAgent {
|
|
|
199
200
|
this.sttProvider = resolveSttProvider(sttProvider);
|
|
200
201
|
const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
|
|
201
202
|
this.stt = makeStt({
|
|
202
|
-
apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
203
|
+
...sttOptions, apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
203
204
|
// `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
|
|
204
205
|
// whose credential differs per provider never has to re-derive the webspeech downgrade
|
|
205
206
|
// (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
|
|
@@ -604,6 +605,19 @@ export class VoiceAgent {
|
|
|
604
605
|
// turn on. Keeps the running session/history alive — no restart needed.
|
|
605
606
|
setSysmsg(sysmsg) { this.sysmsg = composeSysmsg(sysmsg, this._persona, this.sttLang); this._dropPrefetch(); } // a running speculation was built with the OLD sysmsg
|
|
606
607
|
|
|
608
|
+
// Feed the agent something it "looked up" WITHOUT rewriting the system prompt: a tool_use +
|
|
609
|
+
// tool_result pair appended to history, in the same shape as its own past calls. For live host
|
|
610
|
+
// context (a watched chat that keeps producing output) this beats setSysmsg(): the prompt prefix
|
|
611
|
+
// stays byte-identical (provider prompt cache holds), and the model reads it as a fresh
|
|
612
|
+
// observation rather than a silently mutated system message. `name` should be a tool the agent
|
|
613
|
+
// really has, so a later "check again" is a natural call, not an imitation of a phantom.
|
|
614
|
+
pushToolRecord(name, args = {}, result = '') {
|
|
615
|
+
const id = `host_${Date.now().toString(36)}_${(this._hostRecSeq = (this._hostRecSeq ?? 0) + 1)}`;
|
|
616
|
+
this.history.push({ role: 'assistant', content: [{ type: 'tool_use', id, name, input: args }] });
|
|
617
|
+
this.history.push({ role: 'user', content: [toolResultBlock({ id, result })] });
|
|
618
|
+
this._dropPrefetch(); // the speculation's history base is stale (histLen check would refuse it anyway)
|
|
619
|
+
}
|
|
620
|
+
|
|
607
621
|
// Mute/unmute the mic by disabling the stream's audio tracks: the browser then feeds silence to
|
|
608
622
|
// VAD and STT, so nothing the user says reaches the agent — without dropping the mic permission
|
|
609
623
|
// or tearing down the pipeline. Stored so a mute toggled while paused applies on the next start().
|