@todoforai/voiceloop 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +239 -0
- package/examples/simple-browser.html +100 -0
- package/package.json +48 -0
- package/src/deps.js +49 -0
- package/src/index.js +15 -0
- package/src/llm-openai.js +122 -0
- package/src/stt.js +815 -0
- package/src/tts-elevenlabs.js +50 -0
- package/src/tuning.js +182 -0
- package/src/voice-agent.js +1984 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// ── ElevenLabs TTS (cloud, human-grade voices) ─────────────────────────────────────────────────
|
|
2
|
+
// A StreamingTTS provider: only `_synth(text, signal) → audio Blob` — sentence chunking, playback,
|
|
3
|
+
// barge-in, seek and speculative presynth all come from the base class, so switching Piper ↔
|
|
4
|
+
// ElevenLabs changes the VOICE, not the behavior. decodeAudioData handles the returned mp3.
|
|
5
|
+
//
|
|
6
|
+
// Auth, two modes (mirrors the STT providers' "key stays server-side" stance):
|
|
7
|
+
// • ttsUrl — YOUR backend proxy: POST { text, voice_id, model_id } → audio bytes. The proxy adds
|
|
8
|
+
// the xi-api-key and forwards to ElevenLabs. Recommended for production.
|
|
9
|
+
// • apiKey — direct browser→ElevenLabs (dev/bench only: the key is visible in the page).
|
|
10
|
+
//
|
|
11
|
+
// Latency: eleven_flash_v2_5 is EL's low-latency model (~75ms model time); the rest of a clip's
|
|
12
|
+
// cost is network TTFB + transfer, which presynth (during the end-of-turn debounce) and the base
|
|
13
|
+
// class's ahead-of-playback synth queue hide for all but the first clip of a cold turn.
|
|
14
|
+
import { StreamingTTS } from './voice-agent.js';
|
|
15
|
+
|
|
16
|
+
const EL_API = 'https://api.elevenlabs.io/v1/text-to-speech';
|
|
17
|
+
|
|
18
|
+
export class ElevenLabsTTS extends StreamingTTS {
|
|
19
|
+
constructor({ ttsUrl, apiKey, voiceId = 'JBFqnCBsd6RMkjVDRZzb' /* George */,
|
|
20
|
+
modelId = 'eleven_flash_v2_5', format = 'mp3_44100_64' } = {}) {
|
|
21
|
+
super(voiceId);
|
|
22
|
+
if (!ttsUrl && !apiKey) throw new Error('ElevenLabsTTS needs ttsUrl (backend proxy) or apiKey (direct, dev only)');
|
|
23
|
+
this.ttsUrl = ttsUrl; this.apiKey = apiKey; this.modelId = modelId; this.format = format;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Pre-establish the connection (DNS + TLS + any proxy hop) so the first real clip only pays TTFB.
|
|
27
|
+
// A HEAD to the API root is enough to open the socket; failures are non-fatal.
|
|
28
|
+
async warm() {
|
|
29
|
+
const url = this.ttsUrl ?? EL_API;
|
|
30
|
+
try { await fetch(url, { method: 'HEAD', signal: AbortSignal.timeout(3000) }); } catch {}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async _synth(text, signal) {
|
|
34
|
+
text = text?.trim();
|
|
35
|
+
if (signal?.aborted || !text) return null;
|
|
36
|
+
const direct = !this.ttsUrl;
|
|
37
|
+
const url = direct ? `${EL_API}/${this.voiceId}?output_format=${this.format}` : this.ttsUrl;
|
|
38
|
+
const res = await fetch(url, {
|
|
39
|
+
method: 'POST',
|
|
40
|
+
headers: { 'Content-Type': 'application/json', ...(direct ? { 'xi-api-key': this.apiKey } : {}) },
|
|
41
|
+
// The proxy form carries voice/model/format in the body so one backend route serves any voice.
|
|
42
|
+
body: JSON.stringify(direct ? { text, model_id: this.modelId }
|
|
43
|
+
: { text, voice_id: this.voiceId, model_id: this.modelId, output_format: this.format }),
|
|
44
|
+
signal,
|
|
45
|
+
});
|
|
46
|
+
if (!res.ok) throw new Error(`ElevenLabs TTS ${res.status}: ${(await res.text().catch(() => '')).slice(0, 200)}`);
|
|
47
|
+
const blob = await res.blob();
|
|
48
|
+
return signal?.aborted ? null : blob;
|
|
49
|
+
}
|
|
50
|
+
}
|
package/src/tuning.js
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// tuning.js — ONE place to fine-tune the voice agent's speech behaviour. Edit the numbers here;
|
|
2
|
+
// they're the defaults the VoiceAgent reads at construction, so a change applies to the next
|
|
3
|
+
// session you start (press the mic again — no rebuild needed in dev, the module is re-imported).
|
|
4
|
+
//
|
|
5
|
+
// Anything here can still be overridden per-instance via the VoiceAgent constructor options of the
|
|
6
|
+
// same name (constructor wins). This file is just the single, documented source of the defaults so
|
|
7
|
+
// you don't have to hunt through voice-agent.js / stt.js to dial things in.
|
|
8
|
+
//
|
|
9
|
+
// Quick guide to the symptoms ↔ knobs:
|
|
10
|
+
// • The agent interrupts itself / cuts off on noise → raise VAD.positiveSpeechThreshold,
|
|
11
|
+
// VAD.minSpeechFrames, or BARGE_IN_MIN_CHARS.
|
|
12
|
+
// • The agent transcribes ITS OWN voice (speakers, weak AEC) and self-interrupts → lower
|
|
13
|
+
// ECHO_WORD_MATCH (stricter echo filter); headphones remove the problem entirely.
|
|
14
|
+
// • "mhm"/"yeah" backchannels cut the agent off → raise BARGE_IN_MIN_CHARS.
|
|
15
|
+
// • Turn closes too eagerly mid-sentence → raise VAD.redemptionFrames or MAX_PAUSE_MS.
|
|
16
|
+
// • Turn feels sluggish to respond → lower VAD.redemptionFrames / STT max_delay.
|
|
17
|
+
// • Word-starts clipped ("…ello" instead of "hello")→ raise PREROLL_CHUNKS.
|
|
18
|
+
// • Replies too long/short → MAX_TOKENS.
|
|
19
|
+
|
|
20
|
+
export const TUNING = {
|
|
21
|
+
// ── Barge-in (interrupting the agent while it speaks) ────────────────────────────────────────────
|
|
22
|
+
// Barge-in now fires on TRANSCRIBED NOVEL WORDS, not raw VAD energy (so the AI's own echo/noise
|
|
23
|
+
// can't cut a reply). This is the minimum number of transcribed characters — counting ONLY words
|
|
24
|
+
// not (fuzzily) attributable to the reply being spoken, so even misheard echo contributes zero —
|
|
25
|
+
// that must accumulate WHILE the agent is speaking before we treat it as a real interruption.
|
|
26
|
+
// Higher = short backchannels ("mhm", "yeah", "ok") are ignored and the agent keeps talking; lower =
|
|
27
|
+
// snappier interruption. 0 = any single transcribed character interrupts (most aggressive).
|
|
28
|
+
BARGE_IN_MIN_CHARS: 6,
|
|
29
|
+
|
|
30
|
+
// Self-echo filter: on speakers the browser's AEC often fails to cancel our own Web-Audio TTS
|
|
31
|
+
// playout, so the STT transcribes the agent's own voice — real words that cross BARGE_IN_MIN_CHARS
|
|
32
|
+
// and make it interrupt ITSELF. Since we know EXACTLY what he's voicing, interims heard while
|
|
33
|
+
// speaking are word-matched against the current reply: if at least this fraction of the interim's
|
|
34
|
+
// recent words occur in the reply, it's its own echo → ignored, it keeps talking. Real
|
|
35
|
+
// interruptions ("no wait", "stop that") share few words with the reply, so they still cut in.
|
|
36
|
+
// Lower = stricter filtering (more echo ignored, but a real interruption that REUSES the reply's
|
|
37
|
+
// words might be missed); > 1 disables the filter.
|
|
38
|
+
ECHO_WORD_MATCH: 0.7,
|
|
39
|
+
|
|
40
|
+
// ── LLM ───────────────────────────────────────────────────────────────────────────────────────
|
|
41
|
+
MAX_TOKENS: 1024, // cap on the spoken reply length
|
|
42
|
+
|
|
43
|
+
// ── Liveness backstop (host code that never returns) ──────────────────────────────────────────
|
|
44
|
+
// The turn loop is strictly serialized, so anything it AWAITS can wedge it. Host hooks are kept
|
|
45
|
+
// off that path structurally rather than by deadline: tools are dispatched and never joined, and
|
|
46
|
+
// an aborted LLM stream is detached instead of drained. The one hook whose verdict is genuinely
|
|
47
|
+
// REQUIRED to proceed is the turnDetector (it decides whether the turn closes) — bounded by
|
|
48
|
+
// MAX_PAUSE_MS above, as ONE absolute deadline from end-of-speech.
|
|
49
|
+
// Not covered, by design: a custom TTS _synth() that never settles (there is no reply without it),
|
|
50
|
+
// and a muted-TTS turn draining a generator that ignores its abort signal.
|
|
51
|
+
|
|
52
|
+
// ── Turn boundary (when is the user done talking?) ────────────────────────────────────────────
|
|
53
|
+
// Hard cap (ms) a turnDetector may hold a turn open across a pause before it's force-committed —
|
|
54
|
+
// so a user who trails off mid-sentence can't hang the turn forever. Only used when a turnDetector
|
|
55
|
+
// is set; with the default silence-only endpointing the VAD redemptionFrames govern this instead.
|
|
56
|
+
MAX_PAUSE_MS: 4000,
|
|
57
|
+
|
|
58
|
+
// Browser Web Speech end-of-turn debounce (ms). The engine's own isFinal boundaries are per-pause
|
|
59
|
+
// fragments; after a final we wait this long for a new one before closing the turn, so a mid-thought
|
|
60
|
+
// pause keeps one turn together. Raise if turns split mid-sentence; lower to reply sooner.
|
|
61
|
+
// 500 measured comfortable: Chrome's endpointer already waits out its own internal silence before
|
|
62
|
+
// emitting isFinal, so this only needs to bridge restart seams and brief continuations.
|
|
63
|
+
WEBSPEECH_EOT_MS: 500,
|
|
64
|
+
|
|
65
|
+
// Chrome's endpointer can take longer than WEBSPEECH_EOT_MS to promote trailing words to isFinal.
|
|
66
|
+
// Closing while it still holds that un-promoted tail emits a TRUNCATED turn and lets the late final
|
|
67
|
+
// land as a bogus second one (disjoint text, so the seam-dedup can't catch it) — which barges in and
|
|
68
|
+
// kills the reply. So a close waits for the engine to go quiescent; this caps how many extra EOT
|
|
69
|
+
// windows it may wait before giving up on a tail that never resolves (liveness backstop only —
|
|
70
|
+
// reaching it re-exposes the truncation). Worst-case added close latency = this × WEBSPEECH_EOT_MS.
|
|
71
|
+
// Raise if slow engines still split turns; lower to bound the wait harder.
|
|
72
|
+
WEBSPEECH_TAIL_MAX_DEFERS: 3,
|
|
73
|
+
|
|
74
|
+
// Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
|
|
75
|
+
// agent starts generating a reply for the probable final text — overlapping the LLM's first-token
|
|
76
|
+
// latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
|
|
77
|
+
// (instant first token); a changed text just aborts it (cost: a few prompt tokens). Must be
|
|
78
|
+
// comfortably below WEBSPEECH_EOT_MS for the overlap to pay. 0 = prefetch on every interim tick
|
|
79
|
+
// (most aggressive, most wasted calls).
|
|
80
|
+
PREFETCH_MS: 200,
|
|
81
|
+
|
|
82
|
+
// Audio chunks (~256ms each at 16kHz/4096-sample frames; ~1s total at 4) kept BEFORE VAD fires and
|
|
83
|
+
// replayed into STT when speech starts, so the first word isn't clipped. Raise if word-starts are
|
|
84
|
+
// getting cut; lower to shave a touch of latency/cost.
|
|
85
|
+
PREROLL_CHUNKS: 4,
|
|
86
|
+
|
|
87
|
+
// ── VAD (Silero v5) — speech detection sensitivity ────────────────────────────────────────────
|
|
88
|
+
// These directly govern how readily the mic decides "the user is speaking". Tightening them is the
|
|
89
|
+
// main lever against false speech-starts (self-echo, coughs, keyboard, room noise).
|
|
90
|
+
VAD: {
|
|
91
|
+
// Probability (0–1) above which a frame counts as speech. ↑ → needs clearer/louder speech to
|
|
92
|
+
// trigger (fewer false starts, but may miss a soft start). Default 0.5.
|
|
93
|
+
positiveSpeechThreshold: 0.5,
|
|
94
|
+
// Probability below which we consider it NOT speech (hysteresis floor — keep < positive). Default 0.25.
|
|
95
|
+
negativeSpeechThreshold: 0.25,
|
|
96
|
+
// Minimum consecutive speech frames before a speech-START fires. ↑ → ignores very short blips
|
|
97
|
+
// (coughs, clicks). Default 4 (~130ms).
|
|
98
|
+
minSpeechFrames: 4,
|
|
99
|
+
// Silence frames required to declare speech ENDED. ↑ → more patient across natural pauses (won't
|
|
100
|
+
// cut you off mid-sentence) but slower to respond; ↓ → snappier endpointing. Default 24 (~770ms).
|
|
101
|
+
redemptionFrames: 24,
|
|
102
|
+
// Frames of audio padded before detected speech (smooths the onset). Default 10.
|
|
103
|
+
preSpeechPadFrames: 10,
|
|
104
|
+
},
|
|
105
|
+
|
|
106
|
+
// ── Bootstrap VAD (instant, zero-load fallback until Silero is ready) ────────────────────────
|
|
107
|
+
// Silero (MicVAD.new()) needs a CDN fetch + WASM compile + model load (~0.5-2s cold) before it can
|
|
108
|
+
// detect speech at all — start() begins capturing mic audio well before that resolves. Until then,
|
|
109
|
+
// a plain RMS-energy threshold (no model, no async load, gates from frame 1) stands in so the FIRST
|
|
110
|
+
// words spoken right after clicking the mic are never silently dropped from the (size-capped)
|
|
111
|
+
// preroll buffer while waiting for Silero. Swapped out for Silero the moment it's ready, always at
|
|
112
|
+
// an utterance boundary (never mid-speech) — see _swapToSileroWhenReady().
|
|
113
|
+
// NB: these frames are the raw ~256ms worklet capture chunks (4096 samples @16kHz) — much coarser
|
|
114
|
+
// than Silero's own ~32ms internal frames above, so the counts are on a different scale.
|
|
115
|
+
BOOTSTRAP_VAD: {
|
|
116
|
+
// RMS above this (0-1 scale, ~loudness of normal speech close to the mic) counts as speech.
|
|
117
|
+
// Deliberately looser/cruder than Silero — it only needs to survive the first second(s), not
|
|
118
|
+
// replace real speech detection.
|
|
119
|
+
energyThreshold: 0.02,
|
|
120
|
+
// Consecutive ~256ms chunks above threshold before declaring speech-start. 1 = fires on the very
|
|
121
|
+
// first loud chunk — snappiest possible protection for the opening word(s).
|
|
122
|
+
minSpeechFrames: 1,
|
|
123
|
+
// Consecutive ~256ms chunks below threshold before declaring speech-end (~1.5s). Deliberately
|
|
124
|
+
// more patient than Silero's own redemptionFrames above: this stand-in only runs for the first
|
|
125
|
+
// second or two of a session, and the goal here is "never chop a continuous sentence", not tight
|
|
126
|
+
// endpointing — Silero takes over (with its tighter default) as soon as it's ready.
|
|
127
|
+
redemptionFrames: 6,
|
|
128
|
+
},
|
|
129
|
+
|
|
130
|
+
// ── TTS (first-audio latency) ─────────────────────────────────────────────────────────────────
|
|
131
|
+
// How soon the FIRST spoken chunk is cut from the streaming LLM text. Lower = the agent starts talking
|
|
132
|
+
// sooner (less waiting on generation), at a small prosody cost (it may break on a comma/clause rather
|
|
133
|
+
// than a full sentence). Only the FIRST chunk is aggressive; every later sentence still breaks on a
|
|
134
|
+
// whole-sentence ender for natural delivery.
|
|
135
|
+
// • First chunk feels slow / waits for the whole answer → lower firstChunkMaxChars.
|
|
136
|
+
// • First chunk too choppy ("Sure," then a pause) → raise firstChunkMinChars.
|
|
137
|
+
TTS: {
|
|
138
|
+
// Minimum non-space chars in the VERY first spoken chunk (the first-WORD opener). A tiny first
|
|
139
|
+
// word ("I", "Hi", "A") makes a choppy micro-clip with a pause after it, so we keep gluing the
|
|
140
|
+
// next word(s) until the opener reaches this many chars. Lower = even snappier first sound but
|
|
141
|
+
// riskier choppiness; 3-4 keeps one/two-letter openers glued without delaying real words.
|
|
142
|
+
firstWordMinChars: 4,
|
|
143
|
+
// Don't cut the first chunk on a comma/clause break until at least this many chars exist, so a tiny
|
|
144
|
+
// leading filler ("Sure,", "Okay,") doesn't become its own choppy clip with a pause after it. A full
|
|
145
|
+
// sentence ender (. ! ?) ALWAYS breaks immediately regardless. Lower = faster first sound, at the risk
|
|
146
|
+
// of a short choppy opener (cheap on local Piper). 8 keeps one-word fillers glued; ~6 starts sooner.
|
|
147
|
+
firstChunkMinChars: 8,
|
|
148
|
+
// Hard cap: if the opening clause has NO punctuation by this many chars, break it at the last word
|
|
149
|
+
// boundary anyway so a long unpunctuated start still begins playing instead of awaiting the sentence.
|
|
150
|
+
firstChunkMaxChars: 70,
|
|
151
|
+
// Hard cap for every LATER sentence: a long run with no sentence ender (lists, dashed prose)
|
|
152
|
+
// breaks at the last word boundary by this many chars, so audio keeps flowing instead of
|
|
153
|
+
// waiting silently for one monster chunk. Generous — normal sentences end well before it.
|
|
154
|
+
sentenceMaxChars: 200,
|
|
155
|
+
},
|
|
156
|
+
|
|
157
|
+
// ── STT (Speechmatics) — finalization timing ──────────────────────────────────────────────────
|
|
158
|
+
STT: {
|
|
159
|
+
// Seconds Speechmatics waits before LOCKING interim words into a final. ↓ → a shorter flickering
|
|
160
|
+
// tail (words settle faster) at a small accuracy cost; ↑ → more context, steadier finals. The
|
|
161
|
+
// agent picks per operating_point: 'enhanced' has the accuracy headroom for a tight 1s; 'standard'
|
|
162
|
+
// uses 2s (Speechmatics' documented "optimal balance"). Range 0.7–4.
|
|
163
|
+
maxDelayEnhanced: 1,
|
|
164
|
+
maxDelayStandard: 2,
|
|
165
|
+
// Seconds of silence after which Speechmatics closes a turn ITSELF (emits EndOfUtterance).
|
|
166
|
+
// Required for the ForceEndOfUtterance/EndOfUtterance handshake (see stt.js StartRecognition);
|
|
167
|
+
// must be < max_delay. Speechmatics recommends 0.5-0.8s for voice AI. In practice the agent's
|
|
168
|
+
// VAD commit() is the boundary — audio is VAD-gated, so little post-speech silence reaches
|
|
169
|
+
// Speechmatics for this trigger to fire on.
|
|
170
|
+
eouSilenceTrigger: 0.6,
|
|
171
|
+
// Fallback (ms) after a ForceEndOfUtterance: if Speechmatics never sends EndOfUtterance, close the
|
|
172
|
+
// turn anyway after this long so the pipeline can't hang. Rarely needs changing.
|
|
173
|
+
forceEndSafetyMs: 800,
|
|
174
|
+
// Deepgram Flux end-of-turn confidence (0.5–0.9). Flux's model closes the turn itself when its
|
|
175
|
+
// confidence the user is done exceeds this. ↓ → snappier turn ends but may cut mid-thought
|
|
176
|
+
// pauses; ↑ → more patient. Deepgram default 0.7.
|
|
177
|
+
fluxEotThreshold: 0.7,
|
|
178
|
+
// Cap (ms) on the token fetch that gates open() — a hung/slow backend or provider must not stall
|
|
179
|
+
// voice-mode startup indefinitely. onFatal fires past this so the host can surface it and retry.
|
|
180
|
+
tokenFetchTimeoutMs: 8000,
|
|
181
|
+
},
|
|
182
|
+
};
|