@todoforai/voiceloop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,50 @@
1
+ // ── ElevenLabs TTS (cloud, human-grade voices) ─────────────────────────────────────────────────
2
+ // A StreamingTTS provider: only `_synth(text, signal) → audio Blob` — sentence chunking, playback,
3
+ // barge-in, seek and speculative presynth all come from the base class, so switching Piper ↔
4
+ // ElevenLabs changes the VOICE, not the behavior. decodeAudioData handles the returned mp3.
5
+ //
6
+ // Auth, two modes (mirrors the STT providers' "key stays server-side" stance):
7
+ // • ttsUrl — YOUR backend proxy: POST { text, voice_id, model_id } → audio bytes. The proxy adds
8
+ // the xi-api-key and forwards to ElevenLabs. Recommended for production.
9
+ // • apiKey — direct browser→ElevenLabs (dev/bench only: the key is visible in the page).
10
+ //
11
+ // Latency: eleven_flash_v2_5 is EL's low-latency model (~75ms model time); the rest of a clip's
12
+ // cost is network TTFB + transfer, which presynth (during the end-of-turn debounce) and the base
13
+ // class's ahead-of-playback synth queue hide for all but the first clip of a cold turn.
14
+ import { StreamingTTS } from './voice-agent.js';
15
+
16
+ const EL_API = 'https://api.elevenlabs.io/v1/text-to-speech';
17
+
18
+ export class ElevenLabsTTS extends StreamingTTS {
19
+ constructor({ ttsUrl, apiKey, voiceId = 'JBFqnCBsd6RMkjVDRZzb' /* George */,
20
+ modelId = 'eleven_flash_v2_5', format = 'mp3_44100_64' } = {}) {
21
+ super(voiceId);
22
+ if (!ttsUrl && !apiKey) throw new Error('ElevenLabsTTS needs ttsUrl (backend proxy) or apiKey (direct, dev only)');
23
+ this.ttsUrl = ttsUrl; this.apiKey = apiKey; this.modelId = modelId; this.format = format;
24
+ }
25
+
26
+ // Pre-establish the connection (DNS + TLS + any proxy hop) so the first real clip only pays TTFB.
27
+ // A HEAD to the API root is enough to open the socket; failures are non-fatal.
28
+ async warm() {
29
+ const url = this.ttsUrl ?? EL_API;
30
+ try { await fetch(url, { method: 'HEAD', signal: AbortSignal.timeout(3000) }); } catch {}
31
+ }
32
+
33
+ async _synth(text, signal) {
34
+ text = text?.trim();
35
+ if (signal?.aborted || !text) return null;
36
+ const direct = !this.ttsUrl;
37
+ const url = direct ? `${EL_API}/${this.voiceId}?output_format=${this.format}` : this.ttsUrl;
38
+ const res = await fetch(url, {
39
+ method: 'POST',
40
+ headers: { 'Content-Type': 'application/json', ...(direct ? { 'xi-api-key': this.apiKey } : {}) },
41
+ // The proxy form carries voice/model/format in the body so one backend route serves any voice.
42
+ body: JSON.stringify(direct ? { text, model_id: this.modelId }
43
+ : { text, voice_id: this.voiceId, model_id: this.modelId, output_format: this.format }),
44
+ signal,
45
+ });
46
+ if (!res.ok) throw new Error(`ElevenLabs TTS ${res.status}: ${(await res.text().catch(() => '')).slice(0, 200)}`);
47
+ const blob = await res.blob();
48
+ return signal?.aborted ? null : blob;
49
+ }
50
+ }
package/src/tuning.js ADDED
@@ -0,0 +1,182 @@
1
+ // tuning.js — ONE place to fine-tune the voice agent's speech behaviour. Edit the numbers here;
2
+ // they're the defaults the VoiceAgent reads at construction, so a change applies to the next
3
+ // session you start (press the mic again — no rebuild needed in dev, the module is re-imported).
4
+ //
5
+ // Anything here can still be overridden per-instance via the VoiceAgent constructor options of the
6
+ // same name (constructor wins). This file is just the single, documented source of the defaults so
7
+ // you don't have to hunt through voice-agent.js / stt.js to dial things in.
8
+ //
9
+ // Quick guide to the symptoms ↔ knobs:
10
+ // • The agent interrupts itself / cuts off on noise → raise VAD.positiveSpeechThreshold,
11
+ // VAD.minSpeechFrames, or BARGE_IN_MIN_CHARS.
12
+ // • The agent transcribes ITS OWN voice (speakers, weak AEC) and self-interrupts → lower
13
+ // ECHO_WORD_MATCH (stricter echo filter); headphones remove the problem entirely.
14
+ // • "mhm"/"yeah" backchannels cut the agent off → raise BARGE_IN_MIN_CHARS.
15
+ // • Turn closes too eagerly mid-sentence → raise VAD.redemptionFrames or MAX_PAUSE_MS.
16
+ // • Turn feels sluggish to respond → lower VAD.redemptionFrames / STT max_delay.
17
+ // • Word-starts clipped ("…ello" instead of "hello")→ raise PREROLL_CHUNKS.
18
+ // • Replies too long/short → MAX_TOKENS.
19
+
20
+ export const TUNING = {
21
+ // ── Barge-in (interrupting the agent while it speaks) ────────────────────────────────────────────
22
+ // Barge-in now fires on TRANSCRIBED NOVEL WORDS, not raw VAD energy (so the AI's own echo/noise
23
+ // can't cut a reply). This is the minimum number of transcribed characters — counting ONLY words
24
+ // not (fuzzily) attributable to the reply being spoken, so even misheard echo contributes zero —
25
+ // that must accumulate WHILE the agent is speaking before we treat it as a real interruption.
26
+ // Higher = short backchannels ("mhm", "yeah", "ok") are ignored and the agent keeps talking; lower =
27
+ // snappier interruption. 0 = any single transcribed character interrupts (most aggressive).
28
+ BARGE_IN_MIN_CHARS: 6,
29
+
30
+ // Self-echo filter: on speakers the browser's AEC often fails to cancel our own Web-Audio TTS
31
+ // playout, so the STT transcribes the agent's own voice — real words that cross BARGE_IN_MIN_CHARS
32
+ // and make it interrupt ITSELF. Since we know EXACTLY what he's voicing, interims heard while
33
+ // speaking are word-matched against the current reply: if at least this fraction of the interim's
34
+ // recent words occur in the reply, it's its own echo → ignored, it keeps talking. Real
35
+ // interruptions ("no wait", "stop that") share few words with the reply, so they still cut in.
36
+ // Lower = stricter filtering (more echo ignored, but a real interruption that REUSES the reply's
37
+ // words might be missed); > 1 disables the filter.
38
+ ECHO_WORD_MATCH: 0.7,
39
+
40
+ // ── LLM ───────────────────────────────────────────────────────────────────────────────────────
41
+ MAX_TOKENS: 1024, // cap on the spoken reply length
42
+
43
+ // ── Liveness backstop (host code that never returns) ──────────────────────────────────────────
44
+ // The turn loop is strictly serialized, so anything it AWAITS can wedge it. Host hooks are kept
45
+ // off that path structurally rather than by deadline: tools are dispatched and never joined, and
46
+ // an aborted LLM stream is detached instead of drained. The one hook whose verdict is genuinely
47
+ // REQUIRED to proceed is the turnDetector (it decides whether the turn closes) — bounded by
48
+ // MAX_PAUSE_MS above, as ONE absolute deadline from end-of-speech.
49
+ // Not covered, by design: a custom TTS _synth() that never settles (there is no reply without it),
50
+ // and a muted-TTS turn draining a generator that ignores its abort signal.
51
+
52
+ // ── Turn boundary (when is the user done talking?) ────────────────────────────────────────────
53
+ // Hard cap (ms) a turnDetector may hold a turn open across a pause before it's force-committed —
54
+ // so a user who trails off mid-sentence can't hang the turn forever. Only used when a turnDetector
55
+ // is set; with the default silence-only endpointing the VAD redemptionFrames govern this instead.
56
+ MAX_PAUSE_MS: 4000,
57
+
58
+ // Browser Web Speech end-of-turn debounce (ms). The engine's own isFinal boundaries are per-pause
59
+ // fragments; after a final we wait this long for a new one before closing the turn, so a mid-thought
60
+ // pause keeps one turn together. Raise if turns split mid-sentence; lower to reply sooner.
61
+ // 500 measured comfortable: Chrome's endpointer already waits out its own internal silence before
62
+ // emitting isFinal, so this only needs to bridge restart seams and brief continuations.
63
+ WEBSPEECH_EOT_MS: 500,
64
+
65
+ // Chrome's endpointer can take longer than WEBSPEECH_EOT_MS to promote trailing words to isFinal.
66
+ // Closing while it still holds that un-promoted tail emits a TRUNCATED turn and lets the late final
67
+ // land as a bogus second one (disjoint text, so the seam-dedup can't catch it) — which barges in and
68
+ // kills the reply. So a close waits for the engine to go quiescent; this caps how many extra EOT
69
+ // windows it may wait before giving up on a tail that never resolves (liveness backstop only —
70
+ // reaching it re-exposes the truncation). Worst-case added close latency = this × WEBSPEECH_EOT_MS.
71
+ // Raise if slow engines still split turns; lower to bound the wait harder.
72
+ WEBSPEECH_TAIL_MAX_DEFERS: 3,
73
+
74
+ // Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
75
+ // agent starts generating a reply for the probable final text — overlapping the LLM's first-token
76
+ // latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
77
+ // (instant first token); a changed text just aborts it (cost: a few prompt tokens). Must be
78
+ // comfortably below WEBSPEECH_EOT_MS for the overlap to pay. 0 = prefetch on every interim tick
79
+ // (most aggressive, most wasted calls).
80
+ PREFETCH_MS: 200,
81
+
82
+ // Audio chunks (~256ms each at 16kHz/4096-sample frames; ~1s total at 4) kept BEFORE VAD fires and
83
+ // replayed into STT when speech starts, so the first word isn't clipped. Raise if word-starts are
84
+ // getting cut; lower to shave a touch of latency/cost.
85
+ PREROLL_CHUNKS: 4,
86
+
87
+ // ── VAD (Silero v5) — speech detection sensitivity ────────────────────────────────────────────
88
+ // These directly govern how readily the mic decides "the user is speaking". Tightening them is the
89
+ // main lever against false speech-starts (self-echo, coughs, keyboard, room noise).
90
+ VAD: {
91
+ // Probability (0–1) above which a frame counts as speech. ↑ → needs clearer/louder speech to
92
+ // trigger (fewer false starts, but may miss a soft start). Default 0.5.
93
+ positiveSpeechThreshold: 0.5,
94
+ // Probability below which we consider it NOT speech (hysteresis floor — keep < positive). Default 0.25.
95
+ negativeSpeechThreshold: 0.25,
96
+ // Minimum consecutive speech frames before a speech-START fires. ↑ → ignores very short blips
97
+ // (coughs, clicks). Default 4 (~130ms).
98
+ minSpeechFrames: 4,
99
+ // Silence frames required to declare speech ENDED. ↑ → more patient across natural pauses (won't
100
+ // cut you off mid-sentence) but slower to respond; ↓ → snappier endpointing. Default 24 (~770ms).
101
+ redemptionFrames: 24,
102
+ // Frames of audio padded before detected speech (smooths the onset). Default 10.
103
+ preSpeechPadFrames: 10,
104
+ },
105
+
106
+ // ── Bootstrap VAD (instant, zero-load fallback until Silero is ready) ────────────────────────
107
+ // Silero (MicVAD.new()) needs a CDN fetch + WASM compile + model load (~0.5-2s cold) before it can
108
+ // detect speech at all — start() begins capturing mic audio well before that resolves. Until then,
109
+ // a plain RMS-energy threshold (no model, no async load, gates from frame 1) stands in so the FIRST
110
+ // words spoken right after clicking the mic are never silently dropped from the (size-capped)
111
+ // preroll buffer while waiting for Silero. Swapped out for Silero the moment it's ready, always at
112
+ // an utterance boundary (never mid-speech) — see _swapToSileroWhenReady().
113
+ // NB: these frames are the raw ~256ms worklet capture chunks (4096 samples @16kHz) — much coarser
114
+ // than Silero's own ~32ms internal frames above, so the counts are on a different scale.
115
+ BOOTSTRAP_VAD: {
116
+ // RMS above this (0-1 scale, ~loudness of normal speech close to the mic) counts as speech.
117
+ // Deliberately looser/cruder than Silero — it only needs to survive the first second(s), not
118
+ // replace real speech detection.
119
+ energyThreshold: 0.02,
120
+ // Consecutive ~256ms chunks above threshold before declaring speech-start. 1 = fires on the very
121
+ // first loud chunk — snappiest possible protection for the opening word(s).
122
+ minSpeechFrames: 1,
123
+ // Consecutive ~256ms chunks below threshold before declaring speech-end (~1.5s). Deliberately
124
+ // more patient than Silero's own redemptionFrames above: this stand-in only runs for the first
125
+ // second or two of a session, and the goal here is "never chop a continuous sentence", not tight
126
+ // endpointing — Silero takes over (with its tighter default) as soon as it's ready.
127
+ redemptionFrames: 6,
128
+ },
129
+
130
+ // ── TTS (first-audio latency) ─────────────────────────────────────────────────────────────────
131
+ // How soon the FIRST spoken chunk is cut from the streaming LLM text. Lower = the agent starts talking
132
+ // sooner (less waiting on generation), at a small prosody cost (it may break on a comma/clause rather
133
+ // than a full sentence). Only the FIRST chunk is aggressive; every later sentence still breaks on a
134
+ // whole-sentence ender for natural delivery.
135
+ // • First chunk feels slow / waits for the whole answer → lower firstChunkMaxChars.
136
+ // • First chunk too choppy ("Sure," then a pause) → raise firstChunkMinChars.
137
+ TTS: {
138
+ // Minimum non-space chars in the VERY first spoken chunk (the first-WORD opener). A tiny first
139
+ // word ("I", "Hi", "A") makes a choppy micro-clip with a pause after it, so we keep gluing the
140
+ // next word(s) until the opener reaches this many chars. Lower = even snappier first sound but
141
+ // riskier choppiness; 3-4 keeps one/two-letter openers glued without delaying real words.
142
+ firstWordMinChars: 4,
143
+ // Don't cut the first chunk on a comma/clause break until at least this many chars exist, so a tiny
144
+ // leading filler ("Sure,", "Okay,") doesn't become its own choppy clip with a pause after it. A full
145
+ // sentence ender (. ! ?) ALWAYS breaks immediately regardless. Lower = faster first sound, at the risk
146
+ // of a short choppy opener (cheap on local Piper). 8 keeps one-word fillers glued; ~6 starts sooner.
147
+ firstChunkMinChars: 8,
148
+ // Hard cap: if the opening clause has NO punctuation by this many chars, break it at the last word
149
+ // boundary anyway so a long unpunctuated start still begins playing instead of awaiting the sentence.
150
+ firstChunkMaxChars: 70,
151
+ // Hard cap for every LATER sentence: a long run with no sentence ender (lists, dashed prose)
152
+ // breaks at the last word boundary by this many chars, so audio keeps flowing instead of
153
+ // waiting silently for one monster chunk. Generous — normal sentences end well before it.
154
+ sentenceMaxChars: 200,
155
+ },
156
+
157
+ // ── STT (Speechmatics) — finalization timing ──────────────────────────────────────────────────
158
+ STT: {
159
+ // Seconds Speechmatics waits before LOCKING interim words into a final. ↓ → a shorter flickering
160
+ // tail (words settle faster) at a small accuracy cost; ↑ → more context, steadier finals. The
161
+ // agent picks per operating_point: 'enhanced' has the accuracy headroom for a tight 1s; 'standard'
162
+ // uses 2s (Speechmatics' documented "optimal balance"). Range 0.7–4.
163
+ maxDelayEnhanced: 1,
164
+ maxDelayStandard: 2,
165
+ // Seconds of silence after which Speechmatics closes a turn ITSELF (emits EndOfUtterance).
166
+ // Required for the ForceEndOfUtterance/EndOfUtterance handshake (see stt.js StartRecognition);
167
+ // must be < max_delay. Speechmatics recommends 0.5-0.8s for voice AI. In practice the agent's
168
+ // VAD commit() is the boundary — audio is VAD-gated, so little post-speech silence reaches
169
+ // Speechmatics for this trigger to fire on.
170
+ eouSilenceTrigger: 0.6,
171
+ // Fallback (ms) after a ForceEndOfUtterance: if Speechmatics never sends EndOfUtterance, close the
172
+ // turn anyway after this long so the pipeline can't hang. Rarely needs changing.
173
+ forceEndSafetyMs: 800,
174
+ // Deepgram Flux end-of-turn confidence (0.5–0.9). Flux's model closes the turn itself when its
175
+ // confidence the user is done exceeds this. ↓ → snappier turn ends but may cut mid-thought
176
+ // pauses; ↑ → more patient. Deepgram default 0.7.
177
+ fluxEotThreshold: 0.7,
178
+ // Cap (ms) on the token fetch that gates open() — a hung/slow backend or provider must not stall
179
+ // voice-mode startup indefinitely. onFatal fires past this so the host can surface it and retry.
180
+ tokenFetchTimeoutMs: 8000,
181
+ },
182
+ };