@todoforai/voiceloop 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -9
- package/examples/demo.html +246 -44
- package/package.json +2 -2
- package/src/index.d.ts +55 -19
- package/src/stt.js +41 -7
- package/src/tuning.js +17 -0
- package/src/voice-agent.js +85 -14
package/README.md
CHANGED
|
@@ -54,14 +54,24 @@ That's it. Speak, get spoken answers, interrupt at will.
|
|
|
54
54
|
|
|
55
55
|
## Try it in 20 seconds
|
|
56
56
|
|
|
57
|
+
**[todoforai.github.io/voiceloop](https://todoforai.github.io/voiceloop/)** — the live demo, no
|
|
58
|
+
install, no keys and no backend of ours: the default path is your browser's own Web Speech STT,
|
|
59
|
+
local Piper TTS, and an echo stand-in for the LLM. (Web Speech is the *browser's* recognizer —
|
|
60
|
+
on most builds it still sends audio to the browser vendor. Pick a pipeline STT provider in
|
|
61
|
+
settings to see voiceloop's own VAD→STT path.)
|
|
62
|
+
|
|
63
|
+
Or locally, to point it at your own LLM:
|
|
64
|
+
|
|
57
65
|
```sh
|
|
58
66
|
npx serve . # from a clone
|
|
59
67
|
npx serve node_modules/@todoforai/voiceloop # from npm
|
|
60
68
|
```
|
|
61
69
|
|
|
62
|
-
Open **`/examples/demo.html`** — the whole loop behind one orb
|
|
63
|
-
|
|
64
|
-
|
|
70
|
+
Open **`/examples/demo.html`** — the whole loop behind one orb that breathes with the
|
|
71
|
+
mic level (and, while it answers, with how fast its spoken cursor advances), the live
|
|
72
|
+
interim transcript, click-to-interrupt with the unspoken remainder struck through where
|
|
73
|
+
you cut it off, and the two numbers that decide whether it feels alive: **first token**
|
|
74
|
+
and **first sound**. It starts in **echo mode**
|
|
65
75
|
— browser-native Web Speech STT, local Piper TTS and a stand-in "LLM" that
|
|
66
76
|
repeats you — so it runs with no keys, no backend and no cost. Open *settings*
|
|
67
77
|
to point it at any OpenAI-compatible endpoint (Ollama, your proxy) and a cloud
|
|
@@ -76,7 +86,8 @@ the place to start reading.
|
|
|
76
86
|
|
|
77
87
|
```
|
|
78
88
|
you stop speaking ──┐
|
|
79
|
-
│ end-of-turn debounce (~
|
|
89
|
+
│ end-of-turn debounce (500ms configured; ~1.5s in practice
|
|
90
|
+
│ on browser STT — see the webspeech note below)
|
|
80
91
|
LLM prefetch ────┤ ← already streaming since your interim transcript stabilized
|
|
81
92
|
│
|
|
82
93
|
first LLM tokens ───┤ first sentence boundary (~8+ chars)
|
|
@@ -96,6 +107,7 @@ into the mic (no AEC):
|
|
|
96
107
|
| OpenAI Realtime (own LLM)* | 870ms | **0/30** | **17/30** |
|
|
97
108
|
| **voiceloop** (deepgram + ElevenLabs flash) | **980ms** | **0/30** | **0/30** |
|
|
98
109
|
| **voiceloop** (deepgram + Piper, free local TTS) | **970ms** | **0/30** | — |
|
|
110
|
+
| **voiceloop** (webspeech + Piper, zero-key default) | 2110ms | — | — |
|
|
99
111
|
| Pipecat 1.8.1 (same providers) | 1050ms | 2/30 | **20/30** |
|
|
100
112
|
| ElevenLabs ConvAI | 1450ms | **0/30** | 0/30 |
|
|
101
113
|
|
|
@@ -114,7 +126,7 @@ Full tables, method and reproduction steps: [`results/RESULTS.md`](https://githu
|
|
|
114
126
|
## STT providers
|
|
115
127
|
|
|
116
128
|
```js
|
|
117
|
-
new VoiceAgent({ sttProvider: 'webspeech' }) // default:
|
|
129
|
+
new VoiceAgent({ sttProvider: 'webspeech' }) // default: zero-key, browser-native (Chrome/Safari) — but ~1.2s slower to close a turn; see the table below
|
|
118
130
|
new VoiceAgent({ sttProvider: 'elevenlabs', sttTokenUrl: '/api/stt/token' })
|
|
119
131
|
new VoiceAgent({ sttProvider: 'deepgram', sttTokenUrl: '/api/stt/token' })
|
|
120
132
|
new VoiceAgent({ sttProvider: 'speechmatics', sttTokenUrl: '/api/stt/token' })
|
|
@@ -132,9 +144,16 @@ new VoiceAgent({ sttProvider, getSttToken: p => fetch(ROUTE[p], { method: 'POST'
|
|
|
132
144
|
|
|
133
145
|
On browsers without SpeechRecognition (Firefox, WebKitGTK) `webspeech` falls back to `elevenlabs` — which needs a token route configured, so pass `sttTokenUrl`/`getSttToken` if you want the fallback to work.
|
|
134
146
|
|
|
147
|
+
**On iOS, prefer a cloud STT provider.** Safari has shipped `webkitSpeechRecognition` since 14.5, but the
|
|
148
|
+
implementation is unreliable for a continuous agent loop: recognition often goes silent after the first
|
|
149
|
+
utterance without firing `onend` or `onerror`, playing audio in the same page can kill it the same way, and
|
|
150
|
+
iOS ducks the speakers while the recognizer holds the mic — which is exactly what a talking agent needs.
|
|
151
|
+
`webspeech` is a fine zero-key default on desktop Chrome; on phones, point `sttProvider` at `elevenlabs`,
|
|
152
|
+
`deepgram` or `speechmatics` so voiceloop owns the mic and its own VAD/echo path.
|
|
153
|
+
|
|
135
154
|
| Provider | End-of-turn | Notes |
|
|
136
155
|
|---|---|---|
|
|
137
|
-
| `webspeech` | Native (browser-managed) |
|
|
156
|
+
| `webspeech` | Native (browser-managed) | Zero-key (no token endpoint), Chrome/Safari only. **~2.1s voice→voice** — ~1.2s slower to close a turn than cloud STT ([measured](bench/results/RESULTS.md)); it also owns the mic, so voiceloop's VAD/echo pipeline is bypassed. Recognition is handled by the browser, which may use a vendor cloud service — it is not guaranteed on-device |
|
|
138
157
|
| `elevenlabs` | VAD-gated | Scribe v2 realtime, great accuracy |
|
|
139
158
|
| `deepgram` | **Native** (Flux) | Provider's own turn model hears the full stream |
|
|
140
159
|
| `speechmatics` | VAD-gated | Cheapest per second, locked-words-only finals |
|
|
@@ -145,6 +164,7 @@ The default adapter speaks OpenAI's streaming wire format (OpenAI, Groq, Cerebra
|
|
|
145
164
|
|
|
146
165
|
```js
|
|
147
166
|
const agent = new VoiceAgent({
|
|
167
|
+
// TS: annotate as `LLM` (exported) and every parameter is inferred.
|
|
148
168
|
llm: async function* (history, system, signal, { toolGate } = {}) {
|
|
149
169
|
// history: [{ role: 'user'|'assistant', content }], system: composed persona+context
|
|
150
170
|
// toolGate: only for adapters that run tools themselves — see below; ignore it otherwise
|
|
@@ -212,9 +232,42 @@ agent.replay(text, fromNw?, onProgress?) // re-voice a past reply; onProgress(
|
|
|
212
232
|
agent.dumpAudio() // last 30s of mic audio as WAV + stall report (debugging)
|
|
213
233
|
```
|
|
214
234
|
|
|
215
|
-
|
|
235
|
+
### Constructor options
|
|
236
|
+
|
|
237
|
+
Full types (with per-option notes) in [`src/index.d.ts`](src/index.d.ts).
|
|
238
|
+
|
|
239
|
+
| | |
|
|
240
|
+
|---|---|
|
|
241
|
+
| `persona`, `sysmsg` | identity (static) and live context (`setSysmsg()` mid-session) |
|
|
242
|
+
| `llmUrl`, `model`, `apiKey`, `maxTokens` | built-in OpenAI-compatible adapter |
|
|
243
|
+
| `llm` | your own adapter — replaces all of the above |
|
|
244
|
+
| `tools` | what the model may call ([Tools](#tools)) |
|
|
245
|
+
| `tts`, `speed` | TTS engine (default Piper) and Piper's speaking rate |
|
|
246
|
+
| `sttProvider`, `sttLang`, `sttModel`, `sttUrl` | which recognizer, which language |
|
|
247
|
+
| `sttTokenUrl` / `getSttToken`, `sttUsageUrl` | token minting ([STT providers](#stt-providers)); usage metering |
|
|
248
|
+
| `keyterms` | domain words to bias STT towards (max 50, ≤20 chars) |
|
|
249
|
+
| `micDeviceId` | input device; also `start(deviceId)` |
|
|
250
|
+
| `turnDetector`, `maxPauseMs`, `sttEotThreshold` | when a user's turn is over |
|
|
251
|
+
| `bargeInMinChars`, `vadOptions`, `preroll` | interruption sensitivity and mic gating |
|
|
252
|
+
| `fetchFn` | transport seam for the built-in adapter (tests, request wrapping) |
|
|
253
|
+
| `onEvent` | every event below |
|
|
254
|
+
|
|
255
|
+
Events via `onEvent(e)`: `state`, `stt`, `assistant`, `tool`, `vad`, `level`, `echo`, `error`, `diag`.
|
|
216
256
|
A `tool` event carries `{ name, id, args }` plus either `result` (settled) or `running: true` (an [adapter-announced](#adapters-that-run-tools-themselves) call still executing) — render the pair as one chip keyed by `id`.
|
|
217
257
|
|
|
258
|
+
Two of them exist purely so a UI can look alive, and both cost you nothing to ignore:
|
|
259
|
+
|
|
260
|
+
- **`level`** — `{ level }`, mic loudness 0..1 on a perceptual curve, ~every 256ms while capturing.
|
|
261
|
+
Measured off the frames the pipeline already has (no second mic tap, no `AnalyserNode`). Drive a
|
|
262
|
+
meter or a reacting orb with it; decay it over *elapsed time* rather than per frame, or it
|
|
263
|
+
flickers out between samples. Not emitted for a `selfCapture` provider (webspeech owns its mic).
|
|
264
|
+
- **interruption** — there is no "cut short" event: the final `assistant` event carries both what
|
|
265
|
+
was heard (`text`) and what was written (`full`), so a non-empty `full.slice(text.length)` means
|
|
266
|
+
the reply was cut before that remainder was ever spoken. Render it struck through and the
|
|
267
|
+
transcript shows exactly where playback stopped (see `examples/demo.html`). It does not say *why*:
|
|
268
|
+
a voice barge-in, `interrupt()`, a superseding turn and `setTtsMuted(true)` all end a reply the
|
|
269
|
+
same way.
|
|
270
|
+
|
|
218
271
|
## Tuning
|
|
219
272
|
|
|
220
273
|
Every latency/sensitivity knob lives in [`src/tuning.js`](src/tuning.js) — VAD thresholds, barge-in minimum characters, sentence-break aggressiveness, prefetch stability window, echo-match threshold. Constructor options (`bargeInMinChars`, `vadOptions`, `turnDetector`, `maxPauseMs`) override per-agent.
|
|
@@ -245,7 +298,7 @@ Custom engines subclass `StreamingTTS` and implement one method:
|
|
|
245
298
|
|
|
246
299
|
```js
|
|
247
300
|
class MyTTS extends StreamingTTS {
|
|
248
|
-
async _synth(text) { return wavBlob; } // sentence in, audio Blob out
|
|
301
|
+
async _synth(text, signal) { return wavBlob; } // sentence in, audio Blob out (null if aborted)
|
|
249
302
|
}
|
|
250
303
|
```
|
|
251
304
|
|
|
@@ -254,7 +307,7 @@ Sentence chunking, the playback tape, tap-to-seek, barge-in and progress reporti
|
|
|
254
307
|
## Testing
|
|
255
308
|
|
|
256
309
|
```
|
|
257
|
-
npm test #
|
|
310
|
+
npm test # no browser needed
|
|
258
311
|
```
|
|
259
312
|
|
|
260
313
|
The suite locks in turn serialization, hold/release semantics, tool dedup, self-echo classification, prefetch adoption rules, tape seeking, and each STT provider's turn-boundary state machine.
|
package/examples/demo.html
CHANGED
|
@@ -26,61 +26,164 @@
|
|
|
26
26
|
main { max-width:660px; margin:0 auto; padding:36px 18px 48px;
|
|
27
27
|
display:flex; flex-direction:column; align-items:center; gap:16px }
|
|
28
28
|
|
|
29
|
-
/* ── the orb
|
|
29
|
+
/* ── the orb ───────────────────────────────────────────────────────────────
|
|
30
30
|
Plain CSS on purpose. A WebGPU version looks marginally better and is dark
|
|
31
31
|
for every Linux visitor (Chrome blocklists the adapter there), and an SVG
|
|
32
|
-
goo filter costs real CPU per frame for the same silhouette.
|
|
33
|
-
|
|
34
|
-
prefers-reduced-motion.
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
goo filter costs real CPU per frame for the same silhouette. Everything
|
|
33
|
+
below runs on the compositor and degrades to a flat disc under
|
|
34
|
+
prefers-reduced-motion.
|
|
35
|
+
|
|
36
|
+
One knob drives the whole thing: --lvl (0..1, the live mic/voice level from
|
|
37
|
+
the agent's `level` event). Size, glow, rim and halo all read from it, so
|
|
38
|
+
the orb BREATHES with the audio instead of blinking between states. */
|
|
39
|
+
#orb { position:relative; width:230px; height:230px; display:grid; place-items:center;
|
|
40
|
+
--a:#22304a; --b:#3d5274; --cc:#1d2740; --d:#2c3b57; --spin:30s; --glow:.14; --lvl:0 }
|
|
41
|
+
|
|
42
|
+
/* Halo: the light this thing throws on the page. It sits OUTSIDE the clipped orb, so the glow
|
|
43
|
+
blooms past the rim — at video scale that bloom is most of the "alive" read.
|
|
44
|
+
Two stops, not one: a tight inner bloom that hugs the rim (that's the part that reads as the
|
|
45
|
+
object emitting light) and a wide, very faint spill. A single wide gradient just washes the
|
|
46
|
+
whole frame in flat colour and looks like a backdrop instead of a glow. */
|
|
47
|
+
#orb::before { content:''; position:absolute; inset:-34%; border-radius:50%; pointer-events:none;
|
|
48
|
+
background:radial-gradient(circle, var(--b) 0 30%,
|
|
49
|
+
color-mix(in oklab, var(--b) 45%, transparent) 44%,
|
|
50
|
+
transparent 68%);
|
|
51
|
+
opacity:calc(var(--glow) * (.3 + var(--lvl) * .7)); filter:blur(18px);
|
|
52
|
+
transform:scale(calc(.94 + var(--lvl) * .12));
|
|
53
|
+
transition:opacity .12s linear, transform .12s ease-out }
|
|
54
|
+
|
|
55
|
+
/* The ball. A lit sphere, not a flat disc: the base gradient puts the light source up-left and
|
|
56
|
+
falls to near-black at the lower-right, which is what gives it volume before a single blob
|
|
57
|
+
is drawn. */
|
|
58
|
+
#mic { position:relative; width:100%; height:100%; border-radius:50%; border:0; padding:0;
|
|
59
|
+
cursor:pointer; overflow:hidden; display:grid; place-items:center;
|
|
60
|
+
background:radial-gradient(125% 125% at 30% 22%, #202a3d 0%, #131a28 36%, #080b12 70%, #05070b 100%);
|
|
61
|
+
/* level scales the body; the transition is short enough to track speech, long enough
|
|
62
|
+
not to jitter on a single loud frame */
|
|
63
|
+
transform:scale(calc(1 + var(--lvl) * .05));
|
|
64
|
+
transition:transform .12s ease-out }
|
|
39
65
|
/* Each blob is a soft disc smaller than the orb, wandering on its own ellipse.
|
|
40
|
-
Blurred and screen-blended,
|
|
41
|
-
pinches where they overlap — metaballs, minus the shader.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
#mic i
|
|
46
|
-
|
|
66
|
+
Blurred and screen-blended, five of them read as one body that swells and
|
|
67
|
+
pinches where they overlap — metaballs, minus the shader. The blur tightens
|
|
68
|
+
as the level rises, so loud audio makes the core condense and brighten.
|
|
69
|
+
They are deliberately LARGE and travel a short path: small blobs on a wide orbit separate into
|
|
70
|
+
visibly distinct lumps at the extremes, which reads as four stains rather than one substance. */
|
|
71
|
+
#mic i { position:absolute; width:62%; height:62%; left:19%; top:19%; border-radius:50%;
|
|
72
|
+
filter:blur(calc(26px - var(--lvl) * 7px)); mix-blend-mode:screen;
|
|
73
|
+
animation:wander var(--spin) ease-in-out infinite;
|
|
74
|
+
transition:background 1s, filter .12s linear } /* colours cross-fade; the speed change snaps (animation-* isn't animatable) */
|
|
75
|
+
/* Each blob gets its OWN size and resting offset. Sharing one geometry and varying only the
|
|
76
|
+
animation timing makes them sweep the same path and stack into a single horizontal band
|
|
77
|
+
through the middle; staggering the anchors keeps the mass off-axis and organic. */
|
|
78
|
+
/* Anchors sit in DIFFERENT quadrants (up-left, up-right, low-left, low-right) so the combined
|
|
79
|
+
mass stays centred. Clustering them on one side pools all the light there and leaves a heavy
|
|
80
|
+
dead crescent on the other — the sphere stops looking filled. */
|
|
81
|
+
/* Anchors are each offset from the blob's OWN centred position ((100-size)/2) into a different
|
|
82
|
+
quadrant, so the resting pose is balanced. Offsetting from a shared number instead makes the
|
|
83
|
+
bigger blobs sit further out than the small ones and the cluster drifts up-left. */
|
|
84
|
+
#mic i:nth-child(1) { background:radial-gradient(circle, var(--a) 0 24%, transparent 62%);
|
|
85
|
+
top:9%; left:9% } /* 62% → centred 19% */
|
|
86
|
+
#mic i:nth-child(2) { background:radial-gradient(circle, var(--b) 0 22%, transparent 60%);
|
|
87
|
+
width:54%; height:54%; top:13%; left:33%; /* 54% → centred 23% */
|
|
88
|
+
animation-name:wander2;
|
|
47
89
|
animation-duration:calc(var(--spin)*.72); animation-direction:reverse }
|
|
48
|
-
#mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0
|
|
90
|
+
#mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0 26%, transparent 64%);
|
|
91
|
+
width:66%; height:66%; top:27%; left:7%; /* 66% → centred 17% */
|
|
92
|
+
animation-name:wander3;
|
|
49
93
|
animation-duration:calc(var(--spin)*1.4); animation-delay:calc(var(--spin)*-.3) }
|
|
50
|
-
#mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0
|
|
94
|
+
#mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0 20%, transparent 58%);
|
|
95
|
+
width:50%; height:50%; top:35%; left:35%; /* 50% → centred 25% */
|
|
96
|
+
animation-name:wander2;
|
|
51
97
|
animation-duration:calc(var(--spin)*.9); animation-direction:reverse;
|
|
52
98
|
animation-delay:calc(var(--spin)*-.55) }
|
|
53
|
-
/*
|
|
99
|
+
/* A 5th blob pinned near the centre and driven ONLY by level: the core that lights up when you
|
|
100
|
+
speak. Without it the wandering blobs can all be off-centre at the moment you talk, and the
|
|
101
|
+
orb answers your voice from its edge. */
|
|
102
|
+
/* Only a HINT of white at the very centre: screen-blending five layers already drives the overlap
|
|
103
|
+
toward white, so a white-cored fifth blob clips the whole middle to a flat highlight and the
|
|
104
|
+
state colour disappears exactly when the orb is most active. */
|
|
105
|
+
#mic i:nth-child(5) { width:64%; height:64%; left:18%; top:18%; animation:none;
|
|
106
|
+
background:radial-gradient(circle, color-mix(in oklab, var(--b) 88%, white) 0 14%,
|
|
107
|
+
var(--b) 34%, transparent 64%);
|
|
108
|
+
opacity:calc(var(--lvl) * .72); filter:blur(calc(20px - var(--lvl) * 8px));
|
|
109
|
+
transform:scale(calc(.72 + var(--lvl) * .38));
|
|
110
|
+
transition:opacity .12s linear, transform .12s ease-out, filter .12s linear }
|
|
111
|
+
/* Glass shell: the specular cap up-left where the light is, plus a BOUNCE light low-right in the
|
|
112
|
+
state colour. The bounce is what stops the bottom of the sphere reading as a dead black
|
|
113
|
+
crescent — real objects catch light back off their surroundings, and without it the orb looks
|
|
114
|
+
like a lit stain on a black hole. */
|
|
54
115
|
#mic::before { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
55
|
-
background:radial-gradient(
|
|
56
|
-
rgba(255,255,255,.
|
|
57
|
-
|
|
116
|
+
background:radial-gradient(62% 46% at 33% 19%, rgba(255,255,255,.34) 0 8%,
|
|
117
|
+
rgba(255,255,255,.07) 30%, transparent 58%),
|
|
118
|
+
radial-gradient(58% 44% at 70% 86%,
|
|
119
|
+
color-mix(in oklab, var(--b) calc(16% + var(--lvl)*26%), transparent) 0 12%,
|
|
120
|
+
transparent 62%);
|
|
121
|
+
box-shadow:inset 0 0 46px rgba(0,0,0,.7), inset 0 1px 1px rgba(255,255,255,.22) }
|
|
122
|
+
/* Fresnel rim + outer glow. The hairline ring is the polish: a lit sphere has a bright edge where
|
|
123
|
+
it turns away from you, and 1px of state colour there is what separates "orb" from "blurry
|
|
124
|
+
blob". It brightens with level, so the edge rings on loud syllables. */
|
|
58
125
|
#mic::after { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
59
|
-
box-shadow:0 0
|
|
126
|
+
box-shadow:inset 0 0 0 1px color-mix(in oklab, var(--b) calc(20% + var(--lvl) * 45%), transparent),
|
|
127
|
+
inset 0 0 calc(20px + var(--lvl) * 28px) color-mix(in oklab, var(--b) calc(8% + var(--lvl) * 22%), transparent),
|
|
128
|
+
0 0 44px 6px color-mix(in oklab, var(--b) 45%, transparent);
|
|
129
|
+
opacity:calc(.35 + var(--glow) * .65); transition:opacity .8s, box-shadow .12s linear }
|
|
130
|
+
/* Three ORBITS, not one. Sharing a single path (even at different speeds and delays) marches every
|
|
131
|
+
blob along the same diagonal, so the mass stays a bar that slides corner to corner instead of
|
|
132
|
+
circulating. These run the loop in different rotational orders, which keeps the interior
|
|
133
|
+
churning and the silhouette asymmetric. */
|
|
60
134
|
@keyframes wander {
|
|
61
|
-
0%,100% { transform:translate(-
|
|
62
|
-
25% { transform:translate(
|
|
63
|
-
50% { transform:translate(
|
|
64
|
-
75% { transform:translate(-
|
|
135
|
+
0%,100% { transform:translate(-20%,-13%) scale(1) }
|
|
136
|
+
25% { transform:translate( 18%,-20%) scale(1.16) }
|
|
137
|
+
50% { transform:translate( 21%, 17%) scale(.88) }
|
|
138
|
+
75% { transform:translate(-17%, 20%) scale(1.1) }
|
|
139
|
+
}
|
|
140
|
+
@keyframes wander2 {
|
|
141
|
+
0%,100% { transform:translate( 19%,-16%) scale(1.08) }
|
|
142
|
+
25% { transform:translate(-21%,-10%) scale(.9) }
|
|
143
|
+
50% { transform:translate(-14%, 21%) scale(1.18) }
|
|
144
|
+
75% { transform:translate( 20%, 12%) scale(.94) }
|
|
145
|
+
}
|
|
146
|
+
@keyframes wander3 {
|
|
147
|
+
0%,100% { transform:translate( 4%, 22%) scale(.92) }
|
|
148
|
+
25% { transform:translate( 22%, -6%) scale(1.12) }
|
|
149
|
+
50% { transform:translate( -6%,-22%) scale(1) }
|
|
150
|
+
75% { transform:translate(-22%, 6%) scale(1.14) }
|
|
151
|
+
}
|
|
152
|
+
/* Ripple: one ring per turn boundary, fired from JS (see `ripple()`). It marks the two instants
|
|
153
|
+
the transcript can't show — the moment your turn was committed, and the moment a barge-in cut
|
|
154
|
+
the reply — which is exactly what a viewer needs to SEE to believe the latency numbers. */
|
|
155
|
+
#orb u { position:absolute; inset:0; border-radius:50%; border:1.5px solid var(--c);
|
|
156
|
+
opacity:0; pointer-events:none; animation:ripple .75s ease-out forwards }
|
|
157
|
+
@keyframes ripple {
|
|
158
|
+
from { opacity:.55; transform:scale(1) }
|
|
159
|
+
to { opacity:0; transform:scale(1.5) }
|
|
65
160
|
}
|
|
66
161
|
/* the glyph leads while the orb is dark; once the loop is live the orb says it better */
|
|
67
162
|
#glyph { position:relative; font-size:36px; line-height:1; transition:opacity .4s;
|
|
68
163
|
text-shadow:0 2px 12px #000 }
|
|
69
164
|
body:not([data-state="idle"]) #glyph { opacity:0 }
|
|
70
165
|
|
|
71
|
-
/*
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
body[data-
|
|
166
|
+
/* Same object, more energy — the whole state display is these three lines. Each palette is one
|
|
167
|
+
hue family with a neighbour (green→teal, amber→gold, coral→magenta) rather than opposite
|
|
168
|
+
corners of the wheel: four unrelated hues screen-blended average out to a muddy grey-brown
|
|
169
|
+
exactly where the blobs overlap, which is the middle of the orb. */
|
|
170
|
+
body[data-state="listening"] #orb { --a:#0fa5c9; --b:#2ee6a8; --cc:#0d7fa8; --d:#7df0c8; --spin:14s; --glow:.5 }
|
|
171
|
+
body[data-state="thinking"] #orb { --a:#d98324; --b:#ffb547; --cc:#b0621a; --d:#ffd98a; --spin:6s; --glow:.7 }
|
|
172
|
+
body[data-state="speaking"] #orb { --a:#e8355f; --b:#ff7038; --cc:#b8329a; --d:#ff9a5e; --spin:4s; --glow:.95 }
|
|
77
173
|
|
|
78
174
|
body[data-state="listening"] { --c:#39d98a }
|
|
79
175
|
body[data-state="thinking"] { --c:#ffc453 }
|
|
80
176
|
body[data-state="speaking"] { --c:#ff8a3d }
|
|
81
177
|
|
|
178
|
+
/* Reduced motion: hold the orb STILL. Killing the transitions alone isn't enough — the level var
|
|
179
|
+
keeps changing, so every scale would just snap instead of glide. Pin --lvl and the object
|
|
180
|
+
becomes a flat, quietly coloured disc that still reads state by hue. (setLevel() also stops
|
|
181
|
+
writing the var, and ripple() creates no rings — a display:none ring never fires animationend,
|
|
182
|
+
so it would leak one element per turn.) */
|
|
82
183
|
@media (prefers-reduced-motion: reduce) {
|
|
83
|
-
#
|
|
184
|
+
#orb { --lvl:0 !important }
|
|
185
|
+
#mic i { animation:none } #mic, #orb::before, #mic i { transition:none }
|
|
186
|
+
#orb u { display:none }
|
|
84
187
|
}
|
|
85
188
|
|
|
86
189
|
#status { color:var(--c); font-size:14px; min-height:22px; text-align:center; transition:color .3s }
|
|
@@ -92,6 +195,11 @@
|
|
|
92
195
|
.msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
|
|
93
196
|
.msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
|
|
94
197
|
.msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
|
|
198
|
+
/* Barge-in: the reply stopped mid-word because you cut in. Without a mark, an interrupted bubble
|
|
199
|
+
is indistinguishable from a short answer — and interruption is the feature. */
|
|
200
|
+
.msg.ai.cut .tail { text-decoration:line-through; text-decoration-color:#ff8a3d80 }
|
|
201
|
+
.msg.ai.cut::after { content:'interrupted'; display:block; margin-top:5px;
|
|
202
|
+
font-size:10px; letter-spacing:.04em; color:#ff8a3d; opacity:.85 }
|
|
95
203
|
/* An echoed reply is not an answer — mark it so the transcript can't be read as a model talking. */
|
|
96
204
|
.msg.ai.echoed { border-left:2px solid #e0a94a }
|
|
97
205
|
.msg.ai.echoed::after { content:'echo — no model'; display:block; margin-top:5px;
|
|
@@ -109,9 +217,18 @@
|
|
|
109
217
|
border-radius:9px; padding:9px 12px; font:inherit }
|
|
110
218
|
input:focus, select:focus { outline:none; border-color:var(--c) }
|
|
111
219
|
button.btn { cursor:pointer } button.btn:hover { border-color:#3a424e }
|
|
220
|
+
/* iOS Safari auto-zooms the page when you focus a field whose text is under 16px, and it does NOT
|
|
221
|
+
zoom back out on blur — one tap on "type a turn" and the whole demo is left scaled up and
|
|
222
|
+
panned. 16px is the documented threshold, so meet it on touch devices (the 11px settings
|
|
223
|
+
labels stay small; only the FIELDS matter). */
|
|
224
|
+
@media (pointer: coarse) { input, select, textarea { font-size:16px } }
|
|
112
225
|
#typed { flex:1 }
|
|
113
226
|
details { width:100%; border-top:1px solid var(--line); padding-top:10px }
|
|
114
227
|
summary { cursor:pointer; color:var(--dim); font-size:13px; list-style:none }
|
|
228
|
+
/* A 13px line of text is a ~20px tap target; both platforms' guidelines say ~44px. Padding is
|
|
229
|
+
enough — these are the only touch controls on the page that aren't already finger-sized.
|
|
230
|
+
min-height pins the result at 44px instead of trusting 13px text to round up to it. */
|
|
231
|
+
@media (pointer: coarse) { summary { padding:11px 0; min-height:44px; box-sizing:border-box } }
|
|
115
232
|
summary::before { content:'▸ ' } details[open] summary::before { content:'▾ ' }
|
|
116
233
|
.grid { display:flex; flex-wrap:wrap; gap:10px; padding:12px 0 4px }
|
|
117
234
|
label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
|
|
@@ -126,9 +243,11 @@
|
|
|
126
243
|
</head>
|
|
127
244
|
<body>
|
|
128
245
|
<main>
|
|
129
|
-
<
|
|
130
|
-
<
|
|
131
|
-
|
|
246
|
+
<div id="orb">
|
|
247
|
+
<button id="mic" title="click to start / stop · click while it talks to interrupt">
|
|
248
|
+
<i></i><i></i><i></i><i></i><i></i><span id="glyph">🎙</span>
|
|
249
|
+
</button>
|
|
250
|
+
</div>
|
|
132
251
|
<div id="status">click to start · then just talk</div>
|
|
133
252
|
<div id="lat"></div>
|
|
134
253
|
<div id="pipeline"></div>
|
|
@@ -198,7 +317,7 @@ const log = (...a) => {
|
|
|
198
317
|
|
|
199
318
|
// ── transcript ──────────────────────────────────────────────────────────────
|
|
200
319
|
const chat = $('chat');
|
|
201
|
-
let interimEl = null, aiEl = null, vadOff = null;
|
|
320
|
+
let interimEl = null, aiEl = null, vadOff = null, spokenLen = 0;
|
|
202
321
|
const toolEls = new Map();
|
|
203
322
|
const bubble = (cls, text = '') => {
|
|
204
323
|
const el = document.createElement('div');
|
|
@@ -234,6 +353,15 @@ async function* echoLLM(history) {
|
|
|
234
353
|
// ── what is ACTUALLY running ────────────────────────────────────────────────
|
|
235
354
|
// A demo that hides which engine served a reply is worse than no demo: a browser-native
|
|
236
355
|
// recognizer looks identical to voiceloop's pipeline, and an echo stand-in looks identical
|
|
356
|
+
// iPadOS 13+ reports itself as "MacIntel", so the touch-point check is what separates an iPad
|
|
357
|
+
// from a desktop Mac — UA sniffing alone would let the iPad through unwarned.
|
|
358
|
+
const IS_IOS = /iPad|iPhone|iPod/.test(navigator.userAgent) ||
|
|
359
|
+
(navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1);
|
|
360
|
+
// Safari ships webkitSpeechRecognition but it goes deaf after an utterance without firing onend/onerror,
|
|
361
|
+
// and iOS ducks the speakers while it holds the mic. Say so here rather than let the demo look broken.
|
|
362
|
+
const IOS_WEBSPEECH_WARNING = '<span class="warn"><br>⚠️ on iOS this often stops hearing you after the ' +
|
|
363
|
+
'first reply, and quiets the speaker — pick a cloud STT provider in settings for a reliable loop</span>';
|
|
364
|
+
|
|
237
365
|
// to a model. Both get named on screen, always.
|
|
238
366
|
const describePipeline = (a) => {
|
|
239
367
|
const echo = $('mode').value === 'echo';
|
|
@@ -242,7 +370,7 @@ const describePipeline = (a) => {
|
|
|
242
370
|
// pipeline is running when it is exactly what gets bypassed.
|
|
243
371
|
const self = a?.stt ? !!a.stt.selfCapture : sttSelfCaptures($('sttProvider').value);
|
|
244
372
|
const stt = self
|
|
245
|
-
? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span
|
|
373
|
+
? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span>${IS_IOS ? IOS_WEBSPEECH_WARNING : ''}`
|
|
246
374
|
: `<b>${$('sttProvider').value}</b> via voiceloop's pipeline — Silero VAD → feed → commit`;
|
|
247
375
|
const llm = echo
|
|
248
376
|
? '<span class="warn"><b>echo</b> — no model: replies repeat your words back</span>'
|
|
@@ -253,6 +381,51 @@ const showPipeline = () => { $('pipeline').innerHTML = describePipeline(agent);
|
|
|
253
381
|
// Hoisted (function declaration) so the settings listeners above can call it before this point.
|
|
254
382
|
function onSettingChanged() { showPipeline(); prewarmIfPipelined(); }
|
|
255
383
|
|
|
384
|
+
// ── the orb's level ─────────────────────────────────────────────────────────
|
|
385
|
+
// One CSS var, --lvl (0..1), drives size/glow/rim/halo. Three sources feed it, in order of truth:
|
|
386
|
+
// 1. the agent's `level` event — real mic RMS, the only honest one (pipeline STT only);
|
|
387
|
+
// 2. while SPEAKING — the reply's spoken-char rate, so the orb pulses with its own voice
|
|
388
|
+
// (voiceloop plays TTS through an internal AEC sink we can't tap, and re-tapping the mic
|
|
389
|
+
// would just measure the echo it works to suppress);
|
|
390
|
+
// 3. a selfCapture STT (webspeech) emits no level at all — fall back to the VAD-style pulse.
|
|
391
|
+
// Every source decays: a level that only ever rises reads as a stuck meter.
|
|
392
|
+
// Sources are SPARSE (a level event every ~256ms capture chunk), so both the target's decay and the
|
|
393
|
+
// displayed value are functions of ELAPSED TIME, not of frame count: a per-frame multiplier would
|
|
394
|
+
// empty the meter between two events at 60fps and flicker, and would run at a different speed on a
|
|
395
|
+
// 144Hz display.
|
|
396
|
+
const orb = $('orb');
|
|
397
|
+
const stillMotion = matchMedia('(prefers-reduced-motion: reduce)');
|
|
398
|
+
const HALF_LIFE_MS = 260; // target sags to half this long after its last source sample
|
|
399
|
+
const ATTACK_MS = 60, RELEASE_MS = 320; // fast attack, slow release — how a meter reads as "alive"
|
|
400
|
+
const decayed = (v, ms, halfLife) => v * Math.pow(0.5, ms / halfLife);
|
|
401
|
+
let lvl = 0, lvlTarget = 0, lastFrame = 0, running = false;
|
|
402
|
+
const setLevel = (v) => {
|
|
403
|
+
if (!Number.isFinite(v) || stillMotion.matches) return; // a NaN would poison the meter for the page's lifetime
|
|
404
|
+
lvlTarget = Math.max(lvlTarget, Math.min(1, Math.max(0, v)));
|
|
405
|
+
if (!running) { running = true; lastFrame = 0; requestAnimationFrame(tickLevel); }
|
|
406
|
+
};
|
|
407
|
+
// Runs only while there is something to show: sources re-arm it, and it parks itself once the meter
|
|
408
|
+
// has settled at rest — an idle demo shouldn't be writing a style property 60 times a second.
|
|
409
|
+
function tickLevel(now) {
|
|
410
|
+
const dt = Math.min(100, now - (lastFrame || now)); lastFrame = now; // clamp: a backgrounded tab resumes with a huge gap
|
|
411
|
+
lvlTarget = decayed(lvlTarget, dt, HALF_LIFE_MS);
|
|
412
|
+
// Exponential approach toward the target, converging at the same rate regardless of frame rate.
|
|
413
|
+
lvl += (lvlTarget - lvl) * (1 - Math.exp(-dt / (lvlTarget > lvl ? ATTACK_MS : RELEASE_MS)));
|
|
414
|
+
if (lvl < 0.002 && lvlTarget < 0.002) { lvl = lvlTarget = 0; running = false; }
|
|
415
|
+
orb.style.setProperty('--lvl', lvl.toFixed(3));
|
|
416
|
+
if (running) requestAnimationFrame(tickLevel);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// One ring per turn boundary. Marks the two instants the transcript cannot show: your turn being
|
|
420
|
+
// committed, and a barge-in cutting the reply. Under reduced motion the ring is display:none, which
|
|
421
|
+
// never fires animationend — so don't create one at all rather than leak an element per turn.
|
|
422
|
+
const ripple = () => {
|
|
423
|
+
if (stillMotion.matches) return;
|
|
424
|
+
const u = document.createElement('u');
|
|
425
|
+
u.onanimationend = () => u.remove();
|
|
426
|
+
orb.append(u);
|
|
427
|
+
};
|
|
428
|
+
|
|
256
429
|
// A typed turn drives the same state machine without a mic — don't claim we're listening then.
|
|
257
430
|
let micOn = false;
|
|
258
431
|
const STATUS = {
|
|
@@ -261,10 +434,20 @@ const STATUS = {
|
|
|
261
434
|
thinking: () => 'thinking…',
|
|
262
435
|
speaking: () => micOn ? 'speaking — talk over it to interrupt' : 'speaking — click to interrupt',
|
|
263
436
|
};
|
|
437
|
+
// The first spoken reply blocks on a ~60MB Piper voice download (once — it's cached in OPFS after).
|
|
438
|
+
// On mobile data that is a minute during which the status line claimed "speaking", which reads as
|
|
439
|
+
// "the demo is broken". While the model is coming down, the download owns the status line.
|
|
440
|
+
let modelPct = null;
|
|
264
441
|
const setState = s => {
|
|
265
442
|
document.body.dataset.state = micOn || s !== 'listening' ? s : 'idle';
|
|
266
|
-
$('status').textContent = STATUS[s]?.() ?? s;
|
|
443
|
+
if (modelPct === null) $('status').textContent = STATUS[s]?.() ?? s;
|
|
267
444
|
};
|
|
445
|
+
tts.setOnModelProgress?.((loaded, total) => {
|
|
446
|
+
const done = total && loaded >= total;
|
|
447
|
+
modelPct = done ? null : total ? Math.floor((loaded / total) * 100) : 0;
|
|
448
|
+
$('status').textContent = done ? (STATUS[document.body.dataset.state]?.() ?? '')
|
|
449
|
+
: `downloading the voice — ${modelPct}% (${(loaded / 1048576).toFixed(0)}MB${total ? ` of ${(total / 1048576).toFixed(0)}MB` : ''}, once)`;
|
|
450
|
+
});
|
|
268
451
|
|
|
269
452
|
function onEvent(e) {
|
|
270
453
|
if (e.type !== 'stt' || e.turnComplete) log(e);
|
|
@@ -276,19 +459,22 @@ function onEvent(e) {
|
|
|
276
459
|
setState(e.state);
|
|
277
460
|
break;
|
|
278
461
|
case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
|
|
462
|
+
case 'level': setLevel(e.level); break;
|
|
279
463
|
case 'stt':
|
|
280
|
-
// A selfCapture provider runs no agent VAD, so no 'vad' event ever arrives and the orb
|
|
281
|
-
// sit dead still while the browser hears you perfectly — looking exactly like a broken
|
|
282
|
-
// Live partials are the only "I hear you" signal on that path, so
|
|
464
|
+
// A selfCapture provider runs no agent VAD, so no 'vad'/'level' event ever arrives and the orb
|
|
465
|
+
// would sit dead still while the browser hears you perfectly — looking exactly like a broken
|
|
466
|
+
// mic. Live partials are the only "I hear you" signal on that path, so pulse from them.
|
|
283
467
|
if (agent?.stt?.selfCapture) {
|
|
284
468
|
document.body.dataset.vad = e.turnComplete ? '0' : '1';
|
|
469
|
+
if (!e.turnComplete) setLevel(0.55);
|
|
285
470
|
clearTimeout(vadOff);
|
|
286
471
|
if (!e.turnComplete) vadOff = setTimeout(() => { document.body.dataset.vad = '0'; }, 700);
|
|
287
472
|
}
|
|
288
473
|
if (e.turnComplete) {
|
|
474
|
+
ripple(); // your turn closed — the instant the latency clock starts
|
|
289
475
|
interimEl?.remove(); interimEl = null;
|
|
290
476
|
bubble('user', e.text);
|
|
291
|
-
aiEl = null; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
|
|
477
|
+
aiEl = null; spokenLen = 0; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
|
|
292
478
|
} else {
|
|
293
479
|
interimEl ??= bubble('user interim');
|
|
294
480
|
interimEl.textContent = `${e.committed ?? ''} ${e.text}`.trim();
|
|
@@ -298,7 +484,7 @@ function onEvent(e) {
|
|
|
298
484
|
case 'assistant': {
|
|
299
485
|
// Empty and non-final means "that draft never happened" (interrupted before
|
|
300
486
|
// a word was spoken) — drop the bubble rather than blank it.
|
|
301
|
-
if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; break; }
|
|
487
|
+
if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; spokenLen = 0; break; }
|
|
302
488
|
if (!tText && t0) { tText = Date.now(); showLat(); }
|
|
303
489
|
// `full` only exists once the TTS progress cursor is live — i.e. a clip is
|
|
304
490
|
// actually playing. The 'speaking' state is just the first token, not sound.
|
|
@@ -306,9 +492,24 @@ function onEvent(e) {
|
|
|
306
492
|
// Tag the bubble (never the text) when no model produced it — the reply is a verbatim echo,
|
|
307
493
|
// and only the UI should say so.
|
|
308
494
|
aiEl ??= bubble($('mode').value === 'echo' ? 'ai echoed' : 'ai');
|
|
495
|
+
// The agent's own voice has no level event (TTS plays through an internal AEC sink), so drive
|
|
496
|
+
// the orb from how fast the spoken CURSOR advances — the orb rides its own speech. Only events
|
|
497
|
+
// carrying `full` have a cursor; a pre-audio streaming draft would otherwise run spokenLen up
|
|
498
|
+
// to the draft's length and the NEXT reply would sit flat until it grew past that stale mark.
|
|
499
|
+
if (e.full != null) {
|
|
500
|
+
setLevel(0.35 + Math.min(0.6, Math.max(0, e.text.length - spokenLen) * 0.09));
|
|
501
|
+
spokenLen = e.text.length;
|
|
502
|
+
}
|
|
309
503
|
aiEl.textContent = e.text; // spoken so far
|
|
310
504
|
const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
|
|
311
505
|
if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
|
|
506
|
+
// Cut short: the final says what was HEARD (text) and what was written (full) — a gap between
|
|
507
|
+
// them is a barge-in, the one thing the transcript would otherwise render as a short answer.
|
|
508
|
+
// (trim: a natural finish can leave trailing whitespace in `full` — not an interruption)
|
|
509
|
+
if (e.final) {
|
|
510
|
+
if (tail.trim()) { aiEl.classList.add('cut'); ripple(); }
|
|
511
|
+
spokenLen = 0;
|
|
512
|
+
}
|
|
312
513
|
aiEl.scrollIntoView({ block:'nearest' });
|
|
313
514
|
break;
|
|
314
515
|
}
|
|
@@ -404,6 +605,7 @@ navigator.mediaDevices?.enumerateDevices().then(ds => {
|
|
|
404
605
|
}).catch(() => {});
|
|
405
606
|
|
|
406
607
|
setState('idle');
|
|
608
|
+
onSettingChanged(); // the banner must describe the restored settings from the FIRST frame, not only after a click
|
|
407
609
|
window.agent = () => agent; // console poking: agent().history, …
|
|
408
610
|
</script>
|
|
409
611
|
</body>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"type": "git",
|
|
43
43
|
"url": "git+https://github.com/todoforai/voiceloop.git"
|
|
44
44
|
},
|
|
45
|
-
"homepage": "https://github.
|
|
45
|
+
"homepage": "https://todoforai.github.io/voiceloop/",
|
|
46
46
|
"bugs": {
|
|
47
47
|
"url": "https://github.com/todoforai/voiceloop/issues"
|
|
48
48
|
},
|
package/src/index.d.ts
CHANGED
|
@@ -7,12 +7,19 @@ export type VoiceAgentEvent =
|
|
|
7
7
|
| { type: 'stt'; turnComplete: boolean; text: string; ms: number; committed?: string } // text=live tail; committed=locked segments (Speechmatics AddTranscript); turnComplete=user turn ended → send to LLM
|
|
8
8
|
| { type: 'assistant'; text: string; final: boolean; full?: string } // text=spoken-so-far (solid, advances live as TTS plays); full=complete answer → tail full.slice(text.length) not-yet-spoken (dim)
|
|
9
9
|
| { type: 'vad'; active: boolean } // VAD hears speech?
|
|
10
|
+
// Mic loudness, 0..1 on a perceptual curve, ~every 256ms while capturing. Cosmetic only (meters,
|
|
11
|
+
// a reacting orb) — nothing in the pipeline reads it. Never emitted for a selfCapture STT
|
|
12
|
+
// (webspeech owns its own mic, so there are no frames to measure).
|
|
13
|
+
| { type: 'level'; level: number }
|
|
10
14
|
// running:true = loop call announced, executing now (display-only spinner; no result yet).
|
|
11
15
|
// The outcome event follows with the same `id` and replaces it in place.
|
|
12
16
|
| { type: 'tool'; name: string; args: Record<string, unknown>; result?: unknown; id?: string; running?: true }
|
|
13
17
|
| { type: 'echo'; text: string } // dropped self-echo (the agent's own TTS leaked into the mic) — diagnostics only
|
|
14
18
|
| { type: 'diag'; diag: 'aec-fallback'; message: string } // once/session: AEC loopback failed → TTS plays un-cancelled (self-echo only text-filtered)
|
|
15
19
|
| { type: 'diag'; diag: 'presynth'; text: string } // a reply's first clip was synthesized speculatively (latency diagnostics)
|
|
20
|
+
// Did that speculation pay off? hit = the real reply opened with the presynthesized text (clip
|
|
21
|
+
// reused, latency saved); miss = it didn't (clip discarded). `pre` is what we had guessed.
|
|
22
|
+
| { type: 'diag'; diag: 'presynth-hit' | 'presynth-miss'; text: string; pre: string }
|
|
16
23
|
| { type: 'error'; error: string };
|
|
17
24
|
|
|
18
25
|
export interface VoiceTool {
|
|
@@ -34,21 +41,31 @@ export type LLMChunk =
|
|
|
34
41
|
| { text: string }
|
|
35
42
|
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
|
|
36
43
|
// Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
|
|
37
|
-
// The default (
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
// The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; the results are not
|
|
45
|
+
// fed back into it mid-turn, so such calls are terminal for the turn. That is the low-level seam.
|
|
46
|
+
// An agent LOOP that runs tools inside its own generator declares the two flags below.
|
|
47
|
+
export interface LLM {
|
|
48
|
+
(
|
|
49
|
+
history: Array<{ role: string; content: string }>,
|
|
50
|
+
system: string,
|
|
51
|
+
signal: AbortSignal,
|
|
52
|
+
options?: { toolGate?: Promise<unknown> },
|
|
53
|
+
): AsyncIterable<LLMChunk>;
|
|
54
|
+
/** This generator ran the tool itself: chunks carry `result`/`error` and the agent REPORTS them
|
|
55
|
+
* instead of executing again. Without it, an outcome-bearing chunk is re-run. */
|
|
56
|
+
executesTools?: boolean;
|
|
57
|
+
/** This generator awaits `options.toolGate` before ANY tool executes. Required to keep
|
|
58
|
+
* speculative prefetch on: prefetch starts from an uncommitted interim, and the gate is what
|
|
59
|
+
* stops a speculation from firing a real side effect. `executesTools` without this silently
|
|
60
|
+
* disables prefetch (see voice-agent.js:793). */
|
|
61
|
+
acceptsToolGate?: boolean;
|
|
62
|
+
}
|
|
49
63
|
|
|
50
64
|
// Pluggable TTS contract. `speak` resolves with the text actually HEARD (full on completion,
|
|
51
65
|
// a prefix on barge-in). The optional members back the voice picker / switching.
|
|
66
|
+
/** (spokenSoFar, scope) — see VoiceTTS.setOnProgress. */
|
|
67
|
+
export type TtsProgress = (spokenText: string, scope: string) => void;
|
|
68
|
+
|
|
52
69
|
export interface VoiceTTS {
|
|
53
70
|
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
54
71
|
stop?(): void;
|
|
@@ -58,8 +75,10 @@ export interface VoiceTTS {
|
|
|
58
75
|
setVoice?(voiceId: string): void;
|
|
59
76
|
setSpeed?(speed: number): void;
|
|
60
77
|
/** Register a callback fired while a clip plays with the chars spoken SO FAR across the whole
|
|
61
|
-
* reply — drives the host's live spoken cursor. Pass null to clear.
|
|
62
|
-
|
|
78
|
+
* reply — drives the host's live spoken cursor. Pass null to clear.
|
|
79
|
+
* `scope` is that prefix PLUS the whole clip now playing: the cursor lags real audio, so echo
|
|
80
|
+
* suppression judges against this upper bound rather than the spoken text. */
|
|
81
|
+
setOnProgress?(fn: TtsProgress | null): void;
|
|
63
82
|
}
|
|
64
83
|
|
|
65
84
|
export interface VoiceAgentOptions {
|
|
@@ -70,7 +89,8 @@ export interface VoiceAgentOptions {
|
|
|
70
89
|
/** LLM model id sent to the backend `/llm` route; empty → the backend's VOICE_MODEL default. */
|
|
71
90
|
model?: string;
|
|
72
91
|
llm?: LLM;
|
|
73
|
-
/**
|
|
92
|
+
/** Transport seam for the built-in LLM adapter; defaults to global fetch. For tests and hosts
|
|
93
|
+
* that must wrap every request (auth refresh, tracing). Unused when you pass your own `llm`. */
|
|
74
94
|
fetchFn?: typeof fetch;
|
|
75
95
|
apiKey?: string;
|
|
76
96
|
/** Full endpoint URLs, not a base: voiceloop composes no routes, so a host can point the LLM and
|
|
@@ -99,8 +119,15 @@ export interface VoiceAgentOptions {
|
|
|
99
119
|
/** Mint a short-TTL STT token yourself, for auth that doesn't fit one POST route. Called with the
|
|
100
120
|
* provider actually RUNNING (post-downgrade), never the requested one. Wins over sttTokenUrl. */
|
|
101
121
|
getSttToken?: (provider: SttProvider) => Promise<SttToken>;
|
|
122
|
+
/** POST route that receives metered STT seconds: `{ seconds, provider, ... }`, batched and
|
|
123
|
+
* fire-and-forget (failures are swallowed — billing telemetry never breaks the pipeline).
|
|
124
|
+
* Omit it and nothing is reported. Cloud providers only; the browser's own STT is free. */
|
|
102
125
|
sttUsageUrl?: string;
|
|
126
|
+
/** Override the provider's realtime WS endpoint (self-hosted/regional gateways). Default: the
|
|
127
|
+
* provider's public URL. */
|
|
103
128
|
sttUrl?: string;
|
|
129
|
+
/** Provider STT model id (e.g. Deepgram's flux variants); empty → the provider's default.
|
|
130
|
+
* Dropped when the provider downgrades, since a model id is meaningless across providers. */
|
|
104
131
|
sttModel?: string;
|
|
105
132
|
/** Deepgram Flux end-of-turn confidence (0.5–0.9) — its model closes the turn itself once its
|
|
106
133
|
* confidence the user is done exceeds this threshold. Lower = snappier turn ends, higher = more
|
|
@@ -109,8 +136,13 @@ export interface VoiceAgentOptions {
|
|
|
109
136
|
/** Every tool the model may call. voiceloop ships NO built-in tools — a voice library has no
|
|
110
137
|
* business knowing what a "todo" is — so product tools (and their `run`) are supplied here. */
|
|
111
138
|
tools?: Record<string, VoiceTool>;
|
|
139
|
+
/** Domain words to bias cloud STT towards ("Kubernetes", product names) — the fix for jargon
|
|
140
|
+
* transcribed phonetically. Silently trimmed to 50 terms of ≤20 chars. Cloud providers only. */
|
|
112
141
|
keyterms?: string[];
|
|
142
|
+
/** Audio chunks retained BEFORE the VAD fires, prepended to the utterance so the first syllable
|
|
143
|
+
* isn't clipped (VAD confirms speech a beat after it starts). Default TUNING.PREROLL_CHUNKS. */
|
|
113
144
|
preroll?: number;
|
|
145
|
+
/** Passed through to Silero VAD (thresholds, frame sizes) — see tuning.js for the defaults. */
|
|
114
146
|
vadOptions?: Record<string, unknown>;
|
|
115
147
|
/** Semantic end-of-turn hook run when VAD hears end-of-speech: return false to KEEP listening (the
|
|
116
148
|
* user only paused mid-thought), true (default, when omitted) to close the turn. Lets you swap the
|
|
@@ -279,14 +311,18 @@ export function resolveSttProvider(id: string): SttProvider;
|
|
|
279
311
|
// ── TTS ──────────────────────────────────────────────────────────────────────────────────────
|
|
280
312
|
/** Base class for streaming TTS engines: sentence-splits the incoming token stream, synthesizes
|
|
281
313
|
* the next clip while the current one plays, and reports the spoken cursor via setOnProgress.
|
|
282
|
-
* Subclass and implement `_synth
|
|
314
|
+
* Subclass and implement `_synth` to add an engine. */
|
|
283
315
|
export class StreamingTTS implements VoiceTTS {
|
|
284
316
|
constructor(voiceId?: string, speed?: number);
|
|
317
|
+
/** THE subclass contract: synthesize one sentence → audio Blob (null when aborted/empty).
|
|
318
|
+
* Honour `signal` — a barge-in aborts mid-clip and the base class won't wait for you. */
|
|
319
|
+
protected _synth(text: string, signal?: AbortSignal): Promise<Blob | null>;
|
|
285
320
|
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
286
321
|
stop(): void;
|
|
287
|
-
setOnProgress(fn:
|
|
288
|
-
/** Jump playback to a
|
|
289
|
-
* honoured (nothing playing, or past the text
|
|
322
|
+
setOnProgress(fn: TtsProgress | null): void;
|
|
323
|
+
/** Jump playback to a position in the current reply (non-whitespace char count, as everywhere
|
|
324
|
+
* else). Returns false when the seek can't be honoured (nothing playing, or past the text
|
|
325
|
+
* streamed so far) — check it before moving a cursor. */
|
|
290
326
|
seek(nwIndex: number): boolean;
|
|
291
327
|
}
|
|
292
328
|
/** Human-grade cloud voices. Pass `ttsUrl` (your backend proxy, key stays server-side); `apiKey`
|
package/src/stt.js
CHANGED
|
@@ -32,6 +32,16 @@ import { TUNING } from './tuning.js';
|
|
|
32
32
|
|
|
33
33
|
const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at least this often
|
|
34
34
|
|
|
35
|
+
// Cloud STT is a browser WebSocket. Resolved through globalThis at call time, not as a bare
|
|
36
|
+
// identifier: this module is imported under SSR and under node (tests, node <22), where bare
|
|
37
|
+
// `WebSocket` is a ReferenceError instead of undefined. A missing WS is a clear provider error —
|
|
38
|
+
// not a stack trace from inside open().
|
|
39
|
+
const newSocket = (url, protocols) => {
|
|
40
|
+
const WS = globalThis.WebSocket;
|
|
41
|
+
if (!WS) throw new Error('cloud STT needs WebSocket (unavailable here — browser or node >=22)');
|
|
42
|
+
return protocols ? new WS(url, protocols) : new WS(url);
|
|
43
|
+
};
|
|
44
|
+
|
|
35
45
|
// Common opts shape (provided by the VoiceAgent):
|
|
36
46
|
// { apiKey, sttUrl, sttModel, sttLang, keyterms, sttTokenUrl, getToken, sttUsageUrl,
|
|
37
47
|
// onPartial, onFinal, onError, onClose, isClosed }
|
|
@@ -145,7 +155,7 @@ export function makeElevenLabsSTT(opts) {
|
|
|
145
155
|
if (sttLang) p.set('language_code', sttLang);
|
|
146
156
|
for (const k of keyterms) p.append('keyterms', k);
|
|
147
157
|
sttStart = performance.now();
|
|
148
|
-
ws =
|
|
158
|
+
ws = newSocket(`${sttUrl}?${p}`);
|
|
149
159
|
ws.onopen = () => { const q = outbox; outbox = []; for (const it of q) it === 'commit' ? sendCommit() : sendAudio(it); };
|
|
150
160
|
ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('STT ws error', ev); } };
|
|
151
161
|
ws.onclose = ev => { usage.flush(); onClose?.(); if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); console.error('STT ws closed', ev.code, ev.reason); } };
|
|
@@ -156,6 +166,11 @@ export function makeElevenLabsSTT(opts) {
|
|
|
156
166
|
if (m.message_type === 'partial_transcript') onPartial(m.text || '', ms);
|
|
157
167
|
else if (m.message_type?.startsWith('committed') && m.text) onFinal(m.text, ms);
|
|
158
168
|
};
|
|
169
|
+
} catch (e) {
|
|
170
|
+
// open() is called fire-and-forget (pre-open, and lazily from feed()), so anything thrown here
|
|
171
|
+
// would surface as an unhandled rejection and kill the process under node. A socket that can't
|
|
172
|
+
// be built is unrecoverable for this provider — report it as fatal and let the host stop.
|
|
173
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
159
174
|
} finally {
|
|
160
175
|
opening = false;
|
|
161
176
|
}
|
|
@@ -284,7 +299,7 @@ export function makeSpeechmaticsSTT(opts) {
|
|
|
284
299
|
if (!body) { outbox = []; sentSinceCommit = false; return; }
|
|
285
300
|
if (isClosed()) return;
|
|
286
301
|
sttStart = performance.now();
|
|
287
|
-
ws =
|
|
302
|
+
ws = newSocket(`${SM_URL}?jwt=${encodeURIComponent(body.token)}`);
|
|
288
303
|
ws.binaryType = 'arraybuffer';
|
|
289
304
|
ws.onopen = () => {
|
|
290
305
|
// Declare the audio format + transcription config, then flush buffered audio once started.
|
|
@@ -376,6 +391,11 @@ export function makeSpeechmaticsSTT(opts) {
|
|
|
376
391
|
case 'Error': onError(`${m.type || 'error'}: ${m.reason || ''}`); break;
|
|
377
392
|
}
|
|
378
393
|
};
|
|
394
|
+
} catch (e) {
|
|
395
|
+
// open() is called fire-and-forget (pre-open, and lazily from feed()), so anything thrown here
|
|
396
|
+
// would surface as an unhandled rejection and kill the process under node. A socket that can't
|
|
397
|
+
// be built is unrecoverable for this provider — report it as fatal and let the host stop.
|
|
398
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
379
399
|
} finally {
|
|
380
400
|
opening = false;
|
|
381
401
|
}
|
|
@@ -506,7 +526,7 @@ export function makeDeepgramSTT(opts) {
|
|
|
506
526
|
sttStart = performance.now();
|
|
507
527
|
// Browser WS auth: Sec-WebSocket-Protocol "bearer, <token>" (short-TTL access token from
|
|
508
528
|
// /v1/auth/grant). The token only needs to be valid at handshake time; the socket persists.
|
|
509
|
-
ws =
|
|
529
|
+
ws = newSocket(`${DG_URL}?${p}`, ['bearer', token]);
|
|
510
530
|
ws.binaryType = 'arraybuffer';
|
|
511
531
|
ws.onopen = () => { const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c); };
|
|
512
532
|
ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('DG ws error', ev); } };
|
|
@@ -535,6 +555,11 @@ export function makeDeepgramSTT(opts) {
|
|
|
535
555
|
}
|
|
536
556
|
} else if (m.type === 'Error') { onError(`${m.code || 'error'}: ${m.description || ''}`); console.error('DG error msg', m); }
|
|
537
557
|
};
|
|
558
|
+
} catch (e) {
|
|
559
|
+
// open() is called fire-and-forget (pre-open, and lazily from feed()), so anything thrown here
|
|
560
|
+
// would surface as an unhandled rejection and kill the process under node. A socket that can't
|
|
561
|
+
// be built is unrecoverable for this provider — report it as fatal and let the host stop.
|
|
562
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
538
563
|
} finally {
|
|
539
564
|
opening = false;
|
|
540
565
|
}
|
|
@@ -543,6 +568,10 @@ export function makeDeepgramSTT(opts) {
|
|
|
543
568
|
return {
|
|
544
569
|
continuous: true, // agent feeds the full mic stream (Flux needs real silences for its EOT model)
|
|
545
570
|
nativeEOT: true, // agent skips VAD-commit; EndOfTurn → unsolicited onFinal closes the turn
|
|
571
|
+
// Flux ends turns SEMANTICALLY, right behind its own last interim (21ms median, measured) —
|
|
572
|
+
// there is no trailing-silence debounce to hide a stability wait in, so the default 200ms timer
|
|
573
|
+
// never fires and every turn pays full LLM TTFT. Speculate on the interim tick instead.
|
|
574
|
+
prefetchMs: 0,
|
|
546
575
|
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
547
576
|
feed(i16) {
|
|
548
577
|
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
@@ -567,7 +596,9 @@ export function makeDeepgramSTT(opts) {
|
|
|
567
596
|
};
|
|
568
597
|
}
|
|
569
598
|
|
|
570
|
-
// ── Browser Web Speech API (
|
|
599
|
+
// ── Browser Web Speech API (browser-managed, no token) ───────────────────────
|
|
600
|
+
// "Browser-native" is not "on-device": Chrome may route recognition through a vendor speech
|
|
601
|
+
// service. The win here is zero setup (no key, no token endpoint), not guaranteed privacy.
|
|
571
602
|
// The browser's own SpeechRecognition engine (Chrome/Safari `webkitSpeechRecognition`). It OWNS the
|
|
572
603
|
// whole audio path — its own mic capture AND its own silence-based end-of-turn — so the agent builds
|
|
573
604
|
// NO getUserMedia / AudioWorklet / Silero-VAD pipeline for it (`selfCapture: true`, which the agent
|
|
@@ -576,7 +607,7 @@ export function makeDeepgramSTT(opts) {
|
|
|
576
607
|
// interim results → onPartial(tail) live draft + word-based barge-in while the agent is speaking
|
|
577
608
|
// final results → accumulate + (re)arm the end-of-turn timer; if the user keeps talking within
|
|
578
609
|
// the debounce window the fragments merge, else the timer flushes the turn
|
|
579
|
-
// No backend token, no /stt/usage reporting (free
|
|
610
|
+
// No backend token, no /stt/usage reporting (free, zero-setup), no socket. Only where the API exists
|
|
580
611
|
// (else open() reports a fatal so the host can fall back / surface it).
|
|
581
612
|
// Browser Web Speech API STT. The agent hands it nothing (selfCapture) — it owns its mic and its own
|
|
582
613
|
// end-of-turn (nativeEOT). It reports transcripts via onPartial (live draft, drives the user bubble +
|
|
@@ -635,9 +666,12 @@ export function makeWebSpeechSTT(opts) {
|
|
|
635
666
|
let micPending = null; // the single in-flight acquisition — concurrent ensureTrack calls share it
|
|
636
667
|
const acquireMic = () => {
|
|
637
668
|
if (micPending) return micPending;
|
|
638
|
-
|
|
669
|
+
// globalThis.navigator, not bare `navigator`: this module is imported under SSR and under node
|
|
670
|
+
// (tests), where the bare identifier is a ReferenceError rather than undefined — and this path
|
|
671
|
+
// is already written to degrade to the recognizer's own capture when there's no getUserMedia.
|
|
672
|
+
if (micTrack?.readyState === 'live' || !trackArg || !globalThis.navigator?.mediaDevices?.getUserMedia) return Promise.resolve();
|
|
639
673
|
const gen = micGen;
|
|
640
|
-
micPending = navigator.mediaDevices.getUserMedia({ audio: {
|
|
674
|
+
micPending = globalThis.navigator.mediaDevices.getUserMedia({ audio: {
|
|
641
675
|
echoCancellation: true, noiseSuppression: true, autoGainControl: true,
|
|
642
676
|
...(micDeviceId ? { deviceId: { exact: micDeviceId } } : {}),
|
|
643
677
|
} }).then((s) => {
|
package/src/tuning.js
CHANGED
|
@@ -82,12 +82,22 @@ export const TUNING = {
|
|
|
82
82
|
// a dead speech service costs a handful of tries over a few seconds instead of a spin loop.
|
|
83
83
|
WEBSPEECH_ERROR_BACKOFF_MS: 300,
|
|
84
84
|
|
|
85
|
+
// Delay before pre-warming the TTS engine after start() (download+compile the voice model, JIT the
|
|
86
|
+
// WASM inference path). Deferred rather than immediate: Piper's ~60MB model download saturates
|
|
87
|
+
// bandwidth and was measured stretching the Deepgram WS handshake from ~0.5s to ~3s when fired
|
|
88
|
+
// concurrently. TTS isn't needed until the first REPLY (a user utterance + LLM roundtrip away), so
|
|
89
|
+
// waiting costs nothing — while NOT warming at all costs the first reply Piper's full cold start
|
|
90
|
+
// (measured 5.4–10.2s to first audio, vs 637ms warmed).
|
|
91
|
+
TTS_WARM_DELAY_MS: 2500,
|
|
92
|
+
|
|
85
93
|
// Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
|
|
86
94
|
// agent starts generating a reply for the probable final text — overlapping the LLM's first-token
|
|
87
95
|
// latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
|
|
88
96
|
// (instant first token); a changed text just aborts it (cost: a few prompt tokens). Must be
|
|
89
97
|
// comfortably below WEBSPEECH_EOT_MS for the overlap to pay. 0 = prefetch on every interim tick
|
|
90
98
|
// (most aggressive, most wasted calls).
|
|
99
|
+
// Providers whose end-of-turn leaves no room for this wait override it with their own
|
|
100
|
+
// `prefetchMs` (deepgram flux: 0 — see stt.js).
|
|
91
101
|
PREFETCH_MS: 200,
|
|
92
102
|
|
|
93
103
|
// Audio chunks (~256ms each at 16kHz/4096-sample frames; ~1s total at 4) kept BEFORE VAD fires and
|
|
@@ -138,6 +148,13 @@ export const TUNING = {
|
|
|
138
148
|
redemptionFrames: 6,
|
|
139
149
|
},
|
|
140
150
|
|
|
151
|
+
// ── Mic level (host visualisers) ──────────────────────────────────────────────────────────────
|
|
152
|
+
// The 'level' event: a 0..1 loudness of the mic, emitted off the frames capture already produces.
|
|
153
|
+
// Purely cosmetic — nothing in the pipeline reads it — so it's tuned for how a meter LOOKS.
|
|
154
|
+
LEVEL_MIN_MS: 45, // rate limit; capture chunks are ~256ms, so this never throttles today but bounds a finer-grained capture
|
|
155
|
+
LEVEL_NOISE_FLOOR_RMS: 0.004, // below this the meter reads 0 — a quiet room must look silent, not "restless"
|
|
156
|
+
LEVEL_FULL_SCALE_RMS: 0.28, // RMS that reads as "full" — normal close speech peaks near 0.1-0.2, so this leaves headroom without pinning at 1
|
|
157
|
+
|
|
141
158
|
// ── TTS (first-audio latency) ─────────────────────────────────────────────────────────────────
|
|
142
159
|
// How soon the FIRST spoken chunk is cut from the streaming LLM text. Lower = the agent starts talking
|
|
143
160
|
// sooner (less waiting on generation), at a small prosody cost (it may break on a comma/clause rather
|
package/src/voice-agent.js
CHANGED
|
@@ -241,7 +241,7 @@ export class VoiceAgent {
|
|
|
241
241
|
// also bounds a turnDetector that never returns at all.
|
|
242
242
|
// `bargeInMinChars` — min transcribed NOVEL chars heard while speaking to count as a real
|
|
243
243
|
// barge-in; higher ignores short backchannels ("mhm","yeah"). Defaults live in tuning.js.
|
|
244
|
-
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'echo'|'error'|'diag', … }.
|
|
244
|
+
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'level'|'echo'|'error'|'diag', … }.
|
|
245
245
|
constructor({ sysmsg = '', persona, model = '', llm, fetchFn, apiKey = '', llmUrl = '', maxTokens = TUNING.MAX_TOKENS,
|
|
246
246
|
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
247
247
|
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
@@ -314,10 +314,18 @@ export class VoiceAgent {
|
|
|
314
314
|
// stable for PREFETCH_MS (no new words), start generating NOW with the probable final text;
|
|
315
315
|
// if the turn closes with that exact text the reply streams instantly (see _takePrefetch),
|
|
316
316
|
// if the user kept talking the stale request is aborted (cost: a few prompt tokens).
|
|
317
|
+
// The stability wait must fit INSIDE the gap between the last interim and the commit, or the
|
|
318
|
+
// timer never fires and the turn pays the LLM's full TTFT — silently, since a speculation
|
|
319
|
+
// that never STARTED looks identical to one that was never worth starting. Providers that
|
|
320
|
+
// debounce leave room for it (webspeech: ~1.2s of trailing silence, measured); a model that
|
|
321
|
+
// decides end-of-turn semantically does not (deepgram flux: 21ms median, measured) and must
|
|
322
|
+
// speculate on the interim tick instead. Providers set `prefetchMs` when 200ms won't land.
|
|
317
323
|
clearTimeout(this._prefetchTimer); this._prefetchTimer = null;
|
|
318
324
|
if (this._prefetch && interim && this._prefetch.text !== interim) this._dropPrefetch(); // transcript moved on → the running speculation is stale, stop paying for it
|
|
319
|
-
if (interim && this.state === 'listening' && !this._held && !this._echoRef)
|
|
320
|
-
|
|
325
|
+
if (interim && this.state === 'listening' && !this._held && !this._echoRef) {
|
|
326
|
+
const delay = this.stt.prefetchMs ?? TUNING.PREFETCH_MS ?? 200;
|
|
327
|
+
this._prefetchTimer = setTimeout(() => this._startPrefetch(interim), delay);
|
|
328
|
+
}
|
|
321
329
|
},
|
|
322
330
|
onFinal: (text, ms) => {
|
|
323
331
|
clearTimeout(this._prefetchTimer); this._prefetchTimer = null; // the turn is closing — no new speculation may start behind it
|
|
@@ -371,6 +379,7 @@ export class VoiceAgent {
|
|
|
371
379
|
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
372
380
|
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
373
381
|
this._tap = []; this._tapSamples = 0; this._tapTotal = 0; this._tapLastMs = 0; this._tapGaps = [];
|
|
382
|
+
this._levelAt = 0; // last 'level' emit (rate limit) — see _emitLevel
|
|
374
383
|
this._stream = null; this._ctx = null; this._node = null; this._vad = null; this._streamDeviceId = undefined; // pipeline kept warm across stop()/start() for instant resume
|
|
375
384
|
this._muted = false; // mic disabled → browser feeds silence to VAD+STT (nothing reaches the agent)
|
|
376
385
|
this._ttsMuted = false; // AI voice OUTPUT off → skip speaking, still stream the reply as text
|
|
@@ -407,7 +416,17 @@ export class VoiceAgent {
|
|
|
407
416
|
// handled by setMuted() → stt.setEnabled(). No warm-resume machinery: open() is idempotent + cheap.
|
|
408
417
|
// open() can fail synchronously (Web Speech unsupported → onFatal → stop() sets _closed + idle).
|
|
409
418
|
// Guard the listening flip so a fatal isn't overwritten with a fake 'listening' state.
|
|
410
|
-
|
|
419
|
+
// TTS still needs warming though: it's the one cold start a selfCapture STT does NOT own. Same
|
|
420
|
+
// deferral as the pipeline path below — the first reply is a user utterance + LLM roundtrip away,
|
|
421
|
+
// so a late start costs nothing, while skipping it entirely made turn 0 pay Piper's full cold
|
|
422
|
+
// start (measured 5.4–10.2s to first audio, vs 637ms on the warmed pipeline path).
|
|
423
|
+
if (this.stt.selfCapture) {
|
|
424
|
+
this.stt.open?.();
|
|
425
|
+
if (!this._closed) this._set('listening');
|
|
426
|
+
const gen = ++this._pipelineGen;
|
|
427
|
+
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
411
430
|
// WARM RESUME: stop() pauses (not tears down) the pipeline, so a resume of the SAME agent finds the
|
|
412
431
|
// mic stream + audio graph + VAD model still alive — just re-arm them. This skips the getUserMedia
|
|
413
432
|
// permission round-trip AND the MicVAD.new() Silero/ONNX cold start (the bulk of "open→listening"
|
|
@@ -486,7 +505,7 @@ export class VoiceAgent {
|
|
|
486
505
|
await ctx.audioWorklet.addModule(CAPTURE_WORKLET_URL);
|
|
487
506
|
if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); ctx.close(); return; } // superseded/stopped while loading
|
|
488
507
|
const node = new AudioWorkletNode(ctx, 'capture-processor');
|
|
489
|
-
node.port.onmessage = e => this.
|
|
508
|
+
node.port.onmessage = e => this._onCapture(e.data); // Float32Array chunk from the audio thread
|
|
490
509
|
// The processor writes no output — this connection only keeps the node pulled by the graph.
|
|
491
510
|
src.connect(node); node.connect(ctx.destination);
|
|
492
511
|
this._ctx = ctx; this._node = node;
|
|
@@ -509,11 +528,9 @@ export class VoiceAgent {
|
|
|
509
528
|
this._startVad(gen).then(() => { if (gen === this._pipelineGen && !this._closed) this._swapToSileroWhenReady(); })
|
|
510
529
|
.catch(() => { if (gen === this._pipelineGen) this._sileroPending = false; });
|
|
511
530
|
// Warm the TTS engine (Piper: download+compile the ONNX voice model, JIT the WASM path) so the
|
|
512
|
-
// first reply isn't stalled by cold start mid-turn —
|
|
513
|
-
//
|
|
514
|
-
|
|
515
|
-
// first REPLY (user utterance + LLM roundtrip away), so a short delay costs nothing.
|
|
516
|
-
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, 2500);
|
|
531
|
+
// first reply isn't stalled by cold start mid-turn — deferred off the STT connect window (see
|
|
532
|
+
// TTS_WARM_DELAY_MS).
|
|
533
|
+
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
|
|
517
534
|
})();
|
|
518
535
|
}
|
|
519
536
|
|
|
@@ -826,7 +843,16 @@ export class VoiceAgent {
|
|
|
826
843
|
_takePrefetch(text) {
|
|
827
844
|
const p = this._prefetch;
|
|
828
845
|
// histLen is checked BEFORE _runTurn pushes this turn's own user message (see call site).
|
|
829
|
-
if (!p || p.text !== text || p.ctl.signal.aborted || p.histLen !== this.history.length) {
|
|
846
|
+
if (!p || p.text !== text || p.ctl.signal.aborted || p.histLen !== this.history.length) {
|
|
847
|
+
// Which guard rejected: adoption is worth ~600ms (the whole LLM TTFT lands on the critical
|
|
848
|
+
// path without it), so a silent miss is invisible latency. `text` mismatch is the expected
|
|
849
|
+
// one (the user kept talking); the others mean the speculation was raced or aborted.
|
|
850
|
+
this.onEvent?.({ type: 'diag', diag: 'prefetch-miss',
|
|
851
|
+
why: !p ? 'none' : p.text !== text ? 'text' : p.ctl.signal.aborted ? 'aborted' : 'histLen',
|
|
852
|
+
text, pre: p?.text });
|
|
853
|
+
this._dropPrefetch(); return null;
|
|
854
|
+
}
|
|
855
|
+
this.onEvent?.({ type: 'diag', diag: 'prefetch-hit', text });
|
|
830
856
|
this._prefetch = null; // timer already cleared: successful adoption is only reached from a spoken onFinal
|
|
831
857
|
p.commit(); // turn committed → open the tool gate; held-back tool calls execute now
|
|
832
858
|
return { ctl: p.ctl, gen: (async function* () { const f = await p.first; if (!f.done) { yield f.value; yield* p.gen; } })() };
|
|
@@ -1231,6 +1257,37 @@ export class VoiceAgent {
|
|
|
1231
1257
|
this.stt.feed(i16);
|
|
1232
1258
|
}
|
|
1233
1259
|
|
|
1260
|
+
// The capture entry point: everything the pipeline needs, THEN the cosmetic tap. _feed has several
|
|
1261
|
+
// early returns (preroll while silent, continuous providers), so the level can't just live at its
|
|
1262
|
+
// end — and it must not live at the START either, where a host that throws while painting its
|
|
1263
|
+
// meter would take down the frame STT was waiting for.
|
|
1264
|
+
_onCapture(f32) { this._feed(f32); this._emitLevel(f32); }
|
|
1265
|
+
|
|
1266
|
+
// ── mic level ──────────────────────────────────────────────────────────
|
|
1267
|
+
// How loud the user is RIGHT NOW, for host visualisers (a mic meter, a reacting orb). Derived from
|
|
1268
|
+
// the frames already in hand — no second AudioContext, no AnalyserNode, no extra mic tap.
|
|
1269
|
+
// Purely cosmetic, so it is emitted only for a session the host would call LIVE: stop() pauses the
|
|
1270
|
+
// pipeline without closing the mic (warm resume) and the worklet keeps delivering frames, so
|
|
1271
|
+
// without the _closed guard a "stopped" agent would keep driving the host's meter; a muted mic
|
|
1272
|
+
// would likewise stream a flat zero forever instead of going quiet.
|
|
1273
|
+
// Rate-limited to ~LEVEL_MIN_MS so a capture finer than the current 4096-sample (256ms @16k)
|
|
1274
|
+
// chunk can't flood the host. A selfCapture STT (Web Speech) owns its own mic — no frames, no
|
|
1275
|
+
// level events at all.
|
|
1276
|
+
_emitLevel(f32) {
|
|
1277
|
+
if (this._closed || this._muted) return;
|
|
1278
|
+
const now = (globalThis.performance ?? Date).now();
|
|
1279
|
+
if (now - this._levelAt < TUNING.LEVEL_MIN_MS) return;
|
|
1280
|
+
this._levelAt = now;
|
|
1281
|
+
let sum = 0; for (let i = 0; i < f32.length; i++) sum += f32[i] * f32[i];
|
|
1282
|
+
const rms = Math.sqrt(sum / f32.length);
|
|
1283
|
+
// Perceptual curve, floored: linear RMS spends most of its range on levels nobody can hear, so a
|
|
1284
|
+
// meter driven by it barely moves during speech. The floor is what keeps room noise reading as
|
|
1285
|
+
// silence — without it a cube root maps a near-silent room to ~0.15-0.3 and the meter never rests.
|
|
1286
|
+
const norm = (rms - TUNING.LEVEL_NOISE_FLOOR_RMS) / (TUNING.LEVEL_FULL_SCALE_RMS - TUNING.LEVEL_NOISE_FLOOR_RMS);
|
|
1287
|
+
const level = norm <= 0 ? 0 : Math.min(1, Math.cbrt(norm));
|
|
1288
|
+
this.onEvent({ type: 'level', level });
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1234
1291
|
// ── diagnostic tap ─────────────────────────────────────────────────────
|
|
1235
1292
|
// Ring-buffer the frame + detect delivery stalls: a wall-clock gap between _feed() calls beyond
|
|
1236
1293
|
// ~1.5 chunks. With the worklet capturing off-thread that's usually DELAYED delivery rather than
|
|
@@ -1863,13 +1920,23 @@ const ORT_SPECIFIER = 'onnxruntime-web';
|
|
|
1863
1920
|
|
|
1864
1921
|
// OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
|
|
1865
1922
|
// models already downloaded through the lib are reused (and vice versa).
|
|
1866
|
-
|
|
1923
|
+
// `onProgress(loaded, total)` fires while the file is coming down the wire (total is 0 if the server
|
|
1924
|
+
// sends no content-length). A Piper voice is ~60MB: on mobile data that is a minute of apparent
|
|
1925
|
+
// nothing, so a host needs to be able to SAY so — see PiperTTS.setOnModelProgress. Cache hits report
|
|
1926
|
+
// nothing (they're instant).
|
|
1927
|
+
async function opfsCachedFetch(url, onProgress) {
|
|
1867
1928
|
const name = url.split('/').at(-1);
|
|
1868
1929
|
const dir = () => navigator.storage.getDirectory().then((r) => r.getDirectoryHandle('piper', { create: true }));
|
|
1869
1930
|
try { return await (await (await dir()).getFileHandle(name)).getFile(); } catch {}
|
|
1870
1931
|
const res = await fetch(url);
|
|
1871
1932
|
if (!res.ok) throw new Error(`fetch ${url}: ${res.status}`);
|
|
1872
|
-
|
|
1933
|
+
let blob;
|
|
1934
|
+
if (onProgress && res.body) {
|
|
1935
|
+
const total = +(res.headers.get('content-length') || 0);
|
|
1936
|
+
const chunks = []; let loaded = 0;
|
|
1937
|
+
for await (const c of res.body) { chunks.push(c); loaded += c.byteLength; onProgress(loaded, total); }
|
|
1938
|
+
blob = new Blob(chunks);
|
|
1939
|
+
} else blob = await res.blob();
|
|
1873
1940
|
try { const w = await (await (await dir()).getFileHandle(name, { create: true })).createWritable(); await w.write(blob); await w.close(); } catch {}
|
|
1874
1941
|
return blob;
|
|
1875
1942
|
}
|
|
@@ -1888,6 +1955,10 @@ function pcm2wav(f32, sampleRate) {
|
|
|
1888
1955
|
}
|
|
1889
1956
|
export class PiperTTS extends StreamingTTS {
|
|
1890
1957
|
constructor(voiceId = 'en_US-joe-medium', speed = 1.0) { super(voiceId, speed); }
|
|
1958
|
+
// Called with (loadedBytes, totalBytes) while the ~60MB voice model downloads on first use —
|
|
1959
|
+
// that download is one silent minute on mobile data, so a UI can show a real progress bar
|
|
1960
|
+
// instead of a dead button. Fires only on a cache MISS; later sessions read it from OPFS.
|
|
1961
|
+
setOnModelProgress(fn) { this._onModelProgress = fn || null; return this; }
|
|
1891
1962
|
// webpackIgnore is honored by webpack AND Turbopack (vite needs its own marker) — without it
|
|
1892
1963
|
// bundlers rewrite this into their runtime require, which can't load a URL
|
|
1893
1964
|
// ("turbopack_context.x is not a function").
|
|
@@ -1931,7 +2002,7 @@ export class PiperTTS extends StreamingTTS {
|
|
|
1931
2002
|
import(/* webpackIgnore: true */ /* @vite-ignore */ ORT_SPECIFIER),
|
|
1932
2003
|
import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
|
|
1933
2004
|
opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
|
|
1934
|
-
opfsCachedFetch(`${lib.HF_BASE}/${path}
|
|
2005
|
+
opfsCachedFetch(`${lib.HF_BASE}/${path}`, this._onModelProgress),
|
|
1935
2006
|
]);
|
|
1936
2007
|
const cfg = JSON.parse(await cfgBlob.text());
|
|
1937
2008
|
// Match the importmap's onnxruntime-web version: the lib's baked-in default points at a
|