@todoforai/voiceloop 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -4
- package/examples/demo.html +246 -44
- package/package.json +2 -2
- package/src/index.d.ts +4 -0
- package/src/stt.js +37 -5
- package/src/tuning.js +9 -0
- package/src/voice-agent.js +71 -8
package/README.md
CHANGED
|
@@ -54,14 +54,24 @@ That's it. Speak, get spoken answers, interrupt at will.
|
|
|
54
54
|
|
|
55
55
|
## Try it in 20 seconds
|
|
56
56
|
|
|
57
|
+
**[todoforai.github.io/voiceloop](https://todoforai.github.io/voiceloop/)** — the live demo, no
|
|
58
|
+
install, no keys and no backend of ours: the default path is your browser's own Web Speech STT,
|
|
59
|
+
local Piper TTS, and an echo stand-in for the LLM. (Web Speech is the *browser's* recognizer —
|
|
60
|
+
on most builds it still sends audio to the browser vendor. Pick a pipeline STT provider in
|
|
61
|
+
settings to see voiceloop's own VAD→STT path.)
|
|
62
|
+
|
|
63
|
+
Or locally, to point it at your own LLM:
|
|
64
|
+
|
|
57
65
|
```sh
|
|
58
66
|
npx serve . # from a clone
|
|
59
67
|
npx serve node_modules/@todoforai/voiceloop # from npm
|
|
60
68
|
```
|
|
61
69
|
|
|
62
|
-
Open **`/examples/demo.html`** — the whole loop behind one orb
|
|
63
|
-
|
|
64
|
-
|
|
70
|
+
Open **`/examples/demo.html`** — the whole loop behind one orb that breathes with the
|
|
71
|
+
mic level (and, while it answers, with how fast its spoken cursor advances), the live
|
|
72
|
+
interim transcript, click-to-interrupt with the unspoken remainder struck through where
|
|
73
|
+
you cut it off, and the two numbers that decide whether it feels alive: **first token**
|
|
74
|
+
and **first sound**. It starts in **echo mode**
|
|
65
75
|
— browser-native Web Speech STT, local Piper TTS and a stand-in "LLM" that
|
|
66
76
|
repeats you — so it runs with no keys, no backend and no cost. Open *settings*
|
|
67
77
|
to point it at any OpenAI-compatible endpoint (Ollama, your proxy) and a cloud
|
|
@@ -134,6 +144,13 @@ new VoiceAgent({ sttProvider, getSttToken: p => fetch(ROUTE[p], { method: 'POST'
|
|
|
134
144
|
|
|
135
145
|
On browsers without SpeechRecognition (Firefox, WebKitGTK) `webspeech` falls back to `elevenlabs` — which needs a token route configured, so pass `sttTokenUrl`/`getSttToken` if you want the fallback to work.
|
|
136
146
|
|
|
147
|
+
**On iOS, prefer a cloud STT provider.** Safari has shipped `webkitSpeechRecognition` since 14.5, but the
|
|
148
|
+
implementation is unreliable for a continuous agent loop: recognition often goes silent after the first
|
|
149
|
+
utterance without firing `onend` or `onerror`, playing audio in the same page can kill it the same way, and
|
|
150
|
+
iOS ducks the speakers while the recognizer holds the mic — which is exactly what a talking agent needs.
|
|
151
|
+
`webspeech` is a fine zero-key default on desktop Chrome; on phones, point `sttProvider` at `elevenlabs`,
|
|
152
|
+
`deepgram` or `speechmatics` so voiceloop owns the mic and its own VAD/echo path.
|
|
153
|
+
|
|
137
154
|
| Provider | End-of-turn | Notes |
|
|
138
155
|
|---|---|---|
|
|
139
156
|
| `webspeech` | Native (browser-managed) | Zero-key (no token endpoint), Chrome/Safari only. **~2.1s voice→voice** — ~1.2s slower to close a turn than cloud STT ([measured](bench/results/RESULTS.md)); it also owns the mic, so voiceloop's VAD/echo pipeline is bypassed. Recognition is handled by the browser, which may use a vendor cloud service — it is not guaranteed on-device |
|
|
@@ -235,9 +252,22 @@ Full types (with per-option notes) in [`src/index.d.ts`](src/index.d.ts).
|
|
|
235
252
|
| `fetchFn` | transport seam for the built-in adapter (tests, request wrapping) |
|
|
236
253
|
| `onEvent` | every event below |
|
|
237
254
|
|
|
238
|
-
Events via `onEvent(e)`: `state`, `stt`, `assistant`, `tool`, `vad`, `echo`, `error`, `diag`.
|
|
255
|
+
Events via `onEvent(e)`: `state`, `stt`, `assistant`, `tool`, `vad`, `level`, `echo`, `error`, `diag`.
|
|
239
256
|
A `tool` event carries `{ name, id, args }` plus either `result` (settled) or `running: true` (an [adapter-announced](#adapters-that-run-tools-themselves) call still executing) — render the pair as one chip keyed by `id`.
|
|
240
257
|
|
|
258
|
+
Two of them exist purely so a UI can look alive, and both cost you nothing to ignore:
|
|
259
|
+
|
|
260
|
+
- **`level`** — `{ level }`, mic loudness 0..1 on a perceptual curve, ~every 256ms while capturing.
|
|
261
|
+
Measured off the frames the pipeline already has (no second mic tap, no `AnalyserNode`). Drive a
|
|
262
|
+
meter or a reacting orb with it; decay it over *elapsed time* rather than per frame, or it
|
|
263
|
+
flickers out between samples. Not emitted for a `selfCapture` provider (webspeech owns its mic).
|
|
264
|
+
- **interruption** — there is no "cut short" event: the final `assistant` event carries both what
|
|
265
|
+
was heard (`text`) and what was written (`full`), so a non-empty `full.slice(text.length)` means
|
|
266
|
+
the reply was cut before that remainder was ever spoken. Render it struck through and the
|
|
267
|
+
transcript shows exactly where playback stopped (see `examples/demo.html`). It does not say *why*:
|
|
268
|
+
a voice barge-in, `interrupt()`, a superseding turn and `setTtsMuted(true)` all end a reply the
|
|
269
|
+
same way.
|
|
270
|
+
|
|
241
271
|
## Tuning
|
|
242
272
|
|
|
243
273
|
Every latency/sensitivity knob lives in [`src/tuning.js`](src/tuning.js) — VAD thresholds, barge-in minimum characters, sentence-break aggressiveness, prefetch stability window, echo-match threshold. Constructor options (`bargeInMinChars`, `vadOptions`, `turnDetector`, `maxPauseMs`) override per-agent.
|
package/examples/demo.html
CHANGED
|
@@ -26,61 +26,164 @@
|
|
|
26
26
|
main { max-width:660px; margin:0 auto; padding:36px 18px 48px;
|
|
27
27
|
display:flex; flex-direction:column; align-items:center; gap:16px }
|
|
28
28
|
|
|
29
|
-
/* ── the orb
|
|
29
|
+
/* ── the orb ───────────────────────────────────────────────────────────────
|
|
30
30
|
Plain CSS on purpose. A WebGPU version looks marginally better and is dark
|
|
31
31
|
for every Linux visitor (Chrome blocklists the adapter there), and an SVG
|
|
32
|
-
goo filter costs real CPU per frame for the same silhouette.
|
|
33
|
-
|
|
34
|
-
prefers-reduced-motion.
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
goo filter costs real CPU per frame for the same silhouette. Everything
|
|
33
|
+
below runs on the compositor and degrades to a flat disc under
|
|
34
|
+
prefers-reduced-motion.
|
|
35
|
+
|
|
36
|
+
One knob drives the whole thing: --lvl (0..1, the live mic/voice level from
|
|
37
|
+
the agent's `level` event). Size, glow, rim and halo all read from it, so
|
|
38
|
+
the orb BREATHES with the audio instead of blinking between states. */
|
|
39
|
+
#orb { position:relative; width:230px; height:230px; display:grid; place-items:center;
|
|
40
|
+
--a:#22304a; --b:#3d5274; --cc:#1d2740; --d:#2c3b57; --spin:30s; --glow:.14; --lvl:0 }
|
|
41
|
+
|
|
42
|
+
/* Halo: the light this thing throws on the page. It sits OUTSIDE the clipped orb, so the glow
|
|
43
|
+
blooms past the rim — at video scale that bloom is most of the "alive" read.
|
|
44
|
+
Two stops, not one: a tight inner bloom that hugs the rim (that's the part that reads as the
|
|
45
|
+
object emitting light) and a wide, very faint spill. A single wide gradient just washes the
|
|
46
|
+
whole frame in flat colour and looks like a backdrop instead of a glow. */
|
|
47
|
+
#orb::before { content:''; position:absolute; inset:-34%; border-radius:50%; pointer-events:none;
|
|
48
|
+
background:radial-gradient(circle, var(--b) 0 30%,
|
|
49
|
+
color-mix(in oklab, var(--b) 45%, transparent) 44%,
|
|
50
|
+
transparent 68%);
|
|
51
|
+
opacity:calc(var(--glow) * (.3 + var(--lvl) * .7)); filter:blur(18px);
|
|
52
|
+
transform:scale(calc(.94 + var(--lvl) * .12));
|
|
53
|
+
transition:opacity .12s linear, transform .12s ease-out }
|
|
54
|
+
|
|
55
|
+
/* The ball. A lit sphere, not a flat disc: the base gradient puts the light source up-left and
|
|
56
|
+
falls to near-black at the lower-right, which is what gives it volume before a single blob
|
|
57
|
+
is drawn. */
|
|
58
|
+
#mic { position:relative; width:100%; height:100%; border-radius:50%; border:0; padding:0;
|
|
59
|
+
cursor:pointer; overflow:hidden; display:grid; place-items:center;
|
|
60
|
+
background:radial-gradient(125% 125% at 30% 22%, #202a3d 0%, #131a28 36%, #080b12 70%, #05070b 100%);
|
|
61
|
+
/* level scales the body; the transition is short enough to track speech, long enough
|
|
62
|
+
not to jitter on a single loud frame */
|
|
63
|
+
transform:scale(calc(1 + var(--lvl) * .05));
|
|
64
|
+
transition:transform .12s ease-out }
|
|
39
65
|
/* Each blob is a soft disc smaller than the orb, wandering on its own ellipse.
|
|
40
|
-
Blurred and screen-blended,
|
|
41
|
-
pinches where they overlap — metaballs, minus the shader.
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
#mic i
|
|
46
|
-
|
|
66
|
+
Blurred and screen-blended, five of them read as one body that swells and
|
|
67
|
+
pinches where they overlap — metaballs, minus the shader. The blur tightens
|
|
68
|
+
as the level rises, so loud audio makes the core condense and brighten.
|
|
69
|
+
They are deliberately LARGE and travel a short path: small blobs on a wide orbit separate into
|
|
70
|
+
visibly distinct lumps at the extremes, which reads as four stains rather than one substance. */
|
|
71
|
+
#mic i { position:absolute; width:62%; height:62%; left:19%; top:19%; border-radius:50%;
|
|
72
|
+
filter:blur(calc(26px - var(--lvl) * 7px)); mix-blend-mode:screen;
|
|
73
|
+
animation:wander var(--spin) ease-in-out infinite;
|
|
74
|
+
transition:background 1s, filter .12s linear } /* colours cross-fade; the speed change snaps (animation-* isn't animatable) */
|
|
75
|
+
/* Each blob gets its OWN size and resting offset. Sharing one geometry and varying only the
|
|
76
|
+
animation timing makes them sweep the same path and stack into a single horizontal band
|
|
77
|
+
through the middle; staggering the anchors keeps the mass off-axis and organic. */
|
|
78
|
+
/* Anchors sit in DIFFERENT quadrants (up-left, up-right, low-left, low-right) so the combined
|
|
79
|
+
mass stays centred. Clustering them on one side pools all the light there and leaves a heavy
|
|
80
|
+
dead crescent on the other — the sphere stops looking filled. */
|
|
81
|
+
/* Anchors are each offset from the blob's OWN centred position ((100-size)/2) into a different
|
|
82
|
+
quadrant, so the resting pose is balanced. Offsetting from a shared number instead makes the
|
|
83
|
+
bigger blobs sit further out than the small ones and the cluster drifts up-left. */
|
|
84
|
+
#mic i:nth-child(1) { background:radial-gradient(circle, var(--a) 0 24%, transparent 62%);
|
|
85
|
+
top:9%; left:9% } /* 62% → centred 19% */
|
|
86
|
+
#mic i:nth-child(2) { background:radial-gradient(circle, var(--b) 0 22%, transparent 60%);
|
|
87
|
+
width:54%; height:54%; top:13%; left:33%; /* 54% → centred 23% */
|
|
88
|
+
animation-name:wander2;
|
|
47
89
|
animation-duration:calc(var(--spin)*.72); animation-direction:reverse }
|
|
48
|
-
#mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0
|
|
90
|
+
#mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0 26%, transparent 64%);
|
|
91
|
+
width:66%; height:66%; top:27%; left:7%; /* 66% → centred 17% */
|
|
92
|
+
animation-name:wander3;
|
|
49
93
|
animation-duration:calc(var(--spin)*1.4); animation-delay:calc(var(--spin)*-.3) }
|
|
50
|
-
#mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0
|
|
94
|
+
#mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0 20%, transparent 58%);
|
|
95
|
+
width:50%; height:50%; top:35%; left:35%; /* 50% → centred 25% */
|
|
96
|
+
animation-name:wander2;
|
|
51
97
|
animation-duration:calc(var(--spin)*.9); animation-direction:reverse;
|
|
52
98
|
animation-delay:calc(var(--spin)*-.55) }
|
|
53
|
-
/*
|
|
99
|
+
/* A 5th blob pinned near the centre and driven ONLY by level: the core that lights up when you
|
|
100
|
+
speak. Without it the wandering blobs can all be off-centre at the moment you talk, and the
|
|
101
|
+
orb answers your voice from its edge. */
|
|
102
|
+
/* Only a HINT of white at the very centre: screen-blending five layers already drives the overlap
|
|
103
|
+
toward white, so a white-cored fifth blob clips the whole middle to a flat highlight and the
|
|
104
|
+
state colour disappears exactly when the orb is most active. */
|
|
105
|
+
#mic i:nth-child(5) { width:64%; height:64%; left:18%; top:18%; animation:none;
|
|
106
|
+
background:radial-gradient(circle, color-mix(in oklab, var(--b) 88%, white) 0 14%,
|
|
107
|
+
var(--b) 34%, transparent 64%);
|
|
108
|
+
opacity:calc(var(--lvl) * .72); filter:blur(calc(20px - var(--lvl) * 8px));
|
|
109
|
+
transform:scale(calc(.72 + var(--lvl) * .38));
|
|
110
|
+
transition:opacity .12s linear, transform .12s ease-out, filter .12s linear }
|
|
111
|
+
/* Glass shell: the specular cap up-left where the light is, plus a BOUNCE light low-right in the
|
|
112
|
+
state colour. The bounce is what stops the bottom of the sphere reading as a dead black
|
|
113
|
+
crescent — real objects catch light back off their surroundings, and without it the orb looks
|
|
114
|
+
like a lit stain on a black hole. */
|
|
54
115
|
#mic::before { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
55
|
-
background:radial-gradient(
|
|
56
|
-
rgba(255,255,255,.
|
|
57
|
-
|
|
116
|
+
background:radial-gradient(62% 46% at 33% 19%, rgba(255,255,255,.34) 0 8%,
|
|
117
|
+
rgba(255,255,255,.07) 30%, transparent 58%),
|
|
118
|
+
radial-gradient(58% 44% at 70% 86%,
|
|
119
|
+
color-mix(in oklab, var(--b) calc(16% + var(--lvl)*26%), transparent) 0 12%,
|
|
120
|
+
transparent 62%);
|
|
121
|
+
box-shadow:inset 0 0 46px rgba(0,0,0,.7), inset 0 1px 1px rgba(255,255,255,.22) }
|
|
122
|
+
/* Fresnel rim + outer glow. The hairline ring is the polish: a lit sphere has a bright edge where
|
|
123
|
+
it turns away from you, and 1px of state colour there is what separates "orb" from "blurry
|
|
124
|
+
blob". It brightens with level, so the edge rings on loud syllables. */
|
|
58
125
|
#mic::after { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
59
|
-
box-shadow:0 0
|
|
126
|
+
box-shadow:inset 0 0 0 1px color-mix(in oklab, var(--b) calc(20% + var(--lvl) * 45%), transparent),
|
|
127
|
+
inset 0 0 calc(20px + var(--lvl) * 28px) color-mix(in oklab, var(--b) calc(8% + var(--lvl) * 22%), transparent),
|
|
128
|
+
0 0 44px 6px color-mix(in oklab, var(--b) 45%, transparent);
|
|
129
|
+
opacity:calc(.35 + var(--glow) * .65); transition:opacity .8s, box-shadow .12s linear }
|
|
130
|
+
/* Three ORBITS, not one. Sharing a single path (even at different speeds and delays) marches every
|
|
131
|
+
blob along the same diagonal, so the mass stays a bar that slides corner to corner instead of
|
|
132
|
+
circulating. These run the loop in different rotational orders, which keeps the interior
|
|
133
|
+
churning and the silhouette asymmetric. */
|
|
60
134
|
@keyframes wander {
|
|
61
|
-
0%,100% { transform:translate(-
|
|
62
|
-
25% { transform:translate(
|
|
63
|
-
50% { transform:translate(
|
|
64
|
-
75% { transform:translate(-
|
|
135
|
+
0%,100% { transform:translate(-20%,-13%) scale(1) }
|
|
136
|
+
25% { transform:translate( 18%,-20%) scale(1.16) }
|
|
137
|
+
50% { transform:translate( 21%, 17%) scale(.88) }
|
|
138
|
+
75% { transform:translate(-17%, 20%) scale(1.1) }
|
|
139
|
+
}
|
|
140
|
+
@keyframes wander2 {
|
|
141
|
+
0%,100% { transform:translate( 19%,-16%) scale(1.08) }
|
|
142
|
+
25% { transform:translate(-21%,-10%) scale(.9) }
|
|
143
|
+
50% { transform:translate(-14%, 21%) scale(1.18) }
|
|
144
|
+
75% { transform:translate( 20%, 12%) scale(.94) }
|
|
145
|
+
}
|
|
146
|
+
@keyframes wander3 {
|
|
147
|
+
0%,100% { transform:translate( 4%, 22%) scale(.92) }
|
|
148
|
+
25% { transform:translate( 22%, -6%) scale(1.12) }
|
|
149
|
+
50% { transform:translate( -6%,-22%) scale(1) }
|
|
150
|
+
75% { transform:translate(-22%, 6%) scale(1.14) }
|
|
151
|
+
}
|
|
152
|
+
/* Ripple: one ring per turn boundary, fired from JS (see `ripple()`). It marks the two instants
|
|
153
|
+
the transcript can't show — the moment your turn was committed, and the moment a barge-in cut
|
|
154
|
+
the reply — which is exactly what a viewer needs to SEE to believe the latency numbers. */
|
|
155
|
+
#orb u { position:absolute; inset:0; border-radius:50%; border:1.5px solid var(--c);
|
|
156
|
+
opacity:0; pointer-events:none; animation:ripple .75s ease-out forwards }
|
|
157
|
+
@keyframes ripple {
|
|
158
|
+
from { opacity:.55; transform:scale(1) }
|
|
159
|
+
to { opacity:0; transform:scale(1.5) }
|
|
65
160
|
}
|
|
66
161
|
/* the glyph leads while the orb is dark; once the loop is live the orb says it better */
|
|
67
162
|
#glyph { position:relative; font-size:36px; line-height:1; transition:opacity .4s;
|
|
68
163
|
text-shadow:0 2px 12px #000 }
|
|
69
164
|
body:not([data-state="idle"]) #glyph { opacity:0 }
|
|
70
165
|
|
|
71
|
-
/*
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
body[data-
|
|
166
|
+
/* Same object, more energy — the whole state display is these three lines. Each palette is one
|
|
167
|
+
hue family with a neighbour (green→teal, amber→gold, coral→magenta) rather than opposite
|
|
168
|
+
corners of the wheel: four unrelated hues screen-blended average out to a muddy grey-brown
|
|
169
|
+
exactly where the blobs overlap, which is the middle of the orb. */
|
|
170
|
+
body[data-state="listening"] #orb { --a:#0fa5c9; --b:#2ee6a8; --cc:#0d7fa8; --d:#7df0c8; --spin:14s; --glow:.5 }
|
|
171
|
+
body[data-state="thinking"] #orb { --a:#d98324; --b:#ffb547; --cc:#b0621a; --d:#ffd98a; --spin:6s; --glow:.7 }
|
|
172
|
+
body[data-state="speaking"] #orb { --a:#e8355f; --b:#ff7038; --cc:#b8329a; --d:#ff9a5e; --spin:4s; --glow:.95 }
|
|
77
173
|
|
|
78
174
|
body[data-state="listening"] { --c:#39d98a }
|
|
79
175
|
body[data-state="thinking"] { --c:#ffc453 }
|
|
80
176
|
body[data-state="speaking"] { --c:#ff8a3d }
|
|
81
177
|
|
|
178
|
+
/* Reduced motion: hold the orb STILL. Killing the transitions alone isn't enough — the level var
|
|
179
|
+
keeps changing, so every scale would just snap instead of glide. Pin --lvl and the object
|
|
180
|
+
becomes a flat, quietly coloured disc that still reads state by hue. (setLevel() also stops
|
|
181
|
+
writing the var, and ripple() creates no rings — a display:none ring never fires animationend,
|
|
182
|
+
so it would leak one element per turn.) */
|
|
82
183
|
@media (prefers-reduced-motion: reduce) {
|
|
83
|
-
#
|
|
184
|
+
#orb { --lvl:0 !important }
|
|
185
|
+
#mic i { animation:none } #mic, #orb::before, #mic i { transition:none }
|
|
186
|
+
#orb u { display:none }
|
|
84
187
|
}
|
|
85
188
|
|
|
86
189
|
#status { color:var(--c); font-size:14px; min-height:22px; text-align:center; transition:color .3s }
|
|
@@ -92,6 +195,11 @@
|
|
|
92
195
|
.msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
|
|
93
196
|
.msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
|
|
94
197
|
.msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
|
|
198
|
+
/* Barge-in: the reply stopped mid-word because you cut in. Without a mark, an interrupted bubble
|
|
199
|
+
is indistinguishable from a short answer — and interruption is the feature. */
|
|
200
|
+
.msg.ai.cut .tail { text-decoration:line-through; text-decoration-color:#ff8a3d80 }
|
|
201
|
+
.msg.ai.cut::after { content:'interrupted'; display:block; margin-top:5px;
|
|
202
|
+
font-size:10px; letter-spacing:.04em; color:#ff8a3d; opacity:.85 }
|
|
95
203
|
/* An echoed reply is not an answer — mark it so the transcript can't be read as a model talking. */
|
|
96
204
|
.msg.ai.echoed { border-left:2px solid #e0a94a }
|
|
97
205
|
.msg.ai.echoed::after { content:'echo — no model'; display:block; margin-top:5px;
|
|
@@ -109,9 +217,18 @@
|
|
|
109
217
|
border-radius:9px; padding:9px 12px; font:inherit }
|
|
110
218
|
input:focus, select:focus { outline:none; border-color:var(--c) }
|
|
111
219
|
button.btn { cursor:pointer } button.btn:hover { border-color:#3a424e }
|
|
220
|
+
/* iOS Safari auto-zooms the page when you focus a field whose text is under 16px, and it does NOT
|
|
221
|
+
zoom back out on blur — one tap on "type a turn" and the whole demo is left scaled up and
|
|
222
|
+
panned. 16px is the documented threshold, so meet it on touch devices (the 11px settings
|
|
223
|
+
labels stay small; only the FIELDS matter). */
|
|
224
|
+
@media (pointer: coarse) { input, select, textarea { font-size:16px } }
|
|
112
225
|
#typed { flex:1 }
|
|
113
226
|
details { width:100%; border-top:1px solid var(--line); padding-top:10px }
|
|
114
227
|
summary { cursor:pointer; color:var(--dim); font-size:13px; list-style:none }
|
|
228
|
+
/* A 13px line of text is a ~20px tap target; both platforms' guidelines say ~44px. Padding is
|
|
229
|
+
enough — these are the only touch controls on the page that aren't already finger-sized.
|
|
230
|
+
min-height pins the result at 44px instead of trusting 13px text to round up to it. */
|
|
231
|
+
@media (pointer: coarse) { summary { padding:11px 0; min-height:44px; box-sizing:border-box } }
|
|
115
232
|
summary::before { content:'▸ ' } details[open] summary::before { content:'▾ ' }
|
|
116
233
|
.grid { display:flex; flex-wrap:wrap; gap:10px; padding:12px 0 4px }
|
|
117
234
|
label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
|
|
@@ -126,9 +243,11 @@
|
|
|
126
243
|
</head>
|
|
127
244
|
<body>
|
|
128
245
|
<main>
|
|
129
|
-
<
|
|
130
|
-
<
|
|
131
|
-
|
|
246
|
+
<div id="orb">
|
|
247
|
+
<button id="mic" title="click to start / stop · click while it talks to interrupt">
|
|
248
|
+
<i></i><i></i><i></i><i></i><i></i><span id="glyph">🎙</span>
|
|
249
|
+
</button>
|
|
250
|
+
</div>
|
|
132
251
|
<div id="status">click to start · then just talk</div>
|
|
133
252
|
<div id="lat"></div>
|
|
134
253
|
<div id="pipeline"></div>
|
|
@@ -198,7 +317,7 @@ const log = (...a) => {
|
|
|
198
317
|
|
|
199
318
|
// ── transcript ──────────────────────────────────────────────────────────────
|
|
200
319
|
const chat = $('chat');
|
|
201
|
-
let interimEl = null, aiEl = null, vadOff = null;
|
|
320
|
+
let interimEl = null, aiEl = null, vadOff = null, spokenLen = 0;
|
|
202
321
|
const toolEls = new Map();
|
|
203
322
|
const bubble = (cls, text = '') => {
|
|
204
323
|
const el = document.createElement('div');
|
|
@@ -234,6 +353,15 @@ async function* echoLLM(history) {
|
|
|
234
353
|
// ── what is ACTUALLY running ────────────────────────────────────────────────
|
|
235
354
|
// A demo that hides which engine served a reply is worse than no demo: a browser-native
|
|
236
355
|
// recognizer looks identical to voiceloop's pipeline, and an echo stand-in looks identical
|
|
356
|
+
// iPadOS 13+ reports itself as "MacIntel", so the touch-point check is what separates an iPad
|
|
357
|
+
// from a desktop Mac — UA sniffing alone would let the iPad through unwarned.
|
|
358
|
+
const IS_IOS = /iPad|iPhone|iPod/.test(navigator.userAgent) ||
|
|
359
|
+
(navigator.platform === 'MacIntel' && navigator.maxTouchPoints > 1);
|
|
360
|
+
// Safari ships webkitSpeechRecognition but it goes deaf after an utterance without firing onend/onerror,
|
|
361
|
+
// and iOS ducks the speakers while it holds the mic. Say so here rather than let the demo look broken.
|
|
362
|
+
const IOS_WEBSPEECH_WARNING = '<span class="warn"><br>⚠️ on iOS this often stops hearing you after the ' +
|
|
363
|
+
'first reply, and quiets the speaker — pick a cloud STT provider in settings for a reliable loop</span>';
|
|
364
|
+
|
|
237
365
|
// to a model. Both get named on screen, always.
|
|
238
366
|
const describePipeline = (a) => {
|
|
239
367
|
const echo = $('mode').value === 'echo';
|
|
@@ -242,7 +370,7 @@ const describePipeline = (a) => {
|
|
|
242
370
|
// pipeline is running when it is exactly what gets bypassed.
|
|
243
371
|
const self = a?.stt ? !!a.stt.selfCapture : sttSelfCaptures($('sttProvider').value);
|
|
244
372
|
const stt = self
|
|
245
|
-
? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span
|
|
373
|
+
? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span>${IS_IOS ? IOS_WEBSPEECH_WARNING : ''}`
|
|
246
374
|
: `<b>${$('sttProvider').value}</b> via voiceloop's pipeline — Silero VAD → feed → commit`;
|
|
247
375
|
const llm = echo
|
|
248
376
|
? '<span class="warn"><b>echo</b> — no model: replies repeat your words back</span>'
|
|
@@ -253,6 +381,51 @@ const showPipeline = () => { $('pipeline').innerHTML = describePipeline(agent);
|
|
|
253
381
|
// Hoisted (function declaration) so the settings listeners above can call it before this point.
|
|
254
382
|
function onSettingChanged() { showPipeline(); prewarmIfPipelined(); }
|
|
255
383
|
|
|
384
|
+
// ── the orb's level ─────────────────────────────────────────────────────────
|
|
385
|
+
// One CSS var, --lvl (0..1), drives size/glow/rim/halo. Three sources feed it, in order of truth:
|
|
386
|
+
// 1. the agent's `level` event — real mic RMS, the only honest one (pipeline STT only);
|
|
387
|
+
// 2. while SPEAKING — the reply's spoken-char rate, so the orb pulses with its own voice
|
|
388
|
+
// (voiceloop plays TTS through an internal AEC sink we can't tap, and re-tapping the mic
|
|
389
|
+
// would just measure the echo it works to suppress);
|
|
390
|
+
// 3. a selfCapture STT (webspeech) emits no level at all — fall back to the VAD-style pulse.
|
|
391
|
+
// Every source decays: a level that only ever rises reads as a stuck meter.
|
|
392
|
+
// Sources are SPARSE (a level event every ~256ms capture chunk), so both the target's decay and the
|
|
393
|
+
// displayed value are functions of ELAPSED TIME, not of frame count: a per-frame multiplier would
|
|
394
|
+
// empty the meter between two events at 60fps and flicker, and would run at a different speed on a
|
|
395
|
+
// 144Hz display.
|
|
396
|
+
const orb = $('orb');
|
|
397
|
+
const stillMotion = matchMedia('(prefers-reduced-motion: reduce)');
|
|
398
|
+
const HALF_LIFE_MS = 260; // target sags to half this long after its last source sample
|
|
399
|
+
const ATTACK_MS = 60, RELEASE_MS = 320; // fast attack, slow release — how a meter reads as "alive"
|
|
400
|
+
const decayed = (v, ms, halfLife) => v * Math.pow(0.5, ms / halfLife);
|
|
401
|
+
let lvl = 0, lvlTarget = 0, lastFrame = 0, running = false;
|
|
402
|
+
const setLevel = (v) => {
|
|
403
|
+
if (!Number.isFinite(v) || stillMotion.matches) return; // a NaN would poison the meter for the page's lifetime
|
|
404
|
+
lvlTarget = Math.max(lvlTarget, Math.min(1, Math.max(0, v)));
|
|
405
|
+
if (!running) { running = true; lastFrame = 0; requestAnimationFrame(tickLevel); }
|
|
406
|
+
};
|
|
407
|
+
// Runs only while there is something to show: sources re-arm it, and it parks itself once the meter
|
|
408
|
+
// has settled at rest — an idle demo shouldn't be writing a style property 60 times a second.
|
|
409
|
+
function tickLevel(now) {
|
|
410
|
+
const dt = Math.min(100, now - (lastFrame || now)); lastFrame = now; // clamp: a backgrounded tab resumes with a huge gap
|
|
411
|
+
lvlTarget = decayed(lvlTarget, dt, HALF_LIFE_MS);
|
|
412
|
+
// Exponential approach toward the target, converging at the same rate regardless of frame rate.
|
|
413
|
+
lvl += (lvlTarget - lvl) * (1 - Math.exp(-dt / (lvlTarget > lvl ? ATTACK_MS : RELEASE_MS)));
|
|
414
|
+
if (lvl < 0.002 && lvlTarget < 0.002) { lvl = lvlTarget = 0; running = false; }
|
|
415
|
+
orb.style.setProperty('--lvl', lvl.toFixed(3));
|
|
416
|
+
if (running) requestAnimationFrame(tickLevel);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// One ring per turn boundary. Marks the two instants the transcript cannot show: your turn being
|
|
420
|
+
// committed, and a barge-in cutting the reply. Under reduced motion the ring is display:none, which
|
|
421
|
+
// never fires animationend — so don't create one at all rather than leak an element per turn.
|
|
422
|
+
const ripple = () => {
|
|
423
|
+
if (stillMotion.matches) return;
|
|
424
|
+
const u = document.createElement('u');
|
|
425
|
+
u.onanimationend = () => u.remove();
|
|
426
|
+
orb.append(u);
|
|
427
|
+
};
|
|
428
|
+
|
|
256
429
|
// A typed turn drives the same state machine without a mic — don't claim we're listening then.
|
|
257
430
|
let micOn = false;
|
|
258
431
|
const STATUS = {
|
|
@@ -261,10 +434,20 @@ const STATUS = {
|
|
|
261
434
|
thinking: () => 'thinking…',
|
|
262
435
|
speaking: () => micOn ? 'speaking — talk over it to interrupt' : 'speaking — click to interrupt',
|
|
263
436
|
};
|
|
437
|
+
// The first spoken reply blocks on a ~60MB Piper voice download (once — it's cached in OPFS after).
|
|
438
|
+
// On mobile data that is a minute during which the status line claimed "speaking", which reads as
|
|
439
|
+
// "the demo is broken". While the model is coming down, the download owns the status line.
|
|
440
|
+
let modelPct = null;
|
|
264
441
|
const setState = s => {
|
|
265
442
|
document.body.dataset.state = micOn || s !== 'listening' ? s : 'idle';
|
|
266
|
-
$('status').textContent = STATUS[s]?.() ?? s;
|
|
443
|
+
if (modelPct === null) $('status').textContent = STATUS[s]?.() ?? s;
|
|
267
444
|
};
|
|
445
|
+
tts.setOnModelProgress?.((loaded, total) => {
|
|
446
|
+
const done = total && loaded >= total;
|
|
447
|
+
modelPct = done ? null : total ? Math.floor((loaded / total) * 100) : 0;
|
|
448
|
+
$('status').textContent = done ? (STATUS[document.body.dataset.state]?.() ?? '')
|
|
449
|
+
: `downloading the voice — ${modelPct}% (${(loaded / 1048576).toFixed(0)}MB${total ? ` of ${(total / 1048576).toFixed(0)}MB` : ''}, once)`;
|
|
450
|
+
});
|
|
268
451
|
|
|
269
452
|
function onEvent(e) {
|
|
270
453
|
if (e.type !== 'stt' || e.turnComplete) log(e);
|
|
@@ -276,19 +459,22 @@ function onEvent(e) {
|
|
|
276
459
|
setState(e.state);
|
|
277
460
|
break;
|
|
278
461
|
case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
|
|
462
|
+
case 'level': setLevel(e.level); break;
|
|
279
463
|
case 'stt':
|
|
280
|
-
// A selfCapture provider runs no agent VAD, so no 'vad' event ever arrives and the orb
|
|
281
|
-
// sit dead still while the browser hears you perfectly — looking exactly like a broken
|
|
282
|
-
// Live partials are the only "I hear you" signal on that path, so
|
|
464
|
+
// A selfCapture provider runs no agent VAD, so no 'vad'/'level' event ever arrives and the orb
|
|
465
|
+
// would sit dead still while the browser hears you perfectly — looking exactly like a broken
|
|
466
|
+
// mic. Live partials are the only "I hear you" signal on that path, so pulse from them.
|
|
283
467
|
if (agent?.stt?.selfCapture) {
|
|
284
468
|
document.body.dataset.vad = e.turnComplete ? '0' : '1';
|
|
469
|
+
if (!e.turnComplete) setLevel(0.55);
|
|
285
470
|
clearTimeout(vadOff);
|
|
286
471
|
if (!e.turnComplete) vadOff = setTimeout(() => { document.body.dataset.vad = '0'; }, 700);
|
|
287
472
|
}
|
|
288
473
|
if (e.turnComplete) {
|
|
474
|
+
ripple(); // your turn closed — the instant the latency clock starts
|
|
289
475
|
interimEl?.remove(); interimEl = null;
|
|
290
476
|
bubble('user', e.text);
|
|
291
|
-
aiEl = null; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
|
|
477
|
+
aiEl = null; spokenLen = 0; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
|
|
292
478
|
} else {
|
|
293
479
|
interimEl ??= bubble('user interim');
|
|
294
480
|
interimEl.textContent = `${e.committed ?? ''} ${e.text}`.trim();
|
|
@@ -298,7 +484,7 @@ function onEvent(e) {
|
|
|
298
484
|
case 'assistant': {
|
|
299
485
|
// Empty and non-final means "that draft never happened" (interrupted before
|
|
300
486
|
// a word was spoken) — drop the bubble rather than blank it.
|
|
301
|
-
if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; break; }
|
|
487
|
+
if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; spokenLen = 0; break; }
|
|
302
488
|
if (!tText && t0) { tText = Date.now(); showLat(); }
|
|
303
489
|
// `full` only exists once the TTS progress cursor is live — i.e. a clip is
|
|
304
490
|
// actually playing. The 'speaking' state is just the first token, not sound.
|
|
@@ -306,9 +492,24 @@ function onEvent(e) {
|
|
|
306
492
|
// Tag the bubble (never the text) when no model produced it — the reply is a verbatim echo,
|
|
307
493
|
// and only the UI should say so.
|
|
308
494
|
aiEl ??= bubble($('mode').value === 'echo' ? 'ai echoed' : 'ai');
|
|
495
|
+
// The agent's own voice has no level event (TTS plays through an internal AEC sink), so drive
|
|
496
|
+
// the orb from how fast the spoken CURSOR advances — the orb rides its own speech. Only events
|
|
497
|
+
// carrying `full` have a cursor; a pre-audio streaming draft would otherwise run spokenLen up
|
|
498
|
+
// to the draft's length and the NEXT reply would sit flat until it grew past that stale mark.
|
|
499
|
+
if (e.full != null) {
|
|
500
|
+
setLevel(0.35 + Math.min(0.6, Math.max(0, e.text.length - spokenLen) * 0.09));
|
|
501
|
+
spokenLen = e.text.length;
|
|
502
|
+
}
|
|
309
503
|
aiEl.textContent = e.text; // spoken so far
|
|
310
504
|
const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
|
|
311
505
|
if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
|
|
506
|
+
// Cut short: the final says what was HEARD (text) and what was written (full) — a gap between
|
|
507
|
+
// them is a barge-in, the one thing the transcript would otherwise render as a short answer.
|
|
508
|
+
// (trim: a natural finish can leave trailing whitespace in `full` — not an interruption)
|
|
509
|
+
if (e.final) {
|
|
510
|
+
if (tail.trim()) { aiEl.classList.add('cut'); ripple(); }
|
|
511
|
+
spokenLen = 0;
|
|
512
|
+
}
|
|
312
513
|
aiEl.scrollIntoView({ block:'nearest' });
|
|
313
514
|
break;
|
|
314
515
|
}
|
|
@@ -404,6 +605,7 @@ navigator.mediaDevices?.enumerateDevices().then(ds => {
|
|
|
404
605
|
}).catch(() => {});
|
|
405
606
|
|
|
406
607
|
setState('idle');
|
|
608
|
+
onSettingChanged(); // the banner must describe the restored settings from the FIRST frame, not only after a click
|
|
407
609
|
window.agent = () => agent; // console poking: agent().history, …
|
|
408
610
|
</script>
|
|
409
611
|
</body>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
"type": "git",
|
|
43
43
|
"url": "git+https://github.com/todoforai/voiceloop.git"
|
|
44
44
|
},
|
|
45
|
-
"homepage": "https://github.
|
|
45
|
+
"homepage": "https://todoforai.github.io/voiceloop/",
|
|
46
46
|
"bugs": {
|
|
47
47
|
"url": "https://github.com/todoforai/voiceloop/issues"
|
|
48
48
|
},
|
package/src/index.d.ts
CHANGED
|
@@ -7,6 +7,10 @@ export type VoiceAgentEvent =
|
|
|
7
7
|
| { type: 'stt'; turnComplete: boolean; text: string; ms: number; committed?: string } // text=live tail; committed=locked segments (Speechmatics AddTranscript); turnComplete=user turn ended → send to LLM
|
|
8
8
|
| { type: 'assistant'; text: string; final: boolean; full?: string } // text=spoken-so-far (solid, advances live as TTS plays); full=complete answer → tail full.slice(text.length) not-yet-spoken (dim)
|
|
9
9
|
| { type: 'vad'; active: boolean } // VAD hears speech?
|
|
10
|
+
// Mic loudness, 0..1 on a perceptual curve, ~every 256ms while capturing. Cosmetic only (meters,
|
|
11
|
+
// a reacting orb) — nothing in the pipeline reads it. Never emitted for a selfCapture STT
|
|
12
|
+
// (webspeech owns its own mic, so there are no frames to measure).
|
|
13
|
+
| { type: 'level'; level: number }
|
|
10
14
|
// running:true = loop call announced, executing now (display-only spinner; no result yet).
|
|
11
15
|
// The outcome event follows with the same `id` and replaces it in place.
|
|
12
16
|
| { type: 'tool'; name: string; args: Record<string, unknown>; result?: unknown; id?: string; running?: true }
|
package/src/stt.js
CHANGED
|
@@ -32,6 +32,16 @@ import { TUNING } from './tuning.js';
|
|
|
32
32
|
|
|
33
33
|
const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at least this often
|
|
34
34
|
|
|
35
|
+
// Cloud STT is a browser WebSocket. Resolved through globalThis at call time, not as a bare
|
|
36
|
+
// identifier: this module is imported under SSR and under node (tests, node <22), where bare
|
|
37
|
+
// `WebSocket` is a ReferenceError instead of undefined. A missing WS is a clear provider error —
|
|
38
|
+
// not a stack trace from inside open().
|
|
39
|
+
const newSocket = (url, protocols) => {
|
|
40
|
+
const WS = globalThis.WebSocket;
|
|
41
|
+
if (!WS) throw new Error('cloud STT needs WebSocket (unavailable here — browser or node >=22)');
|
|
42
|
+
return protocols ? new WS(url, protocols) : new WS(url);
|
|
43
|
+
};
|
|
44
|
+
|
|
35
45
|
// Common opts shape (provided by the VoiceAgent):
|
|
36
46
|
// { apiKey, sttUrl, sttModel, sttLang, keyterms, sttTokenUrl, getToken, sttUsageUrl,
|
|
37
47
|
// onPartial, onFinal, onError, onClose, isClosed }
|
|
@@ -145,7 +155,7 @@ export function makeElevenLabsSTT(opts) {
|
|
|
145
155
|
if (sttLang) p.set('language_code', sttLang);
|
|
146
156
|
for (const k of keyterms) p.append('keyterms', k);
|
|
147
157
|
sttStart = performance.now();
|
|
148
|
-
ws =
|
|
158
|
+
ws = newSocket(`${sttUrl}?${p}`);
|
|
149
159
|
ws.onopen = () => { const q = outbox; outbox = []; for (const it of q) it === 'commit' ? sendCommit() : sendAudio(it); };
|
|
150
160
|
ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('STT ws error', ev); } };
|
|
151
161
|
ws.onclose = ev => { usage.flush(); onClose?.(); if (!isClosed() && ev.code !== 1000) { onError(`STT closed ${ev.code}: ${ev.reason || 'no reason'}`); console.error('STT ws closed', ev.code, ev.reason); } };
|
|
@@ -156,6 +166,11 @@ export function makeElevenLabsSTT(opts) {
|
|
|
156
166
|
if (m.message_type === 'partial_transcript') onPartial(m.text || '', ms);
|
|
157
167
|
else if (m.message_type?.startsWith('committed') && m.text) onFinal(m.text, ms);
|
|
158
168
|
};
|
|
169
|
+
} catch (e) {
|
|
170
|
+
// open() is called fire-and-forget (pre-open, and lazily from feed()), so anything thrown here
|
|
171
|
+
// would surface as an unhandled rejection and kill the process under node. A socket that can't
|
|
172
|
+
// be built is unrecoverable for this provider — report it as fatal and let the host stop.
|
|
173
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
159
174
|
} finally {
|
|
160
175
|
opening = false;
|
|
161
176
|
}
|
|
@@ -284,7 +299,7 @@ export function makeSpeechmaticsSTT(opts) {
|
|
|
284
299
|
if (!body) { outbox = []; sentSinceCommit = false; return; }
|
|
285
300
|
if (isClosed()) return;
|
|
286
301
|
sttStart = performance.now();
|
|
287
|
-
ws =
|
|
302
|
+
ws = newSocket(`${SM_URL}?jwt=${encodeURIComponent(body.token)}`);
|
|
288
303
|
ws.binaryType = 'arraybuffer';
|
|
289
304
|
ws.onopen = () => {
|
|
290
305
|
// Declare the audio format + transcription config, then flush buffered audio once started.
|
|
@@ -376,6 +391,11 @@ export function makeSpeechmaticsSTT(opts) {
|
|
|
376
391
|
case 'Error': onError(`${m.type || 'error'}: ${m.reason || ''}`); break;
|
|
377
392
|
}
|
|
378
393
|
};
|
|
394
|
+
} catch (e) {
|
|
395
|
+
// open() is called fire-and-forget (pre-open, and lazily from feed()), so anything thrown here
|
|
396
|
+
// would surface as an unhandled rejection and kill the process under node. A socket that can't
|
|
397
|
+
// be built is unrecoverable for this provider — report it as fatal and let the host stop.
|
|
398
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
379
399
|
} finally {
|
|
380
400
|
opening = false;
|
|
381
401
|
}
|
|
@@ -506,7 +526,7 @@ export function makeDeepgramSTT(opts) {
|
|
|
506
526
|
sttStart = performance.now();
|
|
507
527
|
// Browser WS auth: Sec-WebSocket-Protocol "bearer, <token>" (short-TTL access token from
|
|
508
528
|
// /v1/auth/grant). The token only needs to be valid at handshake time; the socket persists.
|
|
509
|
-
ws =
|
|
529
|
+
ws = newSocket(`${DG_URL}?${p}`, ['bearer', token]);
|
|
510
530
|
ws.binaryType = 'arraybuffer';
|
|
511
531
|
ws.onopen = () => { const q = outbox; outbox = []; outboxSamples = 0; for (const c of q) sendAudio(c); };
|
|
512
532
|
ws.onerror = ev => { if (!isClosed()) { onError('STT socket error'); console.error('DG ws error', ev); } };
|
|
@@ -535,6 +555,11 @@ export function makeDeepgramSTT(opts) {
|
|
|
535
555
|
}
|
|
536
556
|
} else if (m.type === 'Error') { onError(`${m.code || 'error'}: ${m.description || ''}`); console.error('DG error msg', m); }
|
|
537
557
|
};
|
|
558
|
+
} catch (e) {
|
|
559
|
+
// open() is called fire-and-forget (pre-open, and lazily from feed()), so anything thrown here
|
|
560
|
+
// would surface as an unhandled rejection and kill the process under node. A socket that can't
|
|
561
|
+
// be built is unrecoverable for this provider — report it as fatal and let the host stop.
|
|
562
|
+
if (!isClosed()) onFatal(`STT open failed: ${e?.message || e}`);
|
|
538
563
|
} finally {
|
|
539
564
|
opening = false;
|
|
540
565
|
}
|
|
@@ -543,6 +568,10 @@ export function makeDeepgramSTT(opts) {
|
|
|
543
568
|
return {
|
|
544
569
|
continuous: true, // agent feeds the full mic stream (Flux needs real silences for its EOT model)
|
|
545
570
|
nativeEOT: true, // agent skips VAD-commit; EndOfTurn → unsolicited onFinal closes the turn
|
|
571
|
+
// Flux ends turns SEMANTICALLY, right behind its own last interim (21ms median, measured) —
|
|
572
|
+
// there is no trailing-silence debounce to hide a stability wait in, so the default 200ms timer
|
|
573
|
+
// never fires and every turn pays full LLM TTFT. Speculate on the interim tick instead.
|
|
574
|
+
prefetchMs: 0,
|
|
546
575
|
open() { if (!isClosed() && (!ws || ws.readyState >= 2)) open(); },
|
|
547
576
|
feed(i16) {
|
|
548
577
|
if (!isClosed() && (!ws || ws.readyState >= 2)) open();
|
|
@@ -637,9 +666,12 @@ export function makeWebSpeechSTT(opts) {
|
|
|
637
666
|
let micPending = null; // the single in-flight acquisition — concurrent ensureTrack calls share it
|
|
638
667
|
const acquireMic = () => {
|
|
639
668
|
if (micPending) return micPending;
|
|
640
|
-
|
|
669
|
+
// globalThis.navigator, not bare `navigator`: this module is imported under SSR and under node
|
|
670
|
+
// (tests), where the bare identifier is a ReferenceError rather than undefined — and this path
|
|
671
|
+
// is already written to degrade to the recognizer's own capture when there's no getUserMedia.
|
|
672
|
+
if (micTrack?.readyState === 'live' || !trackArg || !globalThis.navigator?.mediaDevices?.getUserMedia) return Promise.resolve();
|
|
641
673
|
const gen = micGen;
|
|
642
|
-
micPending = navigator.mediaDevices.getUserMedia({ audio: {
|
|
674
|
+
micPending = globalThis.navigator.mediaDevices.getUserMedia({ audio: {
|
|
643
675
|
echoCancellation: true, noiseSuppression: true, autoGainControl: true,
|
|
644
676
|
...(micDeviceId ? { deviceId: { exact: micDeviceId } } : {}),
|
|
645
677
|
} }).then((s) => {
|
package/src/tuning.js
CHANGED
|
@@ -96,6 +96,8 @@ export const TUNING = {
|
|
|
96
96
|
// (instant first token); a changed text just aborts it (cost: a few prompt tokens). Must be
|
|
97
97
|
// comfortably below WEBSPEECH_EOT_MS for the overlap to pay. 0 = prefetch on every interim tick
|
|
98
98
|
// (most aggressive, most wasted calls).
|
|
99
|
+
// Providers whose end-of-turn leaves no room for this wait override it with their own
|
|
100
|
+
// `prefetchMs` (deepgram flux: 0 — see stt.js).
|
|
99
101
|
PREFETCH_MS: 200,
|
|
100
102
|
|
|
101
103
|
// Audio chunks (~256ms each at 16kHz/4096-sample frames; ~1s total at 4) kept BEFORE VAD fires and
|
|
@@ -146,6 +148,13 @@ export const TUNING = {
|
|
|
146
148
|
redemptionFrames: 6,
|
|
147
149
|
},
|
|
148
150
|
|
|
151
|
+
// ── Mic level (host visualisers) ──────────────────────────────────────────────────────────────
|
|
152
|
+
// The 'level' event: a 0..1 loudness of the mic, emitted off the frames capture already produces.
|
|
153
|
+
// Purely cosmetic — nothing in the pipeline reads it — so it's tuned for how a meter LOOKS.
|
|
154
|
+
LEVEL_MIN_MS: 45, // rate limit; capture chunks are ~256ms, so this never throttles today but bounds a finer-grained capture
|
|
155
|
+
LEVEL_NOISE_FLOOR_RMS: 0.004, // below this the meter reads 0 — a quiet room must look silent, not "restless"
|
|
156
|
+
LEVEL_FULL_SCALE_RMS: 0.28, // RMS that reads as "full" — normal close speech peaks near 0.1-0.2, so this leaves headroom without pinning at 1
|
|
157
|
+
|
|
149
158
|
// ── TTS (first-audio latency) ─────────────────────────────────────────────────────────────────
|
|
150
159
|
// How soon the FIRST spoken chunk is cut from the streaming LLM text. Lower = the agent starts talking
|
|
151
160
|
// sooner (less waiting on generation), at a small prosody cost (it may break on a comma/clause rather
|
package/src/voice-agent.js
CHANGED
|
@@ -241,7 +241,7 @@ export class VoiceAgent {
|
|
|
241
241
|
// also bounds a turnDetector that never returns at all.
|
|
242
242
|
// `bargeInMinChars` — min transcribed NOVEL chars heard while speaking to count as a real
|
|
243
243
|
// barge-in; higher ignores short backchannels ("mhm","yeah"). Defaults live in tuning.js.
|
|
244
|
-
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'echo'|'error'|'diag', … }.
|
|
244
|
+
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'level'|'echo'|'error'|'diag', … }.
|
|
245
245
|
constructor({ sysmsg = '', persona, model = '', llm, fetchFn, apiKey = '', llmUrl = '', maxTokens = TUNING.MAX_TOKENS,
|
|
246
246
|
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
247
247
|
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
@@ -314,10 +314,18 @@ export class VoiceAgent {
|
|
|
314
314
|
// stable for PREFETCH_MS (no new words), start generating NOW with the probable final text;
|
|
315
315
|
// if the turn closes with that exact text the reply streams instantly (see _takePrefetch),
|
|
316
316
|
// if the user kept talking the stale request is aborted (cost: a few prompt tokens).
|
|
317
|
+
// The stability wait must fit INSIDE the gap between the last interim and the commit, or the
|
|
318
|
+
// timer never fires and the turn pays the LLM's full TTFT — silently, since a speculation
|
|
319
|
+
// that never STARTED looks identical to one that was never worth starting. Providers that
|
|
320
|
+
// debounce leave room for it (webspeech: ~1.2s of trailing silence, measured); a model that
|
|
321
|
+
// decides end-of-turn semantically does not (deepgram flux: 21ms median, measured) and must
|
|
322
|
+
// speculate on the interim tick instead. Providers set `prefetchMs` when 200ms won't land.
|
|
317
323
|
clearTimeout(this._prefetchTimer); this._prefetchTimer = null;
|
|
318
324
|
if (this._prefetch && interim && this._prefetch.text !== interim) this._dropPrefetch(); // transcript moved on → the running speculation is stale, stop paying for it
|
|
319
|
-
if (interim && this.state === 'listening' && !this._held && !this._echoRef)
|
|
320
|
-
|
|
325
|
+
if (interim && this.state === 'listening' && !this._held && !this._echoRef) {
|
|
326
|
+
const delay = this.stt.prefetchMs ?? TUNING.PREFETCH_MS ?? 200;
|
|
327
|
+
this._prefetchTimer = setTimeout(() => this._startPrefetch(interim), delay);
|
|
328
|
+
}
|
|
321
329
|
},
|
|
322
330
|
onFinal: (text, ms) => {
|
|
323
331
|
clearTimeout(this._prefetchTimer); this._prefetchTimer = null; // the turn is closing — no new speculation may start behind it
|
|
@@ -371,6 +379,7 @@ export class VoiceAgent {
|
|
|
371
379
|
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
372
380
|
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
373
381
|
this._tap = []; this._tapSamples = 0; this._tapTotal = 0; this._tapLastMs = 0; this._tapGaps = [];
|
|
382
|
+
this._levelAt = 0; // last 'level' emit (rate limit) — see _emitLevel
|
|
374
383
|
this._stream = null; this._ctx = null; this._node = null; this._vad = null; this._streamDeviceId = undefined; // pipeline kept warm across stop()/start() for instant resume
|
|
375
384
|
this._muted = false; // mic disabled → browser feeds silence to VAD+STT (nothing reaches the agent)
|
|
376
385
|
this._ttsMuted = false; // AI voice OUTPUT off → skip speaking, still stream the reply as text
|
|
@@ -496,7 +505,7 @@ export class VoiceAgent {
|
|
|
496
505
|
await ctx.audioWorklet.addModule(CAPTURE_WORKLET_URL);
|
|
497
506
|
if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); ctx.close(); return; } // superseded/stopped while loading
|
|
498
507
|
const node = new AudioWorkletNode(ctx, 'capture-processor');
|
|
499
|
-
node.port.onmessage = e => this.
|
|
508
|
+
node.port.onmessage = e => this._onCapture(e.data); // Float32Array chunk from the audio thread
|
|
500
509
|
// The processor writes no output — this connection only keeps the node pulled by the graph.
|
|
501
510
|
src.connect(node); node.connect(ctx.destination);
|
|
502
511
|
this._ctx = ctx; this._node = node;
|
|
@@ -834,7 +843,16 @@ export class VoiceAgent {
|
|
|
834
843
|
_takePrefetch(text) {
|
|
835
844
|
const p = this._prefetch;
|
|
836
845
|
// histLen is checked BEFORE _runTurn pushes this turn's own user message (see call site).
|
|
837
|
-
if (!p || p.text !== text || p.ctl.signal.aborted || p.histLen !== this.history.length) {
|
|
846
|
+
if (!p || p.text !== text || p.ctl.signal.aborted || p.histLen !== this.history.length) {
|
|
847
|
+
// Which guard rejected: adoption is worth ~600ms (the whole LLM TTFT lands on the critical
|
|
848
|
+
// path without it), so a silent miss is invisible latency. `text` mismatch is the expected
|
|
849
|
+
// one (the user kept talking); the others mean the speculation was raced or aborted.
|
|
850
|
+
this.onEvent?.({ type: 'diag', diag: 'prefetch-miss',
|
|
851
|
+
why: !p ? 'none' : p.text !== text ? 'text' : p.ctl.signal.aborted ? 'aborted' : 'histLen',
|
|
852
|
+
text, pre: p?.text });
|
|
853
|
+
this._dropPrefetch(); return null;
|
|
854
|
+
}
|
|
855
|
+
this.onEvent?.({ type: 'diag', diag: 'prefetch-hit', text });
|
|
838
856
|
this._prefetch = null; // timer already cleared: successful adoption is only reached from a spoken onFinal
|
|
839
857
|
p.commit(); // turn committed → open the tool gate; held-back tool calls execute now
|
|
840
858
|
return { ctl: p.ctl, gen: (async function* () { const f = await p.first; if (!f.done) { yield f.value; yield* p.gen; } })() };
|
|
@@ -1239,6 +1257,37 @@ export class VoiceAgent {
|
|
|
1239
1257
|
this.stt.feed(i16);
|
|
1240
1258
|
}
|
|
1241
1259
|
|
|
1260
|
+
// The capture entry point: everything the pipeline needs, THEN the cosmetic tap. _feed has several
|
|
1261
|
+
// early returns (preroll while silent, continuous providers), so the level can't just live at its
|
|
1262
|
+
// end — and it must not live at the START either, where a host that throws while painting its
|
|
1263
|
+
// meter would take down the frame STT was waiting for.
|
|
1264
|
+
_onCapture(f32) { this._feed(f32); this._emitLevel(f32); }
|
|
1265
|
+
|
|
1266
|
+
// ── mic level ──────────────────────────────────────────────────────────
|
|
1267
|
+
// How loud the user is RIGHT NOW, for host visualisers (a mic meter, a reacting orb). Derived from
|
|
1268
|
+
// the frames already in hand — no second AudioContext, no AnalyserNode, no extra mic tap.
|
|
1269
|
+
// Purely cosmetic, so it is emitted only for a session the host would call LIVE: stop() pauses the
|
|
1270
|
+
// pipeline without closing the mic (warm resume) and the worklet keeps delivering frames, so
|
|
1271
|
+
// without the _closed guard a "stopped" agent would keep driving the host's meter; a muted mic
|
|
1272
|
+
// would likewise stream a flat zero forever instead of going quiet.
|
|
1273
|
+
// Rate-limited to ~LEVEL_MIN_MS so a capture finer than the current 4096-sample (256ms @16k)
|
|
1274
|
+
// chunk can't flood the host. A selfCapture STT (Web Speech) owns its own mic — no frames, no
|
|
1275
|
+
// level events at all.
|
|
1276
|
+
_emitLevel(f32) {
|
|
1277
|
+
if (this._closed || this._muted) return;
|
|
1278
|
+
const now = (globalThis.performance ?? Date).now();
|
|
1279
|
+
if (now - this._levelAt < TUNING.LEVEL_MIN_MS) return;
|
|
1280
|
+
this._levelAt = now;
|
|
1281
|
+
let sum = 0; for (let i = 0; i < f32.length; i++) sum += f32[i] * f32[i];
|
|
1282
|
+
const rms = Math.sqrt(sum / f32.length);
|
|
1283
|
+
// Perceptual curve, floored: linear RMS spends most of its range on levels nobody can hear, so a
|
|
1284
|
+
// meter driven by it barely moves during speech. The floor is what keeps room noise reading as
|
|
1285
|
+
// silence — without it a cube root maps a near-silent room to ~0.15-0.3 and the meter never rests.
|
|
1286
|
+
const norm = (rms - TUNING.LEVEL_NOISE_FLOOR_RMS) / (TUNING.LEVEL_FULL_SCALE_RMS - TUNING.LEVEL_NOISE_FLOOR_RMS);
|
|
1287
|
+
const level = norm <= 0 ? 0 : Math.min(1, Math.cbrt(norm));
|
|
1288
|
+
this.onEvent({ type: 'level', level });
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1242
1291
|
// ── diagnostic tap ─────────────────────────────────────────────────────
|
|
1243
1292
|
// Ring-buffer the frame + detect delivery stalls: a wall-clock gap between _feed() calls beyond
|
|
1244
1293
|
// ~1.5 chunks. With the worklet capturing off-thread that's usually DELAYED delivery rather than
|
|
@@ -1871,13 +1920,23 @@ const ORT_SPECIFIER = 'onnxruntime-web';
|
|
|
1871
1920
|
|
|
1872
1921
|
// OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
|
|
1873
1922
|
// models already downloaded through the lib are reused (and vice versa).
|
|
1874
|
-
|
|
1923
|
+
// `onProgress(loaded, total)` fires while the file is coming down the wire (total is 0 if the server
|
|
1924
|
+
// sends no content-length). A Piper voice is ~60MB: on mobile data that is a minute of apparent
|
|
1925
|
+
// nothing, so a host needs to be able to SAY so — see PiperTTS.setOnModelProgress. Cache hits report
|
|
1926
|
+
// nothing (they're instant).
|
|
1927
|
+
async function opfsCachedFetch(url, onProgress) {
|
|
1875
1928
|
const name = url.split('/').at(-1);
|
|
1876
1929
|
const dir = () => navigator.storage.getDirectory().then((r) => r.getDirectoryHandle('piper', { create: true }));
|
|
1877
1930
|
try { return await (await (await dir()).getFileHandle(name)).getFile(); } catch {}
|
|
1878
1931
|
const res = await fetch(url);
|
|
1879
1932
|
if (!res.ok) throw new Error(`fetch ${url}: ${res.status}`);
|
|
1880
|
-
|
|
1933
|
+
let blob;
|
|
1934
|
+
if (onProgress && res.body) {
|
|
1935
|
+
const total = +(res.headers.get('content-length') || 0);
|
|
1936
|
+
const chunks = []; let loaded = 0;
|
|
1937
|
+
for await (const c of res.body) { chunks.push(c); loaded += c.byteLength; onProgress(loaded, total); }
|
|
1938
|
+
blob = new Blob(chunks);
|
|
1939
|
+
} else blob = await res.blob();
|
|
1881
1940
|
try { const w = await (await (await dir()).getFileHandle(name, { create: true })).createWritable(); await w.write(blob); await w.close(); } catch {}
|
|
1882
1941
|
return blob;
|
|
1883
1942
|
}
|
|
@@ -1896,6 +1955,10 @@ function pcm2wav(f32, sampleRate) {
|
|
|
1896
1955
|
}
|
|
1897
1956
|
export class PiperTTS extends StreamingTTS {
|
|
1898
1957
|
constructor(voiceId = 'en_US-joe-medium', speed = 1.0) { super(voiceId, speed); }
|
|
1958
|
+
// Called with (loadedBytes, totalBytes) while the ~60MB voice model downloads on first use —
|
|
1959
|
+
// that download is one silent minute on mobile data, so a UI can show a real progress bar
|
|
1960
|
+
// instead of a dead button. Fires only on a cache MISS; later sessions read it from OPFS.
|
|
1961
|
+
setOnModelProgress(fn) { this._onModelProgress = fn || null; return this; }
|
|
1899
1962
|
// webpackIgnore is honored by webpack AND Turbopack (vite needs its own marker) — without it
|
|
1900
1963
|
// bundlers rewrite this into their runtime require, which can't load a URL
|
|
1901
1964
|
// ("turbopack_context.x is not a function").
|
|
@@ -1939,7 +2002,7 @@ export class PiperTTS extends StreamingTTS {
|
|
|
1939
2002
|
import(/* webpackIgnore: true */ /* @vite-ignore */ ORT_SPECIFIER),
|
|
1940
2003
|
import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
|
|
1941
2004
|
opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
|
|
1942
|
-
opfsCachedFetch(`${lib.HF_BASE}/${path}
|
|
2005
|
+
opfsCachedFetch(`${lib.HF_BASE}/${path}`, this._onModelProgress),
|
|
1943
2006
|
]);
|
|
1944
2007
|
const cfg = JSON.parse(await cfgBlob.text());
|
|
1945
2008
|
// Match the importmap's onnxruntime-web version: the lib's baked-in default points at a
|