@todoforai/voiceloop 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/examples/demo.html +410 -0
- package/package.json +9 -4
- package/src/index.d.ts +320 -0
- package/src/index.js +2 -2
- package/src/stt.js +52 -12
- package/src/tuning.js +11 -0
- package/src/voice-agent.js +10 -3
package/README.md
CHANGED
|
@@ -52,6 +52,26 @@ button.onclick = async () => {
|
|
|
52
52
|
|
|
53
53
|
That's it. Speak, get spoken answers, interrupt at will.
|
|
54
54
|
|
|
55
|
+
## Try it in 20 seconds
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
npx serve . # from a clone
|
|
59
|
+
npx serve node_modules/@todoforai/voiceloop # from npm
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Open **`/examples/demo.html`** — the whole loop behind one orb, with the live
|
|
63
|
+
interim transcript, click-to-interrupt, and the two numbers that decide whether
|
|
64
|
+
it feels alive: **first token** and **first sound**. It starts in **echo mode**
|
|
65
|
+
— browser-native Web Speech STT, local Piper TTS and a stand-in "LLM" that
|
|
66
|
+
repeats you — so it runs with no keys, no backend and no cost. Open *settings*
|
|
67
|
+
to point it at any OpenAI-compatible endpoint (Ollama, your proxy) and a cloud
|
|
68
|
+
STT provider.
|
|
69
|
+
|
|
70
|
+
`/examples/simple-browser.html` is the same loop with no UI at all — 100 lines,
|
|
71
|
+
the place to start reading.
|
|
72
|
+
|
|
73
|
+
(`file://` won't work — mic and module imports need `http://localhost` or https.)
|
|
74
|
+
|
|
55
75
|
## How the latency adds up
|
|
56
76
|
|
|
57
77
|
```
|
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<!-- voiceloop demo: the whole loop behind one button, plus the numbers that say
|
|
3
|
+
whether it feels seamless. No bundler, no framework, no dependencies.
|
|
4
|
+
|
|
5
|
+
from a clone: npx serve . → /examples/demo.html
|
|
6
|
+
from npm: npx serve node_modules/@todoforai/voiceloop
|
|
7
|
+
|
|
8
|
+
file:// won't work — mic + module imports need http://localhost or https.
|
|
9
|
+
Out of the box it runs in ECHO mode: Web Speech STT + local Piper TTS +
|
|
10
|
+
a fake "LLM" that repeats you. No keys, no backend, no cost. Open
|
|
11
|
+
settings to point it at a real LLM. -->
|
|
12
|
+
<html lang="en">
|
|
13
|
+
<head>
|
|
14
|
+
<meta charset="utf-8">
|
|
15
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
16
|
+
<title>voiceloop</title>
|
|
17
|
+
<style>
|
|
18
|
+
:root {
|
|
19
|
+
color-scheme: dark;
|
|
20
|
+
--bg:#0b0d10; --panel:#14171c; --line:#242932; --fg:#eceef1; --dim:#79818d;
|
|
21
|
+
--c:#4a5361; /* current state colour — everything reads from here */
|
|
22
|
+
}
|
|
23
|
+
* { box-sizing:border-box }
|
|
24
|
+
body { margin:0; background:var(--bg); color:var(--fg);
|
|
25
|
+
font:15px/1.55 ui-sans-serif,system-ui,-apple-system,sans-serif }
|
|
26
|
+
main { max-width:660px; margin:0 auto; padding:36px 18px 48px;
|
|
27
|
+
display:flex; flex-direction:column; align-items:center; gap:16px }
|
|
28
|
+
|
|
29
|
+
/* ── the orb: four radial blobs drifting behind a glass shell ──────────────
|
|
30
|
+
Plain CSS on purpose. A WebGPU version looks marginally better and is dark
|
|
31
|
+
for every Linux visitor (Chrome blocklists the adapter there), and an SVG
|
|
32
|
+
goo filter costs real CPU per frame for the same silhouette. This is ~25
|
|
33
|
+
lines, runs on the compositor, and degrades to a flat disc under
|
|
34
|
+
prefers-reduced-motion. */
|
|
35
|
+
#mic { position:relative; width:210px; height:210px; border-radius:50%; border:0; padding:0;
|
|
36
|
+
cursor:pointer; overflow:hidden; background:#0c0f14; display:grid; place-items:center;
|
|
37
|
+
--a:#2b3d5c; --b:#39506e; --cc:#2a3550; --d:#334463; --spin:30s; --glow:.14;
|
|
38
|
+
transition:transform .18s ease-out }
|
|
39
|
+
/* Each blob is a soft disc smaller than the orb, wandering on its own ellipse.
|
|
40
|
+
Blurred and screen-blended, four of them read as one body that swells and
|
|
41
|
+
pinches where they overlap — metaballs, minus the shader. */
|
|
42
|
+
#mic i { position:absolute; width:52%; height:52%; left:24%; top:24%; border-radius:50%;
|
|
43
|
+
filter:blur(13px); mix-blend-mode:screen; animation:wander var(--spin) ease-in-out infinite;
|
|
44
|
+
transition:background 1s } /* colours cross-fade; the speed change snaps (animation-* isn't animatable) */
|
|
45
|
+
#mic i:nth-child(1) { background:radial-gradient(circle, var(--a) 0 26%, transparent 66%) }
|
|
46
|
+
#mic i:nth-child(2) { background:radial-gradient(circle, var(--b) 0 24%, transparent 64%);
|
|
47
|
+
animation-duration:calc(var(--spin)*.72); animation-direction:reverse }
|
|
48
|
+
#mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0 28%, transparent 68%);
|
|
49
|
+
animation-duration:calc(var(--spin)*1.4); animation-delay:calc(var(--spin)*-.3) }
|
|
50
|
+
#mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0 22%, transparent 62%);
|
|
51
|
+
animation-duration:calc(var(--spin)*.9); animation-direction:reverse;
|
|
52
|
+
animation-delay:calc(var(--spin)*-.55) }
|
|
53
|
+
/* glass shell: specular cap + rim shadow, one pseudo each */
|
|
54
|
+
#mic::before { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
55
|
+
background:radial-gradient(120% 90% at 32% 20%, rgba(255,255,255,.30) 0 10%,
|
|
56
|
+
rgba(255,255,255,.05) 26%, transparent 52%);
|
|
57
|
+
box-shadow:inset 0 0 46px rgba(0,0,0,.75), inset 0 1px 1px rgba(255,255,255,.24) }
|
|
58
|
+
#mic::after { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
59
|
+
box-shadow:0 0 44px 8px var(--b); opacity:var(--glow); transition:opacity .8s }
|
|
60
|
+
@keyframes wander {
|
|
61
|
+
0%,100% { transform:translate(-34%,-22%) scale(1) }
|
|
62
|
+
25% { transform:translate( 30%,-34%) scale(1.18) }
|
|
63
|
+
50% { transform:translate( 36%, 28%) scale(.86) }
|
|
64
|
+
75% { transform:translate(-28%, 34%) scale(1.12) }
|
|
65
|
+
}
|
|
66
|
+
/* the glyph leads while the orb is dark; once the loop is live the orb says it better */
|
|
67
|
+
#glyph { position:relative; font-size:36px; line-height:1; transition:opacity .4s;
|
|
68
|
+
text-shadow:0 2px 12px #000 }
|
|
69
|
+
body:not([data-state="idle"]) #glyph { opacity:0 }
|
|
70
|
+
|
|
71
|
+
/* same object, more energy — the whole state display is these four lines */
|
|
72
|
+
body[data-state="listening"] #mic { --a:#12b87e; --b:#39d98a; --cc:#0e7fb4; --d:#5fd9b4; --spin:14s; --glow:.45 }
|
|
73
|
+
body[data-state="thinking"] #mic { --a:#e0a12e; --b:#ff9f43; --cc:#a63fc0; --d:#ffd166; --spin:6s; --glow:.65 }
|
|
74
|
+
body[data-state="speaking"] #mic { --a:#ff7a2d; --b:#ff5f6d; --cc:#ffa94d; --d:#b83fd0; --spin:4s; --glow:.9 }
|
|
75
|
+
/* VAD: the orb swells the instant the mic hears you — the "it's alive" tell */
|
|
76
|
+
body[data-vad="1"] #mic { transform:scale(1.035) }
|
|
77
|
+
|
|
78
|
+
body[data-state="listening"] { --c:#39d98a }
|
|
79
|
+
body[data-state="thinking"] { --c:#ffc453 }
|
|
80
|
+
body[data-state="speaking"] { --c:#ff8a3d }
|
|
81
|
+
|
|
82
|
+
@media (prefers-reduced-motion: reduce) {
|
|
83
|
+
#mic i { animation:none } #mic { transition:none }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
#status { color:var(--c); font-size:14px; min-height:22px; text-align:center; transition:color .3s }
|
|
87
|
+
#lat { font:12px/1 ui-monospace,monospace; color:var(--dim); min-height:14px }
|
|
88
|
+
|
|
89
|
+
/* ── transcript ─────────────────────────────────────────────────────── */
|
|
90
|
+
#chat { width:100%; display:flex; flex-direction:column; gap:10px }
|
|
91
|
+
#chat:not(:empty) { margin-top:6px }
|
|
92
|
+
.msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
|
|
93
|
+
.msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
|
|
94
|
+
.msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
|
|
95
|
+
/* An echoed reply is not an answer — mark it so the transcript can't be read as a model talking. */
|
|
96
|
+
.msg.ai.echoed { border-left:2px solid #e0a94a }
|
|
97
|
+
.msg.ai.echoed::after { content:'echo — no model'; display:block; margin-top:5px;
|
|
98
|
+
font-size:10px; letter-spacing:.04em; color:#e0a94a; opacity:.85 }
|
|
99
|
+
.msg.interim { opacity:.5 }
|
|
100
|
+
.msg .tail { opacity:.35 } /* generated, not yet spoken */
|
|
101
|
+
.msg.tool { align-self:flex-start; background:#161d17; color:#8fc79a;
|
|
102
|
+
font:12px/1.4 ui-monospace,monospace }
|
|
103
|
+
.msg.note { align-self:center; background:none; color:var(--dim); font-size:12px; padding:0 }
|
|
104
|
+
.msg.err { align-self:center; background:#38141d; color:#ffb0bd; font-size:13px }
|
|
105
|
+
|
|
106
|
+
/* ── everything else, out of the way ────────────────────────────────── */
|
|
107
|
+
.bar { width:100%; display:flex; gap:8px }
|
|
108
|
+
input, select, button.btn { background:#0f1216; color:var(--fg); border:1px solid var(--line);
|
|
109
|
+
border-radius:9px; padding:9px 12px; font:inherit }
|
|
110
|
+
input:focus, select:focus { outline:none; border-color:var(--c) }
|
|
111
|
+
button.btn { cursor:pointer } button.btn:hover { border-color:#3a424e }
|
|
112
|
+
#typed { flex:1 }
|
|
113
|
+
details { width:100%; border-top:1px solid var(--line); padding-top:10px }
|
|
114
|
+
summary { cursor:pointer; color:var(--dim); font-size:13px; list-style:none }
|
|
115
|
+
summary::before { content:'▸ ' } details[open] summary::before { content:'▾ ' }
|
|
116
|
+
.grid { display:flex; flex-wrap:wrap; gap:10px; padding:12px 0 4px }
|
|
117
|
+
label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
|
|
118
|
+
#log { font:11px/1.5 ui-monospace,monospace; color:var(--dim);
|
|
119
|
+
max-height:220px; overflow:auto; white-space:pre-wrap; padding-top:8px }
|
|
120
|
+
/* Which code is actually running — the demo must never let you mistake the browser's own
|
|
121
|
+
recognizer for voiceloop's pipeline, or an echo stand-in for a real LLM. */
|
|
122
|
+
#pipeline { font-size:11.5px; color:var(--dim); text-align:center; line-height:1.7; max-width:46ch }
|
|
123
|
+
#pipeline b { color:#c8d0da; font-weight:600 }
|
|
124
|
+
#pipeline .warn { color:#e0a94a }
|
|
125
|
+
</style>
|
|
126
|
+
</head>
|
|
127
|
+
<body>
|
|
128
|
+
<main>
|
|
129
|
+
<button id="mic" title="click to start / stop · click while it talks to interrupt">
|
|
130
|
+
<i></i><i></i><i></i><i></i><span id="glyph">🎙</span>
|
|
131
|
+
</button>
|
|
132
|
+
<div id="status">click to start · then just talk</div>
|
|
133
|
+
<div id="lat"></div>
|
|
134
|
+
<div id="pipeline"></div>
|
|
135
|
+
|
|
136
|
+
<div id="chat"></div>
|
|
137
|
+
|
|
138
|
+
<div class="bar">
|
|
139
|
+
<input id="typed" placeholder="…or type a turn">
|
|
140
|
+
<button class="btn" id="send">Send</button>
|
|
141
|
+
</div>
|
|
142
|
+
|
|
143
|
+
<details>
|
|
144
|
+
<summary>settings</summary>
|
|
145
|
+
<div class="grid">
|
|
146
|
+
<label>mode<select id="mode">
|
|
147
|
+
<option value="echo">echo — no keys, no backend</option>
|
|
148
|
+
<option value="llm">full loop — your LLM</option>
|
|
149
|
+
</select></label>
|
|
150
|
+
<label>LLM url<input id="llmUrl" size="30" value="http://localhost:11434/v1/chat/completions"></label>
|
|
151
|
+
<label>model<input id="model" size="14" value="llama3.2"></label>
|
|
152
|
+
<label>STT<select id="sttProvider"></select></label>
|
|
153
|
+
<label>STT token url<input id="sttTokenUrl" size="30" placeholder="your backend route (cloud STT only)"></label>
|
|
154
|
+
<label>api key<input id="apiKey" size="18" placeholder="Bearer for LLM · X-API-Key for token url"></label>
|
|
155
|
+
<label>lang<select id="sttLang"></select></label>
|
|
156
|
+
<label>voice<select id="voice"></select></label>
|
|
157
|
+
<label>mic<select id="micDeviceId"><option value="">default</option></select></label>
|
|
158
|
+
</div>
|
|
159
|
+
<div class="bar">
|
|
160
|
+
<button class="btn" id="clear">Clear</button>
|
|
161
|
+
<button class="btn" id="muteMic">Mute mic</button>
|
|
162
|
+
<button class="btn" id="muteTts">Mute voice</button>
|
|
163
|
+
</div>
|
|
164
|
+
</details>
|
|
165
|
+
|
|
166
|
+
<details><summary>log</summary><div id="log"></div></details>
|
|
167
|
+
</main>
|
|
168
|
+
|
|
169
|
+
<script type="module">
|
|
170
|
+
import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice, STT_PROVIDERS, sttSelfCaptures, LANG_NAMES } from '../src/index.js';
|
|
171
|
+
|
|
172
|
+
// Load VAD/WASM + build the Silero session while the page sits idle — but ONLY for a provider that
|
|
173
|
+
// will actually use it. A selfCapture provider (webspeech) builds no pipeline, so prewarming there
|
|
174
|
+
// downloads an ONNX model to feed a detector that never runs.
|
|
175
|
+
const prewarmIfPipelined = () => { if (!sttSelfCaptures($('sttProvider').value)) prewarmVoice(); };
|
|
176
|
+
|
|
177
|
+
const $ = id => document.getElementById(id);
|
|
178
|
+
const fill = (id, entries) => $(id).append(...entries.map(([value, textContent]) =>
|
|
179
|
+
Object.assign(document.createElement('option'), { value, textContent })));
|
|
180
|
+
|
|
181
|
+
// Every choice below is READ from the library, never retyped here: a provider added to STT_PROVIDERS
|
|
182
|
+
// or a language added to LANG_NAMES shows up in this page by itself. A hand-typed copy would be a
|
|
183
|
+
// second source of truth that silently rots — and a demo that lies about what the library supports.
|
|
184
|
+
fill('sttProvider', Object.keys(STT_PROVIDERS).map(p => [p, p === 'webspeech' ? 'webspeech — free, in-browser' : p]));
|
|
185
|
+
fill('sttLang', Object.entries(LANG_NAMES).map(([code, name]) => [code, `${code} — ${name}`]));
|
|
186
|
+
const SETTINGS = ['mode','llmUrl','model','sttProvider','sttTokenUrl','apiKey','sttLang','voice','micDeviceId'];
|
|
187
|
+
for (const id of SETTINGS) {
|
|
188
|
+
const v = localStorage.getItem('vl.' + id); if (v) $(id).value = v;
|
|
189
|
+
$(id).addEventListener('change', () => {
|
|
190
|
+
localStorage.setItem('vl.' + id, $(id).value);
|
|
191
|
+
onSettingChanged(); // the banner must never describe the PREVIOUS choice
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
const log = (...a) => {
|
|
195
|
+
const line = `${new Date().toISOString().slice(11,23)} ${a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ')}`;
|
|
196
|
+
$('log').textContent = (line + '\n' + $('log').textContent).slice(0, 20000);
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
// ── transcript ──────────────────────────────────────────────────────────────
|
|
200
|
+
const chat = $('chat');
|
|
201
|
+
let interimEl = null, aiEl = null, vadOff = null;
|
|
202
|
+
const toolEls = new Map();
|
|
203
|
+
const bubble = (cls, text = '') => {
|
|
204
|
+
const el = document.createElement('div');
|
|
205
|
+
el.className = `msg ${cls}`; el.textContent = text;
|
|
206
|
+
chat.append(el); el.scrollIntoView({ block:'nearest' });
|
|
207
|
+
return el;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
// ── latency: the two numbers that decide whether this feels alive ───────────
|
|
211
|
+
let t0 = 0, tText = 0, tAudio = 0;
|
|
212
|
+
const showLat = () => $('lat').textContent = [
|
|
213
|
+
tText && `first token ${tText - t0}ms`,
|
|
214
|
+
tAudio && `first sound ${tAudio - t0}ms`,
|
|
215
|
+
].filter(Boolean).join(' · ');
|
|
216
|
+
|
|
217
|
+
// undefined (not '') when unset — so PiperTTS applies its OWN default voice instead of this page
|
|
218
|
+
// re-declaring one. The picker is populated async below and adopts whatever it resolved to.
|
|
219
|
+
const tts = new PiperTTS(localStorage.getItem('vl.voice') || undefined);
|
|
220
|
+
let agent = null;
|
|
221
|
+
|
|
222
|
+
// Echo mode: a stand-in "LLM" that repeats the turn VERBATIM — exercises the entire
|
|
223
|
+
// VAD→STT→…→TTS loop, barge-in included, with no key and no backend. Nothing is added to the
|
|
224
|
+
// text: a reply that is exactly the transcript is the whole point, since any prefix would be
|
|
225
|
+
// this page's words in the reply and would blur what the recognizer actually heard. That it's
|
|
226
|
+
// an echo and not a model is stated in the pipeline banner and on the bubble itself.
|
|
227
|
+
async function* echoLLM(history) {
|
|
228
|
+
const last = [...history].reverse().find(m => m.role === 'user')?.content ?? '';
|
|
229
|
+
for (const w of last.match(/\S+\s*/g) ?? []) {
|
|
230
|
+
yield { text: w }; await new Promise(r => setTimeout(r, 25));
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// ── what is ACTUALLY running ────────────────────────────────────────────────
|
|
235
|
+
// A demo that hides which engine served a reply is worse than no demo: a browser-native
|
|
236
|
+
// recognizer looks identical to voiceloop's pipeline, and an echo stand-in looks identical
|
|
237
|
+
// to a model. Both get named on screen, always.
|
|
238
|
+
const describePipeline = (a) => {
|
|
239
|
+
const echo = $('mode').value === 'echo';
|
|
240
|
+
// Before the first click there is no session, so ask the provider statically — reading
|
|
241
|
+
// `a?.stt?.selfCapture` off a null agent silently yields false and the banner claims the
|
|
242
|
+
// pipeline is running when it is exactly what gets bypassed.
|
|
243
|
+
const self = a?.stt ? !!a.stt.selfCapture : sttSelfCaptures($('sttProvider').value);
|
|
244
|
+
const stt = self
|
|
245
|
+
? `<b>${$('sttProvider').value}</b> — the browser's own recognizer owns the mic <span class="warn">(voiceloop's VAD + STT pipeline bypassed)</span>`
|
|
246
|
+
: `<b>${$('sttProvider').value}</b> via voiceloop's pipeline — Silero VAD → feed → commit`;
|
|
247
|
+
const llm = echo
|
|
248
|
+
? '<span class="warn"><b>echo</b> — no model: replies repeat your words back</span>'
|
|
249
|
+
: `<b>${$('model').value || 'your model'}</b> at ${new URL($('llmUrl').value).host}`;
|
|
250
|
+
return `STT: ${stt}<br>LLM: ${llm}`;
|
|
251
|
+
};
|
|
252
|
+
const showPipeline = () => { $('pipeline').innerHTML = describePipeline(agent); };
|
|
253
|
+
// Hoisted (function declaration) so the settings listeners above can call it before this point.
|
|
254
|
+
function onSettingChanged() { showPipeline(); prewarmIfPipelined(); }
|
|
255
|
+
|
|
256
|
+
// A typed turn drives the same state machine without a mic — don't claim we're listening then.
|
|
257
|
+
let micOn = false;
|
|
258
|
+
const STATUS = {
|
|
259
|
+
idle: () => micOn ? 'listening — just talk' : 'click to start · then just talk',
|
|
260
|
+
listening: () => micOn ? 'listening — just talk' : 'ready · click to talk',
|
|
261
|
+
thinking: () => 'thinking…',
|
|
262
|
+
speaking: () => micOn ? 'speaking — talk over it to interrupt' : 'speaking — click to interrupt',
|
|
263
|
+
};
|
|
264
|
+
const setState = s => {
|
|
265
|
+
document.body.dataset.state = micOn || s !== 'listening' ? s : 'idle';
|
|
266
|
+
$('status').textContent = STATUS[s]?.() ?? s;
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
function onEvent(e) {
|
|
270
|
+
if (e.type !== 'stt' || e.turnComplete) log(e);
|
|
271
|
+
switch (e.type) {
|
|
272
|
+
case 'state':
|
|
273
|
+
// The agent can stop itself (mic denied, STT socket died) — the mic light
|
|
274
|
+
// follows the agent, never the other way round.
|
|
275
|
+
if (e.state === 'idle' && agent?.closed) { micOn = false; document.body.dataset.vad = '0'; }
|
|
276
|
+
setState(e.state);
|
|
277
|
+
break;
|
|
278
|
+
case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
|
|
279
|
+
case 'stt':
|
|
280
|
+
// A selfCapture provider runs no agent VAD, so no 'vad' event ever arrives and the orb would
|
|
281
|
+
// sit dead still while the browser hears you perfectly — looking exactly like a broken mic.
|
|
282
|
+
// Live partials are the only "I hear you" signal on that path, so drive the pulse from them.
|
|
283
|
+
if (agent?.stt?.selfCapture) {
|
|
284
|
+
document.body.dataset.vad = e.turnComplete ? '0' : '1';
|
|
285
|
+
clearTimeout(vadOff);
|
|
286
|
+
if (!e.turnComplete) vadOff = setTimeout(() => { document.body.dataset.vad = '0'; }, 700);
|
|
287
|
+
}
|
|
288
|
+
if (e.turnComplete) {
|
|
289
|
+
interimEl?.remove(); interimEl = null;
|
|
290
|
+
bubble('user', e.text);
|
|
291
|
+
aiEl = null; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
|
|
292
|
+
} else {
|
|
293
|
+
interimEl ??= bubble('user interim');
|
|
294
|
+
interimEl.textContent = `${e.committed ?? ''} ${e.text}`.trim();
|
|
295
|
+
interimEl.scrollIntoView({ block:'nearest' });
|
|
296
|
+
}
|
|
297
|
+
break;
|
|
298
|
+
case 'assistant': {
|
|
299
|
+
// Empty and non-final means "that draft never happened" (interrupted before
|
|
300
|
+
// a word was spoken) — drop the bubble rather than blank it.
|
|
301
|
+
if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; break; }
|
|
302
|
+
if (!tText && t0) { tText = Date.now(); showLat(); }
|
|
303
|
+
// `full` only exists once the TTS progress cursor is live — i.e. a clip is
|
|
304
|
+
// actually playing. The 'speaking' state is just the first token, not sound.
|
|
305
|
+
if (e.full != null && t0 && !tAudio) { tAudio = Date.now(); showLat(); }
|
|
306
|
+
// Tag the bubble (never the text) when no model produced it — the reply is a verbatim echo,
|
|
307
|
+
// and only the UI should say so.
|
|
308
|
+
aiEl ??= bubble($('mode').value === 'echo' ? 'ai echoed' : 'ai');
|
|
309
|
+
aiEl.textContent = e.text; // spoken so far
|
|
310
|
+
const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
|
|
311
|
+
if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
|
|
312
|
+
aiEl.scrollIntoView({ block:'nearest' });
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
case 'tool': {
|
|
316
|
+
// running and result arrive as two events with one id: update, don't stack.
|
|
317
|
+
const el = toolEls.get(e.id) ?? bubble('tool');
|
|
318
|
+
if (e.id) toolEls.set(e.id, el);
|
|
319
|
+
el.textContent = `🔧 ${e.name}(${JSON.stringify(e.args)})${e.running ? ' …' : ` → ${JSON.stringify(e.result ?? '')}`.slice(0,200)}`;
|
|
320
|
+
break;
|
|
321
|
+
}
|
|
322
|
+
case 'echo': bubble('note', `self-echo dropped: ${e.text}`); break;
|
|
323
|
+
case 'error': bubble('err', e.error); break;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const muted = id => $(id).dataset.on === '1';
|
|
328
|
+
const makeAgent = () => {
|
|
329
|
+
const a = new VoiceAgent({
|
|
330
|
+
llmUrl: $('llmUrl').value.trim(),
|
|
331
|
+
model: $('model').value.trim(),
|
|
332
|
+
apiKey: $('apiKey').value.trim(),
|
|
333
|
+
sttProvider: $('sttProvider').value,
|
|
334
|
+
sttTokenUrl: $('sttTokenUrl').value.trim(),
|
|
335
|
+
sttLang: $('sttLang').value,
|
|
336
|
+
micDeviceId: $('micDeviceId').value,
|
|
337
|
+
persona: 'You are a concise, friendly voice assistant. Keep answers to a sentence or two.',
|
|
338
|
+
tts,
|
|
339
|
+
llm: $('mode').value === 'echo' ? echoLLM : undefined,
|
|
340
|
+
onEvent,
|
|
341
|
+
});
|
|
342
|
+
a.setMuted(muted('muteMic')); a.setTtsMuted(muted('muteTts')); // the toggles survive a rebuild
|
|
343
|
+
log('agent created; stt =', a.sttProvider); showPipeline(); // now reflects the LIVE session, not just the form
|
|
344
|
+
return a;
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
// ── one button does everything ──────────────────────────────────────────────
|
|
348
|
+
const stop = () => {
|
|
349
|
+
agent?.destroy(); agent = null; micOn = false;
|
|
350
|
+
document.body.dataset.vad = '0'; setState('idle');
|
|
351
|
+
onSettingChanged(); // describe the restored settings, and prewarm only if they'll use the VAD
|
|
352
|
+
};
|
|
353
|
+
$('mic').onclick = async () => {
|
|
354
|
+
unlockAudio(); // must be inside the gesture (autoplay policy)
|
|
355
|
+
// A reply in flight? then the click means "shut up", whether the mic is open
|
|
356
|
+
// or not — it cancels the turn and keeps the session.
|
|
357
|
+
if (agent && (agent.state === 'thinking' || agent.state === 'speaking')) return agent.interrupt();
|
|
358
|
+
if (micOn) return stop();
|
|
359
|
+
agent ??= makeAgent();
|
|
360
|
+
micOn = true; setState('listening');
|
|
361
|
+
try { await agent.start(); }
|
|
362
|
+
catch (e) { bubble('err', String(e?.message ?? e)); stop(); }
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
$('send').onclick = () => {
|
|
366
|
+
const text = $('typed').value.trim(); if (!text) return;
|
|
367
|
+
unlockAudio();
|
|
368
|
+
agent ??= makeAgent();
|
|
369
|
+
agent.sendUserText(text); // emits its own user bubble via the stt turnComplete event
|
|
370
|
+
$('typed').value = '';
|
|
371
|
+
};
|
|
372
|
+
$('typed').onkeydown = e => { if (e.key === 'Enter') $('send').click(); };
|
|
373
|
+
|
|
374
|
+
$('clear').onclick = () => { chat.textContent = ''; $('log').textContent = ''; $('lat').textContent = '';
|
|
375
|
+
interimEl = aiEl = null; toolEls.clear(); };
|
|
376
|
+
const toggle = (id, fn) => $(id).onclick = () => {
|
|
377
|
+
const on = $(id).dataset.on !== '1';
|
|
378
|
+
$(id).dataset.on = on ? '1' : '0';
|
|
379
|
+
$(id).style.borderColor = on ? 'var(--c)' : '';
|
|
380
|
+
fn(on);
|
|
381
|
+
};
|
|
382
|
+
toggle('muteMic', on => agent?.setMuted(on));
|
|
383
|
+
toggle('muteTts', on => agent?.setTtsMuted(on));
|
|
384
|
+
|
|
385
|
+
// Voice switches live. The rest are constructor options, so they can only apply
|
|
386
|
+
// to a NEW agent — drop the current one so the next turn builds it from these.
|
|
387
|
+
$('voice').addEventListener('change', () => { tts.setVoice($('voice').value); log('voice →', $('voice').value); });
|
|
388
|
+
for (const id of SETTINGS.filter(i => i !== 'voice'))
|
|
389
|
+
$(id).addEventListener('change', () => {
|
|
390
|
+
if (micOn) return bubble('note', `stop the mic to apply ${id}`);
|
|
391
|
+
if (agent) { agent.destroy(); agent = null; log(id, '→ agent rebuilt on next turn'); }
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
// ── device + voice lists ────────────────────────────────────────────────────
|
|
395
|
+
tts.voices().then(vs => {
|
|
396
|
+
fill('voice', vs.map(v => [v.id, v.id]));
|
|
397
|
+
$('voice').value = localStorage.getItem('vl.voice') || tts.voiceId; // the library's own default, not a copy of it
|
|
398
|
+
tts.setVoice($('voice').value);
|
|
399
|
+
}).catch(e => log('voice list failed', String(e)));
|
|
400
|
+
navigator.mediaDevices?.enumerateDevices().then(ds => {
|
|
401
|
+
for (const d of ds.filter(d => d.kind === 'audioinput'))
|
|
402
|
+
$('micDeviceId').append(Object.assign(document.createElement('option'),
|
|
403
|
+
{ value: d.deviceId, textContent: d.label || d.deviceId.slice(0,8) }));
|
|
404
|
+
}).catch(() => {});
|
|
405
|
+
|
|
406
|
+
setState('idle');
|
|
407
|
+
window.agent = () => agent; // console poking: agent().history, …
|
|
408
|
+
</script>
|
|
409
|
+
</body>
|
|
410
|
+
</html>
|
package/package.json
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"exports": {
|
|
8
|
-
".":
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./src/index.d.ts",
|
|
10
|
+
"default": "./src/index.js"
|
|
11
|
+
},
|
|
9
12
|
"./stt": "./src/stt.js",
|
|
10
13
|
"./tuning": "./src/tuning.js",
|
|
11
|
-
"./llm-openai": "./src/llm-openai.js"
|
|
14
|
+
"./llm-openai": "./src/llm-openai.js",
|
|
15
|
+
"./package.json": "./package.json"
|
|
12
16
|
},
|
|
13
17
|
"files": [
|
|
14
18
|
"src",
|
|
@@ -48,5 +52,6 @@
|
|
|
48
52
|
},
|
|
49
53
|
"publishConfig": {
|
|
50
54
|
"access": "public"
|
|
51
|
-
}
|
|
55
|
+
},
|
|
56
|
+
"types": "src/index.d.ts"
|
|
52
57
|
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
// Public types for voiceloop — VAD → STT → LLM → TTS with real barge-in.
|
|
2
|
+
// Hand-written (the source is plain JS); see README.md for the prose version of every option.
|
|
3
|
+
|
|
4
|
+
// Events the agent emits on `onEvent` — discriminated on `type` so a switch narrows the rest.
|
|
5
|
+
export type VoiceAgentEvent =
|
|
6
|
+
| { type: 'state'; state: 'idle' | 'listening' | 'thinking' | 'speaking' }
|
|
7
|
+
| { type: 'stt'; turnComplete: boolean; text: string; ms: number; committed?: string } // text=live tail; committed=locked segments (Speechmatics AddTranscript); turnComplete=user turn ended → send to LLM
|
|
8
|
+
| { type: 'assistant'; text: string; final: boolean; full?: string } // text=spoken-so-far (solid, advances live as TTS plays); full=complete answer → tail full.slice(text.length) not-yet-spoken (dim)
|
|
9
|
+
| { type: 'vad'; active: boolean } // VAD hears speech?
|
|
10
|
+
// running:true = loop call announced, executing now (display-only spinner; no result yet).
|
|
11
|
+
// The outcome event follows with the same `id` and replaces it in place.
|
|
12
|
+
| { type: 'tool'; name: string; args: Record<string, unknown>; result?: unknown; id?: string; running?: true }
|
|
13
|
+
| { type: 'echo'; text: string } // dropped self-echo (the agent's own TTS leaked into the mic) — diagnostics only
|
|
14
|
+
| { type: 'diag'; diag: 'aec-fallback'; message: string } // once/session: AEC loopback failed → TTS plays un-cancelled (self-echo only text-filtered)
|
|
15
|
+
| { type: 'diag'; diag: 'presynth'; text: string } // a reply's first clip was synthesized speculatively (latency diagnostics)
|
|
16
|
+
| { type: 'error'; error: string };
|
|
17
|
+
|
|
18
|
+
export interface VoiceTool {
|
|
19
|
+
description?: string;
|
|
20
|
+
params?: Record<string, unknown>; // JSON-schema `properties` for the arguments
|
|
21
|
+
/** Which `params` the model MUST supply. Defaults to ALL of them — set this to make some
|
|
22
|
+
* optional, otherwise the model is forced to invent a value for every field. */
|
|
23
|
+
required?: string[];
|
|
24
|
+
/** A tool with no `run` is dropped, never advertised: the model would call it, nothing would
|
|
25
|
+
* happen, and the transcript would still show it as having run — a silent lie. */
|
|
26
|
+
run?: (args: Record<string, unknown>) => unknown;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// One LLM delta: either spoken text, or a single tool call (args already parsed). Agent-loop tool
|
|
30
|
+
// chunks additionally carry the provider call `id` (two same-args calls, e.g. a retry, stay distinct
|
|
31
|
+
// acts) and the outcome (`result`/`error`) — the loop already executed the tool, so the agent
|
|
32
|
+
// reports it instead of running it again.
|
|
33
|
+
export type LLMChunk =
|
|
34
|
+
| { text: string }
|
|
35
|
+
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
|
|
36
|
+
// Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
|
|
37
|
+
// The default (agent-llm.js, vendored Pi loop) accepts an optional 4th arg: `toolGate` is awaited
|
|
38
|
+
// before ANY tool executes — speculative prefetch passes one that resolves on turn commit, so a
|
|
39
|
+
// speculation streams text early but can never fire a side effect from uncommitted speech.
|
|
40
|
+
// Custom LLMs are a low-level seam (tests, embedding hosts): their bare tool chunks are executed
|
|
41
|
+
// by the agent, but the results are NOT fed back into that LLM mid-turn — such calls are terminal
|
|
42
|
+
// for the turn. They may ignore toolGate.
|
|
43
|
+
export type LLM = (
|
|
44
|
+
history: Array<{ role: string; content: string }>,
|
|
45
|
+
system: string,
|
|
46
|
+
signal: AbortSignal,
|
|
47
|
+
options?: { toolGate?: Promise<unknown> },
|
|
48
|
+
) => AsyncIterable<LLMChunk>;
|
|
49
|
+
|
|
50
|
+
// Pluggable TTS contract. `speak` resolves with the text actually HEARD (full on completion,
|
|
51
|
+
// a prefix on barge-in). The optional members back the voice picker / switching.
|
|
52
|
+
export interface VoiceTTS {
|
|
53
|
+
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
54
|
+
stop?(): void;
|
|
55
|
+
/** Eagerly prepare the synth engine (load models, JIT) so the first utterance has no cold start. */
|
|
56
|
+
warm?(): Promise<void>;
|
|
57
|
+
voices?(): Promise<Array<{ id: string; name: string; language: string }>>;
|
|
58
|
+
setVoice?(voiceId: string): void;
|
|
59
|
+
setSpeed?(speed: number): void;
|
|
60
|
+
/** Register a callback fired while a clip plays with the chars spoken SO FAR across the whole
|
|
61
|
+
* reply — drives the host's live spoken cursor. Pass null to clear. */
|
|
62
|
+
setOnProgress?(fn: ((spokenText: string) => void) | null): void;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface VoiceAgentOptions {
|
|
66
|
+
sysmsg?: string;
|
|
67
|
+
/** Persona/system prompt prepended to the conversation context. Defaults to the built-in
|
|
68
|
+
* voice persona (VOICE_SYSMSG); pass your own agent's system message to override it. */
|
|
69
|
+
persona?: string;
|
|
70
|
+
/** LLM model id sent to the backend `/llm` route; empty → the backend's VOICE_MODEL default. */
|
|
71
|
+
model?: string;
|
|
72
|
+
llm?: LLM;
|
|
73
|
+
/** Test seam for the agent-loop LLM transport; defaults to global fetch. */
|
|
74
|
+
fetchFn?: typeof fetch;
|
|
75
|
+
apiKey?: string;
|
|
76
|
+
/** Full endpoint URLs, not a base: voiceloop composes no routes, so a host can point the LLM and
|
|
77
|
+
* the STT token at unrelated origins (a backend proxy for one, a direct provider for the other).
|
|
78
|
+
* Deriving `${baseUrl}/llm` + `${baseUrl}/stt/<provider>-token` is the HOST's job — and each STT
|
|
79
|
+
* provider mints its own token type, so one shared token URL cannot serve all of them. */
|
|
80
|
+
llmUrl?: string;
|
|
81
|
+
maxTokens?: number;
|
|
82
|
+
tts?: VoiceTTS;
|
|
83
|
+
/** Speaking speed for the default Piper TTS (1.2 → 20% faster). Ignored if `tts` is passed. */
|
|
84
|
+
speed?: number;
|
|
85
|
+
sttLang?: string;
|
|
86
|
+
micDeviceId?: string;
|
|
87
|
+
/** STT backend: 'webspeech' (default — the browser's own on-device SpeechRecognition; free, no
|
|
88
|
+
* token, no socket, captures its own mic) | 'speechmatics' | 'elevenlabs' (Scribe) | 'deepgram'
|
|
89
|
+
* (Flux — native model-based end-of-turn detection; the agent feeds it the full mic stream and
|
|
90
|
+
* lets it close turns itself). The cloud ones connect the browser DIRECTLY to the provider with a
|
|
91
|
+
* short-TTL token the backend mints (raw key stays server-side).
|
|
92
|
+
* 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
|
|
93
|
+
* the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
|
|
94
|
+
sttProvider?: SttProvider;
|
|
95
|
+
/** ONE endpoint that mints the token. When each provider needs a different route or credential,
|
|
96
|
+
* use `getSttToken` instead — it is handed the resolved provider, so the downgrade rule above
|
|
97
|
+
* stays in this library instead of being mirrored by every host. */
|
|
98
|
+
sttTokenUrl?: string;
|
|
99
|
+
/** Mint a short-TTL STT token yourself, for auth that doesn't fit one POST route. Called with the
|
|
100
|
+
* provider actually RUNNING (post-downgrade), never the requested one. Wins over sttTokenUrl. */
|
|
101
|
+
getSttToken?: (provider: SttProvider) => Promise<SttToken>;
|
|
102
|
+
sttUsageUrl?: string;
|
|
103
|
+
sttUrl?: string;
|
|
104
|
+
sttModel?: string;
|
|
105
|
+
/** Deepgram Flux end-of-turn confidence (0.5–0.9) — its model closes the turn itself once its
|
|
106
|
+
* confidence the user is done exceeds this threshold. Lower = snappier turn ends, higher = more
|
|
107
|
+
* patient across mid-thought pauses. Deepgram-only; default from tuning.js (0.7). */
|
|
108
|
+
sttEotThreshold?: number;
|
|
109
|
+
/** Every tool the model may call. voiceloop ships NO built-in tools — a voice library has no
|
|
110
|
+
* business knowing what a "todo" is — so product tools (and their `run`) are supplied here. */
|
|
111
|
+
tools?: Record<string, VoiceTool>;
|
|
112
|
+
keyterms?: string[];
|
|
113
|
+
preroll?: number;
|
|
114
|
+
vadOptions?: Record<string, unknown>;
|
|
115
|
+
/** Semantic end-of-turn hook run when VAD hears end-of-speech: return false to KEEP listening (the
|
|
116
|
+
* user only paused mid-thought), true (default, when omitted) to close the turn. Lets you swap the
|
|
117
|
+
* "is the turn over?" decision (silence-only | SLM | smart-turn) without touching the STT pipeline. */
|
|
118
|
+
turnDetector?: (text: string) => boolean | Promise<boolean>;
|
|
119
|
+
/** Hard cap (ms) a turnDetector may keep a turn open across a pause before it's force-committed, so a
|
|
120
|
+
* user who stops mid-sentence can't hang the turn forever. Default 4000. Ignored without a detector. */
|
|
121
|
+
maxPauseMs?: number;
|
|
122
|
+
/** Min transcribed chars of the user's interim (heard WHILE the agent is speaking) needed to count as a
|
|
123
|
+
* real barge-in and cut the reply. Higher = short backchannels ("mhm","yeah") don't interrupt; 0 =
|
|
124
|
+
* any transcribed char interrupts. Default 3 (see tuning.js TUNING.BARGE_IN_MIN_CHARS). */
|
|
125
|
+
bargeInMinChars?: number;
|
|
126
|
+
onEvent?: (event: VoiceAgentEvent) => void;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Call from a user-gesture handler (e.g. the mic button click) to bless the shared TTS audio
|
|
130
|
+
* element on mobile — without this, iOS rejects the async play() of synthesized speech. */
|
|
131
|
+
export function unlockAudio(): void;
|
|
132
|
+
/** Pre-build the Silero v5 VAD ONNX session (no mic) so the first start() reaches 'listening'
|
|
133
|
+
* much faster. Best-effort + memoized; requires the vad-web UMD bundle already loaded. */
|
|
134
|
+
export function warmVad(): Promise<void>;
|
|
135
|
+
/** Pre-mint + cache the short-TTL Deepgram STT token off the critical path (e.g. at mic-button
|
|
136
|
+
* mount), so a voice start only pays the WS handshake. Best-effort + deduped; no-op while a
|
|
137
|
+
* fresh cached token remains.
|
|
138
|
+
* `sttTokenUrl` is the EXACT endpoint that mints a Deepgram token (e.g. `/api/v1/stt/deepgram-token`),
|
|
139
|
+
* the same value passed as the `sttTokenUrl` option — NOT an API base. A host that mints tokens
|
|
140
|
+
* itself passes its `getSttToken` as `getToken` and can skip the URL entirely (warm for the
|
|
141
|
+
* provider it will actually run: `() => getSttToken('deepgram')`). Warming with the wrong URL
|
|
142
|
+
* silently no-ops the optimization: the real session just mints again on click. */
|
|
143
|
+
export function warmDeepgramToken(sttTokenUrl?: string, apiKey?: string, getToken?: () => Promise<SttToken>): Promise<void>;
|
|
144
|
+
/** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
|
|
145
|
+
* host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
|
|
146
|
+
export const VOICE_SYSMSG: string;
|
|
147
|
+
/** Language codes the agent can instruct the LLM to reply in, code → English name. The `sttLang`
|
|
148
|
+
* values that actually steer the reply language — exported so a host builds its language picker
|
|
149
|
+
* from this instead of hand-listing a subset that silently rots as the map grows. */
|
|
150
|
+
export const LANG_NAMES: Record<string, string>;
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
export class VoiceAgent {
|
|
154
|
+
constructor(options?: VoiceAgentOptions);
|
|
155
|
+
/** The live conversation the LLM sees. A mutable reference the host may read and push into —
|
|
156
|
+
* carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
|
|
157
|
+
history: Array<{ role: string; content: string }>;
|
|
158
|
+
start(deviceId?: string): Promise<void>;
|
|
159
|
+
/** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
|
|
160
|
+
* start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
|
|
161
|
+
* fully release the mic when ending the session. */
|
|
162
|
+
stop(): void;
|
|
163
|
+
/** End for real: pause + release the mic, audio graph and VAD model (turns off the OS mic indicator).
|
|
164
|
+
* Call this when dropping the agent reference so a paused-but-discarded agent never leaks the mic. */
|
|
165
|
+
destroy(): void;
|
|
166
|
+
listVoices(): Promise<Array<{ id: string; name: string; language: string }>>;
|
|
167
|
+
setVoice(voiceId: string): void;
|
|
168
|
+
setSpeed(speed: number): void;
|
|
169
|
+
/** Update the host context (e.g. a re-opened chat); applies from the next turn — no restart. */
|
|
170
|
+
setSysmsg(sysmsg?: string): void;
|
|
171
|
+
/** Mute/unmute the mic (disables the stream's audio tracks → silence to VAD+STT); keeps the pipeline live. */
|
|
172
|
+
setMuted(muted: boolean): void;
|
|
173
|
+
/** Mute/unmute the AI's spoken output (TTS): the reply still streams as text but is never voiced. */
|
|
174
|
+
setTtsMuted(muted: boolean): void;
|
|
175
|
+
/** Move the playhead to a position in the current reply (non-whitespace char count) and play forward
|
|
176
|
+
* from there. Hits any already-streamed sentence (backward replay is cached, forward skips ahead);
|
|
177
|
+
* beyond streamed text is a no-op. Returns true on a hit, false otherwise. */
|
|
178
|
+
seek(nwIndex: number): boolean;
|
|
179
|
+
/** Re-speak a FINISHED reply from `fromNw` (a non-whitespace offset — the tapped word), interruptibly.
|
|
180
|
+
* Pure TTS (no LLM, no history change). `onProgress(spokenSoFar, done)` is driven by the host so it can
|
|
181
|
+
* update the exact transcript message it tapped: spokenSoFar is the spoken prefix, done flips true when
|
|
182
|
+
* the replay ends (played out, barged-in, or superseded) with spokenSoFar = the whole reply. */
|
|
183
|
+
replay(text: string, fromNw?: number, onProgress?: (spokenSoFar: string, done: boolean) => void): Promise<void>;
|
|
184
|
+
/** Cut off the current in-flight turn (abort the LLM, stop TTS) without stopping the agent —
|
|
185
|
+
* the mic keeps listening. No-op when not thinking/speaking. A host-callable barge-in. */
|
|
186
|
+
interrupt(): void;
|
|
187
|
+
/** Hold: keep listening/transcribing into history but DON'T reply. Entering hold silences the
|
|
188
|
+
* current reply (kept as the full streamed answer so far); releasing flushes all accumulated
|
|
189
|
+
* turns through the LLM in one go. */
|
|
190
|
+
setHeld(held: boolean): void;
|
|
191
|
+
/** Inject an out-of-band turn (e.g. a background TODO finished): speaks a reply. No-op once stopped. */
|
|
192
|
+
notify(text: string): Promise<void>;
|
|
193
|
+
/** Send a typed user turn (renders as a user bubble, then the agent replies in the background).
|
|
194
|
+
* Needs no mic — accepts even while paused/stopped (revives the reply loop; start() re-arms audio
|
|
195
|
+
* separately). Returns true when accepted (sent or queued while held), false for blank text or
|
|
196
|
+
* once destroy()ed (terminal). */
|
|
197
|
+
sendUserText(text: string): boolean;
|
|
198
|
+
/** Diagnostic tap: the last ~30s of the exact audio handed to STT as a 16kHz mono WAV, plus any
|
|
199
|
+
* detected delivery stalls (wall-clock gaps between capture chunks; offsets relative to the WAV).
|
|
200
|
+
* Play it back after a bad transcription run: gaps/corrupt audio → capture layer; clean audio →
|
|
201
|
+
* provider-side. */
|
|
202
|
+
dumpAudio(): { wav: Blob; gaps: Array<{ atSample: number; atSec: number; gapMs: number }>; seconds: number };
|
|
203
|
+
state: 'idle' | 'listening' | 'thinking' | 'speaking';
|
|
204
|
+
/** The STT provider actually in use — differs from the requested one when 'webspeech' was
|
|
205
|
+
* downgraded to a cloud provider (browser without SpeechRecognition). */
|
|
206
|
+
readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
207
|
+
/** True once the pipeline has been torn down (stop() or a fatal STT error self-stopped it). */
|
|
208
|
+
readonly closed: boolean;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export class PiperTTS implements VoiceTTS {
|
|
212
|
+
constructor(voiceId?: string, speed?: number);
|
|
213
|
+
voices(): Promise<Array<{ id: string; name: string; language: string }>>;
|
|
214
|
+
setVoice(voiceId: string): void;
|
|
215
|
+
setSpeed(speed: number): void;
|
|
216
|
+
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
217
|
+
stop(): void;
|
|
218
|
+
/** Eagerly download+compile the ONNX model and JIT the WASM path so the first reply has no cold start. */
|
|
219
|
+
warm(): Promise<void>;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ── STT providers ────────────────────────────────────────────────────────────────────────────
|
|
223
|
+
// A provider is a factory: it receives the callbacks below plus its own credentials/tuning, and
|
|
224
|
+
// returns a live session. VoiceAgent wires these itself — call them directly only when embedding
|
|
225
|
+
// STT without the agent.
|
|
226
|
+
export interface STTCallbacks {
|
|
227
|
+
onPartial?: (text: string, ms: number, committed?: string) => void;
|
|
228
|
+
onFinal?: (text: string, ms: number) => void;
|
|
229
|
+
onError?: (error: unknown) => void;
|
|
230
|
+
onFatal?: (error: unknown) => void;
|
|
231
|
+
onClose?: () => void;
|
|
232
|
+
isClosed?: () => boolean;
|
|
233
|
+
}
|
|
234
|
+
/** A live STT session. The capability flags tell the agent how much of the audio path the provider
|
|
235
|
+
* owns, so it builds only the missing half — a provider with its own mic and its own end-of-turn
|
|
236
|
+
* (WebSpeech) needs no VAD/capture pipeline at all. */
|
|
237
|
+
export interface STTSession {
|
|
238
|
+
/** Provider owns the mic → the agent builds no capture pipeline. Implies nativeEOT. */
|
|
239
|
+
selfCapture?: boolean;
|
|
240
|
+
/** Provider closes turns itself (unsolicited onFinal) → the agent runs no VAD-commit. */
|
|
241
|
+
nativeEOT?: boolean;
|
|
242
|
+
/** Provider wants the full mic stream, including silence (its EOT model needs real pauses). */
|
|
243
|
+
continuous?: boolean;
|
|
244
|
+
/** Pre-open the socket (token + handshake) so the first utterance isn't stalled. Idempotent. */
|
|
245
|
+
open?(): void;
|
|
246
|
+
/** Push PCM. Opens the socket lazily if `open()` was never called. */
|
|
247
|
+
feed?(pcm: Int16Array): void;
|
|
248
|
+
/** End of utterance: ask the provider to finalize. No-op providers with nativeEOT. */
|
|
249
|
+
commit?(): void;
|
|
250
|
+
/** Mute/unmute without tearing the session down (selfCapture providers). */
|
|
251
|
+
setEnabled?(enabled: boolean): void;
|
|
252
|
+
close(): void;
|
|
253
|
+
}
|
|
254
|
+
export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
|
|
255
|
+
|
|
256
|
+
/** The built-in STT provider ids — the keys of STT_PROVIDERS. */
|
|
257
|
+
export type SttProvider = 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
258
|
+
/** What a token minter returns: the short-TTL credential, plus its lifetime in seconds when the
|
|
259
|
+
* provider reports one (used to cache and pre-mint just before expiry). */
|
|
260
|
+
export interface SttToken { token: string; expires_in?: number }
|
|
261
|
+
|
|
262
|
+
export const STT_PROVIDERS: Record<SttProvider, STTFactory>;
|
|
263
|
+
/** Does this provider own the mic and its own end-of-turn? Then the agent builds NO capture pipeline
|
|
264
|
+
* and runs no VAD for it — so it emits no `vad` events, and prewarming the VAD/ONNX model is a
|
|
265
|
+
* wasted download. Answerable without constructing a session, so a host can decide both up front. */
|
|
266
|
+
export function sttSelfCaptures(provider: string): boolean;
|
|
267
|
+
export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
268
|
+
export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
269
|
+
export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
270
|
+
export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
271
|
+
/** Web Speech exists only in Chromium/Safari; elsewhere 'webspeech' resolves to this cloud provider. */
|
|
272
|
+
export const WEBSPEECH_FALLBACK: 'elevenlabs';
|
|
273
|
+
export function webSpeechSupported(): boolean;
|
|
274
|
+
/** The provider id actually used: 'webspeech' downgrades to WEBSPEECH_FALLBACK in a browser
|
|
275
|
+
* without SpeechRecognition. Call it to label the UI ("using cloud recognition") — but NOT to pick
|
|
276
|
+
* a token route: `getSttToken` is already handed the resolved id. */
|
|
277
|
+
export function resolveSttProvider(id: string): SttProvider;
|
|
278
|
+
|
|
279
|
+
// ── TTS ──────────────────────────────────────────────────────────────────────────────────────
|
|
280
|
+
/** Base class for streaming TTS engines: sentence-splits the incoming token stream, synthesizes
|
|
281
|
+
* the next clip while the current one plays, and reports the spoken cursor via setOnProgress.
|
|
282
|
+
* Subclass and implement `_synth(text)` to add an engine. */
|
|
283
|
+
export class StreamingTTS implements VoiceTTS {
|
|
284
|
+
constructor(voiceId?: string, speed?: number);
|
|
285
|
+
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
286
|
+
stop(): void;
|
|
287
|
+
setOnProgress(fn: (spoken: string, scope: string) => void): void;
|
|
288
|
+
/** Jump playback to a word index within the current reply. Returns false when the seek can't be
|
|
289
|
+
* honoured (nothing playing, or past the text streamed so far) — check it before moving a cursor. */
|
|
290
|
+
seek(nwIndex: number): boolean;
|
|
291
|
+
}
|
|
292
|
+
/** Human-grade cloud voices. Pass `ttsUrl` (your backend proxy, key stays server-side); `apiKey`
|
|
293
|
+
* talks to ElevenLabs directly and is dev-only. */
|
|
294
|
+
export class ElevenLabsTTS extends StreamingTTS {
|
|
295
|
+
constructor(options?: { ttsUrl?: string; apiKey?: string; voiceId?: string; modelId?: string; format?: string });
|
|
296
|
+
warm(): Promise<void>;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// ── LLM ──────────────────────────────────────────────────────────────────────────────────────
|
|
300
|
+
/** Any OpenAI-compatible /chat/completions endpoint as an LLM adapter, with tool-calling wired
|
|
301
|
+
* to the `tools` map. `ttsUrl`-style proxying applies: prefer a backend route over apiKey. */
|
|
302
|
+
export function makeOpenAILLM(options?: {
|
|
303
|
+
llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
|
|
304
|
+
tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
|
|
305
|
+
}): LLM;
|
|
306
|
+
|
|
307
|
+
// ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
|
|
308
|
+
/** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
|
|
309
|
+
* session — call it while your UI is idle so the first start() is instant. */
|
|
310
|
+
export function loadVoiceDeps(): Promise<unknown>;
|
|
311
|
+
export function prewarmVoice(): void;
|
|
312
|
+
/** Every latency/sensitivity constant (VAD thresholds, barge-in chars, prefetch window, echo
|
|
313
|
+
* match threshold). Mutate before constructing an agent to retune globally. */
|
|
314
|
+
export const TUNING: Record<string, any>;
|
|
315
|
+
/** Echo filtering internals, exported for tests and custom mic gating: `isSelfEcho` decides
|
|
316
|
+
* whether heard text is the agent's own voice, `novelChars` counts what is genuinely new. */
|
|
317
|
+
export function isSelfEcho(heard: string, reply: string, options?: { threshold?: number; window?: number; minHits?: number }): boolean;
|
|
318
|
+
export function novelChars(heard: string, reply: string): number;
|
|
319
|
+
/** Render a tool result the way the agent reports it to the model (and to `tool` events). */
|
|
320
|
+
export function toolResultText(result: unknown): string;
|
package/src/index.js
CHANGED
|
@@ -2,11 +2,11 @@ export {
|
|
|
2
2
|
VoiceAgent, StreamingTTS, PiperTTS,
|
|
3
3
|
unlockAudio, warmVad,
|
|
4
4
|
isSelfEcho, novelChars, toolResultText,
|
|
5
|
-
VOICE_SYSMSG,
|
|
5
|
+
VOICE_SYSMSG, LANG_NAMES,
|
|
6
6
|
} from './voice-agent.js';
|
|
7
7
|
export {
|
|
8
8
|
STT_PROVIDERS, WEBSPEECH_FALLBACK, resolveSttProvider, webSpeechSupported,
|
|
9
|
-
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT,
|
|
9
|
+
makeWebSpeechSTT, makeElevenLabsSTT, makeSpeechmaticsSTT, makeDeepgramSTT, sttSelfCaptures,
|
|
10
10
|
warmDeepgramToken,
|
|
11
11
|
} from './stt.js';
|
|
12
12
|
export { ElevenLabsTTS } from './tts-elevenlabs.js';
|
package/src/stt.js
CHANGED
|
@@ -39,8 +39,10 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
|
|
|
39
39
|
// AUTH (cloud providers): the raw provider key must stay server-side. Either
|
|
40
40
|
// • `sttTokenUrl` — your backend route that mints a short-TTL token: POST, optional X-API-Key
|
|
41
41
|
// (`apiKey`), responds { token, expires_in? }; or
|
|
42
|
-
// • `getToken` — an async () => ({ token, expires_in? }) callback, when your auth doesn't
|
|
43
|
-
// that one route shape. Takes precedence over sttTokenUrl.
|
|
42
|
+
// • `getToken` — an async (provider) => ({ token, expires_in? }) callback, when your auth doesn't
|
|
43
|
+
// fit that one route shape. Takes precedence over sttTokenUrl. VoiceAgent calls it with the
|
|
44
|
+
// provider actually RUNNING (post-downgrade), so a host whose credential differs per provider
|
|
45
|
+
// writes one function instead of mirroring the downgrade rule.
|
|
44
46
|
|
|
45
47
|
// ── Shared socket-lifecycle helpers ─────────────────────────────────────────
|
|
46
48
|
// The three providers below have genuinely different turn machines (Speechmatics' straggler cursor,
|
|
@@ -57,7 +59,14 @@ const USAGE_FLUSH_SECONDS = 30; // report streamed seconds to the backend at l
|
|
|
57
59
|
async function mintSttToken(url, apiKey, onFatal, getToken) {
|
|
58
60
|
if (getToken) {
|
|
59
61
|
try {
|
|
60
|
-
|
|
62
|
+
// HOST code on the critical path: bound it exactly like the built-in fetch below, or a minter
|
|
63
|
+
// that never settles stalls voice start forever with no error. We can't cancel the host's
|
|
64
|
+
// promise — we just stop waiting on it (it may still resolve into the void, which is fine:
|
|
65
|
+
// a token is idempotent to mint).
|
|
66
|
+
const body = await Promise.race([
|
|
67
|
+
getToken(),
|
|
68
|
+
new Promise((_, rej) => setTimeout(() => rej(new Error('timed out')), TUNING.STT.tokenFetchTimeoutMs)),
|
|
69
|
+
]);
|
|
61
70
|
if (!body?.token) throw new Error('getToken returned no token');
|
|
62
71
|
return body;
|
|
63
72
|
} catch (e) { onFatal(`STT token: ${e.message}`); return null; }
|
|
@@ -71,7 +80,10 @@ async function mintSttToken(url, apiKey, onFatal, getToken) {
|
|
|
71
80
|
return null;
|
|
72
81
|
}
|
|
73
82
|
const body = await res.json().catch(() => ({}));
|
|
74
|
-
|
|
83
|
+
// `message` first: a token route's error body usually puts the reason there and the bare status
|
|
84
|
+
// name in `error` ({ error: 'Unauthorized', message: 'API key not found' }) — the reason is what
|
|
85
|
+
// tells the developer their key is wrong rather than their URL.
|
|
86
|
+
if (!res.ok || !body.token) { onFatal(`STT token: ${body.message || body.error || res.statusText}`); return null; }
|
|
75
87
|
return body;
|
|
76
88
|
}
|
|
77
89
|
|
|
@@ -586,10 +598,12 @@ export function makeDeepgramSTT(opts) {
|
|
|
586
598
|
// result window). Blindly appending would double words ("hello there" + "there how" → "there there").
|
|
587
599
|
// Every final is de-overlapped against the turn-so-far before it's appended.
|
|
588
600
|
export function makeWebSpeechSTT(opts) {
|
|
589
|
-
const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onFatal, onClose, isClosed } = opts;
|
|
601
|
+
const { sttLang = 'en', micDeviceId = '', onPartial, onFinal, onError, onFatal, onClose, isClosed } = opts;
|
|
590
602
|
const Rec = typeof window !== 'undefined' && (window.SpeechRecognition || window.webkitSpeechRecognition);
|
|
591
603
|
const EOT_MS = TUNING.WEBSPEECH_EOT_MS ?? 900;
|
|
592
604
|
const TAIL_MAX_DEFERS = TUNING.WEBSPEECH_TAIL_MAX_DEFERS ?? 3;
|
|
605
|
+
const MAX_ERROR_RESTARTS = TUNING.WEBSPEECH_MAX_ERROR_RESTARTS ?? 5;
|
|
606
|
+
const ERROR_BACKOFF_MS = TUNING.WEBSPEECH_ERROR_BACKOFF_MS ?? 300;
|
|
593
607
|
let rec = null; // the live SpeechRecognition instance (null between restarts / when stopped)
|
|
594
608
|
let sttStart = 0; // performance.now() at the current turn's first activity (for onPartial/onFinal ms)
|
|
595
609
|
let turn = ''; // committed (isFinal) text of the CURRENT turn, merged across recognizer restarts
|
|
@@ -600,6 +614,9 @@ export function makeWebSpeechSTT(opts) {
|
|
|
600
614
|
let tailDefers = 0; // consecutive closes deferred waiting on an un-promoted tail (bounded, reset on new text)
|
|
601
615
|
let want = false; // intent to listen — onend auto-restarts while true
|
|
602
616
|
let muted = false; // mic off — stop the recognizer and don't restart
|
|
617
|
+
let lastError = ''; // error of the recognizer currently ending (drives the backoff/give-up path)
|
|
618
|
+
let errorRestarts = 0; // consecutive failed launches with no recognized text between them
|
|
619
|
+
let restartTimer = null; // pending backed-off relaunch
|
|
603
620
|
// AEC'D CAPTURE (SpeechRecognition.start(audioTrack), spec'd + shipped in recent Chromium): by
|
|
604
621
|
// default the engine opens its OWN mic, bypassing every protection the agent's pipeline has — the
|
|
605
622
|
// browser AEC referenced against the WebRTC TTS loopback (voice-agent initAecLoopback) and noise
|
|
@@ -724,6 +741,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
724
741
|
}
|
|
725
742
|
final = final.trim(); interim = interim.trim();
|
|
726
743
|
if (!turn && !interimTail && (final || interim)) sttStart = performance.now(); // first activity of a fresh turn
|
|
744
|
+
if (final || interim) errorRestarts = 0; // the engine is demonstrably working — forget past failures
|
|
727
745
|
commitFinal(final); // de-overlaps the seam, then locks the newly-final words into the turn
|
|
728
746
|
// Keep the live tail as its own state so a recognizer restart (which drops interim) can replay it
|
|
729
747
|
// instead of blanking the draft. A frame that only finalized words (interim === '') clears it —
|
|
@@ -736,8 +754,11 @@ export function makeWebSpeechSTT(opts) {
|
|
|
736
754
|
};
|
|
737
755
|
r.onerror = (event) => {
|
|
738
756
|
if (r !== rec) return;
|
|
739
|
-
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); }
|
|
740
|
-
// 'no-speech' / 'aborted'
|
|
757
|
+
if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { want = false; onFatal?.(`Speech recognition: ${event.error}`); return; }
|
|
758
|
+
// 'no-speech' / 'aborted' are the engine's normal idle churn — onend just relaunches, silently.
|
|
759
|
+
// Anything else ('network', 'audio-capture', …) is a FAILED launch: remembered here so onend
|
|
760
|
+
// backs off instead of hot-looping, and gives up out loud once the engine looks dead.
|
|
761
|
+
if (event.error !== 'no-speech' && event.error !== 'aborted') lastError = event.error;
|
|
741
762
|
};
|
|
742
763
|
r.onend = () => {
|
|
743
764
|
if (r !== rec) return; // superseded instance's late onend — ignore
|
|
@@ -753,8 +774,19 @@ export function makeWebSpeechSTT(opts) {
|
|
|
753
774
|
// untouched — if this really was end-of-turn, that timer fires on its own EOT_MS after the last
|
|
754
775
|
// word (recognizer alive or not); if the user is still talking, the new recognizer's text
|
|
755
776
|
// re-arms it. No flush decision belongs here — that's what makes the boundary restart-immune.
|
|
756
|
-
|
|
757
|
-
|
|
777
|
+
const err = lastError; lastError = '';
|
|
778
|
+
if (!(want && !muted && !isClosed())) { onClose?.(); return; }
|
|
779
|
+
if (!err) { errorRestarts = 0; start(); return; }
|
|
780
|
+
// A launch that only produced an error and no text: the engine may be permanently unable to run
|
|
781
|
+
// here (speech service unreachable, unreadable capture device). Retry with backoff, then stop —
|
|
782
|
+
// an endless silent restart loop is indistinguishable from "listening" and never recovers.
|
|
783
|
+
if (++errorRestarts > MAX_ERROR_RESTARTS) {
|
|
784
|
+
want = false;
|
|
785
|
+
onFatal?.(`Speech recognition: ${err} (${MAX_ERROR_RESTARTS} restarts failed — the browser's speech service is unreachable)`);
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
onError?.(`Speech recognition: ${err} — retrying (${errorRestarts}/${MAX_ERROR_RESTARTS})`);
|
|
789
|
+
restartTimer = setTimeout(() => { restartTimer = null; if (want && !muted && !isClosed()) start(); }, Math.min(ERROR_BACKOFF_MS * 2 ** (errorRestarts - 1), 4000));
|
|
758
790
|
};
|
|
759
791
|
rec = r;
|
|
760
792
|
let ok = false;
|
|
@@ -768,7 +800,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
768
800
|
|
|
769
801
|
// Stop the current recognizer and drop all in-progress state (mute/close). The stopped recognizer's
|
|
770
802
|
// late onend is stale (rec nulled → r !== rec), so it can't auto-restart.
|
|
771
|
-
const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
|
|
803
|
+
const teardown = () => { clearTimeout(eotTimer); eotTimer = null; clearTimeout(justClosedTimer); justClosedTimer = null; clearTimeout(restartTimer); restartTimer = null; justClosed = ''; turn = ''; interimTail = ''; tailDefers = 0; lastError = ''; if (rec) { const r = rec; rec = null; try { r.stop(); } catch { /* already stopped */ } } };
|
|
772
804
|
|
|
773
805
|
return {
|
|
774
806
|
selfCapture: true, // owns its mic → the agent builds no capture pipeline
|
|
@@ -777,7 +809,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
777
809
|
if (!Rec) { onFatal?.('Speech recognition not supported in this browser'); return; }
|
|
778
810
|
// start() first, synchronously — Safari requires SpeechRecognition.start inside the click
|
|
779
811
|
// gesture. The AEC'd track lands async right after and upgrades the recognizer at the seam.
|
|
780
|
-
want = true; start();
|
|
812
|
+
want = true; errorRestarts = 0; start();
|
|
781
813
|
ensureTrack();
|
|
782
814
|
},
|
|
783
815
|
// Mute: Web Speech captures its OWN mic (independent of the agent), so mute must actually stop it.
|
|
@@ -786,7 +818,7 @@ export function makeWebSpeechSTT(opts) {
|
|
|
786
818
|
setEnabled(on) {
|
|
787
819
|
muted = !on;
|
|
788
820
|
if (muted) { teardown(); dropMic(); onPartial('', 0, ''); } // drop the draft AND the mic (privacy light off)
|
|
789
|
-
else if (want && !rec) { start(); ensureTrack(); }
|
|
821
|
+
else if (want && !rec) { errorRestarts = 0; start(); ensureTrack(); }
|
|
790
822
|
},
|
|
791
823
|
close() { want = false; teardown(); dropMic(); },
|
|
792
824
|
};
|
|
@@ -800,6 +832,14 @@ export const STT_PROVIDERS = {
|
|
|
800
832
|
deepgram: makeDeepgramSTT,
|
|
801
833
|
};
|
|
802
834
|
|
|
835
|
+
// Self-capturing providers own the mic AND their end-of-turn, so the agent builds no capture
|
|
836
|
+
// pipeline and runs no VAD for them. Readable WITHOUT constructing a session, so a host can decide
|
|
837
|
+
// up front whether prewarming the VAD/ONNX model is worth the download (it isn't, for these) and
|
|
838
|
+
// whether to expect 'vad' events at all. Kept on the factory so it can't drift from the session's
|
|
839
|
+
// own selfCapture flag.
|
|
840
|
+
makeWebSpeechSTT.selfCapture = true;
|
|
841
|
+
export const sttSelfCaptures = (provider) => !!STT_PROVIDERS[provider]?.selfCapture;
|
|
842
|
+
|
|
803
843
|
// Web Speech only exists in Chromium/Safari — Firefox and Linux WebKitGTK webviews ship no
|
|
804
844
|
// SpeechRecognition at all. There the default 'webspeech' provider can only fatal ("Speech
|
|
805
845
|
// recognition not supported in this browser") and voice mode is simply dead, so resolve it to a
|
package/src/tuning.js
CHANGED
|
@@ -71,6 +71,17 @@ export const TUNING = {
|
|
|
71
71
|
// Raise if slow engines still split turns; lower to bound the wait harder.
|
|
72
72
|
WEBSPEECH_TAIL_MAX_DEFERS: 3,
|
|
73
73
|
|
|
74
|
+
// Web Speech errors that aren't auth ('network', 'audio-capture', …) are retried by relaunching the
|
|
75
|
+
// recognizer — but if the engine is genuinely broken (browser's speech service unreachable, a fake
|
|
76
|
+
// audio device it can't read) EVERY relaunch fails instantly and the old code hot-looped forever,
|
|
77
|
+
// silently, looking like "listening but deaf". So consecutive failures with no recognized text in
|
|
78
|
+
// between are counted: each one waits longer (backoff below), and this many gives up with onFatal
|
|
79
|
+
// instead of spinning. Reset by any result. 'no-speech'/'aborted' are normal and don't count.
|
|
80
|
+
WEBSPEECH_MAX_ERROR_RESTARTS: 5,
|
|
81
|
+
// First retry delay after a failing recognizer; doubles per consecutive failure (capped at 4s), so
|
|
82
|
+
// a dead speech service costs a handful of tries over a few seconds instead of a spin loop.
|
|
83
|
+
WEBSPEECH_ERROR_BACKOFF_MS: 300,
|
|
84
|
+
|
|
74
85
|
// Speculative LLM prefetch: once the live interim has been STABLE this long (no new words), the
|
|
75
86
|
// agent starts generating a reply for the probable final text — overlapping the LLM's first-token
|
|
76
87
|
// latency with the rest of the end-of-turn debounce. Exact-match turns adopt the running stream
|
package/src/voice-agent.js
CHANGED
|
@@ -54,7 +54,7 @@ const CAPTURE_WORKLET_URL = typeof URL !== 'undefined' && URL.createObjectURL
|
|
|
54
54
|
// + the host's conversation context (if any).
|
|
55
55
|
// The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
|
|
56
56
|
// English under the English persona — so state the selected STT language explicitly.
|
|
57
|
-
const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
|
|
57
|
+
export const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
|
|
58
58
|
const langRule = (lang) => {
|
|
59
59
|
const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
|
|
60
60
|
return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
|
|
@@ -267,7 +267,10 @@ export class VoiceAgent {
|
|
|
267
267
|
const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
|
|
268
268
|
this.stt = makeStt({
|
|
269
269
|
apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
270
|
-
|
|
270
|
+
// `getSttToken` is called with the provider actually running — the RESOLVED one, so a host
|
|
271
|
+
// whose credential differs per provider never has to re-derive the webspeech downgrade
|
|
272
|
+
// (that rule lives here, in resolveSttProvider, and must not be mirrored host-side).
|
|
273
|
+
sttTokenUrl, getToken: getSttToken && (() => getSttToken(this.sttProvider)), sttUsageUrl, eotThreshold: sttEotThreshold,
|
|
271
274
|
onPartial: (text, ms, committed = '') => {
|
|
272
275
|
const interim = `${committed} ${text}`.trim();
|
|
273
276
|
this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector
|
|
@@ -1853,6 +1856,10 @@ const CDN_PIPER = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-tts-web.j
|
|
|
1853
1856
|
// per-call module setup is exactly what we bypass (see PiperTTS._session).
|
|
1854
1857
|
const CDN_PIPER_PHONEMIZE = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-o91UDS6e.js`;
|
|
1855
1858
|
const CDN_PIPER_WASM = `${CDN}@diffusionstudio/piper-wasm@1.0.0/build/piper_phonemize`;
|
|
1859
|
+
// Resolved through the importmap at runtime. Kept in a variable, not a string literal, because a
|
|
1860
|
+
// literal specifier is resolved by Vite's import analysis BEFORE it checks /* @vite-ignore */ —
|
|
1861
|
+
// which fails since onnxruntime-web isn't installed as a dependency.
|
|
1862
|
+
const ORT_SPECIFIER = 'onnxruntime-web';
|
|
1856
1863
|
|
|
1857
1864
|
// OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
|
|
1858
1865
|
// models already downloaded through the lib are reused (and vice versa).
|
|
@@ -1921,7 +1928,7 @@ export class PiperTTS extends StreamingTTS {
|
|
|
1921
1928
|
const path = lib.PATH_MAP[voiceId];
|
|
1922
1929
|
if (!path) throw new Error(`unknown Piper voice: ${voiceId}`);
|
|
1923
1930
|
const [ort, phonMod, cfgBlob, modelBlob] = await Promise.all([
|
|
1924
|
-
import(/* webpackIgnore: true */ /* @vite-ignore */
|
|
1931
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ ORT_SPECIFIER),
|
|
1925
1932
|
import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
|
|
1926
1933
|
opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
|
|
1927
1934
|
opfsCachedFetch(`${lib.HF_BASE}/${path}`),
|