@todoforai/voiceloop 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/examples/demo.html +355 -0
- package/package.json +9 -4
- package/src/index.d.ts +298 -0
- package/src/stt.js +4 -1
- package/src/voice-agent.js +5 -1
package/README.md
CHANGED
|
@@ -52,6 +52,26 @@ button.onclick = async () => {
|
|
|
52
52
|
|
|
53
53
|
That's it. Speak, get spoken answers, interrupt at will.
|
|
54
54
|
|
|
55
|
+
## Try it in 20 seconds
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
npx serve . # from a clone
|
|
59
|
+
npx serve node_modules/@todoforai/voiceloop # from npm
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Open **`/examples/demo.html`** — the whole loop behind one orb, with the live
|
|
63
|
+
interim transcript, click-to-interrupt, and the two numbers that decide whether
|
|
64
|
+
it feels alive: **first token** and **first sound**. It starts in **echo mode**
|
|
65
|
+
— browser-native Web Speech STT, local Piper TTS and a stand-in "LLM" that
|
|
66
|
+
repeats you — so it runs with no keys, no backend and no cost. Open *settings*
|
|
67
|
+
to point it at any OpenAI-compatible endpoint (Ollama, your proxy) and a cloud
|
|
68
|
+
STT provider.
|
|
69
|
+
|
|
70
|
+
`/examples/simple-browser.html` is the same loop with no UI at all — 100 lines,
|
|
71
|
+
the place to start reading.
|
|
72
|
+
|
|
73
|
+
(`file://` won't work — mic and module imports need `http://localhost` or https.)
|
|
74
|
+
|
|
55
75
|
## How the latency adds up
|
|
56
76
|
|
|
57
77
|
```
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<!-- voiceloop demo: the whole loop behind one button, plus the numbers that say
|
|
3
|
+
whether it feels seamless. No bundler, no framework, no dependencies.
|
|
4
|
+
|
|
5
|
+
from a clone: npx serve . → /examples/demo.html
|
|
6
|
+
from npm: npx serve node_modules/@todoforai/voiceloop
|
|
7
|
+
|
|
8
|
+
file:// won't work — mic + module imports need http://localhost or https.
|
|
9
|
+
Out of the box it runs in ECHO mode: Web Speech STT + local Piper TTS +
|
|
10
|
+
a fake "LLM" that repeats you. No keys, no backend, no cost. Open
|
|
11
|
+
settings to point it at a real LLM. -->
|
|
12
|
+
<html lang="en">
|
|
13
|
+
<head>
|
|
14
|
+
<meta charset="utf-8">
|
|
15
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
16
|
+
<title>voiceloop</title>
|
|
17
|
+
<style>
|
|
18
|
+
:root {
|
|
19
|
+
color-scheme: dark;
|
|
20
|
+
--bg:#0b0d10; --panel:#14171c; --line:#242932; --fg:#eceef1; --dim:#79818d;
|
|
21
|
+
--c:#4a5361; /* current state colour — everything reads from here */
|
|
22
|
+
}
|
|
23
|
+
* { box-sizing:border-box }
|
|
24
|
+
body { margin:0; background:var(--bg); color:var(--fg);
|
|
25
|
+
font:15px/1.55 ui-sans-serif,system-ui,-apple-system,sans-serif }
|
|
26
|
+
main { max-width:660px; margin:0 auto; padding:36px 18px 48px;
|
|
27
|
+
display:flex; flex-direction:column; align-items:center; gap:16px }
|
|
28
|
+
|
|
29
|
+
/* ── the orb: four radial blobs drifting behind a glass shell ──────────────
|
|
30
|
+
Plain CSS on purpose. A WebGPU version looks marginally better and is dark
|
|
31
|
+
for every Linux visitor (Chrome blocklists the adapter there), and an SVG
|
|
32
|
+
goo filter costs real CPU per frame for the same silhouette. This is ~25
|
|
33
|
+
lines, runs on the compositor, and degrades to a flat disc under
|
|
34
|
+
prefers-reduced-motion. */
|
|
35
|
+
#mic { position:relative; width:210px; height:210px; border-radius:50%; border:0; padding:0;
|
|
36
|
+
cursor:pointer; overflow:hidden; background:#0c0f14; display:grid; place-items:center;
|
|
37
|
+
--a:#2b3d5c; --b:#39506e; --cc:#2a3550; --d:#334463; --spin:30s; --glow:.14;
|
|
38
|
+
transition:transform .18s ease-out }
|
|
39
|
+
/* Each blob is a soft disc smaller than the orb, wandering on its own ellipse.
|
|
40
|
+
Blurred and screen-blended, four of them read as one body that swells and
|
|
41
|
+
pinches where they overlap — metaballs, minus the shader. */
|
|
42
|
+
#mic i { position:absolute; width:52%; height:52%; left:24%; top:24%; border-radius:50%;
|
|
43
|
+
filter:blur(13px); mix-blend-mode:screen; animation:wander var(--spin) ease-in-out infinite;
|
|
44
|
+
transition:background 1s } /* colours cross-fade; the speed change snaps (animation-* isn't animatable) */
|
|
45
|
+
#mic i:nth-child(1) { background:radial-gradient(circle, var(--a) 0 26%, transparent 66%) }
|
|
46
|
+
#mic i:nth-child(2) { background:radial-gradient(circle, var(--b) 0 24%, transparent 64%);
|
|
47
|
+
animation-duration:calc(var(--spin)*.72); animation-direction:reverse }
|
|
48
|
+
#mic i:nth-child(3) { background:radial-gradient(circle, var(--cc) 0 28%, transparent 68%);
|
|
49
|
+
animation-duration:calc(var(--spin)*1.4); animation-delay:calc(var(--spin)*-.3) }
|
|
50
|
+
#mic i:nth-child(4) { background:radial-gradient(circle, var(--d) 0 22%, transparent 62%);
|
|
51
|
+
animation-duration:calc(var(--spin)*.9); animation-direction:reverse;
|
|
52
|
+
animation-delay:calc(var(--spin)*-.55) }
|
|
53
|
+
/* glass shell: specular cap + rim shadow, one pseudo each */
|
|
54
|
+
#mic::before { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
55
|
+
background:radial-gradient(120% 90% at 32% 20%, rgba(255,255,255,.30) 0 10%,
|
|
56
|
+
rgba(255,255,255,.05) 26%, transparent 52%);
|
|
57
|
+
box-shadow:inset 0 0 46px rgba(0,0,0,.75), inset 0 1px 1px rgba(255,255,255,.24) }
|
|
58
|
+
#mic::after { content:''; position:absolute; inset:0; border-radius:50%; pointer-events:none;
|
|
59
|
+
box-shadow:0 0 44px 8px var(--b); opacity:var(--glow); transition:opacity .8s }
|
|
60
|
+
@keyframes wander {
|
|
61
|
+
0%,100% { transform:translate(-34%,-22%) scale(1) }
|
|
62
|
+
25% { transform:translate( 30%,-34%) scale(1.18) }
|
|
63
|
+
50% { transform:translate( 36%, 28%) scale(.86) }
|
|
64
|
+
75% { transform:translate(-28%, 34%) scale(1.12) }
|
|
65
|
+
}
|
|
66
|
+
/* the glyph leads while the orb is dark; once the loop is live the orb says it better */
|
|
67
|
+
#glyph { position:relative; font-size:36px; line-height:1; transition:opacity .4s;
|
|
68
|
+
text-shadow:0 2px 12px #000 }
|
|
69
|
+
body:not([data-state="idle"]) #glyph { opacity:0 }
|
|
70
|
+
|
|
71
|
+
/* same object, more energy — the whole state display is these four lines */
|
|
72
|
+
body[data-state="listening"] #mic { --a:#12b87e; --b:#39d98a; --cc:#0e7fb4; --d:#5fd9b4; --spin:14s; --glow:.45 }
|
|
73
|
+
body[data-state="thinking"] #mic { --a:#e0a12e; --b:#ff9f43; --cc:#a63fc0; --d:#ffd166; --spin:6s; --glow:.65 }
|
|
74
|
+
body[data-state="speaking"] #mic { --a:#ff7a2d; --b:#ff5f6d; --cc:#ffa94d; --d:#b83fd0; --spin:4s; --glow:.9 }
|
|
75
|
+
/* VAD: the orb swells the instant the mic hears you — the "it's alive" tell */
|
|
76
|
+
body[data-vad="1"] #mic { transform:scale(1.035) }
|
|
77
|
+
|
|
78
|
+
body[data-state="listening"] { --c:#39d98a }
|
|
79
|
+
body[data-state="thinking"] { --c:#ffc453 }
|
|
80
|
+
body[data-state="speaking"] { --c:#ff8a3d }
|
|
81
|
+
|
|
82
|
+
@media (prefers-reduced-motion: reduce) {
|
|
83
|
+
#mic i { animation:none } #mic { transition:none }
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
#status { color:var(--c); font-size:14px; min-height:22px; text-align:center; transition:color .3s }
|
|
87
|
+
#lat { font:12px/1 ui-monospace,monospace; color:var(--dim); min-height:14px }
|
|
88
|
+
|
|
89
|
+
/* ── transcript ─────────────────────────────────────────────────────── */
|
|
90
|
+
#chat { width:100%; display:flex; flex-direction:column; gap:10px }
|
|
91
|
+
#chat:not(:empty) { margin-top:6px }
|
|
92
|
+
.msg { padding:10px 14px; border-radius:14px; max-width:82%; white-space:pre-wrap; word-break:break-word }
|
|
93
|
+
.msg.user { align-self:flex-end; background:#212c3a; border-bottom-right-radius:4px }
|
|
94
|
+
.msg.ai { align-self:flex-start; background:#1a1e25; border-bottom-left-radius:4px }
|
|
95
|
+
.msg.interim { opacity:.5 }
|
|
96
|
+
.msg .tail { opacity:.35 } /* generated, not yet spoken */
|
|
97
|
+
.msg.tool { align-self:flex-start; background:#161d17; color:#8fc79a;
|
|
98
|
+
font:12px/1.4 ui-monospace,monospace }
|
|
99
|
+
.msg.note { align-self:center; background:none; color:var(--dim); font-size:12px; padding:0 }
|
|
100
|
+
.msg.err { align-self:center; background:#38141d; color:#ffb0bd; font-size:13px }
|
|
101
|
+
|
|
102
|
+
/* ── everything else, out of the way ────────────────────────────────── */
|
|
103
|
+
.bar { width:100%; display:flex; gap:8px }
|
|
104
|
+
input, select, button.btn { background:#0f1216; color:var(--fg); border:1px solid var(--line);
|
|
105
|
+
border-radius:9px; padding:9px 12px; font:inherit }
|
|
106
|
+
input:focus, select:focus { outline:none; border-color:var(--c) }
|
|
107
|
+
button.btn { cursor:pointer } button.btn:hover { border-color:#3a424e }
|
|
108
|
+
#typed { flex:1 }
|
|
109
|
+
details { width:100%; border-top:1px solid var(--line); padding-top:10px }
|
|
110
|
+
summary { cursor:pointer; color:var(--dim); font-size:13px; list-style:none }
|
|
111
|
+
summary::before { content:'▸ ' } details[open] summary::before { content:'▾ ' }
|
|
112
|
+
.grid { display:flex; flex-wrap:wrap; gap:10px; padding:12px 0 4px }
|
|
113
|
+
label { font-size:11px; color:var(--dim); display:flex; flex-direction:column; gap:4px }
|
|
114
|
+
#log { font:11px/1.5 ui-monospace,monospace; color:var(--dim);
|
|
115
|
+
max-height:220px; overflow:auto; white-space:pre-wrap; padding-top:8px }
|
|
116
|
+
</style>
|
|
117
|
+
</head>
|
|
118
|
+
<body>
|
|
119
|
+
<main>
|
|
120
|
+
<button id="mic" title="click to start / stop · click while it talks to interrupt">
|
|
121
|
+
<i></i><i></i><i></i><i></i><span id="glyph">🎙</span>
|
|
122
|
+
</button>
|
|
123
|
+
<div id="status">click to start · then just talk</div>
|
|
124
|
+
<div id="lat"></div>
|
|
125
|
+
|
|
126
|
+
<div id="chat"></div>
|
|
127
|
+
|
|
128
|
+
<div class="bar">
|
|
129
|
+
<input id="typed" placeholder="…or type a turn">
|
|
130
|
+
<button class="btn" id="send">Send</button>
|
|
131
|
+
</div>
|
|
132
|
+
|
|
133
|
+
<details>
|
|
134
|
+
<summary>settings</summary>
|
|
135
|
+
<div class="grid">
|
|
136
|
+
<label>mode<select id="mode">
|
|
137
|
+
<option value="echo">echo — no keys, no backend</option>
|
|
138
|
+
<option value="llm">full loop — your LLM</option>
|
|
139
|
+
</select></label>
|
|
140
|
+
<label>LLM url<input id="llmUrl" size="30" value="http://localhost:11434/v1/chat/completions"></label>
|
|
141
|
+
<label>model<input id="model" size="14" value="llama3.2"></label>
|
|
142
|
+
<label>STT<select id="sttProvider">
|
|
143
|
+
<option value="webspeech">webspeech — free, in-browser</option>
|
|
144
|
+
<option value="deepgram">deepgram</option>
|
|
145
|
+
<option value="elevenlabs">elevenlabs</option>
|
|
146
|
+
<option value="speechmatics">speechmatics</option>
|
|
147
|
+
</select></label>
|
|
148
|
+
<label>STT token url<input id="sttTokenUrl" size="30" placeholder="your backend route (cloud STT only)"></label>
|
|
149
|
+
<label>api key<input id="apiKey" size="18" placeholder="Bearer for LLM · X-API-Key for token url"></label>
|
|
150
|
+
<label>lang<select id="sttLang">
|
|
151
|
+
<option value="en">en</option><option value="hu">hu</option><option value="de">de</option>
|
|
152
|
+
</select></label>
|
|
153
|
+
<label>voice<select id="voice"><option value="en_US-joe-medium">en_US-joe-medium</option></select></label>
|
|
154
|
+
<label>mic<select id="micDeviceId"><option value="">default</option></select></label>
|
|
155
|
+
</div>
|
|
156
|
+
<div class="bar">
|
|
157
|
+
<button class="btn" id="clear">Clear</button>
|
|
158
|
+
<button class="btn" id="muteMic">Mute mic</button>
|
|
159
|
+
<button class="btn" id="muteTts">Mute voice</button>
|
|
160
|
+
</div>
|
|
161
|
+
</details>
|
|
162
|
+
|
|
163
|
+
<details><summary>log</summary><div id="log"></div></details>
|
|
164
|
+
</main>
|
|
165
|
+
|
|
166
|
+
<script type="module">
|
|
167
|
+
import { VoiceAgent, PiperTTS, unlockAudio, prewarmVoice } from '../src/index.js';
|
|
168
|
+
|
|
169
|
+
prewarmVoice(); // load VAD/WASM + build the Silero session while the page sits idle
|
|
170
|
+
|
|
171
|
+
const $ = id => document.getElementById(id);
|
|
172
|
+
const SETTINGS = ['mode','llmUrl','model','sttProvider','sttTokenUrl','apiKey','sttLang','voice','micDeviceId'];
|
|
173
|
+
for (const id of SETTINGS) {
|
|
174
|
+
const v = localStorage.getItem('vl.' + id); if (v) $(id).value = v;
|
|
175
|
+
$(id).addEventListener('change', () => localStorage.setItem('vl.' + id, $(id).value));
|
|
176
|
+
}
|
|
177
|
+
const log = (...a) => {
|
|
178
|
+
const line = `${new Date().toISOString().slice(11,23)} ${a.map(x => typeof x === 'string' ? x : JSON.stringify(x)).join(' ')}`;
|
|
179
|
+
$('log').textContent = (line + '\n' + $('log').textContent).slice(0, 20000);
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
// ── transcript ──────────────────────────────────────────────────────────────
|
|
183
|
+
const chat = $('chat');
|
|
184
|
+
let interimEl = null, aiEl = null;
|
|
185
|
+
const toolEls = new Map();
|
|
186
|
+
const bubble = (cls, text = '') => {
|
|
187
|
+
const el = document.createElement('div');
|
|
188
|
+
el.className = `msg ${cls}`; el.textContent = text;
|
|
189
|
+
chat.append(el); el.scrollIntoView({ block:'nearest' });
|
|
190
|
+
return el;
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
// ── latency: the two numbers that decide whether this feels alive ───────────
|
|
194
|
+
let t0 = 0, tText = 0, tAudio = 0;
|
|
195
|
+
const showLat = () => $('lat').textContent = [
|
|
196
|
+
tText && `first token ${tText - t0}ms`,
|
|
197
|
+
tAudio && `first sound ${tAudio - t0}ms`,
|
|
198
|
+
].filter(Boolean).join(' · ');
|
|
199
|
+
|
|
200
|
+
const tts = new PiperTTS($('voice').value);
|
|
201
|
+
let agent = null;
|
|
202
|
+
|
|
203
|
+
// Echo mode: a stand-in "LLM" that repeats the turn — exercises the entire
|
|
204
|
+
// VAD→STT→…→TTS loop, barge-in included, with no key and no backend.
|
|
205
|
+
async function* echoLLM(history) {
|
|
206
|
+
const last = [...history].reverse().find(m => m.role === 'user')?.content ?? '';
|
|
207
|
+
for (const w of `You said: ${last}`.match(/\S+\s*/g) ?? []) {
|
|
208
|
+
yield { text: w }; await new Promise(r => setTimeout(r, 25));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// A typed turn drives the same state machine without a mic — don't claim we're listening then.
|
|
213
|
+
let micOn = false;
|
|
214
|
+
const STATUS = {
|
|
215
|
+
idle: () => micOn ? 'listening — just talk' : 'click to start · then just talk',
|
|
216
|
+
listening: () => micOn ? 'listening — just talk' : 'ready · click to talk',
|
|
217
|
+
thinking: () => 'thinking…',
|
|
218
|
+
speaking: () => micOn ? 'speaking — talk over it to interrupt' : 'speaking — click to interrupt',
|
|
219
|
+
};
|
|
220
|
+
const setState = s => {
|
|
221
|
+
document.body.dataset.state = micOn || s !== 'listening' ? s : 'idle';
|
|
222
|
+
$('status').textContent = STATUS[s]?.() ?? s;
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
function onEvent(e) {
|
|
226
|
+
if (e.type !== 'stt' || e.turnComplete) log(e);
|
|
227
|
+
switch (e.type) {
|
|
228
|
+
case 'state':
|
|
229
|
+
// The agent can stop itself (mic denied, STT socket died) — the mic light
|
|
230
|
+
// follows the agent, never the other way round.
|
|
231
|
+
if (e.state === 'idle' && agent?.closed) { micOn = false; document.body.dataset.vad = '0'; }
|
|
232
|
+
setState(e.state);
|
|
233
|
+
break;
|
|
234
|
+
case 'vad': document.body.dataset.vad = e.active ? '1' : '0'; break;
|
|
235
|
+
case 'stt':
|
|
236
|
+
if (e.turnComplete) {
|
|
237
|
+
interimEl?.remove(); interimEl = null;
|
|
238
|
+
bubble('user', e.text);
|
|
239
|
+
aiEl = null; t0 = Date.now(); tText = tAudio = 0; $('lat').textContent = '';
|
|
240
|
+
} else {
|
|
241
|
+
interimEl ??= bubble('user interim');
|
|
242
|
+
interimEl.textContent = `${e.committed ?? ''} ${e.text}`.trim();
|
|
243
|
+
interimEl.scrollIntoView({ block:'nearest' });
|
|
244
|
+
}
|
|
245
|
+
break;
|
|
246
|
+
case 'assistant': {
|
|
247
|
+
// Empty and non-final means "that draft never happened" (interrupted before
|
|
248
|
+
// a word was spoken) — drop the bubble rather than blank it.
|
|
249
|
+
if (!e.text && !e.final) { aiEl?.remove(); aiEl = null; break; }
|
|
250
|
+
if (!tText && t0) { tText = Date.now(); showLat(); }
|
|
251
|
+
// `full` only exists once the TTS progress cursor is live — i.e. a clip is
|
|
252
|
+
// actually playing. The 'speaking' state is just the first token, not sound.
|
|
253
|
+
if (e.full != null && t0 && !tAudio) { tAudio = Date.now(); showLat(); }
|
|
254
|
+
aiEl ??= bubble('ai');
|
|
255
|
+
aiEl.textContent = e.text; // spoken so far
|
|
256
|
+
const tail = (e.full ?? e.text).slice(e.text.length); // generated, not yet heard
|
|
257
|
+
if (tail) aiEl.append(Object.assign(document.createElement('span'), { className:'tail', textContent:tail }));
|
|
258
|
+
aiEl.scrollIntoView({ block:'nearest' });
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
case 'tool': {
|
|
262
|
+
// running and result arrive as two events with one id: update, don't stack.
|
|
263
|
+
const el = toolEls.get(e.id) ?? bubble('tool');
|
|
264
|
+
if (e.id) toolEls.set(e.id, el);
|
|
265
|
+
el.textContent = `🔧 ${e.name}(${JSON.stringify(e.args)})${e.running ? ' …' : ` → ${JSON.stringify(e.result ?? '')}`.slice(0,200)}`;
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
case 'echo': bubble('note', `self-echo dropped: ${e.text}`); break;
|
|
269
|
+
case 'error': bubble('err', e.error); break;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const muted = id => $(id).dataset.on === '1';
|
|
274
|
+
const makeAgent = () => {
|
|
275
|
+
const a = new VoiceAgent({
|
|
276
|
+
llmUrl: $('llmUrl').value.trim(),
|
|
277
|
+
model: $('model').value.trim(),
|
|
278
|
+
apiKey: $('apiKey').value.trim(),
|
|
279
|
+
sttProvider: $('sttProvider').value,
|
|
280
|
+
sttTokenUrl: $('sttTokenUrl').value.trim(),
|
|
281
|
+
sttLang: $('sttLang').value,
|
|
282
|
+
micDeviceId: $('micDeviceId').value,
|
|
283
|
+
persona: 'You are a concise, friendly voice assistant. Keep answers to a sentence or two.',
|
|
284
|
+
tts,
|
|
285
|
+
llm: $('mode').value === 'echo' ? echoLLM : undefined,
|
|
286
|
+
onEvent,
|
|
287
|
+
});
|
|
288
|
+
a.setMuted(muted('muteMic')); a.setTtsMuted(muted('muteTts')); // the toggles survive a rebuild
|
|
289
|
+
log('agent created; stt =', a.sttProvider);
|
|
290
|
+
return a;
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
// ── one button does everything ──────────────────────────────────────────────
|
|
294
|
+
const stop = () => {
|
|
295
|
+
agent?.destroy(); agent = null; micOn = false;
|
|
296
|
+
document.body.dataset.vad = '0'; setState('idle');
|
|
297
|
+
};
|
|
298
|
+
$('mic').onclick = async () => {
|
|
299
|
+
unlockAudio(); // must be inside the gesture (autoplay policy)
|
|
300
|
+
// A reply in flight? then the click means "shut up", whether the mic is open
|
|
301
|
+
// or not — it cancels the turn and keeps the session.
|
|
302
|
+
if (agent && (agent.state === 'thinking' || agent.state === 'speaking')) return agent.interrupt();
|
|
303
|
+
if (micOn) return stop();
|
|
304
|
+
agent ??= makeAgent();
|
|
305
|
+
micOn = true; setState('listening');
|
|
306
|
+
try { await agent.start(); }
|
|
307
|
+
catch (e) { bubble('err', String(e?.message ?? e)); stop(); }
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
$('send').onclick = () => {
|
|
311
|
+
const text = $('typed').value.trim(); if (!text) return;
|
|
312
|
+
unlockAudio();
|
|
313
|
+
agent ??= makeAgent();
|
|
314
|
+
agent.sendUserText(text); // emits its own user bubble via the stt turnComplete event
|
|
315
|
+
$('typed').value = '';
|
|
316
|
+
};
|
|
317
|
+
$('typed').onkeydown = e => { if (e.key === 'Enter') $('send').click(); };
|
|
318
|
+
|
|
319
|
+
$('clear').onclick = () => { chat.textContent = ''; $('log').textContent = ''; $('lat').textContent = '';
|
|
320
|
+
interimEl = aiEl = null; toolEls.clear(); };
|
|
321
|
+
const toggle = (id, fn) => $(id).onclick = () => {
|
|
322
|
+
const on = $(id).dataset.on !== '1';
|
|
323
|
+
$(id).dataset.on = on ? '1' : '0';
|
|
324
|
+
$(id).style.borderColor = on ? 'var(--c)' : '';
|
|
325
|
+
fn(on);
|
|
326
|
+
};
|
|
327
|
+
toggle('muteMic', on => agent?.setMuted(on));
|
|
328
|
+
toggle('muteTts', on => agent?.setTtsMuted(on));
|
|
329
|
+
|
|
330
|
+
// Voice switches live. The rest are constructor options, so they can only apply
|
|
331
|
+
// to a NEW agent — drop the current one so the next turn builds it from these.
|
|
332
|
+
$('voice').addEventListener('change', () => { tts.setVoice($('voice').value); log('voice →', $('voice').value); });
|
|
333
|
+
for (const id of SETTINGS.filter(i => i !== 'voice'))
|
|
334
|
+
$(id).addEventListener('change', () => {
|
|
335
|
+
if (micOn) return bubble('note', `stop the mic to apply ${id}`);
|
|
336
|
+
if (agent) { agent.destroy(); agent = null; log(id, '→ agent rebuilt on next turn'); }
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
// ── device + voice lists ────────────────────────────────────────────────────
|
|
340
|
+
tts.voices().then(vs => {
|
|
341
|
+
$('voice').innerHTML = vs.map(v => `<option value="${v.id}">${v.id}</option>`).join('');
|
|
342
|
+
$('voice').value = localStorage.getItem('vl.voice') || 'en_US-joe-medium';
|
|
343
|
+
tts.setVoice($('voice').value);
|
|
344
|
+
}).catch(e => log('voice list failed', String(e)));
|
|
345
|
+
navigator.mediaDevices?.enumerateDevices().then(ds => {
|
|
346
|
+
for (const d of ds.filter(d => d.kind === 'audioinput'))
|
|
347
|
+
$('micDeviceId').append(Object.assign(document.createElement('option'),
|
|
348
|
+
{ value: d.deviceId, textContent: d.label || d.deviceId.slice(0,8) }));
|
|
349
|
+
}).catch(() => {});
|
|
350
|
+
|
|
351
|
+
setState('idle');
|
|
352
|
+
window.agent = () => agent; // console poking: agent().history, …
|
|
353
|
+
</script>
|
|
354
|
+
</body>
|
|
355
|
+
</html>
|
package/package.json
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"exports": {
|
|
8
|
-
".":
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./src/index.d.ts",
|
|
10
|
+
"default": "./src/index.js"
|
|
11
|
+
},
|
|
9
12
|
"./stt": "./src/stt.js",
|
|
10
13
|
"./tuning": "./src/tuning.js",
|
|
11
|
-
"./llm-openai": "./src/llm-openai.js"
|
|
14
|
+
"./llm-openai": "./src/llm-openai.js",
|
|
15
|
+
"./package.json": "./package.json"
|
|
12
16
|
},
|
|
13
17
|
"files": [
|
|
14
18
|
"src",
|
|
@@ -48,5 +52,6 @@
|
|
|
48
52
|
},
|
|
49
53
|
"publishConfig": {
|
|
50
54
|
"access": "public"
|
|
51
|
-
}
|
|
55
|
+
},
|
|
56
|
+
"types": "src/index.d.ts"
|
|
52
57
|
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
// Public types for voiceloop — VAD → STT → LLM → TTS with real barge-in.
|
|
2
|
+
// Hand-written (the source is plain JS); see README.md for the prose version of every option.
|
|
3
|
+
|
|
4
|
+
// Events the agent emits on `onEvent` — discriminated on `type` so a switch narrows the rest.
|
|
5
|
+
export type VoiceAgentEvent =
|
|
6
|
+
| { type: 'state'; state: 'idle' | 'listening' | 'thinking' | 'speaking' }
|
|
7
|
+
| { type: 'stt'; turnComplete: boolean; text: string; ms: number; committed?: string } // text=live tail; committed=locked segments (Speechmatics AddTranscript); turnComplete=user turn ended → send to LLM
|
|
8
|
+
| { type: 'assistant'; text: string; final: boolean; full?: string } // text=spoken-so-far (solid, advances live as TTS plays); full=complete answer → tail full.slice(text.length) not-yet-spoken (dim)
|
|
9
|
+
| { type: 'vad'; active: boolean } // VAD hears speech?
|
|
10
|
+
// running:true = loop call announced, executing now (display-only spinner; no result yet).
|
|
11
|
+
// The outcome event follows with the same `id` and replaces it in place.
|
|
12
|
+
| { type: 'tool'; name: string; args: Record<string, unknown>; result?: unknown; id?: string; running?: true }
|
|
13
|
+
| { type: 'echo'; text: string } // dropped self-echo (the agent's own TTS leaked into the mic) — diagnostics only
|
|
14
|
+
| { type: 'diag'; diag: 'aec-fallback'; message: string } // once/session: AEC loopback failed → TTS plays un-cancelled (self-echo only text-filtered)
|
|
15
|
+
| { type: 'diag'; diag: 'presynth'; text: string } // a reply's first clip was synthesized speculatively (latency diagnostics)
|
|
16
|
+
| { type: 'error'; error: string };
|
|
17
|
+
|
|
18
|
+
export interface VoiceTool {
|
|
19
|
+
description?: string;
|
|
20
|
+
params?: Record<string, unknown>; // JSON-schema `properties` for the arguments
|
|
21
|
+
/** Which `params` the model MUST supply. Defaults to ALL of them — set this to make some
|
|
22
|
+
* optional, otherwise the model is forced to invent a value for every field. */
|
|
23
|
+
required?: string[];
|
|
24
|
+
/** A tool with no `run` is dropped, never advertised: the model would call it, nothing would
|
|
25
|
+
* happen, and the transcript would still show it as having run — a silent lie. */
|
|
26
|
+
run?: (args: Record<string, unknown>) => unknown;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// One LLM delta: either spoken text, or a single tool call (args already parsed). Agent-loop tool
|
|
30
|
+
// chunks additionally carry the provider call `id` (two same-args calls, e.g. a retry, stay distinct
|
|
31
|
+
// acts) and the outcome (`result`/`error`) — the loop already executed the tool, so the agent
|
|
32
|
+
// reports it instead of running it again.
|
|
33
|
+
export type LLMChunk =
|
|
34
|
+
| { text: string }
|
|
35
|
+
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
|
|
36
|
+
// Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
|
|
37
|
+
// The default (agent-llm.js, vendored Pi loop) accepts an optional 4th arg: `toolGate` is awaited
|
|
38
|
+
// before ANY tool executes — speculative prefetch passes one that resolves on turn commit, so a
|
|
39
|
+
// speculation streams text early but can never fire a side effect from uncommitted speech.
|
|
40
|
+
// Custom LLMs are a low-level seam (tests, embedding hosts): their bare tool chunks are executed
|
|
41
|
+
// by the agent, but the results are NOT fed back into that LLM mid-turn — such calls are terminal
|
|
42
|
+
// for the turn. They may ignore toolGate.
|
|
43
|
+
export type LLM = (
|
|
44
|
+
history: Array<{ role: string; content: string }>,
|
|
45
|
+
system: string,
|
|
46
|
+
signal: AbortSignal,
|
|
47
|
+
options?: { toolGate?: Promise<unknown> },
|
|
48
|
+
) => AsyncIterable<LLMChunk>;
|
|
49
|
+
|
|
50
|
+
// Pluggable TTS contract. `speak` resolves with the text actually HEARD (full on completion,
|
|
51
|
+
// a prefix on barge-in). The optional members back the voice picker / switching.
|
|
52
|
+
export interface VoiceTTS {
|
|
53
|
+
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
54
|
+
stop?(): void;
|
|
55
|
+
/** Eagerly prepare the synth engine (load models, JIT) so the first utterance has no cold start. */
|
|
56
|
+
warm?(): Promise<void>;
|
|
57
|
+
voices?(): Promise<Array<{ id: string; name: string; language: string }>>;
|
|
58
|
+
setVoice?(voiceId: string): void;
|
|
59
|
+
setSpeed?(speed: number): void;
|
|
60
|
+
/** Register a callback fired while a clip plays with the chars spoken SO FAR across the whole
|
|
61
|
+
* reply — drives the host's live spoken cursor. Pass null to clear. */
|
|
62
|
+
setOnProgress?(fn: ((spokenText: string) => void) | null): void;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface VoiceAgentOptions {
|
|
66
|
+
sysmsg?: string;
|
|
67
|
+
/** Persona/system prompt prepended to the conversation context. Defaults to the built-in
|
|
68
|
+
* voice persona (VOICE_SYSMSG); pass your own agent's system message to override it. */
|
|
69
|
+
persona?: string;
|
|
70
|
+
/** LLM model id sent to the backend `/llm` route; empty → the backend's VOICE_MODEL default. */
|
|
71
|
+
model?: string;
|
|
72
|
+
llm?: LLM;
|
|
73
|
+
/** Test seam for the agent-loop LLM transport; defaults to global fetch. */
|
|
74
|
+
fetchFn?: typeof fetch;
|
|
75
|
+
apiKey?: string;
|
|
76
|
+
/** Full endpoint URLs, not a base: voiceloop composes no routes, so a host can point the LLM and
|
|
77
|
+
* the STT token at unrelated origins (a backend proxy for one, a direct provider for the other).
|
|
78
|
+
* Deriving `${baseUrl}/llm` + `${baseUrl}/stt/<provider>-token` is the HOST's job — and each STT
|
|
79
|
+
* provider mints its own token type, so one shared token URL cannot serve all of them. */
|
|
80
|
+
llmUrl?: string;
|
|
81
|
+
maxTokens?: number;
|
|
82
|
+
tts?: VoiceTTS;
|
|
83
|
+
/** Speaking speed for the default Piper TTS (1.2 → 20% faster). Ignored if `tts` is passed. */
|
|
84
|
+
speed?: number;
|
|
85
|
+
sttLang?: string;
|
|
86
|
+
micDeviceId?: string;
|
|
87
|
+
/** STT backend: 'webspeech' (default — the browser's own on-device SpeechRecognition; free, no
|
|
88
|
+
* token, no socket, captures its own mic) | 'speechmatics' | 'elevenlabs' (Scribe) | 'deepgram'
|
|
89
|
+
* (Flux — native model-based end-of-turn detection; the agent feeds it the full mic stream and
|
|
90
|
+
* lets it close turns itself). The cloud ones connect the browser DIRECTLY to the provider with a
|
|
91
|
+
* short-TTL token the backend mints (raw key stays server-side).
|
|
92
|
+
* 'webspeech' auto-downgrades to a cloud provider in browsers with no SpeechRecognition (Firefox,
|
|
93
|
+
* the Linux WebKitGTK desktop webview) — read `agent.sttProvider` for what's actually running. */
|
|
94
|
+
sttProvider?: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
95
|
+
sttTokenUrl?: string;
|
|
96
|
+
sttUsageUrl?: string;
|
|
97
|
+
sttUrl?: string;
|
|
98
|
+
sttModel?: string;
|
|
99
|
+
/** Deepgram Flux end-of-turn confidence (0.5–0.9) — its model closes the turn itself once its
|
|
100
|
+
* confidence the user is done exceeds this threshold. Lower = snappier turn ends, higher = more
|
|
101
|
+
* patient across mid-thought pauses. Deepgram-only; default from tuning.js (0.7). */
|
|
102
|
+
sttEotThreshold?: number;
|
|
103
|
+
/** Every tool the model may call. voiceloop ships NO built-in tools — a voice library has no
|
|
104
|
+
* business knowing what a "todo" is — so product tools (and their `run`) are supplied here. */
|
|
105
|
+
tools?: Record<string, VoiceTool>;
|
|
106
|
+
keyterms?: string[];
|
|
107
|
+
preroll?: number;
|
|
108
|
+
vadOptions?: Record<string, unknown>;
|
|
109
|
+
/** Semantic end-of-turn hook run when VAD hears end-of-speech: return false to KEEP listening (the
|
|
110
|
+
* user only paused mid-thought), true (default, when omitted) to close the turn. Lets you swap the
|
|
111
|
+
* "is the turn over?" decision (silence-only | SLM | smart-turn) without touching the STT pipeline. */
|
|
112
|
+
turnDetector?: (text: string) => boolean | Promise<boolean>;
|
|
113
|
+
/** Hard cap (ms) a turnDetector may keep a turn open across a pause before it's force-committed, so a
|
|
114
|
+
* user who stops mid-sentence can't hang the turn forever. Default 4000. Ignored without a detector. */
|
|
115
|
+
maxPauseMs?: number;
|
|
116
|
+
/** Min transcribed chars of the user's interim (heard WHILE the agent is speaking) needed to count as a
|
|
117
|
+
* real barge-in and cut the reply. Higher = short backchannels ("mhm","yeah") don't interrupt; 0 =
|
|
118
|
+
* any transcribed char interrupts. Default 3 (see tuning.js TUNING.BARGE_IN_MIN_CHARS). */
|
|
119
|
+
bargeInMinChars?: number;
|
|
120
|
+
onEvent?: (event: VoiceAgentEvent) => void;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Call from a user-gesture handler (e.g. the mic button click) to bless the shared TTS audio
|
|
124
|
+
* element on mobile — without this, iOS rejects the async play() of synthesized speech. */
|
|
125
|
+
export function unlockAudio(): void;
|
|
126
|
+
/** Pre-build the Silero v5 VAD ONNX session (no mic) so the first start() reaches 'listening'
|
|
127
|
+
* much faster. Best-effort + memoized; requires the vad-web UMD bundle already loaded. */
|
|
128
|
+
export function warmVad(): Promise<void>;
|
|
129
|
+
/** Pre-mint + cache the short-TTL Deepgram STT token off the critical path (e.g. at mic-button
|
|
130
|
+
* mount), so a voice start only pays the WS handshake. Best-effort + deduped; no-op while a
|
|
131
|
+
* fresh cached token remains.
|
|
132
|
+
* `sttTokenUrl` is the EXACT endpoint that mints a Deepgram token (e.g. `/api/v1/stt/deepgram-token`),
|
|
133
|
+
* the same value passed as the `sttTokenUrl` option — NOT an API base. Pass `getSttToken` here as
|
|
134
|
+
* `getToken` if the host mints tokens itself. Warming with the wrong URL silently no-ops the
|
|
135
|
+
* optimization: the real session just mints again on click. */
|
|
136
|
+
export function warmDeepgramToken(sttTokenUrl: string, apiKey?: string, getToken?: () => Promise<string>): Promise<void>;
|
|
137
|
+
/** The built-in default voice persona (system prompt): terse, speakable answers. Exported so a
|
|
138
|
+
* host can reuse or extend it instead of restating the "you are being spoken aloud" rules. */
|
|
139
|
+
export const VOICE_SYSMSG: string;
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
export class VoiceAgent {
|
|
143
|
+
constructor(options?: VoiceAgentOptions);
|
|
144
|
+
/** The live conversation the LLM sees. A mutable reference the host may read and push into —
|
|
145
|
+
* carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
|
|
146
|
+
history: Array<{ role: string; content: string }>;
|
|
147
|
+
start(deviceId?: string): Promise<void>;
|
|
148
|
+
/** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
|
|
149
|
+
* start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
|
|
150
|
+
* fully release the mic when ending the session. */
|
|
151
|
+
stop(): void;
|
|
152
|
+
/** End for real: pause + release the mic, audio graph and VAD model (turns off the OS mic indicator).
|
|
153
|
+
* Call this when dropping the agent reference so a paused-but-discarded agent never leaks the mic. */
|
|
154
|
+
destroy(): void;
|
|
155
|
+
listVoices(): Promise<Array<{ id: string; name: string; language: string }>>;
|
|
156
|
+
setVoice(voiceId: string): void;
|
|
157
|
+
setSpeed(speed: number): void;
|
|
158
|
+
/** Update the host context (e.g. a re-opened chat); applies from the next turn — no restart. */
|
|
159
|
+
setSysmsg(sysmsg?: string): void;
|
|
160
|
+
/** Mute/unmute the mic (disables the stream's audio tracks → silence to VAD+STT); keeps the pipeline live. */
|
|
161
|
+
setMuted(muted: boolean): void;
|
|
162
|
+
/** Mute/unmute the AI's spoken output (TTS): the reply still streams as text but is never voiced. */
|
|
163
|
+
setTtsMuted(muted: boolean): void;
|
|
164
|
+
/** Move the playhead to a position in the current reply (non-whitespace char count) and play forward
|
|
165
|
+
* from there. Hits any already-streamed sentence (backward replay is cached, forward skips ahead);
|
|
166
|
+
* beyond streamed text is a no-op. Returns true on a hit, false otherwise. */
|
|
167
|
+
seek(nwIndex: number): boolean;
|
|
168
|
+
/** Re-speak a FINISHED reply from `fromNw` (a non-whitespace offset — the tapped word), interruptibly.
|
|
169
|
+
* Pure TTS (no LLM, no history change). `onProgress(spokenSoFar, done)` is driven by the host so it can
|
|
170
|
+
* update the exact transcript message it tapped: spokenSoFar is the spoken prefix, done flips true when
|
|
171
|
+
* the replay ends (played out, barged-in, or superseded) with spokenSoFar = the whole reply. */
|
|
172
|
+
replay(text: string, fromNw?: number, onProgress?: (spokenSoFar: string, done: boolean) => void): Promise<void>;
|
|
173
|
+
/** Cut off the current in-flight turn (abort the LLM, stop TTS) without stopping the agent —
|
|
174
|
+
* the mic keeps listening. No-op when not thinking/speaking. A host-callable barge-in. */
|
|
175
|
+
interrupt(): void;
|
|
176
|
+
/** Hold: keep listening/transcribing into history but DON'T reply. Entering hold silences the
|
|
177
|
+
* current reply (kept as the full streamed answer so far); releasing flushes all accumulated
|
|
178
|
+
* turns through the LLM in one go. */
|
|
179
|
+
setHeld(held: boolean): void;
|
|
180
|
+
/** Inject an out-of-band turn (e.g. a background TODO finished): speaks a reply. No-op once stopped. */
|
|
181
|
+
notify(text: string): Promise<void>;
|
|
182
|
+
/** Send a typed user turn (renders as a user bubble, then the agent replies in the background).
|
|
183
|
+
* Needs no mic — accepts even while paused/stopped (revives the reply loop; start() re-arms audio
|
|
184
|
+
* separately). Returns true when accepted (sent or queued while held), false for blank text or
|
|
185
|
+
* once destroy()ed (terminal). */
|
|
186
|
+
sendUserText(text: string): boolean;
|
|
187
|
+
/** Diagnostic tap: the last ~30s of the exact audio handed to STT as a 16kHz mono WAV, plus any
|
|
188
|
+
* detected delivery stalls (wall-clock gaps between capture chunks; offsets relative to the WAV).
|
|
189
|
+
* Play it back after a bad transcription run: gaps/corrupt audio → capture layer; clean audio →
|
|
190
|
+
* provider-side. */
|
|
191
|
+
dumpAudio(): { wav: Blob; gaps: Array<{ atSample: number; atSec: number; gapMs: number }>; seconds: number };
|
|
192
|
+
state: 'idle' | 'listening' | 'thinking' | 'speaking';
|
|
193
|
+
/** The STT provider actually in use — differs from the requested one when 'webspeech' was
|
|
194
|
+
* downgraded to a cloud provider (browser without SpeechRecognition). */
|
|
195
|
+
readonly sttProvider: 'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram';
|
|
196
|
+
/** True once the pipeline has been torn down (stop() or a fatal STT error self-stopped it). */
|
|
197
|
+
readonly closed: boolean;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export class PiperTTS implements VoiceTTS {
|
|
201
|
+
constructor(voiceId?: string, speed?: number);
|
|
202
|
+
voices(): Promise<Array<{ id: string; name: string; language: string }>>;
|
|
203
|
+
setVoice(voiceId: string): void;
|
|
204
|
+
setSpeed(speed: number): void;
|
|
205
|
+
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
206
|
+
stop(): void;
|
|
207
|
+
/** Eagerly download+compile the ONNX model and JIT the WASM path so the first reply has no cold start. */
|
|
208
|
+
warm(): Promise<void>;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// ── STT providers ────────────────────────────────────────────────────────────────────────────
|
|
212
|
+
// A provider is a factory: it receives the callbacks below plus its own credentials/tuning, and
|
|
213
|
+
// returns a live session. VoiceAgent wires these itself — call them directly only when embedding
|
|
214
|
+
// STT without the agent.
|
|
215
|
+
export interface STTCallbacks {
|
|
216
|
+
onPartial?: (text: string, ms: number, committed?: string) => void;
|
|
217
|
+
onFinal?: (text: string, ms: number) => void;
|
|
218
|
+
onError?: (error: unknown) => void;
|
|
219
|
+
onFatal?: (error: unknown) => void;
|
|
220
|
+
onClose?: () => void;
|
|
221
|
+
isClosed?: () => boolean;
|
|
222
|
+
}
|
|
223
|
+
/** A live STT session. The capability flags tell the agent how much of the audio path the provider
|
|
224
|
+
* owns, so it builds only the missing half — a provider with its own mic and its own end-of-turn
|
|
225
|
+
* (WebSpeech) needs no VAD/capture pipeline at all. */
|
|
226
|
+
export interface STTSession {
|
|
227
|
+
/** Provider owns the mic → the agent builds no capture pipeline. Implies nativeEOT. */
|
|
228
|
+
selfCapture?: boolean;
|
|
229
|
+
/** Provider closes turns itself (unsolicited onFinal) → the agent runs no VAD-commit. */
|
|
230
|
+
nativeEOT?: boolean;
|
|
231
|
+
/** Provider wants the full mic stream, including silence (its EOT model needs real pauses). */
|
|
232
|
+
continuous?: boolean;
|
|
233
|
+
/** Pre-open the socket (token + handshake) so the first utterance isn't stalled. Idempotent. */
|
|
234
|
+
open?(): void;
|
|
235
|
+
/** Push PCM. Opens the socket lazily if `open()` was never called. */
|
|
236
|
+
feed?(pcm: Int16Array): void;
|
|
237
|
+
/** End of utterance: ask the provider to finalize. No-op providers with nativeEOT. */
|
|
238
|
+
commit?(): void;
|
|
239
|
+
/** Mute/unmute without tearing the session down (selfCapture providers). */
|
|
240
|
+
setEnabled?(enabled: boolean): void;
|
|
241
|
+
close(): void;
|
|
242
|
+
}
|
|
243
|
+
export type STTFactory = (opts: STTCallbacks & Record<string, unknown>) => STTSession;
|
|
244
|
+
|
|
245
|
+
export const STT_PROVIDERS: Record<'webspeech' | 'speechmatics' | 'elevenlabs' | 'deepgram', STTFactory>;
|
|
246
|
+
export function makeDeepgramSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
247
|
+
export function makeElevenLabsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
248
|
+
export function makeSpeechmaticsSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
249
|
+
export function makeWebSpeechSTT(opts: STTCallbacks & Record<string, unknown>): STTSession;
|
|
250
|
+
/** Web Speech exists only in Chromium/Safari; elsewhere 'webspeech' resolves to this cloud provider. */
|
|
251
|
+
export const WEBSPEECH_FALLBACK: 'elevenlabs';
|
|
252
|
+
export function webSpeechSupported(): boolean;
|
|
253
|
+
/** The provider id actually used: 'webspeech' downgrades to WEBSPEECH_FALLBACK in a browser
|
|
254
|
+
* without SpeechRecognition. Call it to label the UI ("using cloud recognition"). */
|
|
255
|
+
export function resolveSttProvider(id: string): string;
|
|
256
|
+
|
|
257
|
+
// ── TTS ──────────────────────────────────────────────────────────────────────────────────────
|
|
258
|
+
/** Base class for streaming TTS engines: sentence-splits the incoming token stream, synthesizes
|
|
259
|
+
* the next clip while the current one plays, and reports the spoken cursor via setOnProgress.
|
|
260
|
+
* Subclass and implement `_synth(text)` to add an engine. */
|
|
261
|
+
export class StreamingTTS implements VoiceTTS {
|
|
262
|
+
constructor(voiceId?: string, speed?: number);
|
|
263
|
+
speak(input: AsyncIterable<string> | string, signal?: AbortSignal, fromNw?: number): Promise<string>;
|
|
264
|
+
stop(): void;
|
|
265
|
+
setOnProgress(fn: (spoken: string, scope: string) => void): void;
|
|
266
|
+
/** Jump playback to a word index within the current reply. Returns false when the seek can't be
|
|
267
|
+
* honoured (nothing playing, or past the text streamed so far) — check it before moving a cursor. */
|
|
268
|
+
seek(nwIndex: number): boolean;
|
|
269
|
+
}
|
|
270
|
+
/** Human-grade cloud voices. Pass `ttsUrl` (your backend proxy, key stays server-side); `apiKey`
|
|
271
|
+
* talks to ElevenLabs directly and is dev-only. */
|
|
272
|
+
export class ElevenLabsTTS extends StreamingTTS {
|
|
273
|
+
constructor(options?: { ttsUrl?: string; apiKey?: string; voiceId?: string; modelId?: string; format?: string });
|
|
274
|
+
warm(): Promise<void>;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// ── LLM ──────────────────────────────────────────────────────────────────────────────────────
|
|
278
|
+
/** Any OpenAI-compatible /chat/completions endpoint as an LLM adapter, with tool-calling wired
|
|
279
|
+
* to the `tools` map. `ttsUrl`-style proxying applies: prefer a backend route over apiKey. */
|
|
280
|
+
export function makeOpenAILLM(options?: {
|
|
281
|
+
llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
|
|
282
|
+
tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
|
|
283
|
+
}): LLM;
|
|
284
|
+
|
|
285
|
+
// ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
|
|
286
|
+
/** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
|
|
287
|
+
* session — call it while your UI is idle so the first start() is instant. */
|
|
288
|
+
export function loadVoiceDeps(): Promise<unknown>;
|
|
289
|
+
export function prewarmVoice(): void;
|
|
290
|
+
/** Every latency/sensitivity constant (VAD thresholds, barge-in chars, prefetch window, echo
|
|
291
|
+
* match threshold). Mutate before constructing an agent to retune globally. */
|
|
292
|
+
export const TUNING: Record<string, any>;
|
|
293
|
+
/** Echo filtering internals, exported for tests and custom mic gating: `isSelfEcho` decides
|
|
294
|
+
* whether heard text is the agent's own voice, `novelChars` counts what is genuinely new. */
|
|
295
|
+
export function isSelfEcho(heard: string, reply: string, options?: { threshold?: number; window?: number; minHits?: number }): boolean;
|
|
296
|
+
export function novelChars(heard: string, reply: string): number;
|
|
297
|
+
/** Render a tool result the way the agent reports it to the model (and to `tool` events). */
|
|
298
|
+
export function toolResultText(result: unknown): string;
|
package/src/stt.js
CHANGED
|
@@ -71,7 +71,10 @@ async function mintSttToken(url, apiKey, onFatal, getToken) {
|
|
|
71
71
|
return null;
|
|
72
72
|
}
|
|
73
73
|
const body = await res.json().catch(() => ({}));
|
|
74
|
-
|
|
74
|
+
// `message` first: a token route's error body usually puts the reason there and the bare status
|
|
75
|
+
// name in `error` ({ error: 'Unauthorized', message: 'API key not found' }) — the reason is what
|
|
76
|
+
// tells the developer their key is wrong rather than their URL.
|
|
77
|
+
if (!res.ok || !body.token) { onFatal(`STT token: ${body.message || body.error || res.statusText}`); return null; }
|
|
75
78
|
return body;
|
|
76
79
|
}
|
|
77
80
|
|
package/src/voice-agent.js
CHANGED
|
@@ -1853,6 +1853,10 @@ const CDN_PIPER = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-tts-web.j
|
|
|
1853
1853
|
// per-call module setup is exactly what we bypass (see PiperTTS._session).
|
|
1854
1854
|
const CDN_PIPER_PHONEMIZE = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-o91UDS6e.js`;
|
|
1855
1855
|
const CDN_PIPER_WASM = `${CDN}@diffusionstudio/piper-wasm@1.0.0/build/piper_phonemize`;
|
|
1856
|
+
// Resolved through the importmap at runtime. Kept in a variable, not a string literal, because a
|
|
1857
|
+
// literal specifier is resolved by Vite's import analysis BEFORE it checks /* @vite-ignore */ —
|
|
1858
|
+
// which fails since onnxruntime-web isn't installed as a dependency.
|
|
1859
|
+
const ORT_SPECIFIER = 'onnxruntime-web';
|
|
1856
1860
|
|
|
1857
1861
|
// OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
|
|
1858
1862
|
// models already downloaded through the lib are reused (and vice versa).
|
|
@@ -1921,7 +1925,7 @@ export class PiperTTS extends StreamingTTS {
|
|
|
1921
1925
|
const path = lib.PATH_MAP[voiceId];
|
|
1922
1926
|
if (!path) throw new Error(`unknown Piper voice: ${voiceId}`);
|
|
1923
1927
|
const [ort, phonMod, cfgBlob, modelBlob] = await Promise.all([
|
|
1924
|
-
import(/* webpackIgnore: true */ /* @vite-ignore */
|
|
1928
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ ORT_SPECIFIER),
|
|
1925
1929
|
import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
|
|
1926
1930
|
opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
|
|
1927
1931
|
opfsCachedFetch(`${lib.HF_BASE}/${path}`),
|