@todoforai/voiceloop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1984 @@
1
+ // voice-agent.js — VAD → STT → LLM → TTS, interruptible. One small module.
2
+ //
3
+ // const agent = new VoiceAgent({ llm, tts });
4
+ // agent.start();
5
+ //
6
+ // The whole loop is `_onUserTurn` below:
7
+ // text = vad+stt → history.push(user:text) → llm STREAMS the answer
8
+ // → tts.speak(stream) (barge-in — on transcribed words, not VAD energy — stops tts + aborts llm,
9
+ // and records what was heard so the UI can show where TTS got)
10
+ //
11
+ // First-sentence latency: `tts.speak` takes an async iterator of text deltas. It synthesizes
12
+ // the FIRST sentence as soon as its boundary arrives and starts playing, then streams the rest
13
+ // sentence-by-sentence — synthesizing ONE sentence ahead while the current plays, so there's no
14
+ // growing gap on long answers, and never two clips at once (see StreamingTTS.speak).
15
+ //
16
+ // `llm`, `tts` and `stt` are pluggable; knobs (maxTokens, vadOptions, …) are constructor options.
17
+ // Defaults: stt → browser Web Speech (free, no key), tts → Piper (free local WASM),
18
+ // llm → makeOpenAILLM (any OpenAI-compatible /chat/completions endpoint).
19
+
20
+ import { STT_PROVIDERS, resolveSttProvider } from './stt.js';
21
+ import { TUNING } from './tuning.js';
22
+ import { makeOpenAILLM } from './llm-openai.js';
23
+
24
+ // Default persona: a minimal voice-first system prompt. Hosts replace it via `persona`
25
+ // (who the agent is) and append live context via `sysmsg` / setSysmsg().
26
+ export const VOICE_SYSMSG =
27
+ 'You are a helpful voice assistant. Your words are read aloud by TTS and the user\'s words arrive ' +
28
+ 'via STT — expect mis-hearings and interpret them charitably. Reply in one or two short sentences: ' +
29
+ 'plain words for the ear, no lists, no markdown, no emojis. Lead with the answer; skip preamble. ' +
30
+ 'When a tool fits the request, call it right away instead of describing what you would do.';
31
+
32
+ const CDN = 'https://cdn.jsdelivr.net/npm/';
33
+
34
+ // AudioWorklet processor: accumulate the audio-thread's 128-sample render quanta into ~256ms chunks
35
+ // (4096 samples @16kHz — the cadence the pipeline's bootstrap-VAD/preroll math assumes) and post each
36
+ // to the main thread. Kept as a source string → Blob URL so no separate asset/bundler config is needed.
37
+ const CAPTURE_WORKLET_SRC = `
38
+ registerProcessor('capture-processor', class extends AudioWorkletProcessor {
39
+ constructor() { super(); this._buf = new Float32Array(4096); this._n = 0; }
40
+ process(inputs) {
41
+ const ch = inputs[0]?.[0];
42
+ if (ch) for (let i = 0; i < ch.length; i++) {
43
+ this._buf[this._n++] = ch[i];
44
+ if (this._n === this._buf.length) { this.port.postMessage(this._buf.slice()); this._n = 0; }
45
+ }
46
+ return true;
47
+ }
48
+ });`;
49
+ const CAPTURE_WORKLET_URL = typeof URL !== 'undefined' && URL.createObjectURL
50
+ ? URL.createObjectURL(new Blob([CAPTURE_WORKLET_SRC], { type: 'text/javascript' }))
51
+ : '';
52
+
53
+ // Compose the full system prompt: a persona (base voice persona by default, or a host-supplied one)
54
+ // + the host's conversation context (if any).
55
+ // The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
56
+ // English under the English persona — so state the selected STT language explicitly.
57
+ const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
58
+ const langRule = (lang) => {
59
+ const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
60
+ return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
61
+ };
62
+ // A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
63
+ // what order a provider streamed the keys in (used both for dedup and for the [TOOL CALL] ledger).
64
+ // Identity: the provider call id when present (two calls with identical args are DISTINCT acts,
65
+ // e.g. a legit retry after an error result); name+args otherwise (an id-less identical repeat is a
66
+ // provider re-emitting its buffered call).
67
+ // Stop WAITING on a host promise after `ms` (resolve to `fallback` instead). Used only where a
68
+ // verdict is required to make progress — see _endOfSpeech. It cannot cancel the host's work, so
69
+ // it must never be used to declare an operation failed: elsewhere the fix for host code that
70
+ // hangs is to not await it at all (tools) or to detach it (StreamingTTS.speak).
71
+ const withTimeout = (p, ms, fallback) => {
72
+ let t;
73
+ return Promise.race([
74
+ Promise.resolve(p).finally(() => clearTimeout(t)),
75
+ new Promise((res) => { t = setTimeout(() => res(fallback), ms); }),
76
+ ]);
77
+ };
78
+
79
+ const callArgs = (c) => JSON.stringify(c.args ?? {}, Object.keys(c.args ?? {}).sort());
80
+ const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
81
+ // Render a host tool's return value as model-facing text (used in the history ledger, and by LLM
82
+ // adapters when feeding tool results back to the model). Never throws: the side effect already RAN,
83
+ // so an unserializable result must not retro-fail the call and invite a retry.
84
+ export const toolResultText = (result) => {
85
+ if (result === undefined) return 'ok';
86
+ if (typeof result === 'string') return result;
87
+ if (result && typeof result.text === 'string' && typeof result.ok === 'boolean')
88
+ return result.ok ? result.text : `[failed] ${result.text}`;
89
+ try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
90
+ };
91
+ // Ledger suffix for a call whose outcome is attached to the chunk: keeps the RESULT in the
92
+ // plain-text history so the NEXT turn still knows what the tool said. Bounded — history is
93
+ // spoken-conversation-sized, not a log. Calls without an outcome field contribute nothing.
94
+ const LEDGER_RESULT_MAX = 4000;
95
+ const ledgerOutcome = (c) => {
96
+ if (c.error) return ` → error: ${String(c.error).slice(0, LEDGER_RESULT_MAX)}`;
97
+ if (!('result' in c)) return '';
98
+ let text = toolResultText(c.result);
99
+ if (text.length > LEDGER_RESULT_MAX) text = `${text.slice(0, LEDGER_RESULT_MAX)}… (${text.length} chars)`;
100
+ return ` → ${text}`;
101
+ };
102
+
103
+ // ── [TOOL CALL] mimicry filter ──────────────────────────────────────────────────────────────────
104
+ // The model sees `[TOOL CALL name] {...}` ledger lines in its own history turns, and sometimes
105
+ // IMITATES the format instead of calling the tool natively: it types the line as reply text.
106
+ // Untreated, the raw markup leaks into the spoken/visible reply and the tool never runs. This
107
+ // transform strips such lines from the text stream: a bare call (no → outcome) is what the model
108
+ // MEANT to do → re-emitted as a real { tool, args } chunk (the turn's dedup collapses it if the
109
+ // native call also arrived); a line WITH an → outcome is a hallucinated memory of a past call →
110
+ // dropped. Text before/after the line streams through untouched, held back only while a possible
111
+ // marker/line is still forming so TTS latency is unaffected on normal replies.
112
+ const MIMIC_MARK = '[TOOL CALL ';
113
+ const parseMimicLine = (line) => {
114
+ const m = /^\[TOOL CALL ([\w.-]+)\]\s*(.*)$/.exec(line.trim());
115
+ if (!m) return null; // not the format after all — caller emits it as text
116
+ const [, tool, rest] = m;
117
+ if (!rest) return { tool, args: {} };
118
+ if (!rest.startsWith('{')) return { drop: true }; // `→ outcome` (or junk) with no args — a remembered call
119
+ // Balanced-brace scan (string-aware): the args JSON may be followed by a ` → outcome` tail.
120
+ let depth = 0, inStr = false, escaped = false, end = -1;
121
+ for (let i = 0; i < rest.length; i++) {
122
+ const ch = rest[i];
123
+ if (escaped) { escaped = false; continue; }
124
+ if (inStr) { if (ch === '\\') escaped = true; else if (ch === '"') inStr = false; continue; }
125
+ if (ch === '"') inStr = true;
126
+ else if (ch === '{') depth++;
127
+ else if (ch === '}' && --depth === 0) { end = i + 1; break; }
128
+ }
129
+ if (end < 0) return { drop: true }; // truncated JSON — can't run it faithfully
130
+ if (rest.slice(end).trim()) return { drop: true }; // has an → outcome: a memory, not a new call
131
+ try { return { tool, args: JSON.parse(rest.slice(0, end)) }; } catch { return { drop: true }; }
132
+ };
133
+ async function* stripToolCallMimicry(src) {
134
+ let carry = '';
135
+ // Longest tail of `carry` that could still grow into the marker — held back; the rest is safe to emit.
136
+ const heldLen = () => {
137
+ for (let n = Math.min(carry.length, MIMIC_MARK.length - 1); n > 0; n--)
138
+ if (MIMIC_MARK.startsWith(carry.slice(-n))) return n;
139
+ return 0;
140
+ };
141
+ function* pump(final) {
142
+ for (;;) {
143
+ const at = carry.indexOf(MIMIC_MARK);
144
+ if (at < 0) {
145
+ const hold = final ? 0 : heldLen();
146
+ const out = carry.slice(0, carry.length - hold);
147
+ carry = carry.slice(carry.length - hold);
148
+ if (out) yield { text: out };
149
+ return;
150
+ }
151
+ const nl = carry.indexOf('\n', at);
152
+ if (nl < 0 && !final) { // the line is still streaming — hold from the marker on
153
+ if (at > 0) { yield { text: carry.slice(0, at) }; carry = carry.slice(at); }
154
+ return;
155
+ }
156
+ const end = nl < 0 ? carry.length : nl;
157
+ const before = carry.slice(0, at), line = carry.slice(at, end);
158
+ carry = nl < 0 ? '' : carry.slice(nl + 1); // the line (and its newline) leave the text stream
159
+ if (before) yield { text: before };
160
+ const parsed = parseMimicLine(line);
161
+ if (!parsed) yield { text: nl < 0 ? line : `${line}\n` }; // false alarm — pass it through untouched (newline included)
162
+ else if (parsed.tool) yield { tool: parsed.tool, args: parsed.args };
163
+ }
164
+ }
165
+ for await (const item of src) {
166
+ if (item.tool) { yield item; continue; }
167
+ carry += item.text ?? '';
168
+ yield* pump(false);
169
+ }
170
+ yield* pump(true);
171
+ }
172
+
173
+ // Persona (who the agent is) + language rule + the host's live context block (`sysmsg`, delimited
174
+ // so the model never mistakes injected context for its own conversation).
175
+ const composeSysmsg = (sysmsg, persona = VOICE_SYSMSG, lang = '') => {
176
+ const base = [persona, langRule(lang)].filter(Boolean).join('\n\n');
177
+ return sysmsg
178
+ ? `${base}\n\n=== CONTEXT (not your conversation) ===\n${sysmsg}\n=== END CONTEXT ===`
179
+ : base;
180
+ };
181
+
182
+ // Asset paths for the Silero VAD — MUST match the ones `_startVad` passes to MicVAD.new() so the
183
+ // pre-warm fetches/compiles the SAME artifacts the real session then reuses (HTTP-cached model +
184
+ // already-compiled ONNX-WASM runtime).
185
+ const VAD_ASSET_PATH = `${CDN}@ricky0123/vad-web@0.0.29/dist/`;
186
+ const VAD_WASM_PATH = `${CDN}onnxruntime-web@1.22.0/dist/`;
187
+
188
+ // Pre-build the Silero v5 ONNX session WITHOUT opening the mic, so the costly parts of the cold start
189
+ // — compiling the onnxruntime-web WASM (ort.env.wasm.wasmPaths) and fetching/instantiating the v5
190
+ // model — happen while the panel sits idle, NOT after the user clicks. We run the SAME MicVAD.new()
191
+ // the real session uses, but with `startOnLoad: false` so it never calls getStream (no getUserMedia,
192
+ // no mic indicator) — it only loads the model + builds the FrameProcessor. The real MicVAD.new() in
193
+ // _startVad() then reuses the HTTP-cached v5 model and the already-compiled WASM runtime, so it
194
+ // reaches 'listening' far sooner. NonRealTimeVAD is NOT used here: it ignores `model`/`onnxWASMBasePath`
195
+ // and always loads the LEGACY model, so it would warm the wrong session. Needs `window.vad` (the
196
+ // vad-web UMD bundle) already loaded; call AFTER the deps are injected. Memoized + best-effort: a
197
+ // failure just means start() pays the normal cold cost.
198
+ let warmVadPromise = null;
199
+ export function warmVad() {
200
+ return (warmVadPromise ??= (async () => {
201
+ const d = window.vad;
202
+ if (!d?.MicVAD) return;
203
+ await d.MicVAD.new({ model: 'v5', startOnLoad: false, onnxWASMBasePath: VAD_WASM_PATH, baseAssetPath: VAD_ASSET_PATH });
204
+ })().catch((e) => { warmVadPromise = null; throw e; })); // drop the memo on failure so a later start() can retry
205
+ }
206
+
207
+ export class VoiceAgent {
208
+ // Core options:
209
+ // `llm` — async generator (history, system, signal, { toolGate }) → { text } | { tool, args }
210
+ // chunks. Bring your own, or omit it and pass `llmUrl` (+ `apiKey`, `model`) to talk to any
211
+ // OpenAI-compatible /chat/completions endpoint via the built-in makeOpenAILLM (see
212
+ // llm-openai.js). NEVER put a provider secret key in a public page — point llmUrl at your own
213
+ // proxy route in production.
214
+ //
215
+ // An adapter that RUNS the tools itself (inside the generator, feeding results back to the
216
+ // model for an informed follow-up in the same turn) declares two flags on the function:
217
+ // • `llm.executesTools = true` — chunks arrive with the outcome attached (`result`/`error`),
218
+ // so _runTool reports without executing again. Optionally emit `{ tool, id, args,
219
+ // running: true }` first for a live spinner chip the outcome replaces in place.
220
+ // • `llm.acceptsToolGate = true` — the generator awaits the 4th arg's `toolGate` promise
221
+ // before ANY tool executes. Without this, speculative prefetch is DISABLED for that
222
+ // adapter (a speculation could fire a side effect on speech the user may still change);
223
+ // with it, text streams speculatively and tools wait for the turn to commit.
224
+ // `tts` — a StreamingTTS subclass; default PiperTTS (free, local WASM).
225
+ // `sttProvider` — 'webspeech' (default; free, browser-native) | 'elevenlabs' | 'speechmatics' |
226
+ // 'deepgram'. Cloud providers authenticate via a short-TTL token minted by YOUR backend
227
+ // (`sttTokenUrl`) so the raw key stays server-side. Providers may set `continuous`/`nativeEOT`
228
+ // flags the agent adapts to. 'webspeech' downgrades to `webspeechFallback` where the browser
229
+ // has no SpeechRecognition — `agent.sttProvider` is the one actually running.
230
+ // `sysmsg` — live host context appended to the persona (updatable via setSysmsg()).
231
+ // `persona` — who the agent is (replaces the default VOICE_SYSMSG persona).
232
+ // `tools`: { name: { description, params, run(args) } } — exposed to the LLM; `params` is the
233
+ // JSON-schema `properties` for the arguments. Calls fire during streaming, in parallel with TTS.
234
+ // `sttLang` — STT language code ('en'); also pins the reply language.
235
+ // `keyterms` — domain words to bias STT towards (cloud providers; max 50, ≤20 chars each).
236
+ // `sttUrl`/`sttModel`/`sttTokenUrl`/`sttUsageUrl`/`sttEotThreshold` — STT provider knobs (stt.js).
237
+ // `maxTokens`, `preroll` (chunks kept before VAD fires), `vadOptions` (Silero overrides).
238
+ // `turnDetector(text)` — async/sync predicate run when VAD hears end-of-speech: return false to
239
+ // KEEP listening (the user only paused mid-thought), true (default) to close the turn.
240
+ // `maxPauseMs` — hard cap (default 4000) a turnDetector may hold a turn open before force-commit;
241
+ // also bounds a turnDetector that never returns at all.
242
+ // `bargeInMinChars` — min transcribed NOVEL chars heard while speaking to count as a real
243
+ // barge-in; higher ignores short backchannels ("mhm","yeah"). Defaults live in tuning.js.
244
+ // `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'echo'|'error'|'diag', … }.
245
+ constructor({ sysmsg = '', persona, model = '', llm, fetchFn, apiKey = '', llmUrl = '', maxTokens = TUNING.MAX_TOKENS,
246
+ tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
247
+ sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
248
+ tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
249
+ maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, onEvent = () => {} } = {}) {
250
+ // Each agent gets its own PiperTTS (it owns an AudioContext + playing node) — a shared
251
+ // module-level default would make two agents fight over one audio output. `speed` scales
252
+ // playback rate (1.2 → 20% faster) on the default TTS; ignored if a custom `tts` is passed.
253
+ this.sysmsg = composeSysmsg(sysmsg, persona, sttLang); // persona (base or host-supplied) + language rule + host context
254
+ this._persona = persona; // remembered so setSysmsg() re-composes with the same persona
255
+ this.tts = tts ?? new PiperTTS(undefined, speed); this.sttLang = sttLang; this.onEvent = onEvent;
256
+ this.tts.onEvent = onEvent; // diagnostics channel (e.g. aec-fallback warning from StreamingTTS._playBuf)
257
+ this.apiKey = apiKey;
258
+ this.micDeviceId = micDeviceId; this.prerollMax = preroll; this.vadOptions = vadOptions;
259
+ this.turnDetector = turnDetector; this._turnText = ''; this._turnClosing = false; this.maxPauseMs = maxPauseMs; this._bargeInMinChars = bargeInMinChars; this._maxPause = null; // _bargeInMinChars undefined → live-read tuning.js (see getter) // semantic end-of-turn hook + latest interim it judges + closing latch + hard max-pause failsafe
260
+ this._prefetch = null; this._prefetchTimer = null; // speculative LLM prefetch (see _startPrefetch)
261
+ // STT is a pluggable provider (like tts/llm). The agent owns the mic+VAD and forwards
262
+ // feed/commit/close; the provider opens its own socket and reports transcripts via callbacks.
263
+ // 'webspeech' downgrades to a cloud provider where the browser has no SpeechRecognition (Firefox,
264
+ // the Linux WebKitGTK desktop shell) — otherwise voice mode could only fatal there. The model id
265
+ // belongs to the CHOSEN provider, so a downgrade drops it back to the fallback's own default.
266
+ this.sttProvider = resolveSttProvider(sttProvider);
267
+ const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
268
+ this.stt = makeStt({
269
+ apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
270
+ sttTokenUrl, getToken: getSttToken, sttUsageUrl, eotThreshold: sttEotThreshold,
271
+ onPartial: (text, ms, committed = '') => {
272
+ const interim = `${committed} ${text}`.trim();
273
+ this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector
274
+ this.onEvent({ type: 'stt', turnComplete: false, text, ms, committed });
275
+ // BARGE-IN ON WORDS, not VAD energy: while we're speaking, the user is only "interrupting"
276
+ // once STT actually transcribes words (not a cough or keyboard noise — those produce VAD
277
+ // activity but no text). Full-duplex: the uplink STT keeps transcribing throughout the
278
+ // reply, so this fires the instant words land instead of waiting for a turn boundary.
279
+ // SELF-ECHO GUARD: on speakers the browser's AEC often fails to cancel our own Web-Audio
280
+ // TTS playout, so the STT transcribes the agent's own voice — real words that would pass the
281
+ // char threshold and make it interrupt ITSELF. We track the AUDIBLE prefix of the reply
282
+ // (_replyText, fed by the TTS progress hook — not the generated-ahead answer): a live tail
283
+ // whose recent words are mostly words of that prefix is echo → latch _echoRef (the prefix
284
+ // it echoed, so its final is judged against the right reply) and don't barge in. Notes:
285
+ // • classified on the LIVE tail only — a committed real-user prefix must not tip a mixed
286
+ // turn into "echo", and a fresh real tail after echoed words clears the latch (cuts in);
287
+ // • an EMPTY tail neither reclassifies nor clears — providers emit empty-tail callbacks
288
+ // when words lock into `committed` (same words, just moved), which must not turn a
289
+ // latched echo into a barge-in;
290
+ // • latching runs while speaking AND for a short grace after playback ends (_replyDoneAt):
291
+ // STT latency means echo can land just after the reply finished (while 'listening'),
292
+ // and it must still be latched or its final becomes a user turn the agent then answers.
293
+ // Bounded so the user quoting the reply a while later can never be judged echo.
294
+ const inEchoWindow = this.state === 'speaking' || Date.now() - this._replyDoneAt < ECHO_GRACE_MS;
295
+ // Echo scope: the CURRENT reply's audible prefix PLUS, within the grace window, the PREVIOUS
296
+ // reply's (see _prevReplyText). A new turn clears _replyText before its audio starts, but on
297
+ // speakers the OLD reply's echo tail is still in flight (acoustic delay + STT latency) — with
298
+ // only the cleared reference those words looked novel and barged in on the new reply,
299
+ // cascading self-interruptions. Word-set matching makes concatenating the two safe.
300
+ const cur = this._replyEcho || this._replyText;
301
+ const prevTail = Date.now() - this._replyDoneAt < ECHO_GRACE_MS ? this._prevReplyText : '';
302
+ const echoScope = prevTail && prevTail !== cur ? `${cur} ${prevTail}`.trim() : cur;
303
+ if (text.trim() && inEchoWindow) this._echoRef = isSelfEcho(text, echoScope) ? echoScope : '';
304
+ // Barge-in on NOVEL characters only (not raw interim length): words fuzzily attributable to
305
+ // the audible reply count as echo (misheard echo words land NEAR the reply's words — see
306
+ // fuzzyHas) and contribute nothing, so echo that slipped past the ratio latch above still
307
+ // can't cut the reply; real interruptions are novel words and fire exactly as before.
308
+ if (this.state === 'speaking' && !this._echoRef && novelChars(interim, echoScope) >= this.bargeInMinChars) this._onBargeIn();
309
+ // SPECULATIVE PREFETCH: the turn's end-of-turn debounce is dead time — the LLM request only
310
+ // fires after it elapses, so its TTFT lands ON TOP. Instead, once the interim has been
311
+ // stable for PREFETCH_MS (no new words), start generating NOW with the probable final text;
312
+ // if the turn closes with that exact text the reply streams instantly (see _takePrefetch),
313
+ // if the user kept talking the stale request is aborted (cost: a few prompt tokens).
314
+ clearTimeout(this._prefetchTimer); this._prefetchTimer = null;
315
+ if (this._prefetch && interim && this._prefetch.text !== interim) this._dropPrefetch(); // transcript moved on → the running speculation is stale, stop paying for it
316
+ if (interim && this.state === 'listening' && !this._held && !this._echoRef)
317
+ this._prefetchTimer = setTimeout(() => this._startPrefetch(interim), TUNING.PREFETCH_MS ?? 200);
318
+ },
319
+ onFinal: (text, ms) => {
320
+ clearTimeout(this._prefetchTimer); this._prefetchTimer = null; // the turn is closing — no new speculation may start behind it
321
+ this._turnText = ''; this._turnClosing = false; // turn fully finalized: clear interim + the closing latch
322
+ // Take + clear the echo latch UNCONDITIONALLY (even for the empty finals below) — a
323
+ // leftover latch must never leak into the next STT turn and swallow a real utterance.
324
+ const echoRef = this._echoRef;
325
+ this._echoRef = '';
326
+ // Empty finalization (the STT closed the turn but transcribed nothing — e.g. only the AI's
327
+ // own echo or noise was force-ended): it still clears the closing latch above (so we never
328
+ // wedge waiting on a non-empty onFinal), but it's not a user turn — don't emit/reply.
329
+ if (!text.trim()) { this._dropPrefetch(); return; } // empty close (noise/echo force-end) — the speculation has no turn to serve
330
+ // A turn latched as self-echo (see onPartial): drop it ONLY if the WHOLE final is a strong
331
+ // match against the reply it echoed — a mixed turn (real user words + echo tail) is kept as
332
+ // a user turn. minHits scales with the final's own length (capped at 3): fast-endpointing
333
+ // STTs close echo as SHORT finals ("Yes.", "It is.") that could never reach 3 hits — kept as
334
+ // user turns they'd supersede and cut the playing reply (measured self-interruption cascade
335
+ // on the echo bench). Cost: a ≤2-word latched full-match quote of the reply is swallowed —
336
+ // under acoustic coupling that's overwhelmingly echo. Surfaced as 'echo' for diagnostics.
337
+ if (echoRef && isSelfEcho(text, echoRef, { window: Infinity, minHits: Math.min(3, wordsOf(text).length) })) {
338
+ this._dropPrefetch(); // the speculation was fed by our own echo — never answer it
339
+ this.onEvent({ type: 'echo', text }); return;
340
+ }
341
+ this.onEvent({ type: 'stt', turnComplete: true, text, ms });
342
+ // Held: stack the utterance into history but DON'T reply — _flush() runs them all on release.
343
+ // Wait on the hold barrier first so the turn aborted on hold-entry records its assistant text
344
+ // BEFORE these user turns, keeping history ordered (…, assistant-so-far, user, user, …).
345
+ if (this._held) { this._pushHeld(text); return; } // prefetch already dropped by setHeld(true)
346
+ this._onUserTurn(text, { speech: true }); // only a SPOKEN final may adopt the prefetch it seeded
347
+ },
348
+ onError: (error) => this.onEvent({ type: 'error', error }),
349
+ onFatal: (error) => { this.stop(); this.onEvent({ type: 'error', error }); }, // auth/config — won't self-heal; stop FIRST so the error event sees `closed` (host reconciles `running`)
350
+ onClose: () => {},
351
+ isClosed: () => this._closed,
352
+ });
353
+ // A tool with no runnable `run` is DROPPED, never advertised: the LLM would call it, nothing
354
+ // would happen, and the transcript would still show it as "ran" — a silent lie.
355
+ this.tools = {};
356
+ for (const [name, t] of Object.entries(tools)) {
357
+ if (typeof t.run === 'function') this.tools[name] = t;
358
+ else console.warn(`VoiceAgent: tool "${name}" has no run() — dropped`);
359
+ }
360
+ // Default LLM: any OpenAI-compatible /chat/completions endpoint (llm-openai.js). A custom `llm`
361
+ // generator replaces it entirely.
362
+ this.llm = llm || makeOpenAILLM({ llmUrl, apiKey, model, maxTokens, tools: this.tools, fetchFn });
363
+ this.history = [];
364
+ this.state = 'idle'; // idle | listening | thinking | speaking
365
+ this._abort = null; // aborts the in-flight LLM
366
+ this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
367
+ this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
368
+ // Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
369
+ // dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
370
+ this._tap = []; this._tapSamples = 0; this._tapTotal = 0; this._tapLastMs = 0; this._tapGaps = [];
371
+ this._stream = null; this._ctx = null; this._node = null; this._vad = null; this._streamDeviceId = undefined; // pipeline kept warm across stop()/start() for instant resume
372
+ this._muted = false; // mic disabled → browser feeds silence to VAD+STT (nothing reaches the agent)
373
+ this._ttsMuted = false; // AI voice OUTPUT off → skip speaking, still stream the reply as text
374
+ this._held = false; // HOLD: keep transcribing into history, but don't run the LLM until released
375
+ this._replyText = ''; // AUDIBLE prefix of the reply being (or last) voiced — drives the host's spoken cursor
376
+ this._replyEcho = ''; // echo REFERENCE: prefix + the whole clip now playing (see _playBuf's scope) — the cursor lags real audio, so echo is judged against this upper bound, not _replyText
377
+ this._prevReplyText = ''; // audible text of the PREVIOUS reply — echo reference across a turn boundary (its echo tail outlives _replyText's clear)
378
+ this._echoRef = ''; // echo latch: the _replyText the current STT turn was judged an echo OF ('' = not echo)
379
+ this._replyDoneAt = 0; // when playback last ended — bounds the post-playout echo grace window
380
+ this._holdBarrier = null; // resolves once the turn aborted on hold-entry has finalized; held STT orders after it
381
+ this._heldPushes = new Set(); // in-flight held-utterance append promises; release waits for all before flushing
382
+ // Bootstrap VAD (see _bootstrapVadTick): a zero-load RMS-energy gate that stands in for Silero
383
+ // (this._vad, which needs a CDN fetch + WASM compile + model load) while it's still cold-starting
384
+ // — so the mic is gated (and speech detected) from frame 1 instead of losing/chopping the user's
385
+ // opening words in the size-capped preroll buffer. Swapped to Silero at the next utterance
386
+ // boundary once it's ready (never mid-speech — see _endOfSpeech / onSpeechStart).
387
+ this._useBootstrap = false; this._sileroPending = false; this._bsSpeechFrames = 0; this._bsSilenceFrames = 0;
388
+ // Bumped on every cold start()/_teardownPipeline() — guards the fire-and-forget _startVad() below:
389
+ // a rapid re-click or device switch during the cold-load window can call start() again before the
390
+ // FIRST _startVad() resolves; its late resolution must not stomp the SECOND session's _vad/bootstrap
391
+ // state, so the continuation checks the generation is still current before touching either.
392
+ this._pipelineGen = 0;
393
+ }
394
+
395
+ // `deviceId` pins the mic to listen on — enumerate inputs host-side via
396
+ // navigator.mediaDevices.enumerateDevices() and pass the chosen audioinput's deviceId; omit
397
+ // for the browser default. Start-time choice — switching means stop() + start(id).
398
+ async start(deviceId = this.micDeviceId) {
399
+ this._closed = false;
400
+ // A self-capturing STT (browser Web Speech API) OWNS the whole audio path: its own mic + its own
401
+ // end-of-turn. It needs NO agent pipeline — no getUserMedia, AudioWorklet, preroll or Silero VAD.
402
+ // Just open it (synchronously, inside the click gesture Safari requires for SpeechRecognition.start)
403
+ // and flip to listening. Barge-in still works (word-based, via the provider's onPartial). Mute is
404
+ // handled by setMuted() → stt.setEnabled(). No warm-resume machinery: open() is idempotent + cheap.
405
+ // open() can fail synchronously (Web Speech unsupported → onFatal → stop() sets _closed + idle).
406
+ // Guard the listening flip so a fatal isn't overwritten with a fake 'listening' state.
407
+ if (this.stt.selfCapture) { this.stt.open?.(); if (!this._closed) this._set('listening'); return; }
408
+ // WARM RESUME: stop() pauses (not tears down) the pipeline, so a resume of the SAME agent finds the
409
+ // mic stream + audio graph + VAD model still alive — just re-arm them. This skips the getUserMedia
410
+ // permission round-trip AND the MicVAD.new() Silero/ONNX cold start (the bulk of "open→listening"
411
+ // latency), so reopening a paused voice session feels instant. A device switch (different deviceId)
412
+ // can't reuse the captured stream, so fall through to a fresh build. The VAD's getStream callback
413
+ // captured this._stream at MicVAD.new() time, so we MUST keep the same stream to reuse the model.
414
+ if (this._vad && this._stream?.active && deviceId === this._streamDeviceId) {
415
+ this._ctx?.resume?.(); // a paused-tab AudioContext can be suspended
416
+ this._stream.getAudioTracks().forEach(t => t.enabled = !this._muted); // honor a mute toggled while paused
417
+ // stop() is always a safe boundary (nothing is mid-utterance once stopped) — so if Silero
418
+ // finished loading while paused, resume goes straight to it instead of restarting bootstrap.
419
+ if (this._useBootstrap) { this._useBootstrap = false; this._bsSpeechFrames = 0; this._bsSilenceFrames = 0; }
420
+ this._vad.start();
421
+ this.stt.open?.(); // re-open the STT socket (stop() closed it) so the first utterance doesn't pay a cold connect
422
+ this.tts.warm?.().catch(() => {});
423
+ this._set('listening');
424
+ return;
425
+ }
426
+ // Cold build: a different mic (or first start / post-destroy) — drop any half-built warm pipeline first.
427
+ this._teardownPipeline();
428
+ const gen = ++this._pipelineGen;
429
+ this._streamDeviceId = deviceId;
430
+ // Flip to 'listening' NOW, then open the mic in the background: getUserMedia (mic permission +
431
+ // device open, esp. with the DSP chain below) is the dominant click→listening latency, and it's
432
+ // the only real blocker here. Once the mic actually opens, the bootstrap energy-VAD + preroll ring
433
+ // buffer protect the opening words (audio spoken during the brief permission/device-open window
434
+ // before the graph is wired can't be captured — an accepted tradeoff for an instant-feeling UI).
435
+ // gen (+ _closed) guard the async build: a rapid re-click / device switch bumps _pipelineGen and a
436
+ // stop/destroy sets _closed, so a slow getUserMedia that resolves into a superseded/stopped session
437
+ // tears its own stream down instead of wiring a stale mic into a dead pipeline.
438
+ this._set('listening');
439
+ (async () => {
440
+ let stream;
441
+ try {
442
+ // Dictation-tuned: drop autoGainControl — AGC pumps between words, corrupting what the
443
+ // recognizer sees. Keep echoCancellation (needed for barge-in while TTS is audible) AND
444
+ // noiseSuppression: without NS speaker playout leaks into the mic much stronger, so the STT
445
+ // transcribes the agent's own reply (self-echo) — worse than NS occasionally gating quiet speech.
446
+ const audio = { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: false, ...(deviceId ? { deviceId: { exact: deviceId } } : {}) };
447
+ try {
448
+ stream = await navigator.mediaDevices.getUserMedia({ audio });
449
+ } catch (e) {
450
+ // The saved mic id no longer exists on this machine (unplugged / different device / stale
451
+ // cross-profile localStorage) — `exact` throws OverconstrainedError instead of falling back.
452
+ // Retry once on the system default so a stale preference doesn't brick the whole session.
453
+ if (deviceId && e.name === 'OverconstrainedError') stream = await navigator.mediaDevices.getUserMedia({ audio: { ...audio, deviceId: undefined } });
454
+ else throw e;
455
+ }
456
+ } catch (e) {
457
+ // Mic open failed (permission denied / no device / constraints). We already optimistically
458
+ // flipped to 'listening', so self-stop like a fatal STT error does — stop() sets _closed + goes
459
+ // idle — so the host reconciles `running` against a.closed and the next press restarts cleanly
460
+ // (instead of a fake "listening" session with no mic). Guard on gen: a superseding start already
461
+ // owns the pipeline, so don't stop it out from under it.
462
+ if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: e.message || e.name || 'Microphone access failed' }); }
463
+ return;
464
+ }
465
+ // Superseded (re-click / device switch bumps gen) OR stopped/destroyed while opening (stop() sets
466
+ // _closed but keeps gen — it means to pause a mic that isn't wired yet) → drop the stream, don't
467
+ // wire a live mic into a dead session (would leave the OS mic indicator on).
468
+ if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); return; }
469
+ this._stream = stream;
470
+ this._stream.getAudioTracks().forEach(t => t.enabled = !this._muted); // honor a mute toggled while paused
471
+ // Capture on an AudioWorklet, NOT a ScriptProcessor: ScriptProcessor's onaudioprocess runs on
472
+ // the MAIN thread, so any stall (React renders, ONNX inference, Piper download/compile, SSE
473
+ // parsing) silently DROPS buffers — the recognizer then sees spliced audio and drops the missing
474
+ // span + loops on a token ("eleven eleven eleven"). The worklet captures on the audio rendering
475
+ // thread, so a busy UI can never drop capture; ~256ms chunks match the old cadence.
476
+ // Build on LOCALS and publish to this._ctx/_node only at the end: a supersede/stop during the
477
+ // awaited addModule() may have torn down (and a new session rebuilt) the instance fields — a
478
+ // stale continuation must clean up its own objects only.
479
+ let ctx;
480
+ try {
481
+ ctx = new AudioContext({ sampleRate: 16000 });
482
+ const src = ctx.createMediaStreamSource(stream);
483
+ await ctx.audioWorklet.addModule(CAPTURE_WORKLET_URL);
484
+ if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); ctx.close(); return; } // superseded/stopped while loading
485
+ const node = new AudioWorkletNode(ctx, 'capture-processor');
486
+ node.port.onmessage = e => this._feed(e.data); // Float32Array chunk from the audio thread
487
+ // The processor writes no output — this connection only keeps the node pulled by the graph.
488
+ src.connect(node); node.connect(ctx.destination);
489
+ this._ctx = ctx; this._node = node;
490
+ } catch (e) {
491
+ // Graph build failed (worklet unsupported, CSP-blocked blob:, 16kHz context refused): self-stop
492
+ // like a mic-open failure — never leave a fake 'listening' session holding an open mic.
493
+ stream.getTracks().forEach(t => t.stop()); ctx?.close();
494
+ if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: `audio capture: ${e.message}` }); }
495
+ return;
496
+ }
497
+ // Pre-open the STT socket NOW (token fetch + WS handshake) so it overlaps the MicVAD cold start
498
+ // below — then the FIRST utterance streams immediately instead of paying a cold connect mid-speech.
499
+ this.stt.open?.();
500
+ // Gate on the bootstrap energy-VAD from THIS frame — mic audio can arrive (worklet port above)
501
+ // well before Silero (_startVad, awaited below) resolves, and without a gate that audio would
502
+ // only be protected by the small preroll ring buffer (silently drops the opening words on a slow
503
+ // cold start). _startVad() runs in the background (not awaited) and swaps in Silero the moment
504
+ // it's ready — see _swapToSileroWhenReady.
505
+ this._useBootstrap = true; this._sileroPending = true;
506
+ this._startVad(gen).then(() => { if (gen === this._pipelineGen && !this._closed) this._swapToSileroWhenReady(); })
507
+ .catch(() => { if (gen === this._pipelineGen) this._sileroPending = false; });
508
+ // Warm the TTS engine (Piper: download+compile the ONNX voice model, JIT the WASM path) so the
509
+ // first reply isn't stalled by cold start mid-turn — but DEFERRED off the STT connect window:
510
+ // Piper's ~60MB HuggingFace download saturates bandwidth and was measured stretching the
511
+ // Deepgram WS handshake from ~0.5s to ~3s when fired concurrently. TTS isn't needed until the
512
+ // first REPLY (user utterance + LLM roundtrip away), so a short delay costs nothing.
513
+ setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, 2500);
514
+ })();
515
+ }
516
+
517
+ // True once the pipeline has been torn down (stop() / a fatal STT error). The host reconciles its
518
+ // own "running" flag against this after an error so a self-stopped agent isn't treated as live.
519
+ get closed() { return this._closed; }
520
+
521
+ stop() {
522
+ this._closed = true; this._abort?.abort(); this.tts.stop?.(); this._dropPrefetch();
523
+ clearTimeout(this._maxPause); this._turnText = ''; this._turnClosing = false; this._echoRef = ''; this._prevReplyText = ''; this._replyEcho = '';
524
+ // Clear the echo grace window: a resume within ECHO_GRACE_MS of the last reply must not judge
525
+ // the user's fresh opening words against a reply that stopped playing before the pause.
526
+ this._replyDoneAt = 0;
527
+ // Drop buffered-but-unsent audio: flushing it into the next session's socket on resume
528
+ // would re-transcribe old speech (duplicate user turns).
529
+ this._preroll = []; this._wasSpeaking = false; this._speaking = false;
530
+ // PAUSE, don't tear down: keep the mic stream + audio graph + VAD model alive so a resume of THIS
531
+ // agent (start() again) is instant — no getUserMedia prompt, no MicVAD.new() cold start. The STT
532
+ // socket DOES close (it's cheap to reopen lazily on the next utterance, and a stale socket would
533
+ // re-transcribe old audio). The host releases the mic by calling destroy() when it ends the session.
534
+ this._vad?.pause(); this.stt.close();
535
+ this._bsSpeechFrames = 0; this._bsSilenceFrames = 0;
536
+ this._tapLastMs = 0; // the pause gap isn't a capture dropout — don't flag it on resume
537
+ this._set('idle');
538
+ }
539
+
540
+ // Full teardown: release the mic, audio graph and VAD model. Called by destroy() (host ends the
541
+ // session) and before a cold rebuild (mic device switch). After this, start() pays the full cold
542
+ // cost again. Split out so stop() can pause cheaply without duplicating this.
543
+ _teardownPipeline() {
544
+ this._vad?.destroy?.(); this._vad = null;
545
+ this._node?.disconnect(); this._node = null;
546
+ this._ctx?.close(); this._ctx = null;
547
+ this._stream?.getTracks().forEach(t => t.stop()); this._stream = null;
548
+ this._streamDeviceId = undefined;
549
+ this._tap = []; this._tapSamples = 0; this._tapTotal = 0; this._tapLastMs = 0; this._tapGaps = []; // fresh diagnostics per pipeline
550
+ }
551
+
552
+ // End the session for real: pause logic + release the mic/VAD (turns off the OS mic indicator). The
553
+ // host calls this when it drops the agent (end/close/reset/new-chat) so a paused-but-discarded agent
554
+ // never leaks the microphone.
555
+ destroy() { this._destroyed = true; this.stop(); this.stt.close(); this._teardownPipeline(); }
556
+
557
+ // Host-triggered barge-in: cut off the current in-flight turn (abort the LLM, stop TTS) WITHOUT
558
+ // tearing down the pipeline — the mic keeps listening. Mirrors `_onBargeIn` but callable from a
559
+ // UI "stop" control. No-op when nothing is in flight (already idle/listening).
560
+ interrupt() {
561
+ if (this._closed || (this.state !== 'thinking' && this.state !== 'speaking')) return;
562
+ this._onBargeIn();
563
+ }
564
+
565
+ // Re-speak a FINISHED reply from `fromNw` (a non-whitespace offset — the word the user tapped),
566
+ // interruptibly. Pure TTS: no LLM, no history change (re-listening isn't a new turn). `onProgress`
567
+ // is driven by the host with (spokenSoFarText, done) so it can update the EXACT transcript message
568
+ // it tapped (identity stays host-side — no reliance on draft-matching). Supersedes any in-flight
569
+ // turn/replay (shares _turn/_abort), barge-in cuts it like a live reply, and it returns to listening.
570
+ replay(text, fromNw = 0, onProgress = () => {}) {
571
+ if (this._closed || !text?.trim()) return Promise.resolve();
572
+ const prev = this._turn;
573
+ this._abort?.abort(); // supersede the in-flight reply/replay
574
+ this.tts.stop?.();
575
+ return (this._turn = (async () => {
576
+ await prev?.catch(() => {});
577
+ if (this._closed) return;
578
+ if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText; // keep as the cross-turn echo reference (grace-bounded)
579
+ this._replyText = ''; this._replyEcho = ''; // stale previous reply must not classify pre-audio speech as echo
580
+ this._set('speaking');
581
+ const signal = (this._abort = new AbortController()).signal;
582
+ // TTS reports normalized spoken-so-far; map back to a raw prefix of `text` by non-whitespace count
583
+ // so the host's cursor lands exactly (text keeps the model's original spacing). The audible
584
+ // prefix doubles as the self-echo reference (see onPartial) — a replay leaks into the mic too.
585
+ // Echo reference = ONLY what this replay actually voices: a mid-reply replay (fromNw > 0)
586
+ // never plays the skipped head, so it must not be echo-eligible (a user quoting those words
587
+ // would be swallowed). skipLen = raw chars of that skipped head, cut from every reference.
588
+ const skipLen = fromNw > 0 ? sliceNw(text, fromNw).length : 0;
589
+ this.tts.setOnProgress?.((spoken, scope) => {
590
+ if (signal.aborted) return;
591
+ this._replyText = sliceNw(text, nw(spoken));
592
+ this._replyEcho = (scope ? sliceNw(text, nw(scope)) : this._replyText).slice(skipLen); // clip-bounded echo reference (cursor lags real audio), minus the unplayed head
593
+ onProgress(this._replyText, false);
594
+ });
595
+ try {
596
+ await this.tts.speak((async function* () { yield text; })(), signal, fromNw);
597
+ } catch (e) {
598
+ if (e.name !== 'AbortError') this.onEvent({ type: 'error', error: e.message }); // barge-in/supersede surfaces as AbortError → ignore
599
+ } finally {
600
+ this.tts.setOnProgress?.(null);
601
+ // Clamp the clip-bounded reference to what was actually heard (see _speakTurn's finally):
602
+ // the unspoken remainder of a barged-in clip never hit the speakers.
603
+ this._replyEcho = this._replyText.slice(skipLen);
604
+ this._replyDoneAt = Date.now(); // start the post-playout echo grace window
605
+ onProgress(text, true); // solidify the message back to its whole text (played out, barged-in, or superseded)
606
+ if (!this._closed && this.state !== 'idle' && !signal.aborted) this._set('listening');
607
+ }
608
+ })());
609
+ }
610
+
611
+ // Live barge-in threshold: when the host didn't pass an explicit bargeInMinChars, re-read tuning.js
612
+ // each utterance so editing it (with HMR) applies to the in-flight session; an explicit value is fixed.
613
+ get bargeInMinChars() { return this._bargeInMinChars ?? TUNING.BARGE_IN_MIN_CHARS; }
614
+
615
+ _set(s) { this.state = s; this.onEvent({ type: 'state', state: s }); }
616
+
617
+ // List the TTS voices for a picker (if the active TTS supports it): [{ id, name, language }].
618
+ listVoices() { return this.tts.voices ? this.tts.voices() : Promise.resolve([]); }
619
+ // Switch TTS voice (takes effect on the next utterance).
620
+ setVoice(voiceId) { this.tts.setVoice?.(voiceId); }
621
+ // Speaking speed (no-op under Web Audio playback — kept for API compatibility).
622
+ setSpeed(speed) { this.tts.setSpeed?.(speed); }
623
+
624
+ // Update the host context (e.g. a re-opened chat with fresh conversation); applies from the next
625
+ // turn on. Keeps the running session/history alive — no restart needed.
626
+ setSysmsg(sysmsg) { this.sysmsg = composeSysmsg(sysmsg, this._persona, this.sttLang); this._dropPrefetch(); } // a running speculation was built with the OLD sysmsg
627
+
628
+ // Mute/unmute the mic by disabling the stream's audio tracks: the browser then feeds silence to
629
+ // VAD and STT, so nothing the user says reaches the agent — without dropping the mic permission
630
+ // or tearing down the pipeline. Stored so a mute toggled while paused applies on the next start().
631
+ setMuted(muted) {
632
+ this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted);
633
+ if (muted) this._dropPrefetch(); // the utterance that seeded it just got cut off — no final will claim it
634
+ // A self-capturing STT (Web Speech) has no agent stream to gate — it listens on its OWN mic, so
635
+ // stop it explicitly via setEnabled or a "muted" agent keeps transcribing.
636
+ if (this.stt.selfCapture) this.stt.setEnabled?.(!muted);
637
+ }
638
+
639
+ // Mute/unmute the AI's spoken OUTPUT (TTS). When muted, the reply still streams into the
640
+ // transcript but is never voiced; an utterance already playing is stopped. Stored so it carries
641
+ // across pauses and applies to the next turn.
642
+ setTtsMuted(muted) { this._ttsMuted = muted; if (muted) this.tts.stop?.(); }
643
+
644
+ // Move the playhead to a position in the current reply (non-whitespace char count) and play forward
645
+ // from there — the transcript UI's tap-to-seek. Hits any sentence already streamed (backward replays
646
+ // the cached clip, forward skips ahead); a tap beyond streamed text is a no-op. Returns true on a hit.
647
+ seek(nwIndex) { return this.tts.seek?.(nwIndex) ?? false; }
648
+
649
+ // HOLD: stop responding but keep listening. Entering hold silences the current reply NOW (kept
650
+ // in history as the full streamed answer so far) and from then on every utterance is stacked
651
+ // into history without a reply. Releasing flushes — one turn processes everything accumulated.
652
+ setHeld(held) {
653
+ if (held === this._held) return;
654
+ this._held = held;
655
+ if (held) {
656
+ this._dropPrefetch(); // held turns queue for later — a running speculation would outlive its transcript
657
+ // If a reply is in flight, silence it now and tag ITS signal to record the FULL streamed text
658
+ // (per-turn, so a quick Resume can't flip it back). The barrier resolves once that turn has
659
+ // finalized — held utterances order strictly after the assistant text it kept.
660
+ const inFlight = this.state === 'thinking' || this.state === 'speaking';
661
+ if (inFlight && this._abort) {
662
+ this._abort.signal.keepFull = true;
663
+ this._holdBarrier = (this._turn ?? Promise.resolve()).catch(() => {});
664
+ this._abort.abort();
665
+ this.tts.stop?.();
666
+ this._set('listening');
667
+ } else {
668
+ this._holdBarrier = Promise.resolve(); // nothing in flight — held STT can push immediately
669
+ }
670
+ } else {
671
+ this._flush().catch((e) => this.onEvent({ type: 'error', error: e.message }));
672
+ }
673
+ }
674
+
675
+ // Queue a held user utterance: append it AFTER the hold-entry turn has finalized (the barrier), so
676
+ // history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
677
+ // release (_flush) owns starting the single turn, and waits for all of these to settle first.
678
+ _pushHeld(text) {
679
+ const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
680
+ this._heldPushes.add(p);
681
+ p.finally(() => this._heldPushes.delete(p));
682
+ }
683
+
684
+ // Release a hold: after the hold-entry turn AND every queued held push have settled, run ONE turn
685
+ // over the accumulated history. No-op when nothing is queued (last entry isn't a user turn). The
686
+ // held pushes already appended their user messages, so the turn's prep is empty — _runTurn just
687
+ // serializes and replies over the accumulated history.
688
+ async _flush() {
689
+ if (this._closed || this._held) return;
690
+ await this._holdBarrier;
691
+ if (this._closed || this._held) return; // stopped / re-held while waiting -> abandon
692
+ await Promise.allSettled([...this._heldPushes]);
693
+ if (this._closed || this._held) return;
694
+ const last = this.history[this.history.length - 1];
695
+ if (!last || last.role !== 'user') return;
696
+ await this._runTurn(''); // '' = no new utterance; reply over the accumulated held history (revalidated in _runTurn)
697
+ }
698
+
699
+ // Inject an out-of-band turn: feed `text` to the LLM as if the user had said it, then speak the
700
+ // reply. Used by the host to surface external events (a tool result, a background job finishing)
701
+ // — only meaningful while the agent is running, so it's a no-op once stopped.
702
+ //
703
+ // SOFT turn (supersede: false): unlike a user utterance this must NOT cut the reply in flight.
704
+ // A tool result arrives WHILE the turn that called the tool is still streaming/speaking, so
705
+ // aborting here would kill that turn mid-answer and re-run the LLM over a history whose last
706
+ // entry is the request again — the model then re-calls the same tool (duplicate-action spam).
707
+ // Instead it queues behind the running turn, which by then has recorded both its answer and the
708
+ // [TOOL CALL] it made, so the model sees the work is already done.
709
+ async notify(text) { if (!this._closed) await this._runTurn(text, { supersede: false }); }
710
+
711
+ // Send a TYPED user turn: like speaking, but the text comes from the host's input instead of STT.
712
+ // Emits the same turn-complete `stt` event a spoken turn would (so it renders as a finished user
713
+ // bubble in the transcript), then runs it through the normal turn loop.
714
+ // Returns true the moment the turn is ACCEPTED into the conversation (emitted/queued), false only
715
+ // for blank text — the host clears the typed draft on true and keeps it on false. The reply runs in
716
+ // the BACKGROUND (not awaited), exactly like a spoken turn finalized in onFinal: the user shouldn't
717
+ // wait for the whole answer before the box clears.
718
+ sendUserText(text) {
719
+ if (this._destroyed || !text.trim()) return false; // destroy() is terminal — never resurrect
720
+ // Typed turns need no mic: revive a paused/stopped (_closed) agent's reply loop — stop() only
721
+ // parks the AUDIO pipeline, and the LLM/tools/history don't need it. Also un-wedges a session
722
+ // whose mic was denied or whose STT fataled: voice is dead, typing still works. start() re-arms
723
+ // the audio pipeline separately (it sets _closed = false itself).
724
+ this._closed = false;
725
+ this.onEvent({ type: 'stt', turnComplete: true, text, ms: 0 });
726
+ // Held: stack the typed turn into history but DON'T reply — same as a spoken turn (see onFinal).
727
+ // Releasing the hold (Resume) runs everything accumulated. Typing must NOT silently un-hold.
728
+ if (this._held) { this._pushHeld(text); return true; }
729
+ this._onUserTurn(text);
730
+ return true;
731
+ }
732
+
733
+ // ── the loop ──────────────────────────────────────────────────────────
734
+ // State drives everything the UI needs: listening = VAD/STT, thinking = LLM,
735
+ // speaking = TTS. No separate 'stage'/'user' events — derive the pipeline from state.
736
+ //
737
+ // Fire-and-forget a user turn (spoken onFinal AND typed sendUserText both land here): record the
738
+ // message, then reply — serialized via _runTurn so rapid turns can't overlap. Never rejects to the
739
+ // caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
740
+ _onUserTurn(text, { speech = false } = {}) {
741
+ if (!text.trim()) return Promise.resolve();
742
+ return this._runTurn(text, { speech });
743
+ }
744
+
745
+ // ── Speculative LLM prefetch ──────────────────────────────────────────────────────────────────
746
+ // Start generating a reply for `text` BEFORE the turn officially closes, overlapping the LLM's
747
+ // TTFT with the end-of-turn debounce. The generator is created against history + the probable
748
+ // user turn and its first chunk is PULLED immediately (this.llm is lazy — the fetch only fires on
749
+ // the first .next()), so by the time the turn closes the first tokens are usually already here.
750
+ // Never speaks, never touches this.history — _speakTurn consumes it via _takePrefetch() only when
751
+ // the closed turn's text matches exactly; any mismatch/staleness just aborts it.
752
+ // SPECULATIVE FIRST-CLIP SYNTH: the same dead-time trick as the LLM prefetch, one stage deeper.
753
+ // By the time the end-of-turn debounce closes, the prefetch usually holds the reply's opening
754
+ // words — but Piper's first predict() (the whole post-commit critical path, ~300-900ms of WASM
755
+ // inference) only started AFTER the commit. Here we pull JUST the first chunk out of the running
756
+ // speculation and hand it to the TTS to pre-synthesize during the debounce, so when the turn
757
+ // closes the first clip is already (or nearly) done and playback starts almost immediately.
758
+ // Safe where tools are not: synth has zero side effects — a stale speculation wastes one tiny
759
+ // predict, nothing more. Serial-session cost is bounded: the chunk is the reply's first WORDS
760
+ // (stage-0 breaker, ~a dozen chars), so even a wasted predict delays a real turn only briefly.
761
+ async _presynthFirstClip(p) {
762
+ if (!this.tts.presynth) return;
763
+ try {
764
+ const f = await p.first; // opening delta of the speculation
765
+ if (f.done || p.ctl.signal.aborted || this._prefetch !== p) return;
766
+ p.first = Promise.resolve(f); // re-stitch: first was consumed here, adoption must still see it
767
+ // LLM chunks are { text } / { tool, … } objects (or raw strings from simple custom llms) —
768
+ // only text is voiceable.
769
+ const chunk = typeof f.value === 'string' ? f.value : (f.value?.text ?? '');
770
+ const end = firstWordEnd(chunk);
771
+ if (end > 0) this.tts.presynth(chunk.slice(0, end).trim()); // enough for the stage-0 chunk → warm the exact clip
772
+ } catch { /* speculation failed/aborted — the real turn re-runs it */ }
773
+ }
774
+
775
+ _startPrefetch(text) {
776
+ // Speculating is only safe if pulling the first chunk cannot fire a real side effect on a
777
+ // transcript the user is still speaking and may yet change — abort cannot undo a sent email.
778
+ // The default LLM's chunks are inert (tools run later, in _speakTurn). An LLM that executes
779
+ // tools INSIDE its generator (llm.executesTools) is safe only if it also honors `toolGate`
780
+ // (llm.acceptsToolGate): we hand it a promise it must await before ANY tool runs, so text
781
+ // streams speculatively while tools wait for the turn to commit. Neither → don't speculate.
782
+ if (this.llm.executesTools && !this.llm.acceptsToolGate) return;
783
+ // The 200ms timer raced real events — revalidate EVERYTHING now, not at scheduling time: the
784
+ // agent must still be listening to this exact live transcript (a final/typed turn/replay/hold
785
+ // in the gap cleared or changed _turnText; speculating past that would answer a stale turn).
786
+ if (this._closed || this._held || this._echoRef || this.state !== 'listening' || this._turnText !== text) return;
787
+ if (this._prefetch?.text === text) return; // same speculation already in flight
788
+ this._dropPrefetch();
789
+ const ctl = new AbortController();
790
+ // COMMIT GATE: handed to tool-executing LLMs so every tool waits at the door until this turn
791
+ // COMMITS (_takePrefetch fulfills it) or the speculation is DROPPED (_dropPrefetch rejects it).
792
+ // It always settles, on both terminal paths — a gate that merely stayed pending on drop would
793
+ // park the adapter's `await toolGate` forever, so its finally never runs and the generator +
794
+ // fetch leak. Drop REJECTS rather than fulfilling so the safe reading is also the naive one: a
795
+ // plain `await toolGate` throws instead of silently authorizing a tool the user may have
796
+ // changed their mind about. Adapters need no abort race; awaiting the gate is enough.
797
+ let commit, cancel;
798
+ const committed = new Promise((res, rej) => { commit = res; cancel = rej; });
799
+ committed.catch(() => {}); // an adapter that ignores the gate must not raise an unhandled rejection
800
+ const gen = this.llm([...this.history, { role: 'user', content: text }], this.sysmsg, ctl.signal, { toolGate: committed });
801
+ const p = { text, ctl, gen, commit, cancel, first: gen.next(), histLen: this.history.length }; // pull NOW → fetch fires during the debounce; histLen pins the base the speculation saw
802
+ p.first.catch(() => {}); // aborted/failed speculation must not be an unhandled rejection
803
+ this._prefetch = p;
804
+ this._presynthFirstClip(p); // fire-and-forget: warm the first TTS clip off the same speculation
805
+ }
806
+ _dropPrefetch() {
807
+ clearTimeout(this._prefetchTimer); this._prefetchTimer = null;
808
+ const p = this._prefetch; this._prefetch = null;
809
+ if (p) {
810
+ try { p.ctl.abort(); } catch {}
811
+ // Reject the gate BEFORE touching the iterator: a generator parked on `await toolGate` is not
812
+ // woken by the abort signal alone, so without this its finally never runs and `first` never
813
+ // settles — leaking the generator and its fetch (and stranding the return() below forever).
814
+ p.cancel?.(new DOMException('speculation dropped', 'AbortError'));
815
+ p.first.then(() => p.gen.return?.()).catch(() => {}); // abort the fetch AND close the iterator (custom llm impls may clean up in finally)
816
+ }
817
+ }
818
+ // Claim the prefetch for a closing SPOKEN turn: exact text match AND unchanged history base (the
819
+ // request was built on [...history, user]; if a raced turn appended anything since — e.g. the
820
+ // barged-in reply's assistant prefix committed while we awaited prev — the speculation is missing
821
+ // that context and must be regenerated). Hand over the already-streaming generator (re-stitched so
822
+ // the pre-pulled first chunk isn't lost) + its own AbortController, so barge-in aborts the right fetch.
823
+ _takePrefetch(text) {
824
+ const p = this._prefetch;
825
+ // histLen is checked BEFORE _runTurn pushes this turn's own user message (see call site).
826
+ if (!p || p.text !== text || p.ctl.signal.aborted || p.histLen !== this.history.length) { this._dropPrefetch(); return null; }
827
+ this._prefetch = null; // timer already cleared: successful adoption is only reached from a spoken onFinal
828
+ p.commit(); // turn committed → open the tool gate; held-back tool calls execute now
829
+ return { ctl: p.ctl, gen: (async function* () { const f = await p.first; if (!f.done) { yield f.value; yield* p.gen; } })() };
830
+ }
831
+
832
+ // The ONE serialized turn runner. Every reply (typed, spoken, held-release, notify) goes through
833
+ // here, so turns can NEVER overlap regardless of how fast callers fire. It chains onto the prior
834
+ // turn and reassigns `this._turn` SYNCHRONOUSLY before awaiting anything — so a second caller that
835
+ // arrives in the same tick chains onto THIS turn, not the one it's superseding.
836
+ //
837
+ // `text` (the new user utterance, or '' for a held-release that replies over already-queued history)
838
+ // is appended AFTER the predecessor settles — so its heard prefix lands first, keeping history
839
+ // chronological (…, assistant-heard-so-far, user-new). If a hold was entered while we waited, the
840
+ // utterance is routed into the held queue instead of dropped (acceptance must never be lost). The
841
+ // reply only runs when the latest history entry is a user turn. Never throws (errors -> onEvent).
842
+ //
843
+ // `supersede: false` = a SOFT turn (notify: tool results, board events): record it and reply only
844
+ // if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
845
+ // sees this message in history and answers over the fuller picture — so a burst of tool results
846
+ // collapses into ONE reply instead of one interrupting monologue per result.
847
+ _runTurn(text, { supersede = true, speech = false } = {}) {
848
+ const prev = this._turn;
849
+ if (supersede) {
850
+ this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
851
+ this.tts.stop?.();
852
+ }
853
+ // Reply-generating turns only. `this._turn` can't stand in for this: replay() chains onto it too,
854
+ // and a replay is pure TTS — it never answers pending history, so treating it as "someone newer
855
+ // owns the reply" would strand the turn (a held release or a tool result would go unanswered).
856
+ const seq = ++this._replySeq;
857
+ return (this._turn = (async () => {
858
+ await prev?.catch(() => {}); // let the predecessor settle (records its heard prefix)
859
+ if (this._closed) return; // stopped while we waited -> abandon (stop() dropped the prefetch)
860
+ // Held while we waited: don't drop an accepted utterance — queue it for the next release.
861
+ // (setHeld(true) already dropped any speculation.)
862
+ if (this._held) { if (text) this._pushHeld(text); return; }
863
+ // Superseded while we waited: the newer turn now owns the reply and sees our message in
864
+ // history — generating here too would talk over it (and re-run the same tools). Decided BEFORE
865
+ // claiming the prefetch, so we never adopt a stream just to abort it.
866
+ if (this._replySeq !== seq) { this._dropPrefetch(); if (text) this.history.push({ role: 'user', content: text }); return; }
867
+ // Prefetch handover — decided BEFORE pushing our user message so _takePrefetch can verify the
868
+ // history base is exactly what the speculation was built on (a raced turn that appended its
869
+ // assistant message while we awaited prev fails the histLen check → regenerate with context).
870
+ // Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
871
+ const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
872
+ if (text) this.history.push({ role: 'user', content: text });
873
+ // Reply only if the conversation now ends on a user turn (a raced turn may have already
874
+ // replied). pf is always null here: if we pushed a user turn it IS the last entry.
875
+ if (this.history[this.history.length - 1]?.role !== 'user') return;
876
+ this._set('thinking'); // LLM stage
877
+ this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
878
+ await this._speakTurn(this._abort.signal, pf?.gen); // _speakTurn never throws — errors surface via onEvent
879
+ })());
880
+ }
881
+
882
+ // One assistant turn: stream the LLM → TTS, then record what was actually HEARD (the barge-in
883
+ // prefix, not the full generated answer) plus a ledger of the tools it dispatched. Stored as
884
+ // `this._turn` so the next user turn can await it. Never throws (errors surface via onEvent) so
885
+ // the await is safe — and never awaits a tool, so a hung one can't hold the next turn hostage.
886
+ async _speakTurn(signal, prefetched = null) {
887
+ // Fresh turn → drop the previous reply's audible text so user speech heard before THIS turn's
888
+ // first audio plays can't be misjudged as echo of the OLD reply — but PARK it in _prevReplyText:
889
+ // its speakers→mic echo tail is still in flight for a moment (acoustic delay + STT latency), and
890
+ // the grace-bounded scope in onPartial must still recognize it. The TTS progress hook below
891
+ // repopulates _replyText with the audible prefix as playback advances.
892
+ if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText;
893
+ this._replyText = ''; this._replyEcho = '';
894
+ // Tap the LLM stream once: yield speech text to TTS, and record the tool calls it makes (they
895
+ // fire on arrival; `calls` is the ledger). First delta flips us to 'speaking'; accumulate for the
896
+ // heard-vs-unheard report. `this.llm(...)` is a lazy generator — its fetch fires only when
897
+ // tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
898
+ let answer = '', calls = [];
899
+ // The tools this turn dispatched, as a ledger line recorded INTO its assistant message. The
900
+ // runtime's tool calls are fire-and-forget (no tool_use/tool_result pair on the wire, and history
901
+ // is provider-neutral {role, content} text), so without this the history reads as if it never
902
+ // called anything and the next turn happily calls it AGAIN — duplicate tool-call spam. Relayed
903
+ // calls: the outcome arrives separately as the host's `[TOOL RESULT <name>]` notify() turn.
904
+ // Custom LLMs that execute tools inside their own generator (llm.executesTools) attach the
905
+ // outcome directly, so the ledger line carries it — bounded — or the next turn would see the call with no result and lose what
906
+ // the model learned (a barge-in cuts the spoken follow-up but not the fact). Read late (the
907
+ // stream may still be appending) and on EVERY exit path, including a failed reply.
908
+ const toolNote = () => calls.map((c) => `[TOOL CALL ${c.tool}] ${callArgs(c)}${ledgerOutcome(c)}`).join('\n');
909
+ // Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
910
+ // announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
911
+ // that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
912
+ // outcome arrives finalizes it as interrupted — DISPLAY truth only: an announcement never reaches
913
+ // the ledger (`calls`), so the model treats it as never having happened.
914
+ const announced = new Map();
915
+ // Latched once this turn has finalized its chips (finishInterrupted ran). A barge-in detaches the
916
+ // LLM producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
917
+ // announcement would open a spinner nothing will ever close. Past the latch, announcements are
918
+ // dropped — outcomes still flow (a tool the adapter really ran is truth worth reporting).
919
+ let finalized = false;
920
+ const finishInterrupted = () => {
921
+ finalized = true;
922
+ for (const r of announced.values())
923
+ this.onEvent({ type: 'tool', name: r.tool, id: r.id, args: r.args, result: { ok: false, text: 'interrupted — did not finish' } });
924
+ announced.clear();
925
+ };
926
+ const tapped = (async function* (self, src) {
927
+ for await (const item of src) {
928
+ // tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
929
+ // act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
930
+ if (item.tool) {
931
+ if (item.running) { // display-only announcement; the outcome chunk below does the work
932
+ // An id-less announcement can never be replaced in place NOR finalized (both key on id),
933
+ // so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
934
+ if (finalized || !item.id) continue;
935
+ // Already settled this turn (adapter re-emitting its buffered pair): re-announcing would
936
+ // reopen a spinner over a finished chip, and the duplicate outcome is deduped below and
937
+ // never re-reported — so it would hang there, then finalize as falsely "interrupted".
938
+ if (calls.some((c) => c.id === item.id)) continue;
939
+ announced.set(item.id, item);
940
+ self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
941
+ continue;
942
+ }
943
+ if (item.id) announced.delete(item.id);
944
+ // Same key = same act (provider re-emit / model asking twice). A MIMIC chunk carries no id,
945
+ // so also match by name+args across the id boundary — a typed echo of an id-carrying native
946
+ // call (llm.executesTools mode) must collapse too, or it would execute the tool a second time.
947
+ const dup = calls.some((c) => callKey(c) === callKey(item) ||
948
+ ((!c.id || !item.id) && c.tool === item.tool && callArgs(c) === callArgs(item)));
949
+ if (!dup) { calls.push(item); self._runTool(item); }
950
+ continue;
951
+ }
952
+ if (self.state === 'thinking') self._set('speaking'); // first token → TTS stage
953
+ answer += item.text;
954
+ // While audio plays, the TTS progress hook owns the draft (text = spoken-so-far, full = answer,
955
+ // read live each tick), so streaming the whole answer here would yank the cursor to the end.
956
+ // Once the cursor is live we only grow `answer` (the hook reads it); until then (muted, or the
957
+ // first sentence is still synthesizing) we stream the full answer so the bubble isn't empty.
958
+ if (!self._cursorLive) self.onEvent({ type: 'assistant', text: answer, final: false });
959
+ yield item.text;
960
+ }
961
+ })(this, stripToolCallMimicry(prefetched ?? this.llm(this.history, this.sysmsg, signal))); // prefetched stream already carries this turn's user text (see _startPrefetch)
962
+
963
+ let spoken = '';
964
+ // Live spoken cursor: while a clip plays the TTS reports chars heard so far, which we mirror as a
965
+ // streaming assistant draft — `text` = spoken-so-far, `full` = whole answer, so the host renders the
966
+ // not-yet-spoken remainder dimmed and the cursor follows the voice. Cleared (finally) after the turn.
967
+ // The TTS reports spoken text as normalized sentence chunks (trimmed, space-joined), which is NOT a
968
+ // raw prefix of `answer` (the LLM text keeps newlines/double spaces). Map it back by non-whitespace
969
+ // count: walk `answer` until it has the same number of non-whitespace chars as what's been spoken,
970
+ // so `text` is always an exact prefix of `full` → the UI's full.slice(text.length) tail is correct.
971
+ const rawPrefix = (spoken) => {
972
+ let need = 0; for (const c of spoken) if (!/\s/.test(c)) need++;
973
+ if (!need) return '';
974
+ let seen = 0, i = 0;
975
+ for (; i < answer.length; i++) { if (!/\s/.test(answer[i])) { if (++seen === need) { i++; break; } } }
976
+ return answer.slice(0, i);
977
+ };
978
+ this._cursorLive = false;
979
+ this.tts.setOnProgress?.((spokenSoFar, scope) => {
980
+ if (signal.aborted) return;
981
+ this._cursorLive = true; // hook now drives the draft → LLM stream stops emitting full text
982
+ const heard = rawPrefix(spokenSoFar);
983
+ this._replyText = heard; // AUDIBLE prefix — drives the spoken cursor
984
+ this._replyEcho = scope ? rawPrefix(scope) : heard; // clip-bounded echo reference (cursor lags real audio)
985
+ this.onEvent({ type: 'assistant', text: heard, full: answer, final: false });
986
+ });
987
+ try {
988
+ // TTS muted: drain the same stream so the transcript + tool calls still run, but never voice
989
+ // it. The whole reply counts as "heard" (history) since there's no barge-in without audio.
990
+ if (this._ttsMuted) {
991
+ for await (const _ of tapped) { /* consume: drives answer + assistant events */ }
992
+ spoken = answer;
993
+ } else {
994
+ spoken = await this.tts.speak(tapped, signal); // resolves with text actually heard (prefix on barge-in)
995
+ }
996
+ } catch (e) {
997
+ if (e.name !== 'AbortError') { // real failure (not barge-in) → surface it
998
+ this.tts.setOnProgress?.(null); this._cursorLive = false;
999
+ this.onEvent({ type: 'error', error: e.message });
1000
+ // Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
1001
+ // Record it anyway: the ledger is about what was DONE, not about what got spoken — dropping
1002
+ // it would let the next turn call the same tool again.
1003
+ if (toolNote()) this.history.push({ role: 'assistant', content: toolNote() });
1004
+ finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
1005
+ if (this.state !== 'idle') this._set('listening');
1006
+ return;
1007
+ }
1008
+ // barge-in surfaced as AbortError → fall through and record whatever was heard
1009
+ } finally {
1010
+ this.tts.setOnProgress?.(null); this._cursorLive = false; // drop the cursor hook — next turn re-registers
1011
+ // Clamp the echo reference back to what was actually HEARD: the clip-bounded upper bound
1012
+ // (_replyEcho) exists only because a PLAYING clip is ahead of the cursor — once playback
1013
+ // stops (finished or barged-in), the unspoken remainder never hit the speakers, and keeping
1014
+ // it in scope through the grace window would swallow a user quoting those exact words.
1015
+ this._replyEcho = spoken ? rawPrefix(spoken) : this._replyText;
1016
+ this._replyDoneAt = Date.now(); // start the post-playout echo grace window
1017
+ }
1018
+
1019
+
1020
+ // Normally we record only the HEARD prefix (barge-in: the user reacted to what they heard). But a
1021
+ // turn aborted on hold-entry was tagged signal.keepFull, so we keep the whole streamed answer —
1022
+ // on release the LLM sees everything it had formed. Per-turn tag → a quick Resume can't flip it.
1023
+ const kept = signal.keepFull ? answer : spoken;
1024
+ if (kept || toolNote()) {
1025
+ this.history.push({ role: 'assistant', content: [kept, toolNote()].filter(Boolean).join('\n') });
1026
+ }
1027
+ if (kept) {
1028
+ // History keeps the normalized `kept`; the EVENT emits a raw prefix of `answer` so the host's
1029
+ // full.slice(text.length) tail aligns. keepFull/normal-finish → whole answer (no tail); a barge-in
1030
+ // → the heard prefix solid + the unspoken remainder dimmed, showing exactly where TTS got.
1031
+ const shown = signal.keepFull ? answer : rawPrefix(spoken);
1032
+ this.onEvent({ type: 'assistant', text: shown, full: answer, final: true });
1033
+ } else if (answer) {
1034
+ // Interrupted before anything was heard (nothing voiced yet): drop the streamed draft so the
1035
+ // transcript doesn't keep a dangling unfinished bubble. Empty text → host clears the draft.
1036
+ this.onEvent({ type: 'assistant', text: '', final: false });
1037
+ }
1038
+ // Tools are NOT joined here. They were dispatched during streaming (in parallel with TTS) and
1039
+ // _runTool reports its own outcome via onEvent, so waiting adds nothing — while a tool that
1040
+ // never settles would wedge the turn serializer, and every turn queued behind it, for good. It
1041
+ // also breaks a real deadlock: a tool whose run() awaits agent.notify() (delivering its result
1042
+ // as a turn) would wait on the turn that waits on it. Slow work belongs in a tool that returns
1043
+ // promptly and reports the outcome later via notify().
1044
+ finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
1045
+ if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
1046
+ }
1047
+
1048
+ // Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
1049
+ // parallel with TTS, and NEVER joined: the turn records the dispatch in its ledger and finishes
1050
+ // when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
1051
+ // else. Its outcome reaches the host via onEvent; the model learns it via notify(). Must not
1052
+ // reject — nothing awaits it, so a rejection here would be unhandled (hence the catch-all).
1053
+ // `call.ran` memoizes: the same call object reaching _runTool twice must not execute twice.
1054
+ _runTool(call) {
1055
+ if (!call) return;
1056
+ return (call.ran ??= (async () => {
1057
+ try {
1058
+ // Pre-executed call (agent loop ran it inside the reply): report the attached outcome,
1059
+ // never execute a second time. An in-loop failure was already fed back to the model as an
1060
+ // is_error tool_result and the model narrates it — here it surfaces as ONE failed tool
1061
+ // event (the transcript's failed chip). No extra 'error' event: that channel is for
1062
+ // runtime/transport failures, not a handled domain-level tool miss.
1063
+ // `id` rides along on every tool event so a host can replace THIS call's running chip in
1064
+ // place, and two same-args calls (a legit retry) stay distinct acts in the transcript.
1065
+ if ('result' in call || 'error' in call) {
1066
+ const result = call.error ? { text: call.error, ok: false } : call.result;
1067
+ this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
1068
+ return;
1069
+ }
1070
+ // Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
1071
+ // a deadline could only stop WAITING — reporting "failed" for an operation still on its way
1072
+ // to committing its side effect, inviting a duplicate retry. Nothing awaits this call
1073
+ // (see _speakTurn), so a hung tool costs its own result, never the turn loop.
1074
+ const result = await this.tools[call.tool]?.run?.(call.args);
1075
+ this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
1076
+ } catch (e) {
1077
+ this.onEvent({ type: 'error', error: `tool ${call.tool}: ${e.message}` });
1078
+ }
1079
+ })());
1080
+ }
1081
+
1082
+ _onBargeIn() { // user spoke while we were talking
1083
+ this._abort?.abort(); // stop LLM
1084
+ this.tts.stop?.(); // stop TTS (tts.speak resolves with the heard prefix)
1085
+ this._set('listening');
1086
+ }
1087
+
1088
+ // VAD heard end-of-speech. The detector decides if the turn is truly OVER (commit) or the user just
1089
+ // paused mid-thought (keep listening). Without a detector this is plain VAD endpointing: always commit.
1090
+ // Guarded by a generation token so a late async verdict from a superseded pause can't commit a turn
1091
+ // the user already continued (a new onSpeechStart bumps the token).
1092
+ async _endOfSpeech() {
1093
+ clearTimeout(this._maxPause);
1094
+ // Native end-of-turn provider (e.g. Deepgram Flux): the STT's own model decides when the turn is
1095
+ // over and emits an unsolicited onFinal — VAD end-of-speech is NOT a boundary here (the provider
1096
+ // must hear the silence to judge it), so don't commit and don't run the turnDetector. No agent-side
1097
+ // failsafe either: Flux transcripts are CUMULATIVE per turn, so force-flushing a still-open turn
1098
+ // emits its prefix, which the next flush then repeats ("it is it is …"). Turn-hang protection is
1099
+ // the provider's own eot_timeout_ms (server force-ends the turn after ~5s of silence).
1100
+ if (this.stt.nativeEOT) return;
1101
+ // _close: mark the turn as CLOSING (commit() starts ForceEndOfUtterance) but DON'T clear _turnText —
1102
+ // onFinal clears it when the final actually lands. This keeps a "closing" turn distinct from a fresh
1103
+ // idle state, so a fast next onSpeechStart won't reset() the STT mid-finalization (lost/merged turn).
1104
+ const close = () => { this._turnClosing = true; this.stt.commit(); };
1105
+ if (!this.turnDetector || !this._turnText) { close(); return; }
1106
+ const gen = this._turnGen; // snapshot; a new onSpeechStart bumps it → our verdict goes stale
1107
+ let done = true;
1108
+ // maxPauseMs is ONE absolute deadline from end-of-speech — the wall-clock cap on how long a
1109
+ // detector may hold this turn open, whether it answers "not yet" or never answers at all. Both
1110
+ // legs below draw from the same budget, so a detector that deliberates 3.9s can't then buy a
1111
+ // fresh 4s failsafe on top (which would blow the documented cap to nearly 2×).
1112
+ // The verdict leg needs a deadline of its own because it's HOST code on the only path that can
1113
+ // close the turn: a hang there never reaches the failsafe, and the turn stays open forever.
1114
+ const deadline = Date.now() + this.maxPauseMs;
1115
+ const left = () => Math.max(0, deadline - Date.now());
1116
+ const HUNG = Symbol('hung');
1117
+ try { done = await withTimeout(this.turnDetector(this._turnText), left(), HUNG); }
1118
+ catch (e) { this.onEvent({ type: 'error', error: `turnDetector: ${e.message}` }); }
1119
+ if (gen !== this._turnGen) return; // user resumed speaking while we judged → verdict is stale
1120
+ if (this._speaking) return; // already talking again → let the next end-of-speech decide
1121
+ // Report the hang only once it's established the verdict still MATTERED (stale checks above):
1122
+ // a detector abandoned because the user resumed isn't an error worth surfacing.
1123
+ if (done === HUNG) { this.onEvent({ type: 'error', error: `turnDetector: no verdict in ${this.maxPauseMs}ms — closing turn` }); done = true; }
1124
+ if (done) { close(); return; }
1125
+ // Keep listening: the pause was mid-thought. Failsafe so a turn the detector never closes (user
1126
+ // walked away mid-sentence) can't hang forever — commit anyway at the deadline.
1127
+ this._maxPause = setTimeout(() => { if (gen === this._turnGen && !this._speaking && this._turnText) close(); }, left());
1128
+ }
1129
+
1130
+ // ── VAD ───────────────────────────────────────────────────────────────
1131
+ // Shared speech-start/end handling: called by BOTH the bootstrap energy-VAD (_bootstrapVadTick)
1132
+ // and Silero (onSpeechStart/onSpeechEnd below) — a detector swap never changes turn semantics.
1133
+ _onSpeechStart() {
1134
+ this._speaking = true; this._turnGen = (this._turnGen || 0) + 1; clearTimeout(this._maxPause);
1135
+ if (!this._turnText && !this._turnClosing) this.stt.reset?.();
1136
+ this.onEvent({ type: 'vad', active: true });
1137
+ }
1138
+ _onSpeechEnd() { this._speaking = false; this.onEvent({ type: 'vad', active: false }); this._endOfSpeech(); }
1139
+
1140
+ async _startVad(gen) {
1141
+ // Self-serve the UMD deps (onnxruntime + vad-web) if the host never called prewarmVoice() —
1142
+ // prewarming is an optimization, not a setup requirement.
1143
+ if (!window.vad) { const { loadVoiceDeps } = await import('./deps.js'); await loadVoiceDeps(); }
1144
+ const d = window.vad;
1145
+ const vad = await d.MicVAD.new({
1146
+ model: 'v5',
1147
+ // MicVAD.new() auto-starts listening itself when startOnLoad (default true) — we need it INERT
1148
+ // until we explicitly .start() it below (either right away, or later at a safe utterance
1149
+ // boundary via _swapToSileroWhenReady/_bootstrapVadTick) so it never runs concurrently with the
1150
+ // bootstrap energy-VAD.
1151
+ startOnLoad: false,
1152
+ // Listen on the SAME stream we opened in start() (the chosen mic) — not a second
1153
+ // default-mic getUserMedia. Keeps VAD and STT on one input, and one permission prompt.
1154
+ getStream: async () => this._stream,
1155
+ // Same asset paths warmVad() pre-fetched/compiled, so this reuses the cached v5 model + WASM.
1156
+ onnxWASMBasePath: VAD_WASM_PATH,
1157
+ baseAssetPath: VAD_ASSET_PATH,
1158
+ ...TUNING.VAD, // speech-detection sensitivity (positive/negativeSpeechThreshold, minSpeechFrames, redemptionFrames, preSpeechPadFrames) — see tuning.js
1159
+ ...this.vadOptions, // host tuning overrides the defaults above
1160
+ // NB: barge-in is NOT triggered here anymore — VAD energy alone (the AI's own echo, noise) must not
1161
+ // cut a reply; the first TRANSCRIBED word does, in onPartial. The uplink STT runs through the reply.
1162
+ onSpeechStart: () => this._onSpeechStart(),
1163
+ onSpeechEnd: () => this._onSpeechEnd(),
1164
+ });
1165
+ // A stale build (the pipeline was superseded/torn down while MicVAD.new() resolved) must not
1166
+ // stomp the current session's _vad — destroy its own instance and bail (the caller's .then()
1167
+ // gen-guard skips the swap too; this guard covers the assignment itself).
1168
+ if (gen !== undefined && (gen !== this._pipelineGen || this._closed)) { vad.destroy?.(); throw new Error('superseded'); }
1169
+ this._vad = vad;
1170
+ // Only actually start listening on Silero right away if the bootstrap VAD has already been
1171
+ // swapped out (or never engaged) — otherwise _swapToSileroWhenReady (called after this resolves)
1172
+ // arms it at the next safe utterance boundary instead of running both detectors at once.
1173
+ if (!this._useBootstrap) this._vad.start();
1174
+ }
1175
+
1176
+ // Silero finished loading (this._vad is ready) while the bootstrap energy-VAD was standing in.
1177
+ // Swap over ONLY at a safe boundary — never mid-utterance, so we don't yank the detector while the
1178
+ // user is mid-sentence (which could double-fire or miss the true end-of-speech). If speech is in
1179
+ // progress right now, defer: _onSpeechEnd() (bootstrap path) checks _sileroPending and swaps once
1180
+ // the current utterance closes.
1181
+ _swapToSileroWhenReady() {
1182
+ this._sileroPending = false;
1183
+ if (this._speaking) return; // mid-utterance — _bootstrapVadTick's speech-end path swaps after
1184
+ this._useBootstrap = false; this._bsSpeechFrames = 0; this._bsSilenceFrames = 0;
1185
+ this._vad?.start();
1186
+ }
1187
+
1188
+ // Zero-load RMS-energy stand-in for Silero, active only during `this._useBootstrap` (cold-start
1189
+ // window before Silero has loaded). Deliberately crude — it only needs to survive the first
1190
+ // second(s) of a session without losing/chopping the user's opening words; see tuning.js
1191
+ // BOOTSTRAP_VAD for the thresholds. Ticks once per ~256ms worklet capture chunk (_feed below).
1192
+ _bootstrapVadTick(f32) {
1193
+ const cfg = TUNING.BOOTSTRAP_VAD;
1194
+ let sum = 0; for (let i = 0; i < f32.length; i++) sum += f32[i] * f32[i];
1195
+ const rms = Math.sqrt(sum / f32.length);
1196
+ if (rms >= cfg.energyThreshold) {
1197
+ this._bsSilenceFrames = 0;
1198
+ if (!this._speaking && ++this._bsSpeechFrames >= cfg.minSpeechFrames) this._onSpeechStart();
1199
+ } else {
1200
+ this._bsSpeechFrames = 0;
1201
+ if (this._speaking && ++this._bsSilenceFrames >= cfg.redemptionFrames) {
1202
+ this._bsSilenceFrames = 0;
1203
+ this._onSpeechEnd();
1204
+ // The utterance that was masking the swap (see _swapToSileroWhenReady) just closed — hand
1205
+ // off to Silero now, before the NEXT one starts, instead of waiting for another tick.
1206
+ if (!this._sileroPending && this._vad) { this._useBootstrap = false; this._vad.start(); }
1207
+ }
1208
+ }
1209
+ }
1210
+
1211
+ // ── STT (VAD-gated) — capture mic, gate on VAD, forward to the active provider ──────────
1212
+ // The provider (this.stt, chosen by `sttProvider`) owns its socket + wire protocol and reports
1213
+ // transcripts via the callbacks wired in the constructor. Here we only convert float→int16,
1214
+ // keep a small preroll so word-starts aren't clipped, and feed/commit on VAD boundaries.
1215
+ _feed(f32) {
1216
+ if (this._useBootstrap) this._bootstrapVadTick(f32);
1217
+ const i16 = new Int16Array(f32.length);
1218
+ for (let i = 0; i < f32.length; i++) i16[i] = Math.max(-1, Math.min(1, f32[i])) * 32767;
1219
+ this._tapPush(i16); // diagnostic ring buffer of the exact frames handed to STT (see dumpAudio)
1220
+ // Continuous provider (e.g. Deepgram Flux): its end-of-turn model needs to hear the real
1221
+ // silences, so feed the FULL stream — no VAD gating, no preroll replay. VAD still runs above
1222
+ // (state events + speech-start reset + the nativeEOT failsafe), it just doesn't gate the audio.
1223
+ if (this.stt.continuous) { this.stt.feed(i16); return; }
1224
+ if (!this._speaking) { this._preroll.push(i16); if (this._preroll.length > this.prerollMax) this._preroll.shift(); this._wasSpeaking = false; return; }
1225
+ if (!this._wasSpeaking) {
1226
+ for (const c of this._preroll) this.stt.feed(c); this._preroll = []; this._wasSpeaking = true;
1227
+ }
1228
+ this.stt.feed(i16);
1229
+ }
1230
+
1231
+ // ── diagnostic tap ─────────────────────────────────────────────────────
1232
+ // Ring-buffer the frame + detect delivery stalls: a wall-clock gap between _feed() calls beyond
1233
+ // ~1.5 chunks. With the worklet capturing off-thread that's usually DELAYED delivery rather than
1234
+ // lost samples, but it timestamps exactly where a corrupted run got starved. atSample is an
1235
+ // ABSOLUTE stream offset (samples since pipeline build), so gaps stay addressable after trimming.
1236
+ _tapPush(i16) {
1237
+ const now = (globalThis.performance ?? Date).now();
1238
+ const chunkMs = (i16.length / 16000) * 1000;
1239
+ if (this._tapLastMs && now - this._tapLastMs > chunkMs * 1.5) {
1240
+ this._tapGaps.push({ atSample: this._tapTotal, gapMs: Math.round(now - this._tapLastMs) });
1241
+ }
1242
+ this._tapLastMs = now;
1243
+ this._tap.push(i16); this._tapSamples += i16.length; this._tapTotal += i16.length;
1244
+ const max = 16000 * 30;
1245
+ while (this._tapSamples - this._tap[0].length >= max) this._tapSamples -= this._tap.shift().length;
1246
+ // Keep only gaps still inside (or near) the retained window so the list can't grow unbounded.
1247
+ const windowStart = this._tapTotal - this._tapSamples;
1248
+ if (this._tapGaps.length && this._tapGaps[0].atSample < windowStart) this._tapGaps = this._tapGaps.filter(g => g.atSample >= windowStart);
1249
+ }
1250
+
1251
+ // Dump the tap as a 16kHz mono WAV Blob + detected stall gaps: { wav, gaps, seconds }. Play it back
1252
+ // after a failed run — gapped/corrupt audio → capture layer; clean audio → provider-side.
1253
+ dumpAudio() {
1254
+ const total = this._tapSamples;
1255
+ const windowStart = this._tapTotal - total; // absolute offset of the WAV's first sample
1256
+ const pcm = new Int16Array(total);
1257
+ let off = 0; for (const c of this._tap) { pcm.set(c, off); off += c.length; }
1258
+ const bytes = pcm.length * 2, buf = new ArrayBuffer(44 + bytes), dv = new DataView(buf);
1259
+ const str = (o, s) => { for (let i = 0; i < s.length; i++) dv.setUint8(o + i, s.charCodeAt(i)); };
1260
+ str(0, 'RIFF'); dv.setUint32(4, 36 + bytes, true); str(8, 'WAVE'); str(12, 'fmt ');
1261
+ dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, 1, true);
1262
+ dv.setUint32(24, 16000, true); dv.setUint32(28, 16000 * 2, true); dv.setUint16(32, 2, true); dv.setUint16(34, 16, true);
1263
+ str(36, 'data'); dv.setUint32(40, bytes, true);
1264
+ for (let i = 0; i < pcm.length; i++) dv.setInt16(44 + i * 2, pcm[i], true);
1265
+ // Gaps re-based to WAV-relative sample offsets (atSec for quick seeking during playback).
1266
+ const gaps = this._tapGaps.filter(g => g.atSample >= windowStart)
1267
+ .map(g => ({ atSample: g.atSample - windowStart, atSec: (g.atSample - windowStart) / 16000, gapMs: g.gapMs }));
1268
+ return { wav: new Blob([buf], { type: 'audio/wav' }), gaps, seconds: total / 16000 };
1269
+ }
1270
+ }
1271
+
1272
+
1273
+ // Strip markdown to plain prose for the EAR: the persona prompt already forbids markdown, so this is
1274
+ // a safety net — if formatting ever slips into a reply, the synthesizer must not voice the syntax
1275
+ // ("star star", "hash", "backtick"). We only feed this to _synth; the tape/cursor keep the RAW text
1276
+ // so tap-to-seek alignment is untouched. Drops fences/inline-code, image/link syntax (keeping link
1277
+ // text), heading/quote/list markers, and emphasis/strikethrough runs.
1278
+ const stripMd = s => s
1279
+ .replace(/```[^\n]*\n?([\s\S]*?)```/g, '$1') // fenced code -> its contents
1280
+ .replace(/`([^`]+)`/g, '$1') // inline code
1281
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1') // images -> alt text
1282
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') // links -> link text
1283
+ .replace(/^\s{0,3}#{1,6}\s+/gm, '') // ATX headings
1284
+ .replace(/^\s{0,3}>\s?/gm, '') // blockquotes
1285
+ .replace(/^\s*([-*+]|\d+[.)])\s+/gm, '') // list markers
1286
+ .replace(/(\*\*\*|___)(.+?)\1/g, '$2') // bold+italic
1287
+ .replace(/(\*\*|__)(.+?)\1/g, '$2') // bold
1288
+ .replace(/(\*|_)(.+?)\1/g, '$2') // italic
1289
+ .replace(/~~(.+?)~~/g, '$1') // strikethrough
1290
+ // Streaming splits a reply into word/clause chunks BEFORE stripMd runs, so an emphasis span can
1291
+ // straddle two chunks, leaving one chunk with an UNPAIRED marker the paired rules above can't match.
1292
+ // Sweep any stray emphasis/code markers so the synth never voices "star"/"backtick".
1293
+ .replace(/[*_~`]/g, '')
1294
+ .trim();
1295
+
1296
+ // ── Self-echo detection ─────────────────────────────────────────────────────────────────────────
1297
+ // Is `heard` (transcribed while the agent speaks) just its own voice leaking speakers → mic? We know
1298
+ // the audible reply prefix, so word-match `heard`'s RECENT tail (last `window` words — old echo
1299
+ // followed by a fresh real interruption must still cut in) against the reply's word set: mostly
1300
+ // reply words → echo. Word-set based (unordered, unicode-aware for accented text): tolerant of
1301
+ // STT dropping/reordering echoed words, NOT of substituting them — a badly misheard echo can slip
1302
+ // through (acceptable; it just means the old behavior for that turn). `minHits` = minimum matched
1303
+ // words required (the barge-in suppression check is lenient at 1; the final-drop check demands 3,
1304
+ // so a user briefly QUOTING the reply — "pricing?" — is never swallowed as echo).
1305
+ // Threshold TUNING.ECHO_WORD_MATCH; > 1 disables the filter.
1306
+ // ECHO_GRACE_MS: STT lags the audio, so the tail of the echo can be transcribed just AFTER playback
1307
+ // ended — keep classifying against the finished reply for this long past _replyDoneAt.
1308
+ const ECHO_GRACE_MS = 2000;
1309
+ const wordsOf = (s) => (s.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? []);
1310
+ // Bounded Levenshtein — true if editDistance(a,b) ≤ max. Early-exits a row whose minimum already
1311
+ // exceeds max, so the cost stays tiny on the short words we compare.
1312
+ const editLe = (a, b, max) => {
1313
+ if (a === b) return true;
1314
+ if (Math.abs(a.length - b.length) > max) return false;
1315
+ let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
1316
+ for (let i = 1; i <= a.length; i++) {
1317
+ const cur = [i]; let best = i;
1318
+ for (let j = 1; j <= b.length; j++) {
1319
+ cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
1320
+ if (cur[j] < best) best = cur[j];
1321
+ }
1322
+ if (best > max) return false;
1323
+ prev = cur;
1324
+ }
1325
+ return prev[b.length] <= max;
1326
+ };
1327
+ // FUZZY word hit: STT frequently MIS-hears the synthetic TTS voice ("sure i will create" heard as
1328
+ // "shure i wil kreate"), and an exact word-set match then scores the echo as novel speech — the
1329
+ // filter's main failure mode. A heard word counts as a reply word if it matches exactly OR within an
1330
+ // edit distance scaled by the LONGER of the pair (≥4 chars → 1 edit, ≥8 → 2) — so "wil" still
1331
+ // matches "will", while tiny words ("no" vs "now": longer side 3) must match exactly, or real
1332
+ // interruptions would get swallowed.
1333
+ const fuzzyHas = (replyWords, w) => {
1334
+ for (const r of replyWords) {
1335
+ if (r === w) return true;
1336
+ const len = Math.max(w.length, r.length);
1337
+ const max = len >= 8 ? 2 : len >= 4 ? 1 : 0;
1338
+ if (max && editLe(w, r, max)) return true;
1339
+ }
1340
+ return false;
1341
+ };
1342
+ export function isSelfEcho(heard, reply, { threshold = TUNING.ECHO_WORD_MATCH, window = 8, minHits = 1 } = {}) {
1343
+ if (!(threshold <= 1) || !reply) return false;
1344
+ const hw = window === Infinity ? wordsOf(heard) : wordsOf(heard).slice(-window);
1345
+ if (!hw.length) return false;
1346
+ const rw = wordsOf(reply);
1347
+ let hit = 0; for (const w of hw) if (fuzzyHas(rw, w)) hit++;
1348
+ return hit >= minHits && hit / hw.length >= threshold;
1349
+ }
1350
+ // Chars of `heard` NOT attributable (fuzzily) to `reply` — the barge-in evidence. While the agent
1351
+ // speaks, only these count toward the barge-in threshold: echo words (even misheard ones, via the
1352
+ // fuzzy match) contribute ZERO, so a leaked echo that dodged the ratio-based latch still can't
1353
+ // accumulate enough "user speech" to cut the reply. Real interruptions are made of words the reply
1354
+ // doesn't contain, so they hit the threshold as fast as before. No reply text → everything is novel.
1355
+ export function novelChars(heard, reply) {
1356
+ const rw = wordsOf(reply ?? '');
1357
+ let n = 0;
1358
+ for (const w of wordsOf(heard)) if (!rw.length || !fuzzyHas(rw, w)) n += w.length;
1359
+ return n;
1360
+ }
1361
+
1362
+ // Count non-whitespace chars — the whitespace-invariant position key shared by the cursor and seek().
1363
+ const nw = s => { let n = 0; for (const c of s) if (!/\s/.test(c)) n++; return n; };
1364
+ // Prefix of `s` holding exactly `n` non-whitespace chars (keeps the original whitespace) — the inverse
1365
+ // of nw(), used to turn a non-whitespace cursor position back into a real text slice.
1366
+ const sliceNw = (s, n) => { if (n <= 0) return ''; let seen = 0, i = 0; for (; i < s.length; i++) { if (!/\s/.test(s[i])) { if (++seen === n) { i++; break; } } } return s.slice(0, i); };
1367
+
1368
+ // Earliest break to end the FIRST spoken chunk — minimizing first-audio latency (we don't wait for a
1369
+ // whole sentence to start talking; voice replies are often ONE sentence, so a sentence-only break would
1370
+ // stall TTS until generation finishes). Priority:
1371
+ // 1. a real sentence ender (. ! ? …) or a dash/colon clause break → break immediately;
1372
+ // 2. else, once we have >= firstChunkMinChars, the first comma/semicolon clause pause → break there;
1373
+ // 3. else, if the opener runs past firstChunkMaxChars with no punctuation, break at the last word
1374
+ // boundary anyway so a long unpunctuated start still begins playing.
1375
+ // The VERY first audible unit: break on the first complete word so playback starts the instant the
1376
+ // LLM has emitted one word (Piper synthesizes a whole chunk at once, so a one-word chunk is the
1377
+ // smallest possible first-audio latency). Requires a trailing space — a word still being typed
1378
+ // ("Hel") doesn't break; "Hello " does, returning the index after "Hello". -1 until a word completes.
1379
+ // On free local Piper a slightly choppy opener is worth the instant start; the next chunks below get
1380
+ // whole-clause/sentence prosody, and one-ahead synthesis hides their cost.
1381
+ function firstWordEnd(s) {
1382
+ // Extend across words until the chunk holds >= firstWordMinChars non-space chars, so a tiny opener
1383
+ // ("I", "Hi", "A ...") glues to the next word instead of becoming a choppy 1-2 char micro-clip.
1384
+ const re = /\S\s/g;
1385
+ let m, end = -1, chars = 0;
1386
+ while ((m = re.exec(s))) {
1387
+ end = m.index + 1;
1388
+ chars = nw(s.slice(0, end));
1389
+ if (chars >= TUNING.TTS.firstWordMinChars) return end; // long enough → break here
1390
+ }
1391
+ return -1; // no completed word yet, or all completed words still under the floor (wait for more)
1392
+ }
1393
+
1394
+ // All require a trailing space/end so we don't split inside "3.14", "pl.", or "1,000".
1395
+ function firstSentenceEnd(s) {
1396
+ const hard = /[.!?…]+(?=\s|$)|\s[–—-](?=\s)|:(?=\s)/.exec(s);
1397
+ if (hard) return hard.index + hard[0].trimEnd().length;
1398
+ if (s.length >= TUNING.TTS.firstChunkMinChars) {
1399
+ const soft = /[,;](?=\s)/.exec(s);
1400
+ if (soft && soft.index + 1 >= TUNING.TTS.firstChunkMinChars) return soft.index + 1;
1401
+ }
1402
+ if (s.length >= TUNING.TTS.firstChunkMaxChars) {
1403
+ const sp = s.lastIndexOf(' ', TUNING.TTS.firstChunkMaxChars);
1404
+ if (sp > 0) return sp;
1405
+ }
1406
+ return -1;
1407
+ }
1408
+
1409
+ // First sentence ender (. ! ? …, or a trailing-space colon) in `s`, or -1 — used for every
1410
+ // sentence after the first, where we prefer whole sentences (fewer cuts, better prosody) over
1411
+ // the earliest break. A colon ends a clause often enough to be a good, natural TTS break.
1412
+ // Capped: a long run with no ender (lists, dashed prose) breaks at the last word boundary by
1413
+ // sentenceMaxChars so audio keeps flowing instead of stalling on one monster chunk.
1414
+ function sentenceEnd(s) {
1415
+ const m = /[.!?…]+(?=\s|$)|:(?=\s)/.exec(s);
1416
+ if (m) return m.index + m[0].trimEnd().length;
1417
+ if (s.length >= TUNING.TTS.sentenceMaxChars) {
1418
+ const sp = s.lastIndexOf(' ', TUNING.TTS.sentenceMaxChars);
1419
+ if (sp > 0) return sp;
1420
+ }
1421
+ return -1;
1422
+ }
1423
+
1424
+ // ── streaming TTS base: sentence pipeline + Web Audio playback. Providers (Piper, …)
1425
+ // extend it and implement only `_synth(text, signal) → audio Blob` (and
1426
+ // optionally `voices()`); `speak` below does the rest:
1427
+ // Takes an async iterator of text deltas. Streams sentence-by-sentence for low latency
1428
+ // AND small inter-sentence gaps on long answers:
1429
+ // • re-chunk the deltas into sentences (sentence 1 on the earliest natural break for
1430
+ // fast first audio; later ones on whole-sentence enders for better prosody).
1431
+ // • synth ONE sentence ahead while the current one plays, so the next clip is ready (or
1432
+ // nearly) the moment the current finishes — no growing gap as the answer gets longer.
1433
+ // • play STRICTLY in order, one clip at a time (never two voices at once).
1434
+ // Whole clips (no char timings) → on barge-in the heard prefix is estimated proportionally
1435
+ // within the playing sentence.
1436
+ // ── Web Audio playback ───────────────────────────────────────────────────────
1437
+ // We play TTS via the Web Audio API (AudioContext + AudioBufferSourceNode), NOT an <audio> element:
1438
+ // a buffer source starts the instant its data is decoded — no blob-URL assignment, no
1439
+ // loadedmetadata round-trip — shaving per-clip start latency off EVERY sentence. The tradeoff is no
1440
+ // pitch-preserving rate change, so playback runs at native 1.0x (see StreamingTTS default speed).
1441
+ //
1442
+ // iOS Safari starts the AudioContext "suspended" and only lets it resume() inside a user gesture.
1443
+ // TTS audio arrives async (after STT→LLM), far outside any gesture, so we create + resume ONE shared
1444
+ // context from the host's mic-button click (unlockAudio) and reuse it for all later playback.
1445
+ let sharedCtx = null;
1446
+ const audioCtx = () => (sharedCtx ??= new (window.AudioContext || window.webkitAudioContext)());
1447
+
1448
+ // ── AEC loopback sink ────────────────────────────────────────────────────────
1449
+ // THE self-interruption root cause: the browser's echo canceller (getUserMedia
1450
+ // echoCancellation:true) does NOT use raw Web-Audio output (node.connect(ctx.destination)) as its
1451
+ // cancellation reference — Chrome only cancels audio that plays out via a WebRTC remote track (or a
1452
+ // media element fed by one). So on speakers the mic captures our own TTS verbatim, STT transcribes
1453
+ // it (often MIS-hears the synthetic voice, which defeats the word-match echo filter), and the agent
1454
+ // barge-ins on itself. Fix: route ALL TTS playback through a MediaStreamAudioDestinationNode, pipe
1455
+ // that stream through a loopback RTCPeerConnection pair (pc1→pc2, same page), and play the REMOTE
1456
+ // track in a hidden <audio> element. To the browser that playout is now "WebRTC remote audio", so
1457
+ // the mic-side AEC subtracts it — the echo never reaches VAD/STT at all. Costs a few tens of ms of
1458
+ // playout latency (jitter buffer), which the spoken-word cursor tolerates.
1459
+ // Setup is kicked from unlockAudio() (the mic-button gesture: <audio>.play() needs the same
1460
+ // transient activation the AudioContext resume does). On ANY failure (no RTCPeerConnection —
1461
+ // WebKitGTK builds, negotiation error, autoplay-blocked play()) we fall back to direct
1462
+ // ctx.destination playback — old behavior, still guarded by the isSelfEcho text filter — and allow
1463
+ // a retry on the next unlockAudio() call.
1464
+ let ttsSink = null; // MediaStreamAudioDestinationNode all clips connect to (null → ctx.destination)
1465
+ let sinkAudioEl = null; // keeps the playout element referenced (GC would silence the loopback)
1466
+ let sinkPcs = null; // keeps the loopback peer connections referenced (a GC'd pc closes → silence)
1467
+ let sinkStarting = false;
1468
+ let sinkBuild = null; // resolves when the in-flight loopback build settles — _playBuf awaits it (bounded) so the FIRST clip doesn't race the build and play direct/un-cancelled
1469
+ async function initAecLoopback(ctx) {
1470
+ if (ttsSink || sinkStarting || typeof RTCPeerConnection === 'undefined') return;
1471
+ sinkStarting = true;
1472
+ let settle; sinkBuild = new Promise((r) => (settle = r));
1473
+ const pc1 = new RTCPeerConnection(), pc2 = new RTCPeerConnection();
1474
+ try {
1475
+ const dest = ctx.createMediaStreamDestination();
1476
+ pc1.onicecandidate = (e) => { if (e.candidate) pc2.addIceCandidate(e.candidate).catch(() => {}); };
1477
+ pc2.onicecandidate = (e) => { if (e.candidate) pc1.addIceCandidate(e.candidate).catch(() => {}); };
1478
+ const played = new Promise((res, rej) => {
1479
+ const t = setTimeout(() => rej(new Error('loopback timeout')), 3000);
1480
+ pc2.ontrack = (e) => {
1481
+ const el = new Audio();
1482
+ el.srcObject = e.streams[0] ?? new MediaStream([e.track]);
1483
+ el.play().then(() => { clearTimeout(t); res(el); }, (err) => { clearTimeout(t); rej(err); });
1484
+ };
1485
+ });
1486
+ dest.stream.getTracks().forEach((t) => pc1.addTrack(t, dest.stream));
1487
+ const offer = await pc1.createOffer();
1488
+ await pc1.setLocalDescription(offer);
1489
+ await pc2.setRemoteDescription(offer);
1490
+ const answer = await pc2.createAnswer();
1491
+ await pc2.setLocalDescription(answer);
1492
+ await pc1.setRemoteDescription(answer);
1493
+ sinkAudioEl = await played;
1494
+ sinkPcs = [pc1, pc2];
1495
+ // If the loopback ever dies mid-session (pc failed/closed), clips routed into the dead sink
1496
+ // would play SILENCE — unpublish it so playback falls back to direct output (and a later
1497
+ // unlockAudio() gesture can rebuild it).
1498
+ const onDead = () => {
1499
+ if (pc1.connectionState === 'failed' || pc2.connectionState === 'failed' ||
1500
+ pc1.connectionState === 'closed' || pc2.connectionState === 'closed') {
1501
+ ttsSink = null; sinkAudioEl = null; sinkPcs = null;
1502
+ try { pc1.close(); pc2.close(); } catch {}
1503
+ }
1504
+ };
1505
+ pc1.onconnectionstatechange = pc2.onconnectionstatechange = onDead;
1506
+ ttsSink = dest; // publish only once playout is confirmed — clips route through it from now on
1507
+ } catch {
1508
+ try { pc1.close(); pc2.close(); } catch {}
1509
+ sinkAudioEl = null; // failed (no WebRTC / autoplay-blocked) → direct playback; retry on next gesture
1510
+ } finally {
1511
+ sinkStarting = false;
1512
+ settle(); sinkBuild = null;
1513
+ }
1514
+ }
1515
+ export function unlockAudio() {
1516
+ try { const ctx = audioCtx(); ctx.resume(); initAecLoopback(ctx); } catch {} // gesture context → blesses/resumes the shared AudioContext + builds the AEC sink
1517
+ }
1518
+
1519
+ // A pre-decoded clip playing through a single AudioBufferSourceNode, exposing the small slice of the
1520
+ // HTMLAudioElement surface that _playBuf / stop() / seek() rely on: paused, currentTime, duration,
1521
+ // pause(), and the onended/onpause callbacks. currentTime is computed from the context clock and the
1522
+ // offset the clip started at, so the live cursor and tap-to-seek math are unchanged. A buffer source
1523
+ // can't be restarted, so a seek/replay just creates a new one (seek() already re-plays clips anyway).
1524
+ class WebAudioClip {
1525
+ constructor(ctx, buffer, offsetSec = 0) {
1526
+ this.ctx = ctx; this.buffer = buffer; this.duration = buffer.duration;
1527
+ this.paused = false; this.ended = false; this.onended = null; this.onpause = null;
1528
+ this._fired = false; this._stopped = false;
1529
+ this._startCtxTime = ctx.currentTime; this._offset = offsetSec;
1530
+ this._node = ctx.createBufferSource();
1531
+ this._node.buffer = buffer;
1532
+ this._node.connect(ttsSink ?? ctx.destination); // via the AEC loopback sink when available (see initAecLoopback)
1533
+ // The native onended fires once — on natural end AND on our stop(). Classify with _stopped, but
1534
+ // treat a stop() at/near the buffer's end as a natural end (a pause() racing the final samples
1535
+ // shouldn't be reported as a barge-in). _fired guards the single-shot callback (not `ended`, which
1536
+ // stays false for a stop()).
1537
+ this._node.onended = () => {
1538
+ if (this._fired) return; this._fired = true;
1539
+ const natural = !this._stopped || this.currentTime >= this.duration - 0.02;
1540
+ this.ended = natural; this.paused = true;
1541
+ try { this._node.disconnect(); } catch {}
1542
+ (natural ? this.onended : this.onpause)?.();
1543
+ };
1544
+ this._node.start(0, offsetSec);
1545
+ }
1546
+ get currentTime() { return Math.min(this._offset + (this.ctx.currentTime - this._startCtxTime), this.duration); }
1547
+ pause() { if (this.paused || this._fired) return; this._stopped = true; this.paused = true; try { this._node.stop(); } catch {} }
1548
+ }
1549
+
1550
+ // rAF with a setTimeout fallback (non-browser/SSR/test runtimes lack requestAnimationFrame).
1551
+ const raf = globalThis.requestAnimationFrame?.bind(globalThis) ?? (fn => setTimeout(() => fn(Date.now()), 16));
1552
+ const caf = globalThis.cancelAnimationFrame?.bind(globalThis) ?? clearTimeout;
1553
+
1554
+ export class StreamingTTS {
1555
+ // `speed` is retained for API compatibility but NO LONGER applied: Web Audio playback runs at the
1556
+ // synthesized native rate (1.0) — a buffer source can't change rate without altering pitch. Tune
1557
+ // delivery speed in the synth engine instead if needed.
1558
+ // `onProgress(spokenText)` (set via setOnProgress) fires while a clip plays with the chars heard
1559
+ // SO FAR across the whole reply — drives the host's live "spoken cursor".
1560
+ constructor(voiceId, speed = 1.0) { this.voiceId = voiceId; this.speed = speed; this._audio = null; this._onProgress = null;
1561
+ this._tape = null; this._speaking = false; this._seekTarget = null; this._seekPending = false; this._curIdx = -1;
1562
+ this._synthQ = Promise.resolve(); this._preClip = null; }
1563
+
1564
+ // Serialize EVERY synth through one instance-level queue: a single ONNX session cannot run two
1565
+ // predict()s, and presynth() below may fire while a previous reply's chain is still draining.
1566
+ // Returns a promise that RESOLVES (wav or null-on-error) — callers surface errors per-entry.
1567
+ _enqueueSynth(text, signal) {
1568
+ const run = this._synthQ.then(() => this._synth(stripMd(text) || text, signal), () => null); // || raw: a pure-syntax chunk strips to empty — voice the raw rather than abort
1569
+ this._synthQ = run.catch(() => null);
1570
+ return run;
1571
+ }
1572
+
1573
+ // SPECULATIVE first-clip synth (see VoiceAgent._presynthFirstClip): pre-render the expected first
1574
+ // chunk of the NEXT reply during the end-of-turn debounce, so speak() finds the clip already made.
1575
+ // Single-slot cache keyed by exact chunk text; a mismatched or unused clip is simply dropped.
1576
+ presynth(text) {
1577
+ if (!text || this._preClip?.text === text) return;
1578
+ this.onEvent?.({ type: 'diag', diag: 'presynth', text });
1579
+ this._preClip = { text, blobP: this._enqueueSynth(text).catch(() => null) };
1580
+ }
1581
+ setSpeed(speed) { this.speed = speed; }
1582
+ setVoice(voiceId) { this.voiceId = voiceId; }
1583
+ setOnProgress(fn) { this._onProgress = fn; }
1584
+ // Eagerly prepare the synth engine (load models, JIT the inference path) so the FIRST utterance
1585
+ // doesn't pay cold-start latency mid-reply. No-op by default; providers with a heavy warm-up
1586
+ // (e.g. Piper's ONNX model download + WASM compile) override it. Safe to call repeatedly.
1587
+ async warm() {}
1588
+
1589
+ // Play a pre-synthesized clip via Web Audio (decode → AudioBufferSourceNode); resolve with the chars
1590
+ // of `text` actually heard. A buffer source starts the instant the blob is decoded (no <audio> src
1591
+ // assignment / loadedmetadata wait), cutting per-clip start latency on every sentence — at the cost
1592
+ // of a pitch-preserving rate change, so playback is native 1.0x (StreamingTTS default speed).
1593
+ // `prefix` is everything already spoken in earlier sentences of this reply; we emit `prefix + heard`
1594
+ // on a rAF tick so the host gets the live spoken-so-far text for its cursor.
1595
+ // Resolves with { reason, heard }: reason is why the clip stopped — 'ended' (natural finish),
1596
+ // 'abort' (signal/barge-in), or 'seek' (a tap repositioned the playhead, see seek()); `heard` is the
1597
+ // spoken-so-far text (prefix + this clip's heard part). `startNw` starts the clip mid-sentence (a
1598
+ // forward/backward tap inside it), as a non-whitespace offset into `text`. abort always dominates seek.
1599
+ async _playBuf(wav, text, signal, prefix = '', startNw = 0) {
1600
+ if (signal?.aborted || !wav) return { reason: 'abort', heard: prefix };
1601
+ const ctx = audioCtx();
1602
+ // Decode first (works while suspended) so we don't block synthesis on resume.
1603
+ let buffer;
1604
+ try { buffer = await ctx.decodeAudioData(await wav.arrayBuffer()); }
1605
+ catch (e) { throw new Error(`Audio decode: ${e?.message || e}`); }
1606
+ // A tap or barge-in during decode has no clip to act on yet — honor it now, before we start audio.
1607
+ if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
1608
+ if (this._seekPending) return { reason: 'seek', heard: prefix + sliceNw(text, startNw) };
1609
+ // Now ensure the context can actually run; resume() must happen in/after a gesture (unlockAudio).
1610
+ // Bound it so a policy-blocked resume can't hang the turn forever — if it won't run, surface it.
1611
+ if (ctx.state !== 'running') {
1612
+ try { await Promise.race([ctx.resume(), new Promise((_, rej) => setTimeout(() => rej(new Error('resume timeout')), 1500))]); } catch {}
1613
+ if (ctx.state !== 'running') throw new Error('AudioContext is not running (needs a user gesture)');
1614
+ }
1615
+ // The AEC loopback build (kicked from unlockAudio) races the first reply: if the sink isn't
1616
+ // published yet, this clip would wire to ctx.destination and play UN-cancelled — the mic hears
1617
+ // it and the agent can self-interrupt on its opening sentence. Wait (bounded) for the in-flight
1618
+ // build; on timeout/failure fall through to direct playback exactly as before.
1619
+ if (!ttsSink && sinkBuild) { await Promise.race([sinkBuild, new Promise((r) => setTimeout(r, 1500))]); }
1620
+ // NO silent fallback: if this clip is about to play direct (un-cancelled — the mic will hear
1621
+ // it), tell the host once per session so the user sees WHY self-echo may happen.
1622
+ if (!ttsSink && !this._aecWarned) {
1623
+ this._aecWarned = true;
1624
+ this.onEvent?.({ type: 'diag', diag: 'aec-fallback', message: 'AEC loopback unavailable — TTS plays un-cancelled; echo is only text-filtered' });
1625
+ }
1626
+ if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
1627
+ const dur = buffer.duration;
1628
+ const offsetSec = startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0; // tap-into-clip → start offset
1629
+ const clip = new WebAudioClip(ctx, buffer, offsetSec);
1630
+ this._audio = clip;
1631
+ return new Promise((resolve) => {
1632
+ // Proportional position in NON-WHITESPACE space (the same domain seek() uses) → slice `text` to that
1633
+ // many non-whitespace chars, so a seek to nw-index N lands the cursor exactly on N.
1634
+ const heard = () => sliceNw(text, Math.round(nw(text) * clip.currentTime / (dur || 1)));
1635
+ // 2nd arg = the AUDIBLE-SCOPE bound: prefix + the WHOLE clip now playing. The proportional
1636
+ // cursor above LAGS the real audio (linear time→text estimate, rAF granularity, mid-word
1637
+ // slicing), so an STT echo partial can contain words just AHEAD of it — judged against the
1638
+ // cursor alone those words look novel and un-latch the echo filter (measured self-interruption
1639
+ // on the echo bench). Echo can never be ahead of the clip being played, so the clip's full text
1640
+ // is a safe upper bound for the echo reference; the UI cursor keeps the precise estimate.
1641
+ const scope = prefix + text;
1642
+ let settled = false, frame = 0;
1643
+ const tick = () => { if (settled) return; this._onProgress?.(prefix + heard(), scope); frame = raf(tick); };
1644
+ const cleanup = () => { if (frame) caf(frame); signal?.removeEventListener('abort', onAbort); clip.onpause = clip.onended = null; };
1645
+ const finish = (reason) => { if (settled) return; settled = true; cleanup(); resolve({ reason, heard: prefix + heard() }); };
1646
+ const onAbort = () => { clip.pause(); finish('abort'); }; // barge-in wins over a pending seek
1647
+ // Natural end → onended; stop()/seek() pause() → onpause. A pending seek flag distinguishes a
1648
+ // reposition from a real interruption; abort got there first if it fired.
1649
+ clip.onpause = () => finish((this._seekPending && !signal?.aborted) ? 'seek' : 'abort'); // a barge-in (signal) that races a pending seek must still win → 'abort'
1650
+ clip.onended = () => finish('ended');
1651
+ signal?.addEventListener('abort', onAbort);
1652
+ if (signal?.aborted) return onAbort(); // aborted between start() and listener
1653
+ if (this._onProgress) frame = raf(tick);
1654
+ // Emit the spoken-so-far prefix the moment this clip starts, so the host's cursor takes over at
1655
+ // playback start — no window where the whole answer shows solid.
1656
+ this._onProgress?.(prefix + sliceNw(text, startNw), scope);
1657
+ });
1658
+ }
1659
+
1660
+ // subclass contract — synthesize one sentence → audio Blob (null when aborted/empty)
1661
+ async _synth(_text, _signal) { throw new Error('not implemented'); }
1662
+
1663
+ // Re-chunk an async iterable of text deltas into sentences. The FIRST sentence breaks on
1664
+ // the earliest natural boundary (fast first audio); the rest on whole-sentence enders
1665
+ // (better prosody). Flushes whatever's left when the stream ends.
1666
+ async *_sentences(src, signal) {
1667
+ const it = src[Symbol.asyncIterator]();
1668
+ const next = async () => { try { return await it.next(); } catch (e) { if (e.name === 'AbortError') return { done: true }; throw e; } };
1669
+ // Three-stage break progression for minimum first-audio latency on local (whole-chunk) TTS:
1670
+ // stage 0 → first complete WORD (start talking the instant one word exists);
1671
+ // stage 1 → rest of the first clause/sentence (earliest natural break);
1672
+ // stage 2+ → whole sentences (best prosody; one-ahead synth hides their cost).
1673
+ const breakers = [firstWordEnd, firstSentenceEnd, sentenceEnd];
1674
+ let acc = '', stage = 0;
1675
+ for (let r = await next(); !r.done; r = await next()) {
1676
+ if (signal?.aborted) return;
1677
+ acc += r.value;
1678
+ let end;
1679
+ while ((end = breakers[Math.min(stage, breakers.length - 1)](acc)) >= 0) {
1680
+ const s = acc.slice(0, end).trim();
1681
+ acc = acc.slice(end).trimStart();
1682
+ if (s) { yield s; stage++; }
1683
+ }
1684
+ }
1685
+ const tail = acc.trim();
1686
+ if (tail) yield tail;
1687
+ }
1688
+
1689
+ // `input` is an async iterable of text deltas (or a plain string).
1690
+ // Builds a TAPE — one entry per sentence ({ text, blobP, nwStart }) appended at playback pace —
1691
+ // and walks it by a playhead index. Forward play is the same low-latency, one-sentence-ahead
1692
+ // pipeline as before (synth the next clip while the current plays, one voice at a time). The tape
1693
+ // adds tap-to-seek (see seek()): a tap repositions the playhead to any sentence ALREADY on the
1694
+ // tape and plays forward from there — backward replays the cached clip (no re-synth), forward
1695
+ // skips ahead. Resolves with the heard text (high-water in non-whitespace space → history): a full
1696
+ // play returns the whole reply, a barge-in returns the prefix heard before the user cut in.
1697
+ // `fromNw` (non-whitespace offset) begins playback partway through — used to REPLAY a finished reply
1698
+ // from a tapped word. The whole reply still tapes (we synth the skipped prefix too, lazily), so a
1699
+ // backward tap within the replay hits cached clips; replies are short so the extra synth is cheap.
1700
+ async speak(input, signal, fromNw = 0) {
1701
+ if (signal?.aborted) return '';
1702
+ const src = typeof input === 'string' ? (async function* () { yield input; })() : input;
1703
+ const it = this._sentences(src, signal)[Symbol.asyncIterator]();
1704
+
1705
+ const tape = this._tape = []; // { text, blobP, nwStart } — nwStart = non-whitespace chars before it
1706
+ // Generation token: an aborted speak() DETACHES its producer (see the finally block) rather than
1707
+ // waiting for a host stream that may never yield again. The detached producer stays parked in
1708
+ // it.next() and can wake up long after a NEWER speak() owns this instance — so every step that
1709
+ // touches shared state re-checks that it's still the current generation.
1710
+ const gen = this._speakGen = (this._speakGen || 0) + 1;
1711
+ const current = () => gen === this._speakGen && !signal?.aborted;
1712
+ let nwTotal = 0, streamErr = null, streamDone = false, wake = null;
1713
+ // Single waiter slot; woken when the tape grows, the stream ends, OR we abort — so a consumer
1714
+ // parked in awaitEntry() never hangs past an abort/barge-in even if the LLM stream is slow to react.
1715
+ const bump = () => { const w = wake; wake = null; w?.(); };
1716
+ this._ttsWake = bump; // let seek() wake a consumer parked in a stream gap (see seek())
1717
+ // Synth is serial: ONE ONNX session can't run concurrent predict()s. Every _synth goes through
1718
+ // the instance-level _synthQ (see _enqueueSynth) so audio is produced one-at-a-time, in order —
1719
+ // but the TEXT stream (it.next(), which drives the LLM) is drained as fast as tokens arrive,
1720
+ // decoupled from both synth pace AND playback pace: generation never stalls sentence-by-sentence.
1721
+ const pull = async () => {
1722
+ const r = await it.next();
1723
+ // Host code just returned. If we were abandoned while parked in it.next(), stop HERE — before
1724
+ // touching _preClip (which now belongs to a newer reply), the synth queue or the event tap.
1725
+ if (!current() || r.done) return null;
1726
+ const entry = { text: r.value, nwStart: nwTotal };
1727
+ // Speculative hit: the first chunk was already rendered during the end-of-turn debounce
1728
+ // (presynth) with EXACTLY this text — reuse the clip instead of predicting again. Single-shot:
1729
+ // hit or miss, the slot is cleared (a stale speculation must not leak into a later reply).
1730
+ const pre = tape.length === 0 ? this._preClip : null;
1731
+ if (pre) this.onEvent?.({ type: 'diag', diag: pre.text === r.value ? 'presynth-hit' : 'presynth-miss', text: r.value, pre: pre.text });
1732
+ this._preClip = null;
1733
+ // blobP always RESOLVES — to the wav, or null carrying the error on the entry — so an
1734
+ // ahead-of-playback synth failure never raises unhandledrejection; the queue keeps going for
1735
+ // the rest of the reply, and the consumer re-throws entry.err only if it actually reaches it.
1736
+ const p = pre && pre.text === r.value ? pre.blobP : this._enqueueSynth(r.value, signal);
1737
+ entry.blobP = p.catch((e) => { entry.err = e; return null; });
1738
+ tape.push(entry);
1739
+ nwTotal += nw(r.value);
1740
+ return entry;
1741
+ };
1742
+ // Background PRODUCER: keep draining the LLM stream independent of playback so token generation
1743
+ // never stalls sentence-by-sentence behind audio. It runs ahead, synth queues serially behind it.
1744
+ const produce = (async () => {
1745
+ try { while (!signal?.aborted) { if (!(await pull())) break; bump(); } }
1746
+ catch (e) { if (e?.name !== 'AbortError') streamErr = e; } // LLM/iterator failure → surface via awaitEntry, not a silent truncation
1747
+ finally { streamDone = true; bump(); }
1748
+ })();
1749
+ if (signal) signal.addEventListener('abort', bump, { once: true }); // wake a parked consumer the instant we abort
1750
+ // Wait until the tape has an entry at `i` — or the stream ends, we abort, or a seek arrives (a
1751
+ // backward tap while we're parked waiting for a FUTURE clip must be honored now, not after the
1752
+ // stream advances). Producer runs ahead, so this usually returns immediately; it only blocks when
1753
+ // playback has caught up to generation.
1754
+ const awaitEntry = async (i) => {
1755
+ while (i >= tape.length && !streamDone && !signal?.aborted && !this._seekPending) await new Promise((r) => { wake = r; });
1756
+ if (streamErr) throw streamErr;
1757
+ };
1758
+
1759
+ let idx = 0, startNw = 0, heardMax = 0;
1760
+ // A tap that lands while we're awaiting (synthesis or the LLM stream) has no clip to pause, so it
1761
+ // only sets _seekTarget — honor it at the next safe point. Returns true if it moved the playhead.
1762
+ const takePending = () => {
1763
+ if (!this._seekPending || !this._seekTarget || signal?.aborted) return false;
1764
+ idx = this._seekTarget.idx; startNw = this._seekTarget.startNw;
1765
+ this._seekPending = false; this._seekTarget = null;
1766
+ return true;
1767
+ };
1768
+ try {
1769
+ this._speaking = true;
1770
+ // Replay-from-word: tape forward until fromNw falls inside a sentence, then start playing there
1771
+ // (startNw = the offset within that sentence). heardMax seeds to fromNw so the replayed prefix
1772
+ // already counts as heard (a forward replay never un-hears earlier text).
1773
+ if (fromNw > 0) {
1774
+ while (!signal?.aborted) {
1775
+ const last = tape[tape.length - 1];
1776
+ if (last && fromNw < last.nwStart + nw(last.text)) break;
1777
+ if (streamDone) break; // streamed everything and fromNw is past the end → play from the last
1778
+ await awaitEntry(tape.length); // let the producer append the next sentence
1779
+ }
1780
+ const i = tape.findIndex(e => fromNw < e.nwStart + nw(e.text));
1781
+ if (i >= 0) { idx = i; startNw = Math.max(0, fromNw - tape[i].nwStart); heardMax = fromNw; }
1782
+ }
1783
+ while (true) {
1784
+ if (idx >= tape.length) { await awaitEntry(idx); if (takePending()) continue; if (idx >= tape.length) break; } // wait for the producer to reach this sentence (stream end → done)
1785
+ const e = tape[idx];
1786
+ const buf = await e.blobP;
1787
+ if (e.err) throw e.err; // this clip's synth failed → surface it (only when we actually reach it)
1788
+ if (takePending()) continue; // a tap during synth/stream gap → reposition
1789
+ this._curIdx = idx;
1790
+ const prefix = tape.slice(0, idx).map(x => x.text).join(' ') + (idx ? ' ' : '');
1791
+ this._seekPending = false; this._seekTarget = null; // the clip about to play owns the seek channel
1792
+ const playP = this._playBuf(buf, e.text, signal, prefix, startNw); // start (don't await yet)
1793
+ const { reason, heard } = await playP; // producer keeps synthesizing ahead while this clip plays
1794
+ heardMax = Math.max(heardMax, nw(heard)); // monotonic: a backward replay never un-hears
1795
+ if (reason === 'abort' || signal?.aborted) break; // real barge-in → stop, record heard
1796
+ if (reason === 'seek') { idx = this._seekTarget.idx; startNw = this._seekTarget.startNw; this._seekPending = false; this._seekTarget = null; continue; }
1797
+ idx++; startNw = 0; // natural end → next sentence
1798
+ }
1799
+ } finally {
1800
+ signal?.removeEventListener('abort', bump);
1801
+ // A finished stream is joined (it's already done — this just collects a trailing synth). An
1802
+ // ABORTED one is DETACHED: the producer sits in it.next() on the HOST's llm generator, which
1803
+ // may ignore the abort signal and never yield again, so awaiting it here would never return
1804
+ // from speak() — wedging _speakTurn and every turn queued behind it.
1805
+ //
1806
+ // Detaching is safe on both counts the old join protected:
1807
+ // • concurrent predict() — every synth chains on the instance-level _synthQ (_enqueueSynth is
1808
+ // the ONLY path to _synth), so the next turn's synth queues behind an in-flight one instead
1809
+ // of racing it. It waits on its own entry.blobP, not on the whole turn loop.
1810
+ // • stale writes — pull() returns at the generation check above before touching any shared
1811
+ // state, so a producer that wakes up late can't clear a newer reply's _preClip or enqueue.
1812
+ // return() asks a well-behaved generator to wind down (it can't interrupt a pending next(), and
1813
+ // a custom iterator may throw synchronously — hence the guard).
1814
+ if (signal?.aborted) { try { Promise.resolve(it.return?.()).catch(() => {}); } catch {} }
1815
+ else { await produce.catch(() => {}); await this._synthQ.catch(() => {}); }
1816
+ this._ttsWake = null;
1817
+ this._speaking = false; this._tape = null; this._seekTarget = null; this._seekPending = false; this._curIdx = -1;
1818
+ }
1819
+ return sliceNw(tape.map(x => x.text).join(' '), heardMax); // heard prefix in non-whitespace space
1820
+ }
1821
+ stop() { this._seekPending = false; this._seekTarget = null; try { this._audio?.pause(); } catch {} }
1822
+
1823
+ // Move the playhead to a position in the whole reply (a non-whitespace char count — the same
1824
+ // whitespace-invariant key the host's cursor uses, so the rendered answer and the clips agree
1825
+ // regardless of the model's spacing) and play forward from there. Any tap (even inside the clip now
1826
+ // playing) interrupts the current clip (reason 'seek') so speak() repositions and re-plays from the
1827
+ // tapped offset — a Web Audio buffer source can't seek in place. Backward hits the
1828
+ // cached blob, forward skips ahead. Returns false when the target isn't on the tape yet
1829
+ // (not-yet-streamed/partial text) — the host leaves playback as-is.
1830
+ seek(nwIndex) {
1831
+ const tape = this._tape;
1832
+ if (!this._speaking || !tape || !tape.length || nwIndex < 0) return false;
1833
+ const i = tape.findIndex(e => nwIndex < e.nwStart + nw(e.text));
1834
+ if (i < 0) return false; // beyond streamed text
1835
+ const e = tape[i], localNw = Math.max(0, nwIndex - e.nwStart);
1836
+ const a = this._audio;
1837
+ // Every seek (even within the clip now playing) interrupts + repositions: a Web Audio buffer
1838
+ // source can't be re-positioned in place, so we re-play the target clip from the tapped offset
1839
+ // (startNw). speak() catches the 'seek' reason, sets idx/startNw, and _playBuf starts at that offset.
1840
+ this._seekTarget = { idx: i, startNw: localNw };
1841
+ this._seekPending = true;
1842
+ try { a?.pause(); } catch {} // → onpause resolves current clip as 'seek'
1843
+ this._ttsWake?.(); // a tap during a stream gap (no clip playing) → wake the parked consumer to honor it
1844
+ return true;
1845
+ }
1846
+ }
1847
+
1848
+ // ── Piper TTS (local WASM, free, has Hungarian) via @mintplex-labs/piper-tts-web.
1849
+ // Default voice: en_US-joe-medium.
1850
+ const CDN_PIPER = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-tts-web.js`;
1851
+ // The espeak phonemizer chunk (createPiperPhonemize) — a build artifact of the pinned 1.0.4, not a
1852
+ // public export; imported directly because the lib only exposes it via TtsSession.predict(), whose
1853
+ // per-call module setup is exactly what we bypass (see PiperTTS._session).
1854
+ const CDN_PIPER_PHONEMIZE = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-o91UDS6e.js`;
1855
+ const CDN_PIPER_WASM = `${CDN}@diffusionstudio/piper-wasm@1.0.0/build/piper_phonemize`;
1856
+
1857
+ // OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
1858
+ // models already downloaded through the lib are reused (and vice versa).
1859
+ async function opfsCachedFetch(url) {
1860
+ const name = url.split('/').at(-1);
1861
+ const dir = () => navigator.storage.getDirectory().then((r) => r.getDirectoryHandle('piper', { create: true }));
1862
+ try { return await (await (await dir()).getFileHandle(name)).getFile(); } catch {}
1863
+ const res = await fetch(url);
1864
+ if (!res.ok) throw new Error(`fetch ${url}: ${res.status}`);
1865
+ const blob = await res.blob();
1866
+ try { const w = await (await (await dir()).getFileHandle(name, { create: true })).createWritable(); await w.write(blob); await w.close(); } catch {}
1867
+ return blob;
1868
+ }
1869
+
1870
+ // float32 PCM → 16-bit mono WAV (the lib's pcm2wav isn't exported).
1871
+ function pcm2wav(f32, sampleRate) {
1872
+ const n = f32.length, v = new DataView(new ArrayBuffer(44 + n * 2));
1873
+ const str = (o, s) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)); };
1874
+ str(0, 'RIFF'); v.setUint32(4, 36 + n * 2, true); str(8, 'WAVEfmt ');
1875
+ v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
1876
+ v.setUint32(24, sampleRate, true); v.setUint32(28, sampleRate * 2, true);
1877
+ v.setUint16(32, 2, true); v.setUint16(34, 16, true);
1878
+ str(36, 'data'); v.setUint32(40, n * 2, true);
1879
+ for (let i = 0, p = 44; i < n; i++, p += 2) v.setInt16(p, Math.max(-1, Math.min(1, f32[i])) * 32767 | 0, true);
1880
+ return v.buffer;
1881
+ }
1882
+ export class PiperTTS extends StreamingTTS {
1883
+ constructor(voiceId = 'en_US-joe-medium', speed = 1.0) { super(voiceId, speed); }
1884
+ // webpackIgnore is honored by webpack AND Turbopack (vite needs its own marker) — without it
1885
+ // bundlers rewrite this into their runtime require, which can't load a URL
1886
+ // ("turbopack_context.x is not a function").
1887
+ async _tts() {
1888
+ return this._lib ??= (async () => {
1889
+ // Piper imports a bare "onnxruntime-web" specifier resolved via the importmap that
1890
+ // loadVoiceDeps() installs — ensure it exists before the module loads.
1891
+ if (typeof document !== 'undefined' && !document.getElementById('vl-importmap')) {
1892
+ const { loadVoiceDeps } = await import('./deps.js'); await loadVoiceDeps();
1893
+ }
1894
+ return import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER);
1895
+ })();
1896
+ }
1897
+
1898
+ // List installable Piper voices: [{ id, name, language }]. `id` is what you pass as
1899
+ // `voiceId` (lib calls it `key`). Lets the UI build a voice picker; switch with setVoice.
1900
+ async voices() {
1901
+ const list = await (await this._tts()).voices(); // [{ key, name, language: { code, … }, … }]
1902
+ return list.map(v => ({ id: v.key, name: v.name || v.key, language: v.language?.code || v.language || '' }));
1903
+ }
1904
+
1905
+ // Our OWN synth pipeline instead of lib.TtsSession.predict(): the lib instantiates a FRESH espeak
1906
+ // phonemizer WASM module (fetch + compile + espeak-data mount) on EVERY predict() — measured
1907
+ // 220–360ms per call, dwarfing the ~10ms phonemize itself. We build the phonemizer ONCE per
1908
+ // session and reuse it (callMain is re-entrant — verified over repeated calls), then run the ONNX
1909
+ // inference directly. Measured first-word synth: 500–575ms (lib) → ~185ms (this path).
1910
+ // Cached as a PROMISE keyed by voice so concurrent callers build one session, not two. Only
1911
+ // called from _synth (which every path — speak, presynth, warm — reaches through the serialized
1912
+ // _synthQ), so when a voice change builds a new session the old one is idle → release it.
1913
+ async _session() {
1914
+ const voiceId = this.voiceId; // snapshot: setVoice() mid-build must not mix voices
1915
+ if (this._sessP && this._sessVoice === voiceId) return this._sessP;
1916
+ this._sessVoice = voiceId;
1917
+ const prev = this._sessP;
1918
+ const p = this._sessP = (async () => {
1919
+ prev?.then((s) => s.onnx.release()).catch(() => {}); // free the old voice's model (worker-side memory isn't GC'd with the JS ref)
1920
+ const lib = await this._tts(); // also installs the importmap that resolves 'onnxruntime-web' below
1921
+ const path = lib.PATH_MAP[voiceId];
1922
+ if (!path) throw new Error(`unknown Piper voice: ${voiceId}`);
1923
+ const [ort, phonMod, cfgBlob, modelBlob] = await Promise.all([
1924
+ import(/* webpackIgnore: true */ /* @vite-ignore */ 'onnxruntime-web'),
1925
+ import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
1926
+ opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
1927
+ opfsCachedFetch(`${lib.HF_BASE}/${path}`),
1928
+ ]);
1929
+ const cfg = JSON.parse(await cfgBlob.text());
1930
+ // Match the importmap's onnxruntime-web version: the lib's baked-in default points at a
1931
+ // 1.18.0 CDN folder missing 1.22.0's threaded loader files (404 → "no available backend").
1932
+ ort.env.allowLocalModels = false;
1933
+ ort.env.wasm.numThreads = navigator.hardwareConcurrency || 4; // >1 needs crossOriginIsolated; ort falls back to 1 otherwise
1934
+ ort.env.wasm.proxy = true; // run inference in a worker: session.run computes ON the calling thread otherwise, so an ahead-of-playback synth would starve the clip that's trying to start (measured ~450ms added first-audio delay)
1935
+ ort.env.wasm.wasmPaths = `${CDN}onnxruntime-web@1.22.0/dist/`;
1936
+ const onnx = await ort.InferenceSession.create(await modelBlob.arrayBuffer());
1937
+ let ids = null;
1938
+ const phonemizer = await phonMod.createPiperPhonemize({
1939
+ print: (d) => { ids ??= JSON.parse(d).phoneme_ids; }, // first line only (parity with the lib)
1940
+ printErr: (e) => { throw new Error(e); },
1941
+ locateFile: (u) => u.endsWith('.wasm') ? `${CDN_PIPER_WASM}.wasm` : u.endsWith('.data') ? `${CDN_PIPER_WASM}.data` : u,
1942
+ });
1943
+ // Synchronous; safe to reuse because _synthQ already serializes every synth on this instance.
1944
+ const phonemize = (text) => {
1945
+ ids = null;
1946
+ phonemizer.callMain(['-l', cfg.espeak.voice, '--input', JSON.stringify([{ text }]), '--espeak_data', '/espeak-ng-data']);
1947
+ if (!ids) throw new Error('phonemize produced no output');
1948
+ return ids;
1949
+ };
1950
+ const sid = Object.keys(cfg.speaker_id_map ?? {}).length ? [0] : null;
1951
+ return { ort, onnx, cfg, phonemize, sid };
1952
+ })();
1953
+ p.catch(() => { if (this._sessP === p) this._sessP = null; }); // failed init must not poison the cache — next call retries (only clear if still current: a stale failure must not evict a newer voice's build)
1954
+ return p;
1955
+ }
1956
+
1957
+ // Eagerly download+compile the ONNX model and JIT the WASM inference path, so the first real reply
1958
+ // isn't stalled by Piper's cold start (model fetch + compile + first-inference warm-up) AFTER the LLM
1959
+ // has already produced text. Called when the agent starts listening — the model loads while the user
1960
+ // is still speaking. A throwaway predict primes the inference path; failures are non-fatal (the first
1961
+ // real _synth retries). Idempotent: _session() caches, so a second warm() is cheap.
1962
+ async warm() {
1963
+ try { await this._enqueueSynth('.'); } catch {} // through the queue: a late warm-up must not run concurrently with (or ahead of) a real first clip
1964
+ }
1965
+
1966
+ // Synthesize text → WAV Blob (the slow part; run it ahead of playback).
1967
+ async _synth(text, signal) {
1968
+ text = text?.trim();
1969
+ if (signal?.aborted || !text) return null;
1970
+ const s = await this._session();
1971
+ if (signal?.aborted) return null; // barge-in during cold model download/compile → skip the wasted inference
1972
+ const phonemeIds = s.phonemize(text);
1973
+ const { noise_scale, length_scale, noise_w } = s.cfg.inference;
1974
+ const feeds = {
1975
+ input: new s.ort.Tensor('int64', phonemeIds, [1, phonemeIds.length]),
1976
+ input_lengths: new s.ort.Tensor('int64', [phonemeIds.length]),
1977
+ scales: new s.ort.Tensor('float32', [noise_scale, length_scale, noise_w]),
1978
+ };
1979
+ if (s.sid) feeds.sid = new s.ort.Tensor('int64', s.sid);
1980
+ const { output } = await s.onnx.run(feeds);
1981
+ if (signal?.aborted) return null;
1982
+ return new Blob([pcm2wav(output.data, s.cfg.audio.sample_rate)], { type: 'audio/x-wav' });
1983
+ }
1984
+ }