@todoforai/voiceloop 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +239 -0
- package/examples/simple-browser.html +100 -0
- package/package.json +48 -0
- package/src/deps.js +49 -0
- package/src/index.js +15 -0
- package/src/llm-openai.js +122 -0
- package/src/stt.js +815 -0
- package/src/tts-elevenlabs.js +50 -0
- package/src/tuning.js +182 -0
- package/src/voice-agent.js +1984 -0
|
@@ -0,0 +1,1984 @@
|
|
|
1
|
+
// voice-agent.js — VAD → STT → LLM → TTS, interruptible. One small module.
|
|
2
|
+
//
|
|
3
|
+
// const agent = new VoiceAgent({ llm, tts });
|
|
4
|
+
// agent.start();
|
|
5
|
+
//
|
|
6
|
+
// The whole loop is `_onUserTurn` below:
|
|
7
|
+
// text = vad+stt → history.push(user:text) → llm STREAMS the answer
|
|
8
|
+
// → tts.speak(stream) (barge-in — on transcribed words, not VAD energy — stops tts + aborts llm,
|
|
9
|
+
// and records what was heard so the UI can show where TTS got)
|
|
10
|
+
//
|
|
11
|
+
// First-sentence latency: `tts.speak` takes an async iterator of text deltas. It synthesizes
|
|
12
|
+
// the FIRST sentence as soon as its boundary arrives and starts playing, then streams the rest
|
|
13
|
+
// sentence-by-sentence — synthesizing ONE sentence ahead while the current plays, so there's no
|
|
14
|
+
// growing gap on long answers, and never two clips at once (see StreamingTTS.speak).
|
|
15
|
+
//
|
|
16
|
+
// `llm`, `tts` and `stt` are pluggable; knobs (maxTokens, vadOptions, …) are constructor options.
|
|
17
|
+
// Defaults: stt → browser Web Speech (free, no key), tts → Piper (free local WASM),
|
|
18
|
+
// llm → makeOpenAILLM (any OpenAI-compatible /chat/completions endpoint).
|
|
19
|
+
|
|
20
|
+
import { STT_PROVIDERS, resolveSttProvider } from './stt.js';
|
|
21
|
+
import { TUNING } from './tuning.js';
|
|
22
|
+
import { makeOpenAILLM } from './llm-openai.js';
|
|
23
|
+
|
|
24
|
+
// Default persona: a minimal voice-first system prompt. Hosts replace it via `persona`
|
|
25
|
+
// (who the agent is) and append live context via `sysmsg` / setSysmsg().
|
|
26
|
+
export const VOICE_SYSMSG =
|
|
27
|
+
'You are a helpful voice assistant. Your words are read aloud by TTS and the user\'s words arrive ' +
|
|
28
|
+
'via STT — expect mis-hearings and interpret them charitably. Reply in one or two short sentences: ' +
|
|
29
|
+
'plain words for the ear, no lists, no markdown, no emojis. Lead with the answer; skip preamble. ' +
|
|
30
|
+
'When a tool fits the request, call it right away instead of describing what you would do.';
|
|
31
|
+
|
|
32
|
+
const CDN = 'https://cdn.jsdelivr.net/npm/';
|
|
33
|
+
|
|
34
|
+
// AudioWorklet processor: accumulate the audio-thread's 128-sample render quanta into ~256ms chunks
|
|
35
|
+
// (4096 samples @16kHz — the cadence the pipeline's bootstrap-VAD/preroll math assumes) and post each
|
|
36
|
+
// to the main thread. Kept as a source string → Blob URL so no separate asset/bundler config is needed.
|
|
37
|
+
const CAPTURE_WORKLET_SRC = `
|
|
38
|
+
registerProcessor('capture-processor', class extends AudioWorkletProcessor {
|
|
39
|
+
constructor() { super(); this._buf = new Float32Array(4096); this._n = 0; }
|
|
40
|
+
process(inputs) {
|
|
41
|
+
const ch = inputs[0]?.[0];
|
|
42
|
+
if (ch) for (let i = 0; i < ch.length; i++) {
|
|
43
|
+
this._buf[this._n++] = ch[i];
|
|
44
|
+
if (this._n === this._buf.length) { this.port.postMessage(this._buf.slice()); this._n = 0; }
|
|
45
|
+
}
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
});`;
|
|
49
|
+
const CAPTURE_WORKLET_URL = typeof URL !== 'undefined' && URL.createObjectURL
|
|
50
|
+
? URL.createObjectURL(new Blob([CAPTURE_WORKLET_SRC], { type: 'text/javascript' }))
|
|
51
|
+
: '';
|
|
52
|
+
|
|
53
|
+
// Compose the full system prompt: a persona (base voice persona by default, or a host-supplied one)
|
|
54
|
+
// + the host's conversation context (if any).
|
|
55
|
+
// The LLM otherwise guesses reply language from the (often short/garbled) transcript and drifts to
|
|
56
|
+
// English under the English persona — so state the selected STT language explicitly.
|
|
57
|
+
const LANG_NAMES = { en: 'English', hu: 'Hungarian', de: 'German', fr: 'French', es: 'Spanish', it: 'Italian', pt: 'Portuguese', nl: 'Dutch', pl: 'Polish', ja: 'Japanese', ko: 'Korean', zh: 'Chinese' };
|
|
58
|
+
const langRule = (lang) => {
|
|
59
|
+
const name = LANG_NAMES[String(lang || '').split(/[-_]/)[0].toLowerCase()];
|
|
60
|
+
return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
|
|
61
|
+
};
|
|
62
|
+
// A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
|
|
63
|
+
// what order a provider streamed the keys in (used both for dedup and for the [TOOL CALL] ledger).
|
|
64
|
+
// Identity: the provider call id when present (two calls with identical args are DISTINCT acts,
|
|
65
|
+
// e.g. a legit retry after an error result); name+args otherwise (an id-less identical repeat is a
|
|
66
|
+
// provider re-emitting its buffered call).
|
|
67
|
+
// Stop WAITING on a host promise after `ms` (resolve to `fallback` instead). Used only where a
|
|
68
|
+
// verdict is required to make progress — see _endOfSpeech. It cannot cancel the host's work, so
|
|
69
|
+
// it must never be used to declare an operation failed: elsewhere the fix for host code that
|
|
70
|
+
// hangs is to not await it at all (tools) or to detach it (StreamingTTS.speak).
|
|
71
|
+
const withTimeout = (p, ms, fallback) => {
|
|
72
|
+
let t;
|
|
73
|
+
return Promise.race([
|
|
74
|
+
Promise.resolve(p).finally(() => clearTimeout(t)),
|
|
75
|
+
new Promise((res) => { t = setTimeout(() => res(fallback), ms); }),
|
|
76
|
+
]);
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const callArgs = (c) => JSON.stringify(c.args ?? {}, Object.keys(c.args ?? {}).sort());
|
|
80
|
+
const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
|
|
81
|
+
// Render a host tool's return value as model-facing text (used in the history ledger, and by LLM
|
|
82
|
+
// adapters when feeding tool results back to the model). Never throws: the side effect already RAN,
|
|
83
|
+
// so an unserializable result must not retro-fail the call and invite a retry.
|
|
84
|
+
export const toolResultText = (result) => {
|
|
85
|
+
if (result === undefined) return 'ok';
|
|
86
|
+
if (typeof result === 'string') return result;
|
|
87
|
+
if (result && typeof result.text === 'string' && typeof result.ok === 'boolean')
|
|
88
|
+
return result.ok ? result.text : `[failed] ${result.text}`;
|
|
89
|
+
try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
|
|
90
|
+
};
|
|
91
|
+
// Ledger suffix for a call whose outcome is attached to the chunk: keeps the RESULT in the
|
|
92
|
+
// plain-text history so the NEXT turn still knows what the tool said. Bounded — history is
|
|
93
|
+
// spoken-conversation-sized, not a log. Calls without an outcome field contribute nothing.
|
|
94
|
+
const LEDGER_RESULT_MAX = 4000;
|
|
95
|
+
const ledgerOutcome = (c) => {
|
|
96
|
+
if (c.error) return ` → error: ${String(c.error).slice(0, LEDGER_RESULT_MAX)}`;
|
|
97
|
+
if (!('result' in c)) return '';
|
|
98
|
+
let text = toolResultText(c.result);
|
|
99
|
+
if (text.length > LEDGER_RESULT_MAX) text = `${text.slice(0, LEDGER_RESULT_MAX)}… (${text.length} chars)`;
|
|
100
|
+
return ` → ${text}`;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
// ── [TOOL CALL] mimicry filter ──────────────────────────────────────────────────────────────────
|
|
104
|
+
// The model sees `[TOOL CALL name] {...}` ledger lines in its own history turns, and sometimes
|
|
105
|
+
// IMITATES the format instead of calling the tool natively: it types the line as reply text.
|
|
106
|
+
// Untreated, the raw markup leaks into the spoken/visible reply and the tool never runs. This
|
|
107
|
+
// transform strips such lines from the text stream: a bare call (no → outcome) is what the model
|
|
108
|
+
// MEANT to do → re-emitted as a real { tool, args } chunk (the turn's dedup collapses it if the
|
|
109
|
+
// native call also arrived); a line WITH an → outcome is a hallucinated memory of a past call →
|
|
110
|
+
// dropped. Text before/after the line streams through untouched, held back only while a possible
|
|
111
|
+
// marker/line is still forming so TTS latency is unaffected on normal replies.
|
|
112
|
+
const MIMIC_MARK = '[TOOL CALL ';
|
|
113
|
+
const parseMimicLine = (line) => {
|
|
114
|
+
const m = /^\[TOOL CALL ([\w.-]+)\]\s*(.*)$/.exec(line.trim());
|
|
115
|
+
if (!m) return null; // not the format after all — caller emits it as text
|
|
116
|
+
const [, tool, rest] = m;
|
|
117
|
+
if (!rest) return { tool, args: {} };
|
|
118
|
+
if (!rest.startsWith('{')) return { drop: true }; // `→ outcome` (or junk) with no args — a remembered call
|
|
119
|
+
// Balanced-brace scan (string-aware): the args JSON may be followed by a ` → outcome` tail.
|
|
120
|
+
let depth = 0, inStr = false, escaped = false, end = -1;
|
|
121
|
+
for (let i = 0; i < rest.length; i++) {
|
|
122
|
+
const ch = rest[i];
|
|
123
|
+
if (escaped) { escaped = false; continue; }
|
|
124
|
+
if (inStr) { if (ch === '\\') escaped = true; else if (ch === '"') inStr = false; continue; }
|
|
125
|
+
if (ch === '"') inStr = true;
|
|
126
|
+
else if (ch === '{') depth++;
|
|
127
|
+
else if (ch === '}' && --depth === 0) { end = i + 1; break; }
|
|
128
|
+
}
|
|
129
|
+
if (end < 0) return { drop: true }; // truncated JSON — can't run it faithfully
|
|
130
|
+
if (rest.slice(end).trim()) return { drop: true }; // has an → outcome: a memory, not a new call
|
|
131
|
+
try { return { tool, args: JSON.parse(rest.slice(0, end)) }; } catch { return { drop: true }; }
|
|
132
|
+
};
|
|
133
|
+
async function* stripToolCallMimicry(src) {
|
|
134
|
+
let carry = '';
|
|
135
|
+
// Longest tail of `carry` that could still grow into the marker — held back; the rest is safe to emit.
|
|
136
|
+
const heldLen = () => {
|
|
137
|
+
for (let n = Math.min(carry.length, MIMIC_MARK.length - 1); n > 0; n--)
|
|
138
|
+
if (MIMIC_MARK.startsWith(carry.slice(-n))) return n;
|
|
139
|
+
return 0;
|
|
140
|
+
};
|
|
141
|
+
function* pump(final) {
|
|
142
|
+
for (;;) {
|
|
143
|
+
const at = carry.indexOf(MIMIC_MARK);
|
|
144
|
+
if (at < 0) {
|
|
145
|
+
const hold = final ? 0 : heldLen();
|
|
146
|
+
const out = carry.slice(0, carry.length - hold);
|
|
147
|
+
carry = carry.slice(carry.length - hold);
|
|
148
|
+
if (out) yield { text: out };
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const nl = carry.indexOf('\n', at);
|
|
152
|
+
if (nl < 0 && !final) { // the line is still streaming — hold from the marker on
|
|
153
|
+
if (at > 0) { yield { text: carry.slice(0, at) }; carry = carry.slice(at); }
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
const end = nl < 0 ? carry.length : nl;
|
|
157
|
+
const before = carry.slice(0, at), line = carry.slice(at, end);
|
|
158
|
+
carry = nl < 0 ? '' : carry.slice(nl + 1); // the line (and its newline) leave the text stream
|
|
159
|
+
if (before) yield { text: before };
|
|
160
|
+
const parsed = parseMimicLine(line);
|
|
161
|
+
if (!parsed) yield { text: nl < 0 ? line : `${line}\n` }; // false alarm — pass it through untouched (newline included)
|
|
162
|
+
else if (parsed.tool) yield { tool: parsed.tool, args: parsed.args };
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
for await (const item of src) {
|
|
166
|
+
if (item.tool) { yield item; continue; }
|
|
167
|
+
carry += item.text ?? '';
|
|
168
|
+
yield* pump(false);
|
|
169
|
+
}
|
|
170
|
+
yield* pump(true);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Persona (who the agent is) + language rule + the host's live context block (`sysmsg`, delimited
|
|
174
|
+
// so the model never mistakes injected context for its own conversation).
|
|
175
|
+
const composeSysmsg = (sysmsg, persona = VOICE_SYSMSG, lang = '') => {
|
|
176
|
+
const base = [persona, langRule(lang)].filter(Boolean).join('\n\n');
|
|
177
|
+
return sysmsg
|
|
178
|
+
? `${base}\n\n=== CONTEXT (not your conversation) ===\n${sysmsg}\n=== END CONTEXT ===`
|
|
179
|
+
: base;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
// Asset paths for the Silero VAD — MUST match the ones `_startVad` passes to MicVAD.new() so the
|
|
183
|
+
// pre-warm fetches/compiles the SAME artifacts the real session then reuses (HTTP-cached model +
|
|
184
|
+
// already-compiled ONNX-WASM runtime).
|
|
185
|
+
const VAD_ASSET_PATH = `${CDN}@ricky0123/vad-web@0.0.29/dist/`;
|
|
186
|
+
const VAD_WASM_PATH = `${CDN}onnxruntime-web@1.22.0/dist/`;
|
|
187
|
+
|
|
188
|
+
// Pre-build the Silero v5 ONNX session WITHOUT opening the mic, so the costly parts of the cold start
|
|
189
|
+
// — compiling the onnxruntime-web WASM (ort.env.wasm.wasmPaths) and fetching/instantiating the v5
|
|
190
|
+
// model — happen while the panel sits idle, NOT after the user clicks. We run the SAME MicVAD.new()
|
|
191
|
+
// the real session uses, but with `startOnLoad: false` so it never calls getStream (no getUserMedia,
|
|
192
|
+
// no mic indicator) — it only loads the model + builds the FrameProcessor. The real MicVAD.new() in
|
|
193
|
+
// _startVad() then reuses the HTTP-cached v5 model and the already-compiled WASM runtime, so it
|
|
194
|
+
// reaches 'listening' far sooner. NonRealTimeVAD is NOT used here: it ignores `model`/`onnxWASMBasePath`
|
|
195
|
+
// and always loads the LEGACY model, so it would warm the wrong session. Needs `window.vad` (the
|
|
196
|
+
// vad-web UMD bundle) already loaded; call AFTER the deps are injected. Memoized + best-effort: a
|
|
197
|
+
// failure just means start() pays the normal cold cost.
|
|
198
|
+
let warmVadPromise = null;
|
|
199
|
+
export function warmVad() {
|
|
200
|
+
return (warmVadPromise ??= (async () => {
|
|
201
|
+
const d = window.vad;
|
|
202
|
+
if (!d?.MicVAD) return;
|
|
203
|
+
await d.MicVAD.new({ model: 'v5', startOnLoad: false, onnxWASMBasePath: VAD_WASM_PATH, baseAssetPath: VAD_ASSET_PATH });
|
|
204
|
+
})().catch((e) => { warmVadPromise = null; throw e; })); // drop the memo on failure so a later start() can retry
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export class VoiceAgent {
|
|
208
|
+
// Core options:
|
|
209
|
+
// `llm` — async generator (history, system, signal, { toolGate }) → { text } | { tool, args }
|
|
210
|
+
// chunks. Bring your own, or omit it and pass `llmUrl` (+ `apiKey`, `model`) to talk to any
|
|
211
|
+
// OpenAI-compatible /chat/completions endpoint via the built-in makeOpenAILLM (see
|
|
212
|
+
// llm-openai.js). NEVER put a provider secret key in a public page — point llmUrl at your own
|
|
213
|
+
// proxy route in production.
|
|
214
|
+
//
|
|
215
|
+
// An adapter that RUNS the tools itself (inside the generator, feeding results back to the
|
|
216
|
+
// model for an informed follow-up in the same turn) declares two flags on the function:
|
|
217
|
+
// • `llm.executesTools = true` — chunks arrive with the outcome attached (`result`/`error`),
|
|
218
|
+
// so _runTool reports without executing again. Optionally emit `{ tool, id, args,
|
|
219
|
+
// running: true }` first for a live spinner chip the outcome replaces in place.
|
|
220
|
+
// • `llm.acceptsToolGate = true` — the generator awaits the 4th arg's `toolGate` promise
|
|
221
|
+
// before ANY tool executes. Without this, speculative prefetch is DISABLED for that
|
|
222
|
+
// adapter (a speculation could fire a side effect on speech the user may still change);
|
|
223
|
+
// with it, text streams speculatively and tools wait for the turn to commit.
|
|
224
|
+
// `tts` — a StreamingTTS subclass; default PiperTTS (free, local WASM).
|
|
225
|
+
// `sttProvider` — 'webspeech' (default; free, browser-native) | 'elevenlabs' | 'speechmatics' |
|
|
226
|
+
// 'deepgram'. Cloud providers authenticate via a short-TTL token minted by YOUR backend
|
|
227
|
+
// (`sttTokenUrl`) so the raw key stays server-side. Providers may set `continuous`/`nativeEOT`
|
|
228
|
+
// flags the agent adapts to. 'webspeech' downgrades to `webspeechFallback` where the browser
|
|
229
|
+
// has no SpeechRecognition — `agent.sttProvider` is the one actually running.
|
|
230
|
+
// `sysmsg` — live host context appended to the persona (updatable via setSysmsg()).
|
|
231
|
+
// `persona` — who the agent is (replaces the default VOICE_SYSMSG persona).
|
|
232
|
+
// `tools`: { name: { description, params, run(args) } } — exposed to the LLM; `params` is the
|
|
233
|
+
// JSON-schema `properties` for the arguments. Calls fire during streaming, in parallel with TTS.
|
|
234
|
+
// `sttLang` — STT language code ('en'); also pins the reply language.
|
|
235
|
+
// `keyterms` — domain words to bias STT towards (cloud providers; max 50, ≤20 chars each).
|
|
236
|
+
// `sttUrl`/`sttModel`/`sttTokenUrl`/`sttUsageUrl`/`sttEotThreshold` — STT provider knobs (stt.js).
|
|
237
|
+
// `maxTokens`, `preroll` (chunks kept before VAD fires), `vadOptions` (Silero overrides).
|
|
238
|
+
// `turnDetector(text)` — async/sync predicate run when VAD hears end-of-speech: return false to
|
|
239
|
+
// KEEP listening (the user only paused mid-thought), true (default) to close the turn.
|
|
240
|
+
// `maxPauseMs` — hard cap (default 4000) a turnDetector may hold a turn open before force-commit;
|
|
241
|
+
// also bounds a turnDetector that never returns at all.
|
|
242
|
+
// `bargeInMinChars` — min transcribed NOVEL chars heard while speaking to count as a real
|
|
243
|
+
// barge-in; higher ignores short backchannels ("mhm","yeah"). Defaults live in tuning.js.
|
|
244
|
+
// `onEvent(e)` — the event tap: { type: 'state'|'stt'|'assistant'|'tool'|'vad'|'echo'|'error'|'diag', … }.
|
|
245
|
+
constructor({ sysmsg = '', persona, model = '', llm, fetchFn, apiKey = '', llmUrl = '', maxTokens = TUNING.MAX_TOKENS,
|
|
246
|
+
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
247
|
+
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
248
|
+
tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
|
|
249
|
+
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, onEvent = () => {} } = {}) {
|
|
250
|
+
// Each agent gets its own PiperTTS (it owns an AudioContext + playing node) — a shared
|
|
251
|
+
// module-level default would make two agents fight over one audio output. `speed` scales
|
|
252
|
+
// playback rate (1.2 → 20% faster) on the default TTS; ignored if a custom `tts` is passed.
|
|
253
|
+
this.sysmsg = composeSysmsg(sysmsg, persona, sttLang); // persona (base or host-supplied) + language rule + host context
|
|
254
|
+
this._persona = persona; // remembered so setSysmsg() re-composes with the same persona
|
|
255
|
+
this.tts = tts ?? new PiperTTS(undefined, speed); this.sttLang = sttLang; this.onEvent = onEvent;
|
|
256
|
+
this.tts.onEvent = onEvent; // diagnostics channel (e.g. aec-fallback warning from StreamingTTS._playBuf)
|
|
257
|
+
this.apiKey = apiKey;
|
|
258
|
+
this.micDeviceId = micDeviceId; this.prerollMax = preroll; this.vadOptions = vadOptions;
|
|
259
|
+
this.turnDetector = turnDetector; this._turnText = ''; this._turnClosing = false; this.maxPauseMs = maxPauseMs; this._bargeInMinChars = bargeInMinChars; this._maxPause = null; // _bargeInMinChars undefined → live-read tuning.js (see getter) // semantic end-of-turn hook + latest interim it judges + closing latch + hard max-pause failsafe
|
|
260
|
+
this._prefetch = null; this._prefetchTimer = null; // speculative LLM prefetch (see _startPrefetch)
|
|
261
|
+
// STT is a pluggable provider (like tts/llm). The agent owns the mic+VAD and forwards
|
|
262
|
+
// feed/commit/close; the provider opens its own socket and reports transcripts via callbacks.
|
|
263
|
+
// 'webspeech' downgrades to a cloud provider where the browser has no SpeechRecognition (Firefox,
|
|
264
|
+
// the Linux WebKitGTK desktop shell) — otherwise voice mode could only fatal there. The model id
|
|
265
|
+
// belongs to the CHOSEN provider, so a downgrade drops it back to the fallback's own default.
|
|
266
|
+
this.sttProvider = resolveSttProvider(sttProvider);
|
|
267
|
+
const makeStt = STT_PROVIDERS[this.sttProvider] ?? STT_PROVIDERS.webspeech;
|
|
268
|
+
this.stt = makeStt({
|
|
269
|
+
apiKey, sttUrl, sttModel: this.sttProvider === sttProvider ? sttModel : '', sttLang, micDeviceId, keyterms: keyterms.filter(k => k && k.length <= 20).slice(0, 50),
|
|
270
|
+
sttTokenUrl, getToken: getSttToken, sttUsageUrl, eotThreshold: sttEotThreshold,
|
|
271
|
+
onPartial: (text, ms, committed = '') => {
|
|
272
|
+
const interim = `${committed} ${text}`.trim();
|
|
273
|
+
this._turnText = interim; // latest interim (committed prefix + live tail) for the turn detector
|
|
274
|
+
this.onEvent({ type: 'stt', turnComplete: false, text, ms, committed });
|
|
275
|
+
// BARGE-IN ON WORDS, not VAD energy: while we're speaking, the user is only "interrupting"
|
|
276
|
+
// once STT actually transcribes words (not a cough or keyboard noise — those produce VAD
|
|
277
|
+
// activity but no text). Full-duplex: the uplink STT keeps transcribing throughout the
|
|
278
|
+
// reply, so this fires the instant words land instead of waiting for a turn boundary.
|
|
279
|
+
// SELF-ECHO GUARD: on speakers the browser's AEC often fails to cancel our own Web-Audio
|
|
280
|
+
// TTS playout, so the STT transcribes the agent's own voice — real words that would pass the
|
|
281
|
+
// char threshold and make it interrupt ITSELF. We track the AUDIBLE prefix of the reply
|
|
282
|
+
// (_replyText, fed by the TTS progress hook — not the generated-ahead answer): a live tail
|
|
283
|
+
// whose recent words are mostly words of that prefix is echo → latch _echoRef (the prefix
|
|
284
|
+
// it echoed, so its final is judged against the right reply) and don't barge in. Notes:
|
|
285
|
+
// • classified on the LIVE tail only — a committed real-user prefix must not tip a mixed
|
|
286
|
+
// turn into "echo", and a fresh real tail after echoed words clears the latch (cuts in);
|
|
287
|
+
// • an EMPTY tail neither reclassifies nor clears — providers emit empty-tail callbacks
|
|
288
|
+
// when words lock into `committed` (same words, just moved), which must not turn a
|
|
289
|
+
// latched echo into a barge-in;
|
|
290
|
+
// • latching runs while speaking AND for a short grace after playback ends (_replyDoneAt):
|
|
291
|
+
// STT latency means echo can land just after the reply finished (while 'listening'),
|
|
292
|
+
// and it must still be latched or its final becomes a user turn the agent then answers.
|
|
293
|
+
// Bounded so the user quoting the reply a while later can never be judged echo.
|
|
294
|
+
const inEchoWindow = this.state === 'speaking' || Date.now() - this._replyDoneAt < ECHO_GRACE_MS;
|
|
295
|
+
// Echo scope: the CURRENT reply's audible prefix PLUS, within the grace window, the PREVIOUS
|
|
296
|
+
// reply's (see _prevReplyText). A new turn clears _replyText before its audio starts, but on
|
|
297
|
+
// speakers the OLD reply's echo tail is still in flight (acoustic delay + STT latency) — with
|
|
298
|
+
// only the cleared reference those words looked novel and barged in on the new reply,
|
|
299
|
+
// cascading self-interruptions. Word-set matching makes concatenating the two safe.
|
|
300
|
+
const cur = this._replyEcho || this._replyText;
|
|
301
|
+
const prevTail = Date.now() - this._replyDoneAt < ECHO_GRACE_MS ? this._prevReplyText : '';
|
|
302
|
+
const echoScope = prevTail && prevTail !== cur ? `${cur} ${prevTail}`.trim() : cur;
|
|
303
|
+
if (text.trim() && inEchoWindow) this._echoRef = isSelfEcho(text, echoScope) ? echoScope : '';
|
|
304
|
+
// Barge-in on NOVEL characters only (not raw interim length): words fuzzily attributable to
|
|
305
|
+
// the audible reply count as echo (misheard echo words land NEAR the reply's words — see
|
|
306
|
+
// fuzzyHas) and contribute nothing, so echo that slipped past the ratio latch above still
|
|
307
|
+
// can't cut the reply; real interruptions are novel words and fire exactly as before.
|
|
308
|
+
if (this.state === 'speaking' && !this._echoRef && novelChars(interim, echoScope) >= this.bargeInMinChars) this._onBargeIn();
|
|
309
|
+
// SPECULATIVE PREFETCH: the turn's end-of-turn debounce is dead time — the LLM request only
|
|
310
|
+
// fires after it elapses, so its TTFT lands ON TOP. Instead, once the interim has been
|
|
311
|
+
// stable for PREFETCH_MS (no new words), start generating NOW with the probable final text;
|
|
312
|
+
// if the turn closes with that exact text the reply streams instantly (see _takePrefetch),
|
|
313
|
+
// if the user kept talking the stale request is aborted (cost: a few prompt tokens).
|
|
314
|
+
clearTimeout(this._prefetchTimer); this._prefetchTimer = null;
|
|
315
|
+
if (this._prefetch && interim && this._prefetch.text !== interim) this._dropPrefetch(); // transcript moved on → the running speculation is stale, stop paying for it
|
|
316
|
+
if (interim && this.state === 'listening' && !this._held && !this._echoRef)
|
|
317
|
+
this._prefetchTimer = setTimeout(() => this._startPrefetch(interim), TUNING.PREFETCH_MS ?? 200);
|
|
318
|
+
},
|
|
319
|
+
onFinal: (text, ms) => {
|
|
320
|
+
clearTimeout(this._prefetchTimer); this._prefetchTimer = null; // the turn is closing — no new speculation may start behind it
|
|
321
|
+
this._turnText = ''; this._turnClosing = false; // turn fully finalized: clear interim + the closing latch
|
|
322
|
+
// Take + clear the echo latch UNCONDITIONALLY (even for the empty finals below) — a
|
|
323
|
+
// leftover latch must never leak into the next STT turn and swallow a real utterance.
|
|
324
|
+
const echoRef = this._echoRef;
|
|
325
|
+
this._echoRef = '';
|
|
326
|
+
// Empty finalization (the STT closed the turn but transcribed nothing — e.g. only the AI's
|
|
327
|
+
// own echo or noise was force-ended): it still clears the closing latch above (so we never
|
|
328
|
+
// wedge waiting on a non-empty onFinal), but it's not a user turn — don't emit/reply.
|
|
329
|
+
if (!text.trim()) { this._dropPrefetch(); return; } // empty close (noise/echo force-end) — the speculation has no turn to serve
|
|
330
|
+
// A turn latched as self-echo (see onPartial): drop it ONLY if the WHOLE final is a strong
|
|
331
|
+
// match against the reply it echoed — a mixed turn (real user words + echo tail) is kept as
|
|
332
|
+
// a user turn. minHits scales with the final's own length (capped at 3): fast-endpointing
|
|
333
|
+
// STTs close echo as SHORT finals ("Yes.", "It is.") that could never reach 3 hits — kept as
|
|
334
|
+
// user turns they'd supersede and cut the playing reply (measured self-interruption cascade
|
|
335
|
+
// on the echo bench). Cost: a ≤2-word latched full-match quote of the reply is swallowed —
|
|
336
|
+
// under acoustic coupling that's overwhelmingly echo. Surfaced as 'echo' for diagnostics.
|
|
337
|
+
if (echoRef && isSelfEcho(text, echoRef, { window: Infinity, minHits: Math.min(3, wordsOf(text).length) })) {
|
|
338
|
+
this._dropPrefetch(); // the speculation was fed by our own echo — never answer it
|
|
339
|
+
this.onEvent({ type: 'echo', text }); return;
|
|
340
|
+
}
|
|
341
|
+
this.onEvent({ type: 'stt', turnComplete: true, text, ms });
|
|
342
|
+
// Held: stack the utterance into history but DON'T reply — _flush() runs them all on release.
|
|
343
|
+
// Wait on the hold barrier first so the turn aborted on hold-entry records its assistant text
|
|
344
|
+
// BEFORE these user turns, keeping history ordered (…, assistant-so-far, user, user, …).
|
|
345
|
+
if (this._held) { this._pushHeld(text); return; } // prefetch already dropped by setHeld(true)
|
|
346
|
+
this._onUserTurn(text, { speech: true }); // only a SPOKEN final may adopt the prefetch it seeded
|
|
347
|
+
},
|
|
348
|
+
onError: (error) => this.onEvent({ type: 'error', error }),
|
|
349
|
+
onFatal: (error) => { this.stop(); this.onEvent({ type: 'error', error }); }, // auth/config — won't self-heal; stop FIRST so the error event sees `closed` (host reconciles `running`)
|
|
350
|
+
onClose: () => {},
|
|
351
|
+
isClosed: () => this._closed,
|
|
352
|
+
});
|
|
353
|
+
// A tool with no runnable `run` is DROPPED, never advertised: the LLM would call it, nothing
|
|
354
|
+
// would happen, and the transcript would still show it as "ran" — a silent lie.
|
|
355
|
+
this.tools = {};
|
|
356
|
+
for (const [name, t] of Object.entries(tools)) {
|
|
357
|
+
if (typeof t.run === 'function') this.tools[name] = t;
|
|
358
|
+
else console.warn(`VoiceAgent: tool "${name}" has no run() — dropped`);
|
|
359
|
+
}
|
|
360
|
+
// Default LLM: any OpenAI-compatible /chat/completions endpoint (llm-openai.js). A custom `llm`
|
|
361
|
+
// generator replaces it entirely.
|
|
362
|
+
this.llm = llm || makeOpenAILLM({ llmUrl, apiKey, model, maxTokens, tools: this.tools, fetchFn });
|
|
363
|
+
this.history = [];
|
|
364
|
+
this.state = 'idle'; // idle | listening | thinking | speaking
|
|
365
|
+
this._abort = null; // aborts the in-flight LLM
|
|
366
|
+
this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
|
|
367
|
+
this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
|
|
368
|
+
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
369
|
+
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
370
|
+
this._tap = []; this._tapSamples = 0; this._tapTotal = 0; this._tapLastMs = 0; this._tapGaps = [];
|
|
371
|
+
this._stream = null; this._ctx = null; this._node = null; this._vad = null; this._streamDeviceId = undefined; // pipeline kept warm across stop()/start() for instant resume
|
|
372
|
+
this._muted = false; // mic disabled → browser feeds silence to VAD+STT (nothing reaches the agent)
|
|
373
|
+
this._ttsMuted = false; // AI voice OUTPUT off → skip speaking, still stream the reply as text
|
|
374
|
+
this._held = false; // HOLD: keep transcribing into history, but don't run the LLM until released
|
|
375
|
+
this._replyText = ''; // AUDIBLE prefix of the reply being (or last) voiced — drives the host's spoken cursor
|
|
376
|
+
this._replyEcho = ''; // echo REFERENCE: prefix + the whole clip now playing (see _playBuf's scope) — the cursor lags real audio, so echo is judged against this upper bound, not _replyText
|
|
377
|
+
this._prevReplyText = ''; // audible text of the PREVIOUS reply — echo reference across a turn boundary (its echo tail outlives _replyText's clear)
|
|
378
|
+
this._echoRef = ''; // echo latch: the _replyText the current STT turn was judged an echo OF ('' = not echo)
|
|
379
|
+
this._replyDoneAt = 0; // when playback last ended — bounds the post-playout echo grace window
|
|
380
|
+
this._holdBarrier = null; // resolves once the turn aborted on hold-entry has finalized; held STT orders after it
|
|
381
|
+
this._heldPushes = new Set(); // in-flight held-utterance append promises; release waits for all before flushing
|
|
382
|
+
// Bootstrap VAD (see _bootstrapVadTick): a zero-load RMS-energy gate that stands in for Silero
|
|
383
|
+
// (this._vad, which needs a CDN fetch + WASM compile + model load) while it's still cold-starting
|
|
384
|
+
// — so the mic is gated (and speech detected) from frame 1 instead of losing/chopping the user's
|
|
385
|
+
// opening words in the size-capped preroll buffer. Swapped to Silero at the next utterance
|
|
386
|
+
// boundary once it's ready (never mid-speech — see _endOfSpeech / onSpeechStart).
|
|
387
|
+
this._useBootstrap = false; this._sileroPending = false; this._bsSpeechFrames = 0; this._bsSilenceFrames = 0;
|
|
388
|
+
// Bumped on every cold start()/_teardownPipeline() — guards the fire-and-forget _startVad() below:
|
|
389
|
+
// a rapid re-click or device switch during the cold-load window can call start() again before the
|
|
390
|
+
// FIRST _startVad() resolves; its late resolution must not stomp the SECOND session's _vad/bootstrap
|
|
391
|
+
// state, so the continuation checks the generation is still current before touching either.
|
|
392
|
+
this._pipelineGen = 0;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// `deviceId` pins the mic to listen on — enumerate inputs host-side via
|
|
396
|
+
// navigator.mediaDevices.enumerateDevices() and pass the chosen audioinput's deviceId; omit
|
|
397
|
+
// for the browser default. Start-time choice — switching means stop() + start(id).
|
|
398
|
+
async start(deviceId = this.micDeviceId) {
|
|
399
|
+
this._closed = false;
|
|
400
|
+
// A self-capturing STT (browser Web Speech API) OWNS the whole audio path: its own mic + its own
|
|
401
|
+
// end-of-turn. It needs NO agent pipeline — no getUserMedia, AudioWorklet, preroll or Silero VAD.
|
|
402
|
+
// Just open it (synchronously, inside the click gesture Safari requires for SpeechRecognition.start)
|
|
403
|
+
// and flip to listening. Barge-in still works (word-based, via the provider's onPartial). Mute is
|
|
404
|
+
// handled by setMuted() → stt.setEnabled(). No warm-resume machinery: open() is idempotent + cheap.
|
|
405
|
+
// open() can fail synchronously (Web Speech unsupported → onFatal → stop() sets _closed + idle).
|
|
406
|
+
// Guard the listening flip so a fatal isn't overwritten with a fake 'listening' state.
|
|
407
|
+
if (this.stt.selfCapture) { this.stt.open?.(); if (!this._closed) this._set('listening'); return; }
|
|
408
|
+
// WARM RESUME: stop() pauses (not tears down) the pipeline, so a resume of the SAME agent finds the
|
|
409
|
+
// mic stream + audio graph + VAD model still alive — just re-arm them. This skips the getUserMedia
|
|
410
|
+
// permission round-trip AND the MicVAD.new() Silero/ONNX cold start (the bulk of "open→listening"
|
|
411
|
+
// latency), so reopening a paused voice session feels instant. A device switch (different deviceId)
|
|
412
|
+
// can't reuse the captured stream, so fall through to a fresh build. The VAD's getStream callback
|
|
413
|
+
// captured this._stream at MicVAD.new() time, so we MUST keep the same stream to reuse the model.
|
|
414
|
+
if (this._vad && this._stream?.active && deviceId === this._streamDeviceId) {
|
|
415
|
+
this._ctx?.resume?.(); // a paused-tab AudioContext can be suspended
|
|
416
|
+
this._stream.getAudioTracks().forEach(t => t.enabled = !this._muted); // honor a mute toggled while paused
|
|
417
|
+
// stop() is always a safe boundary (nothing is mid-utterance once stopped) — so if Silero
|
|
418
|
+
// finished loading while paused, resume goes straight to it instead of restarting bootstrap.
|
|
419
|
+
if (this._useBootstrap) { this._useBootstrap = false; this._bsSpeechFrames = 0; this._bsSilenceFrames = 0; }
|
|
420
|
+
this._vad.start();
|
|
421
|
+
this.stt.open?.(); // re-open the STT socket (stop() closed it) so the first utterance doesn't pay a cold connect
|
|
422
|
+
this.tts.warm?.().catch(() => {});
|
|
423
|
+
this._set('listening');
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
// Cold build: a different mic (or first start / post-destroy) — drop any half-built warm pipeline first.
|
|
427
|
+
this._teardownPipeline();
|
|
428
|
+
const gen = ++this._pipelineGen;
|
|
429
|
+
this._streamDeviceId = deviceId;
|
|
430
|
+
// Flip to 'listening' NOW, then open the mic in the background: getUserMedia (mic permission +
|
|
431
|
+
// device open, esp. with the DSP chain below) is the dominant click→listening latency, and it's
|
|
432
|
+
// the only real blocker here. Once the mic actually opens, the bootstrap energy-VAD + preroll ring
|
|
433
|
+
// buffer protect the opening words (audio spoken during the brief permission/device-open window
|
|
434
|
+
// before the graph is wired can't be captured — an accepted tradeoff for an instant-feeling UI).
|
|
435
|
+
// gen (+ _closed) guard the async build: a rapid re-click / device switch bumps _pipelineGen and a
|
|
436
|
+
// stop/destroy sets _closed, so a slow getUserMedia that resolves into a superseded/stopped session
|
|
437
|
+
// tears its own stream down instead of wiring a stale mic into a dead pipeline.
|
|
438
|
+
this._set('listening');
|
|
439
|
+
(async () => {
|
|
440
|
+
let stream;
|
|
441
|
+
try {
|
|
442
|
+
// Dictation-tuned: drop autoGainControl — AGC pumps between words, corrupting what the
|
|
443
|
+
// recognizer sees. Keep echoCancellation (needed for barge-in while TTS is audible) AND
|
|
444
|
+
// noiseSuppression: without NS speaker playout leaks into the mic much stronger, so the STT
|
|
445
|
+
// transcribes the agent's own reply (self-echo) — worse than NS occasionally gating quiet speech.
|
|
446
|
+
const audio = { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: false, ...(deviceId ? { deviceId: { exact: deviceId } } : {}) };
|
|
447
|
+
try {
|
|
448
|
+
stream = await navigator.mediaDevices.getUserMedia({ audio });
|
|
449
|
+
} catch (e) {
|
|
450
|
+
// The saved mic id no longer exists on this machine (unplugged / different device / stale
|
|
451
|
+
// cross-profile localStorage) — `exact` throws OverconstrainedError instead of falling back.
|
|
452
|
+
// Retry once on the system default so a stale preference doesn't brick the whole session.
|
|
453
|
+
if (deviceId && e.name === 'OverconstrainedError') stream = await navigator.mediaDevices.getUserMedia({ audio: { ...audio, deviceId: undefined } });
|
|
454
|
+
else throw e;
|
|
455
|
+
}
|
|
456
|
+
} catch (e) {
|
|
457
|
+
// Mic open failed (permission denied / no device / constraints). We already optimistically
|
|
458
|
+
// flipped to 'listening', so self-stop like a fatal STT error does — stop() sets _closed + goes
|
|
459
|
+
// idle — so the host reconciles `running` against a.closed and the next press restarts cleanly
|
|
460
|
+
// (instead of a fake "listening" session with no mic). Guard on gen: a superseding start already
|
|
461
|
+
// owns the pipeline, so don't stop it out from under it.
|
|
462
|
+
if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: e.message || e.name || 'Microphone access failed' }); }
|
|
463
|
+
return;
|
|
464
|
+
}
|
|
465
|
+
// Superseded (re-click / device switch bumps gen) OR stopped/destroyed while opening (stop() sets
|
|
466
|
+
// _closed but keeps gen — it means to pause a mic that isn't wired yet) → drop the stream, don't
|
|
467
|
+
// wire a live mic into a dead session (would leave the OS mic indicator on).
|
|
468
|
+
if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); return; }
|
|
469
|
+
this._stream = stream;
|
|
470
|
+
this._stream.getAudioTracks().forEach(t => t.enabled = !this._muted); // honor a mute toggled while paused
|
|
471
|
+
// Capture on an AudioWorklet, NOT a ScriptProcessor: ScriptProcessor's onaudioprocess runs on
|
|
472
|
+
// the MAIN thread, so any stall (React renders, ONNX inference, Piper download/compile, SSE
|
|
473
|
+
// parsing) silently DROPS buffers — the recognizer then sees spliced audio and drops the missing
|
|
474
|
+
// span + loops on a token ("eleven eleven eleven"). The worklet captures on the audio rendering
|
|
475
|
+
// thread, so a busy UI can never drop capture; ~256ms chunks match the old cadence.
|
|
476
|
+
// Build on LOCALS and publish to this._ctx/_node only at the end: a supersede/stop during the
|
|
477
|
+
// awaited addModule() may have torn down (and a new session rebuilt) the instance fields — a
|
|
478
|
+
// stale continuation must clean up its own objects only.
|
|
479
|
+
let ctx;
|
|
480
|
+
try {
|
|
481
|
+
ctx = new AudioContext({ sampleRate: 16000 });
|
|
482
|
+
const src = ctx.createMediaStreamSource(stream);
|
|
483
|
+
await ctx.audioWorklet.addModule(CAPTURE_WORKLET_URL);
|
|
484
|
+
if (gen !== this._pipelineGen || this._closed) { stream.getTracks().forEach(t => t.stop()); ctx.close(); return; } // superseded/stopped while loading
|
|
485
|
+
const node = new AudioWorkletNode(ctx, 'capture-processor');
|
|
486
|
+
node.port.onmessage = e => this._feed(e.data); // Float32Array chunk from the audio thread
|
|
487
|
+
// The processor writes no output — this connection only keeps the node pulled by the graph.
|
|
488
|
+
src.connect(node); node.connect(ctx.destination);
|
|
489
|
+
this._ctx = ctx; this._node = node;
|
|
490
|
+
} catch (e) {
|
|
491
|
+
// Graph build failed (worklet unsupported, CSP-blocked blob:, 16kHz context refused): self-stop
|
|
492
|
+
// like a mic-open failure — never leave a fake 'listening' session holding an open mic.
|
|
493
|
+
stream.getTracks().forEach(t => t.stop()); ctx?.close();
|
|
494
|
+
if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: `audio capture: ${e.message}` }); }
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
// Pre-open the STT socket NOW (token fetch + WS handshake) so it overlaps the MicVAD cold start
|
|
498
|
+
// below — then the FIRST utterance streams immediately instead of paying a cold connect mid-speech.
|
|
499
|
+
this.stt.open?.();
|
|
500
|
+
// Gate on the bootstrap energy-VAD from THIS frame — mic audio can arrive (worklet port above)
|
|
501
|
+
// well before Silero (_startVad, awaited below) resolves, and without a gate that audio would
|
|
502
|
+
// only be protected by the small preroll ring buffer (silently drops the opening words on a slow
|
|
503
|
+
// cold start). _startVad() runs in the background (not awaited) and swaps in Silero the moment
|
|
504
|
+
// it's ready — see _swapToSileroWhenReady.
|
|
505
|
+
this._useBootstrap = true; this._sileroPending = true;
|
|
506
|
+
this._startVad(gen).then(() => { if (gen === this._pipelineGen && !this._closed) this._swapToSileroWhenReady(); })
|
|
507
|
+
.catch(() => { if (gen === this._pipelineGen) this._sileroPending = false; });
|
|
508
|
+
// Warm the TTS engine (Piper: download+compile the ONNX voice model, JIT the WASM path) so the
|
|
509
|
+
// first reply isn't stalled by cold start mid-turn — but DEFERRED off the STT connect window:
|
|
510
|
+
// Piper's ~60MB HuggingFace download saturates bandwidth and was measured stretching the
|
|
511
|
+
// Deepgram WS handshake from ~0.5s to ~3s when fired concurrently. TTS isn't needed until the
|
|
512
|
+
// first REPLY (user utterance + LLM roundtrip away), so a short delay costs nothing.
|
|
513
|
+
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, 2500);
|
|
514
|
+
})();
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// True once the pipeline has been torn down (stop() / a fatal STT error). The host reconciles its
|
|
518
|
+
// own "running" flag against this after an error so a self-stopped agent isn't treated as live.
|
|
519
|
+
get closed() { return this._closed; }
|
|
520
|
+
|
|
521
|
+
stop() {
|
|
522
|
+
this._closed = true; this._abort?.abort(); this.tts.stop?.(); this._dropPrefetch();
|
|
523
|
+
clearTimeout(this._maxPause); this._turnText = ''; this._turnClosing = false; this._echoRef = ''; this._prevReplyText = ''; this._replyEcho = '';
|
|
524
|
+
// Clear the echo grace window: a resume within ECHO_GRACE_MS of the last reply must not judge
|
|
525
|
+
// the user's fresh opening words against a reply that stopped playing before the pause.
|
|
526
|
+
this._replyDoneAt = 0;
|
|
527
|
+
// Drop buffered-but-unsent audio: flushing it into the next session's socket on resume
|
|
528
|
+
// would re-transcribe old speech (duplicate user turns).
|
|
529
|
+
this._preroll = []; this._wasSpeaking = false; this._speaking = false;
|
|
530
|
+
// PAUSE, don't tear down: keep the mic stream + audio graph + VAD model alive so a resume of THIS
|
|
531
|
+
// agent (start() again) is instant — no getUserMedia prompt, no MicVAD.new() cold start. The STT
|
|
532
|
+
// socket DOES close (it's cheap to reopen lazily on the next utterance, and a stale socket would
|
|
533
|
+
// re-transcribe old audio). The host releases the mic by calling destroy() when it ends the session.
|
|
534
|
+
this._vad?.pause(); this.stt.close();
|
|
535
|
+
this._bsSpeechFrames = 0; this._bsSilenceFrames = 0;
|
|
536
|
+
this._tapLastMs = 0; // the pause gap isn't a capture dropout — don't flag it on resume
|
|
537
|
+
this._set('idle');
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// Full teardown: release the mic, audio graph and VAD model. Called by destroy() (host ends the
|
|
541
|
+
// session) and before a cold rebuild (mic device switch). After this, start() pays the full cold
|
|
542
|
+
// cost again. Split out so stop() can pause cheaply without duplicating this.
|
|
543
|
+
_teardownPipeline() {
|
|
544
|
+
this._vad?.destroy?.(); this._vad = null;
|
|
545
|
+
this._node?.disconnect(); this._node = null;
|
|
546
|
+
this._ctx?.close(); this._ctx = null;
|
|
547
|
+
this._stream?.getTracks().forEach(t => t.stop()); this._stream = null;
|
|
548
|
+
this._streamDeviceId = undefined;
|
|
549
|
+
this._tap = []; this._tapSamples = 0; this._tapTotal = 0; this._tapLastMs = 0; this._tapGaps = []; // fresh diagnostics per pipeline
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// End the session for real: pause logic + release the mic/VAD (turns off the OS mic indicator). The
|
|
553
|
+
// host calls this when it drops the agent (end/close/reset/new-chat) so a paused-but-discarded agent
|
|
554
|
+
// never leaks the microphone.
|
|
555
|
+
destroy() { this._destroyed = true; this.stop(); this.stt.close(); this._teardownPipeline(); }
|
|
556
|
+
|
|
557
|
+
// Host-triggered barge-in: cut off the current in-flight turn (abort the LLM, stop TTS) WITHOUT
|
|
558
|
+
// tearing down the pipeline — the mic keeps listening. Mirrors `_onBargeIn` but callable from a
|
|
559
|
+
// UI "stop" control. No-op when nothing is in flight (already idle/listening).
|
|
560
|
+
interrupt() {
|
|
561
|
+
if (this._closed || (this.state !== 'thinking' && this.state !== 'speaking')) return;
|
|
562
|
+
this._onBargeIn();
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// Re-speak a FINISHED reply from `fromNw` (a non-whitespace offset — the word the user tapped),
|
|
566
|
+
// interruptibly. Pure TTS: no LLM, no history change (re-listening isn't a new turn). `onProgress`
|
|
567
|
+
// is driven by the host with (spokenSoFarText, done) so it can update the EXACT transcript message
|
|
568
|
+
// it tapped (identity stays host-side — no reliance on draft-matching). Supersedes any in-flight
|
|
569
|
+
// turn/replay (shares _turn/_abort), barge-in cuts it like a live reply, and it returns to listening.
|
|
570
|
+
replay(text, fromNw = 0, onProgress = () => {}) {
|
|
571
|
+
if (this._closed || !text?.trim()) return Promise.resolve();
|
|
572
|
+
const prev = this._turn;
|
|
573
|
+
this._abort?.abort(); // supersede the in-flight reply/replay
|
|
574
|
+
this.tts.stop?.();
|
|
575
|
+
return (this._turn = (async () => {
|
|
576
|
+
await prev?.catch(() => {});
|
|
577
|
+
if (this._closed) return;
|
|
578
|
+
if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText; // keep as the cross-turn echo reference (grace-bounded)
|
|
579
|
+
this._replyText = ''; this._replyEcho = ''; // stale previous reply must not classify pre-audio speech as echo
|
|
580
|
+
this._set('speaking');
|
|
581
|
+
const signal = (this._abort = new AbortController()).signal;
|
|
582
|
+
// TTS reports normalized spoken-so-far; map back to a raw prefix of `text` by non-whitespace count
|
|
583
|
+
// so the host's cursor lands exactly (text keeps the model's original spacing). The audible
|
|
584
|
+
// prefix doubles as the self-echo reference (see onPartial) — a replay leaks into the mic too.
|
|
585
|
+
// Echo reference = ONLY what this replay actually voices: a mid-reply replay (fromNw > 0)
|
|
586
|
+
// never plays the skipped head, so it must not be echo-eligible (a user quoting those words
|
|
587
|
+
// would be swallowed). skipLen = raw chars of that skipped head, cut from every reference.
|
|
588
|
+
const skipLen = fromNw > 0 ? sliceNw(text, fromNw).length : 0;
|
|
589
|
+
this.tts.setOnProgress?.((spoken, scope) => {
|
|
590
|
+
if (signal.aborted) return;
|
|
591
|
+
this._replyText = sliceNw(text, nw(spoken));
|
|
592
|
+
this._replyEcho = (scope ? sliceNw(text, nw(scope)) : this._replyText).slice(skipLen); // clip-bounded echo reference (cursor lags real audio), minus the unplayed head
|
|
593
|
+
onProgress(this._replyText, false);
|
|
594
|
+
});
|
|
595
|
+
try {
|
|
596
|
+
await this.tts.speak((async function* () { yield text; })(), signal, fromNw);
|
|
597
|
+
} catch (e) {
|
|
598
|
+
if (e.name !== 'AbortError') this.onEvent({ type: 'error', error: e.message }); // barge-in/supersede surfaces as AbortError → ignore
|
|
599
|
+
} finally {
|
|
600
|
+
this.tts.setOnProgress?.(null);
|
|
601
|
+
// Clamp the clip-bounded reference to what was actually heard (see _speakTurn's finally):
|
|
602
|
+
// the unspoken remainder of a barged-in clip never hit the speakers.
|
|
603
|
+
this._replyEcho = this._replyText.slice(skipLen);
|
|
604
|
+
this._replyDoneAt = Date.now(); // start the post-playout echo grace window
|
|
605
|
+
onProgress(text, true); // solidify the message back to its whole text (played out, barged-in, or superseded)
|
|
606
|
+
if (!this._closed && this.state !== 'idle' && !signal.aborted) this._set('listening');
|
|
607
|
+
}
|
|
608
|
+
})());
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// Live barge-in threshold: when the host didn't pass an explicit bargeInMinChars, re-read tuning.js
|
|
612
|
+
// each utterance so editing it (with HMR) applies to the in-flight session; an explicit value is fixed.
|
|
613
|
+
get bargeInMinChars() { return this._bargeInMinChars ?? TUNING.BARGE_IN_MIN_CHARS; }
|
|
614
|
+
|
|
615
|
+
_set(s) { this.state = s; this.onEvent({ type: 'state', state: s }); }
|
|
616
|
+
|
|
617
|
+
// List the TTS voices for a picker (if the active TTS supports it): [{ id, name, language }].
|
|
618
|
+
listVoices() { return this.tts.voices ? this.tts.voices() : Promise.resolve([]); }
|
|
619
|
+
// Switch TTS voice (takes effect on the next utterance).
|
|
620
|
+
setVoice(voiceId) { this.tts.setVoice?.(voiceId); }
|
|
621
|
+
// Speaking speed (no-op under Web Audio playback — kept for API compatibility).
|
|
622
|
+
setSpeed(speed) { this.tts.setSpeed?.(speed); }
|
|
623
|
+
|
|
624
|
+
// Update the host context (e.g. a re-opened chat with fresh conversation); applies from the next
|
|
625
|
+
// turn on. Keeps the running session/history alive — no restart needed.
|
|
626
|
+
setSysmsg(sysmsg) { this.sysmsg = composeSysmsg(sysmsg, this._persona, this.sttLang); this._dropPrefetch(); } // a running speculation was built with the OLD sysmsg
|
|
627
|
+
|
|
628
|
+
// Mute/unmute the mic by disabling the stream's audio tracks: the browser then feeds silence to
|
|
629
|
+
// VAD and STT, so nothing the user says reaches the agent — without dropping the mic permission
|
|
630
|
+
// or tearing down the pipeline. Stored so a mute toggled while paused applies on the next start().
|
|
631
|
+
setMuted(muted) {
|
|
632
|
+
this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted);
|
|
633
|
+
if (muted) this._dropPrefetch(); // the utterance that seeded it just got cut off — no final will claim it
|
|
634
|
+
// A self-capturing STT (Web Speech) has no agent stream to gate — it listens on its OWN mic, so
|
|
635
|
+
// stop it explicitly via setEnabled or a "muted" agent keeps transcribing.
|
|
636
|
+
if (this.stt.selfCapture) this.stt.setEnabled?.(!muted);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
// Mute/unmute the AI's spoken OUTPUT (TTS). When muted, the reply still streams into the
|
|
640
|
+
// transcript but is never voiced; an utterance already playing is stopped. Stored so it carries
|
|
641
|
+
// across pauses and applies to the next turn.
|
|
642
|
+
setTtsMuted(muted) { this._ttsMuted = muted; if (muted) this.tts.stop?.(); }
|
|
643
|
+
|
|
644
|
+
// Move the playhead to a position in the current reply (non-whitespace char count) and play forward
|
|
645
|
+
// from there — the transcript UI's tap-to-seek. Hits any sentence already streamed (backward replays
|
|
646
|
+
// the cached clip, forward skips ahead); a tap beyond streamed text is a no-op. Returns true on a hit.
|
|
647
|
+
seek(nwIndex) { return this.tts.seek?.(nwIndex) ?? false; }
|
|
648
|
+
|
|
649
|
+
// HOLD: stop responding but keep listening. Entering hold silences the current reply NOW (kept
|
|
650
|
+
// in history as the full streamed answer so far) and from then on every utterance is stacked
|
|
651
|
+
// into history without a reply. Releasing flushes — one turn processes everything accumulated.
|
|
652
|
+
setHeld(held) {
|
|
653
|
+
if (held === this._held) return;
|
|
654
|
+
this._held = held;
|
|
655
|
+
if (held) {
|
|
656
|
+
this._dropPrefetch(); // held turns queue for later — a running speculation would outlive its transcript
|
|
657
|
+
// If a reply is in flight, silence it now and tag ITS signal to record the FULL streamed text
|
|
658
|
+
// (per-turn, so a quick Resume can't flip it back). The barrier resolves once that turn has
|
|
659
|
+
// finalized — held utterances order strictly after the assistant text it kept.
|
|
660
|
+
const inFlight = this.state === 'thinking' || this.state === 'speaking';
|
|
661
|
+
if (inFlight && this._abort) {
|
|
662
|
+
this._abort.signal.keepFull = true;
|
|
663
|
+
this._holdBarrier = (this._turn ?? Promise.resolve()).catch(() => {});
|
|
664
|
+
this._abort.abort();
|
|
665
|
+
this.tts.stop?.();
|
|
666
|
+
this._set('listening');
|
|
667
|
+
} else {
|
|
668
|
+
this._holdBarrier = Promise.resolve(); // nothing in flight — held STT can push immediately
|
|
669
|
+
}
|
|
670
|
+
} else {
|
|
671
|
+
this._flush().catch((e) => this.onEvent({ type: 'error', error: e.message }));
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
// Queue a held user utterance: append it AFTER the hold-entry turn has finalized (the barrier), so
|
|
676
|
+
// history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
|
|
677
|
+
// release (_flush) owns starting the single turn, and waits for all of these to settle first.
|
|
678
|
+
_pushHeld(text) {
|
|
679
|
+
const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
|
|
680
|
+
this._heldPushes.add(p);
|
|
681
|
+
p.finally(() => this._heldPushes.delete(p));
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
// Release a hold: after the hold-entry turn AND every queued held push have settled, run ONE turn
|
|
685
|
+
// over the accumulated history. No-op when nothing is queued (last entry isn't a user turn). The
|
|
686
|
+
// held pushes already appended their user messages, so the turn's prep is empty — _runTurn just
|
|
687
|
+
// serializes and replies over the accumulated history.
|
|
688
|
+
async _flush() {
|
|
689
|
+
if (this._closed || this._held) return;
|
|
690
|
+
await this._holdBarrier;
|
|
691
|
+
if (this._closed || this._held) return; // stopped / re-held while waiting -> abandon
|
|
692
|
+
await Promise.allSettled([...this._heldPushes]);
|
|
693
|
+
if (this._closed || this._held) return;
|
|
694
|
+
const last = this.history[this.history.length - 1];
|
|
695
|
+
if (!last || last.role !== 'user') return;
|
|
696
|
+
await this._runTurn(''); // '' = no new utterance; reply over the accumulated held history (revalidated in _runTurn)
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// Inject an out-of-band turn: feed `text` to the LLM as if the user had said it, then speak the
|
|
700
|
+
// reply. Used by the host to surface external events (a tool result, a background job finishing)
|
|
701
|
+
// — only meaningful while the agent is running, so it's a no-op once stopped.
|
|
702
|
+
//
|
|
703
|
+
// SOFT turn (supersede: false): unlike a user utterance this must NOT cut the reply in flight.
|
|
704
|
+
// A tool result arrives WHILE the turn that called the tool is still streaming/speaking, so
|
|
705
|
+
// aborting here would kill that turn mid-answer and re-run the LLM over a history whose last
|
|
706
|
+
// entry is the request again — the model then re-calls the same tool (duplicate-action spam).
|
|
707
|
+
// Instead it queues behind the running turn, which by then has recorded both its answer and the
|
|
708
|
+
// [TOOL CALL] it made, so the model sees the work is already done.
|
|
709
|
+
async notify(text) { if (!this._closed) await this._runTurn(text, { supersede: false }); }
|
|
710
|
+
|
|
711
|
+
// Send a TYPED user turn: like speaking, but the text comes from the host's input instead of STT.
|
|
712
|
+
// Emits the same turn-complete `stt` event a spoken turn would (so it renders as a finished user
|
|
713
|
+
// bubble in the transcript), then runs it through the normal turn loop.
|
|
714
|
+
// Returns true the moment the turn is ACCEPTED into the conversation (emitted/queued), false only
|
|
715
|
+
// for blank text — the host clears the typed draft on true and keeps it on false. The reply runs in
|
|
716
|
+
// the BACKGROUND (not awaited), exactly like a spoken turn finalized in onFinal: the user shouldn't
|
|
717
|
+
// wait for the whole answer before the box clears.
|
|
718
|
+
sendUserText(text) {
|
|
719
|
+
if (this._destroyed || !text.trim()) return false; // destroy() is terminal — never resurrect
|
|
720
|
+
// Typed turns need no mic: revive a paused/stopped (_closed) agent's reply loop — stop() only
|
|
721
|
+
// parks the AUDIO pipeline, and the LLM/tools/history don't need it. Also un-wedges a session
|
|
722
|
+
// whose mic was denied or whose STT fataled: voice is dead, typing still works. start() re-arms
|
|
723
|
+
// the audio pipeline separately (it sets _closed = false itself).
|
|
724
|
+
this._closed = false;
|
|
725
|
+
this.onEvent({ type: 'stt', turnComplete: true, text, ms: 0 });
|
|
726
|
+
// Held: stack the typed turn into history but DON'T reply — same as a spoken turn (see onFinal).
|
|
727
|
+
// Releasing the hold (Resume) runs everything accumulated. Typing must NOT silently un-hold.
|
|
728
|
+
if (this._held) { this._pushHeld(text); return true; }
|
|
729
|
+
this._onUserTurn(text);
|
|
730
|
+
return true;
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
// ── the loop ──────────────────────────────────────────────────────────
|
|
734
|
+
// State drives everything the UI needs: listening = VAD/STT, thinking = LLM,
|
|
735
|
+
// speaking = TTS. No separate 'stage'/'user' events — derive the pipeline from state.
|
|
736
|
+
//
|
|
737
|
+
// Fire-and-forget a user turn (spoken onFinal AND typed sendUserText both land here): record the
|
|
738
|
+
// message, then reply — serialized via _runTurn so rapid turns can't overlap. Never rejects to the
|
|
739
|
+
// caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
|
|
740
|
+
_onUserTurn(text, { speech = false } = {}) {
|
|
741
|
+
if (!text.trim()) return Promise.resolve();
|
|
742
|
+
return this._runTurn(text, { speech });
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
// ── Speculative LLM prefetch ──────────────────────────────────────────────────────────────────
|
|
746
|
+
// Start generating a reply for `text` BEFORE the turn officially closes, overlapping the LLM's
|
|
747
|
+
// TTFT with the end-of-turn debounce. The generator is created against history + the probable
|
|
748
|
+
// user turn and its first chunk is PULLED immediately (this.llm is lazy — the fetch only fires on
|
|
749
|
+
// the first .next()), so by the time the turn closes the first tokens are usually already here.
|
|
750
|
+
// Never speaks, never touches this.history — _speakTurn consumes it via _takePrefetch() only when
|
|
751
|
+
// the closed turn's text matches exactly; any mismatch/staleness just aborts it.
|
|
752
|
+
// SPECULATIVE FIRST-CLIP SYNTH: the same dead-time trick as the LLM prefetch, one stage deeper.
|
|
753
|
+
// By the time the end-of-turn debounce closes, the prefetch usually holds the reply's opening
|
|
754
|
+
// words — but Piper's first predict() (the whole post-commit critical path, ~300-900ms of WASM
|
|
755
|
+
// inference) only started AFTER the commit. Here we pull JUST the first chunk out of the running
|
|
756
|
+
// speculation and hand it to the TTS to pre-synthesize during the debounce, so when the turn
|
|
757
|
+
// closes the first clip is already (or nearly) done and playback starts almost immediately.
|
|
758
|
+
// Safe where tools are not: synth has zero side effects — a stale speculation wastes one tiny
|
|
759
|
+
// predict, nothing more. Serial-session cost is bounded: the chunk is the reply's first WORDS
|
|
760
|
+
// (stage-0 breaker, ~a dozen chars), so even a wasted predict delays a real turn only briefly.
|
|
761
|
+
async _presynthFirstClip(p) {
|
|
762
|
+
if (!this.tts.presynth) return;
|
|
763
|
+
try {
|
|
764
|
+
const f = await p.first; // opening delta of the speculation
|
|
765
|
+
if (f.done || p.ctl.signal.aborted || this._prefetch !== p) return;
|
|
766
|
+
p.first = Promise.resolve(f); // re-stitch: first was consumed here, adoption must still see it
|
|
767
|
+
// LLM chunks are { text } / { tool, … } objects (or raw strings from simple custom llms) —
|
|
768
|
+
// only text is voiceable.
|
|
769
|
+
const chunk = typeof f.value === 'string' ? f.value : (f.value?.text ?? '');
|
|
770
|
+
const end = firstWordEnd(chunk);
|
|
771
|
+
if (end > 0) this.tts.presynth(chunk.slice(0, end).trim()); // enough for the stage-0 chunk → warm the exact clip
|
|
772
|
+
} catch { /* speculation failed/aborted — the real turn re-runs it */ }
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
_startPrefetch(text) {
|
|
776
|
+
// Speculating is only safe if pulling the first chunk cannot fire a real side effect on a
|
|
777
|
+
// transcript the user is still speaking and may yet change — abort cannot undo a sent email.
|
|
778
|
+
// The default LLM's chunks are inert (tools run later, in _speakTurn). An LLM that executes
|
|
779
|
+
// tools INSIDE its generator (llm.executesTools) is safe only if it also honors `toolGate`
|
|
780
|
+
// (llm.acceptsToolGate): we hand it a promise it must await before ANY tool runs, so text
|
|
781
|
+
// streams speculatively while tools wait for the turn to commit. Neither → don't speculate.
|
|
782
|
+
if (this.llm.executesTools && !this.llm.acceptsToolGate) return;
|
|
783
|
+
// The 200ms timer raced real events — revalidate EVERYTHING now, not at scheduling time: the
|
|
784
|
+
// agent must still be listening to this exact live transcript (a final/typed turn/replay/hold
|
|
785
|
+
// in the gap cleared or changed _turnText; speculating past that would answer a stale turn).
|
|
786
|
+
if (this._closed || this._held || this._echoRef || this.state !== 'listening' || this._turnText !== text) return;
|
|
787
|
+
if (this._prefetch?.text === text) return; // same speculation already in flight
|
|
788
|
+
this._dropPrefetch();
|
|
789
|
+
const ctl = new AbortController();
|
|
790
|
+
// COMMIT GATE: handed to tool-executing LLMs so every tool waits at the door until this turn
|
|
791
|
+
// COMMITS (_takePrefetch fulfills it) or the speculation is DROPPED (_dropPrefetch rejects it).
|
|
792
|
+
// It always settles, on both terminal paths — a gate that merely stayed pending on drop would
|
|
793
|
+
// park the adapter's `await toolGate` forever, so its finally never runs and the generator +
|
|
794
|
+
// fetch leak. Drop REJECTS rather than fulfilling so the safe reading is also the naive one: a
|
|
795
|
+
// plain `await toolGate` throws instead of silently authorizing a tool the user may have
|
|
796
|
+
// changed their mind about. Adapters need no abort race; awaiting the gate is enough.
|
|
797
|
+
let commit, cancel;
|
|
798
|
+
const committed = new Promise((res, rej) => { commit = res; cancel = rej; });
|
|
799
|
+
committed.catch(() => {}); // an adapter that ignores the gate must not raise an unhandled rejection
|
|
800
|
+
const gen = this.llm([...this.history, { role: 'user', content: text }], this.sysmsg, ctl.signal, { toolGate: committed });
|
|
801
|
+
const p = { text, ctl, gen, commit, cancel, first: gen.next(), histLen: this.history.length }; // pull NOW → fetch fires during the debounce; histLen pins the base the speculation saw
|
|
802
|
+
p.first.catch(() => {}); // aborted/failed speculation must not be an unhandled rejection
|
|
803
|
+
this._prefetch = p;
|
|
804
|
+
this._presynthFirstClip(p); // fire-and-forget: warm the first TTS clip off the same speculation
|
|
805
|
+
}
|
|
806
|
+
_dropPrefetch() {
|
|
807
|
+
clearTimeout(this._prefetchTimer); this._prefetchTimer = null;
|
|
808
|
+
const p = this._prefetch; this._prefetch = null;
|
|
809
|
+
if (p) {
|
|
810
|
+
try { p.ctl.abort(); } catch {}
|
|
811
|
+
// Reject the gate BEFORE touching the iterator: a generator parked on `await toolGate` is not
|
|
812
|
+
// woken by the abort signal alone, so without this its finally never runs and `first` never
|
|
813
|
+
// settles — leaking the generator and its fetch (and stranding the return() below forever).
|
|
814
|
+
p.cancel?.(new DOMException('speculation dropped', 'AbortError'));
|
|
815
|
+
p.first.then(() => p.gen.return?.()).catch(() => {}); // abort the fetch AND close the iterator (custom llm impls may clean up in finally)
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
// Claim the prefetch for a closing SPOKEN turn: exact text match AND unchanged history base (the
|
|
819
|
+
// request was built on [...history, user]; if a raced turn appended anything since — e.g. the
|
|
820
|
+
// barged-in reply's assistant prefix committed while we awaited prev — the speculation is missing
|
|
821
|
+
// that context and must be regenerated). Hand over the already-streaming generator (re-stitched so
|
|
822
|
+
// the pre-pulled first chunk isn't lost) + its own AbortController, so barge-in aborts the right fetch.
|
|
823
|
+
_takePrefetch(text) {
|
|
824
|
+
const p = this._prefetch;
|
|
825
|
+
// histLen is checked BEFORE _runTurn pushes this turn's own user message (see call site).
|
|
826
|
+
if (!p || p.text !== text || p.ctl.signal.aborted || p.histLen !== this.history.length) { this._dropPrefetch(); return null; }
|
|
827
|
+
this._prefetch = null; // timer already cleared: successful adoption is only reached from a spoken onFinal
|
|
828
|
+
p.commit(); // turn committed → open the tool gate; held-back tool calls execute now
|
|
829
|
+
return { ctl: p.ctl, gen: (async function* () { const f = await p.first; if (!f.done) { yield f.value; yield* p.gen; } })() };
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
// The ONE serialized turn runner. Every reply (typed, spoken, held-release, notify) goes through
|
|
833
|
+
// here, so turns can NEVER overlap regardless of how fast callers fire. It chains onto the prior
|
|
834
|
+
// turn and reassigns `this._turn` SYNCHRONOUSLY before awaiting anything — so a second caller that
|
|
835
|
+
// arrives in the same tick chains onto THIS turn, not the one it's superseding.
|
|
836
|
+
//
|
|
837
|
+
// `text` (the new user utterance, or '' for a held-release that replies over already-queued history)
|
|
838
|
+
// is appended AFTER the predecessor settles — so its heard prefix lands first, keeping history
|
|
839
|
+
// chronological (…, assistant-heard-so-far, user-new). If a hold was entered while we waited, the
|
|
840
|
+
// utterance is routed into the held queue instead of dropped (acceptance must never be lost). The
|
|
841
|
+
// reply only runs when the latest history entry is a user turn. Never throws (errors -> onEvent).
|
|
842
|
+
//
|
|
843
|
+
// `supersede: false` = a SOFT turn (notify: tool results, board events): record it and reply only
|
|
844
|
+
// if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
|
|
845
|
+
// sees this message in history and answers over the fuller picture — so a burst of tool results
|
|
846
|
+
// collapses into ONE reply instead of one interrupting monologue per result.
|
|
847
|
+
_runTurn(text, { supersede = true, speech = false } = {}) {
|
|
848
|
+
const prev = this._turn;
|
|
849
|
+
if (supersede) {
|
|
850
|
+
this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
|
|
851
|
+
this.tts.stop?.();
|
|
852
|
+
}
|
|
853
|
+
// Reply-generating turns only. `this._turn` can't stand in for this: replay() chains onto it too,
|
|
854
|
+
// and a replay is pure TTS — it never answers pending history, so treating it as "someone newer
|
|
855
|
+
// owns the reply" would strand the turn (a held release or a tool result would go unanswered).
|
|
856
|
+
const seq = ++this._replySeq;
|
|
857
|
+
return (this._turn = (async () => {
|
|
858
|
+
await prev?.catch(() => {}); // let the predecessor settle (records its heard prefix)
|
|
859
|
+
if (this._closed) return; // stopped while we waited -> abandon (stop() dropped the prefetch)
|
|
860
|
+
// Held while we waited: don't drop an accepted utterance — queue it for the next release.
|
|
861
|
+
// (setHeld(true) already dropped any speculation.)
|
|
862
|
+
if (this._held) { if (text) this._pushHeld(text); return; }
|
|
863
|
+
// Superseded while we waited: the newer turn now owns the reply and sees our message in
|
|
864
|
+
// history — generating here too would talk over it (and re-run the same tools). Decided BEFORE
|
|
865
|
+
// claiming the prefetch, so we never adopt a stream just to abort it.
|
|
866
|
+
if (this._replySeq !== seq) { this._dropPrefetch(); if (text) this.history.push({ role: 'user', content: text }); return; }
|
|
867
|
+
// Prefetch handover — decided BEFORE pushing our user message so _takePrefetch can verify the
|
|
868
|
+
// history base is exactly what the speculation was built on (a raced turn that appended its
|
|
869
|
+
// assistant message while we awaited prev fails the histLen check → regenerate with context).
|
|
870
|
+
// Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
|
|
871
|
+
const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
|
|
872
|
+
if (text) this.history.push({ role: 'user', content: text });
|
|
873
|
+
// Reply only if the conversation now ends on a user turn (a raced turn may have already
|
|
874
|
+
// replied). pf is always null here: if we pushed a user turn it IS the last entry.
|
|
875
|
+
if (this.history[this.history.length - 1]?.role !== 'user') return;
|
|
876
|
+
this._set('thinking'); // LLM stage
|
|
877
|
+
this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
|
|
878
|
+
await this._speakTurn(this._abort.signal, pf?.gen); // _speakTurn never throws — errors surface via onEvent
|
|
879
|
+
})());
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// One assistant turn: stream the LLM → TTS, then record what was actually HEARD (the barge-in
|
|
883
|
+
// prefix, not the full generated answer) plus a ledger of the tools it dispatched. Stored as
|
|
884
|
+
// `this._turn` so the next user turn can await it. Never throws (errors surface via onEvent) so
|
|
885
|
+
// the await is safe — and never awaits a tool, so a hung one can't hold the next turn hostage.
|
|
886
|
+
async _speakTurn(signal, prefetched = null) {
|
|
887
|
+
// Fresh turn → drop the previous reply's audible text so user speech heard before THIS turn's
|
|
888
|
+
// first audio plays can't be misjudged as echo of the OLD reply — but PARK it in _prevReplyText:
|
|
889
|
+
// its speakers→mic echo tail is still in flight for a moment (acoustic delay + STT latency), and
|
|
890
|
+
// the grace-bounded scope in onPartial must still recognize it. The TTS progress hook below
|
|
891
|
+
// repopulates _replyText with the audible prefix as playback advances.
|
|
892
|
+
if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText;
|
|
893
|
+
this._replyText = ''; this._replyEcho = '';
|
|
894
|
+
// Tap the LLM stream once: yield speech text to TTS, and record the tool calls it makes (they
|
|
895
|
+
// fire on arrival; `calls` is the ledger). First delta flips us to 'speaking'; accumulate for the
|
|
896
|
+
// heard-vs-unheard report. `this.llm(...)` is a lazy generator — its fetch fires only when
|
|
897
|
+
// tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
|
|
898
|
+
let answer = '', calls = [];
|
|
899
|
+
// The tools this turn dispatched, as a ledger line recorded INTO its assistant message. The
|
|
900
|
+
// runtime's tool calls are fire-and-forget (no tool_use/tool_result pair on the wire, and history
|
|
901
|
+
// is provider-neutral {role, content} text), so without this the history reads as if it never
|
|
902
|
+
// called anything and the next turn happily calls it AGAIN — duplicate tool-call spam. Relayed
|
|
903
|
+
// calls: the outcome arrives separately as the host's `[TOOL RESULT <name>]` notify() turn.
|
|
904
|
+
// Custom LLMs that execute tools inside their own generator (llm.executesTools) attach the
|
|
905
|
+
// outcome directly, so the ledger line carries it — bounded — or the next turn would see the call with no result and lose what
|
|
906
|
+
// the model learned (a barge-in cuts the spoken follow-up but not the fact). Read late (the
|
|
907
|
+
// stream may still be appending) and on EVERY exit path, including a failed reply.
|
|
908
|
+
const toolNote = () => calls.map((c) => `[TOOL CALL ${c.tool}] ${callArgs(c)}${ledgerOutcome(c)}`).join('\n');
|
|
909
|
+
// Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
|
|
910
|
+
// announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
|
|
911
|
+
// that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
|
|
912
|
+
// outcome arrives finalizes it as interrupted — DISPLAY truth only: an announcement never reaches
|
|
913
|
+
// the ledger (`calls`), so the model treats it as never having happened.
|
|
914
|
+
const announced = new Map();
|
|
915
|
+
// Latched once this turn has finalized its chips (finishInterrupted ran). A barge-in detaches the
|
|
916
|
+
// LLM producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
|
|
917
|
+
// announcement would open a spinner nothing will ever close. Past the latch, announcements are
|
|
918
|
+
// dropped — outcomes still flow (a tool the adapter really ran is truth worth reporting).
|
|
919
|
+
let finalized = false;
|
|
920
|
+
const finishInterrupted = () => {
|
|
921
|
+
finalized = true;
|
|
922
|
+
for (const r of announced.values())
|
|
923
|
+
this.onEvent({ type: 'tool', name: r.tool, id: r.id, args: r.args, result: { ok: false, text: 'interrupted — did not finish' } });
|
|
924
|
+
announced.clear();
|
|
925
|
+
};
|
|
926
|
+
const tapped = (async function* (self, src) {
|
|
927
|
+
for await (const item of src) {
|
|
928
|
+
// tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
|
|
929
|
+
// act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
|
|
930
|
+
if (item.tool) {
|
|
931
|
+
if (item.running) { // display-only announcement; the outcome chunk below does the work
|
|
932
|
+
// An id-less announcement can never be replaced in place NOR finalized (both key on id),
|
|
933
|
+
// so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
|
|
934
|
+
if (finalized || !item.id) continue;
|
|
935
|
+
// Already settled this turn (adapter re-emitting its buffered pair): re-announcing would
|
|
936
|
+
// reopen a spinner over a finished chip, and the duplicate outcome is deduped below and
|
|
937
|
+
// never re-reported — so it would hang there, then finalize as falsely "interrupted".
|
|
938
|
+
if (calls.some((c) => c.id === item.id)) continue;
|
|
939
|
+
announced.set(item.id, item);
|
|
940
|
+
self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
|
|
941
|
+
continue;
|
|
942
|
+
}
|
|
943
|
+
if (item.id) announced.delete(item.id);
|
|
944
|
+
// Same key = same act (provider re-emit / model asking twice). A MIMIC chunk carries no id,
|
|
945
|
+
// so also match by name+args across the id boundary — a typed echo of an id-carrying native
|
|
946
|
+
// call (llm.executesTools mode) must collapse too, or it would execute the tool a second time.
|
|
947
|
+
const dup = calls.some((c) => callKey(c) === callKey(item) ||
|
|
948
|
+
((!c.id || !item.id) && c.tool === item.tool && callArgs(c) === callArgs(item)));
|
|
949
|
+
if (!dup) { calls.push(item); self._runTool(item); }
|
|
950
|
+
continue;
|
|
951
|
+
}
|
|
952
|
+
if (self.state === 'thinking') self._set('speaking'); // first token → TTS stage
|
|
953
|
+
answer += item.text;
|
|
954
|
+
// While audio plays, the TTS progress hook owns the draft (text = spoken-so-far, full = answer,
|
|
955
|
+
// read live each tick), so streaming the whole answer here would yank the cursor to the end.
|
|
956
|
+
// Once the cursor is live we only grow `answer` (the hook reads it); until then (muted, or the
|
|
957
|
+
// first sentence is still synthesizing) we stream the full answer so the bubble isn't empty.
|
|
958
|
+
if (!self._cursorLive) self.onEvent({ type: 'assistant', text: answer, final: false });
|
|
959
|
+
yield item.text;
|
|
960
|
+
}
|
|
961
|
+
})(this, stripToolCallMimicry(prefetched ?? this.llm(this.history, this.sysmsg, signal))); // prefetched stream already carries this turn's user text (see _startPrefetch)
|
|
962
|
+
|
|
963
|
+
let spoken = '';
|
|
964
|
+
// Live spoken cursor: while a clip plays the TTS reports chars heard so far, which we mirror as a
|
|
965
|
+
// streaming assistant draft — `text` = spoken-so-far, `full` = whole answer, so the host renders the
|
|
966
|
+
// not-yet-spoken remainder dimmed and the cursor follows the voice. Cleared (finally) after the turn.
|
|
967
|
+
// The TTS reports spoken text as normalized sentence chunks (trimmed, space-joined), which is NOT a
|
|
968
|
+
// raw prefix of `answer` (the LLM text keeps newlines/double spaces). Map it back by non-whitespace
|
|
969
|
+
// count: walk `answer` until it has the same number of non-whitespace chars as what's been spoken,
|
|
970
|
+
// so `text` is always an exact prefix of `full` → the UI's full.slice(text.length) tail is correct.
|
|
971
|
+
const rawPrefix = (spoken) => {
|
|
972
|
+
let need = 0; for (const c of spoken) if (!/\s/.test(c)) need++;
|
|
973
|
+
if (!need) return '';
|
|
974
|
+
let seen = 0, i = 0;
|
|
975
|
+
for (; i < answer.length; i++) { if (!/\s/.test(answer[i])) { if (++seen === need) { i++; break; } } }
|
|
976
|
+
return answer.slice(0, i);
|
|
977
|
+
};
|
|
978
|
+
this._cursorLive = false;
|
|
979
|
+
this.tts.setOnProgress?.((spokenSoFar, scope) => {
|
|
980
|
+
if (signal.aborted) return;
|
|
981
|
+
this._cursorLive = true; // hook now drives the draft → LLM stream stops emitting full text
|
|
982
|
+
const heard = rawPrefix(spokenSoFar);
|
|
983
|
+
this._replyText = heard; // AUDIBLE prefix — drives the spoken cursor
|
|
984
|
+
this._replyEcho = scope ? rawPrefix(scope) : heard; // clip-bounded echo reference (cursor lags real audio)
|
|
985
|
+
this.onEvent({ type: 'assistant', text: heard, full: answer, final: false });
|
|
986
|
+
});
|
|
987
|
+
try {
|
|
988
|
+
// TTS muted: drain the same stream so the transcript + tool calls still run, but never voice
|
|
989
|
+
// it. The whole reply counts as "heard" (history) since there's no barge-in without audio.
|
|
990
|
+
if (this._ttsMuted) {
|
|
991
|
+
for await (const _ of tapped) { /* consume: drives answer + assistant events */ }
|
|
992
|
+
spoken = answer;
|
|
993
|
+
} else {
|
|
994
|
+
spoken = await this.tts.speak(tapped, signal); // resolves with text actually heard (prefix on barge-in)
|
|
995
|
+
}
|
|
996
|
+
} catch (e) {
|
|
997
|
+
if (e.name !== 'AbortError') { // real failure (not barge-in) → surface it
|
|
998
|
+
this.tts.setOnProgress?.(null); this._cursorLive = false;
|
|
999
|
+
this.onEvent({ type: 'error', error: e.message });
|
|
1000
|
+
// Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
|
|
1001
|
+
// Record it anyway: the ledger is about what was DONE, not about what got spoken — dropping
|
|
1002
|
+
// it would let the next turn call the same tool again.
|
|
1003
|
+
if (toolNote()) this.history.push({ role: 'assistant', content: toolNote() });
|
|
1004
|
+
finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
|
|
1005
|
+
if (this.state !== 'idle') this._set('listening');
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
// barge-in surfaced as AbortError → fall through and record whatever was heard
|
|
1009
|
+
} finally {
|
|
1010
|
+
this.tts.setOnProgress?.(null); this._cursorLive = false; // drop the cursor hook — next turn re-registers
|
|
1011
|
+
// Clamp the echo reference back to what was actually HEARD: the clip-bounded upper bound
|
|
1012
|
+
// (_replyEcho) exists only because a PLAYING clip is ahead of the cursor — once playback
|
|
1013
|
+
// stops (finished or barged-in), the unspoken remainder never hit the speakers, and keeping
|
|
1014
|
+
// it in scope through the grace window would swallow a user quoting those exact words.
|
|
1015
|
+
this._replyEcho = spoken ? rawPrefix(spoken) : this._replyText;
|
|
1016
|
+
this._replyDoneAt = Date.now(); // start the post-playout echo grace window
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
// Normally we record only the HEARD prefix (barge-in: the user reacted to what they heard). But a
|
|
1021
|
+
// turn aborted on hold-entry was tagged signal.keepFull, so we keep the whole streamed answer —
|
|
1022
|
+
// on release the LLM sees everything it had formed. Per-turn tag → a quick Resume can't flip it.
|
|
1023
|
+
const kept = signal.keepFull ? answer : spoken;
|
|
1024
|
+
if (kept || toolNote()) {
|
|
1025
|
+
this.history.push({ role: 'assistant', content: [kept, toolNote()].filter(Boolean).join('\n') });
|
|
1026
|
+
}
|
|
1027
|
+
if (kept) {
|
|
1028
|
+
// History keeps the normalized `kept`; the EVENT emits a raw prefix of `answer` so the host's
|
|
1029
|
+
// full.slice(text.length) tail aligns. keepFull/normal-finish → whole answer (no tail); a barge-in
|
|
1030
|
+
// → the heard prefix solid + the unspoken remainder dimmed, showing exactly where TTS got.
|
|
1031
|
+
const shown = signal.keepFull ? answer : rawPrefix(spoken);
|
|
1032
|
+
this.onEvent({ type: 'assistant', text: shown, full: answer, final: true });
|
|
1033
|
+
} else if (answer) {
|
|
1034
|
+
// Interrupted before anything was heard (nothing voiced yet): drop the streamed draft so the
|
|
1035
|
+
// transcript doesn't keep a dangling unfinished bubble. Empty text → host clears the draft.
|
|
1036
|
+
this.onEvent({ type: 'assistant', text: '', final: false });
|
|
1037
|
+
}
|
|
1038
|
+
// Tools are NOT joined here. They were dispatched during streaming (in parallel with TTS) and
|
|
1039
|
+
// _runTool reports its own outcome via onEvent, so waiting adds nothing — while a tool that
|
|
1040
|
+
// never settles would wedge the turn serializer, and every turn queued behind it, for good. It
|
|
1041
|
+
// also breaks a real deadlock: a tool whose run() awaits agent.notify() (delivering its result
|
|
1042
|
+
// as a turn) would wait on the turn that waits on it. Slow work belongs in a tool that returns
|
|
1043
|
+
// promptly and reports the outcome later via notify().
|
|
1044
|
+
finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
|
|
1045
|
+
if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
// Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
|
|
1049
|
+
// parallel with TTS, and NEVER joined: the turn records the dispatch in its ledger and finishes
|
|
1050
|
+
// when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
|
|
1051
|
+
// else. Its outcome reaches the host via onEvent; the model learns it via notify(). Must not
|
|
1052
|
+
// reject — nothing awaits it, so a rejection here would be unhandled (hence the catch-all).
|
|
1053
|
+
// `call.ran` memoizes: the same call object reaching _runTool twice must not execute twice.
|
|
1054
|
+
_runTool(call) {
|
|
1055
|
+
if (!call) return;
|
|
1056
|
+
return (call.ran ??= (async () => {
|
|
1057
|
+
try {
|
|
1058
|
+
// Pre-executed call (agent loop ran it inside the reply): report the attached outcome,
|
|
1059
|
+
// never execute a second time. An in-loop failure was already fed back to the model as an
|
|
1060
|
+
// is_error tool_result and the model narrates it — here it surfaces as ONE failed tool
|
|
1061
|
+
// event (the transcript's failed chip). No extra 'error' event: that channel is for
|
|
1062
|
+
// runtime/transport failures, not a handled domain-level tool miss.
|
|
1063
|
+
// `id` rides along on every tool event so a host can replace THIS call's running chip in
|
|
1064
|
+
// place, and two same-args calls (a legit retry) stay distinct acts in the transcript.
|
|
1065
|
+
if ('result' in call || 'error' in call) {
|
|
1066
|
+
const result = call.error ? { text: call.error, ok: false } : call.result;
|
|
1067
|
+
this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1070
|
+
// Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
|
|
1071
|
+
// a deadline could only stop WAITING — reporting "failed" for an operation still on its way
|
|
1072
|
+
// to committing its side effect, inviting a duplicate retry. Nothing awaits this call
|
|
1073
|
+
// (see _speakTurn), so a hung tool costs its own result, never the turn loop.
|
|
1074
|
+
const result = await this.tools[call.tool]?.run?.(call.args);
|
|
1075
|
+
this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
|
|
1076
|
+
} catch (e) {
|
|
1077
|
+
this.onEvent({ type: 'error', error: `tool ${call.tool}: ${e.message}` });
|
|
1078
|
+
}
|
|
1079
|
+
})());
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
_onBargeIn() { // user spoke while we were talking
|
|
1083
|
+
this._abort?.abort(); // stop LLM
|
|
1084
|
+
this.tts.stop?.(); // stop TTS (tts.speak resolves with the heard prefix)
|
|
1085
|
+
this._set('listening');
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
// VAD heard end-of-speech. The detector decides if the turn is truly OVER (commit) or the user just
|
|
1089
|
+
// paused mid-thought (keep listening). Without a detector this is plain VAD endpointing: always commit.
|
|
1090
|
+
// Guarded by a generation token so a late async verdict from a superseded pause can't commit a turn
|
|
1091
|
+
// the user already continued (a new onSpeechStart bumps the token).
|
|
1092
|
+
async _endOfSpeech() {
|
|
1093
|
+
clearTimeout(this._maxPause);
|
|
1094
|
+
// Native end-of-turn provider (e.g. Deepgram Flux): the STT's own model decides when the turn is
|
|
1095
|
+
// over and emits an unsolicited onFinal — VAD end-of-speech is NOT a boundary here (the provider
|
|
1096
|
+
// must hear the silence to judge it), so don't commit and don't run the turnDetector. No agent-side
|
|
1097
|
+
// failsafe either: Flux transcripts are CUMULATIVE per turn, so force-flushing a still-open turn
|
|
1098
|
+
// emits its prefix, which the next flush then repeats ("it is it is …"). Turn-hang protection is
|
|
1099
|
+
// the provider's own eot_timeout_ms (server force-ends the turn after ~5s of silence).
|
|
1100
|
+
if (this.stt.nativeEOT) return;
|
|
1101
|
+
// _close: mark the turn as CLOSING (commit() starts ForceEndOfUtterance) but DON'T clear _turnText —
|
|
1102
|
+
// onFinal clears it when the final actually lands. This keeps a "closing" turn distinct from a fresh
|
|
1103
|
+
// idle state, so a fast next onSpeechStart won't reset() the STT mid-finalization (lost/merged turn).
|
|
1104
|
+
const close = () => { this._turnClosing = true; this.stt.commit(); };
|
|
1105
|
+
if (!this.turnDetector || !this._turnText) { close(); return; }
|
|
1106
|
+
const gen = this._turnGen; // snapshot; a new onSpeechStart bumps it → our verdict goes stale
|
|
1107
|
+
let done = true;
|
|
1108
|
+
// maxPauseMs is ONE absolute deadline from end-of-speech — the wall-clock cap on how long a
|
|
1109
|
+
// detector may hold this turn open, whether it answers "not yet" or never answers at all. Both
|
|
1110
|
+
// legs below draw from the same budget, so a detector that deliberates 3.9s can't then buy a
|
|
1111
|
+
// fresh 4s failsafe on top (which would blow the documented cap to nearly 2×).
|
|
1112
|
+
// The verdict leg needs a deadline of its own because it's HOST code on the only path that can
|
|
1113
|
+
// close the turn: a hang there never reaches the failsafe, and the turn stays open forever.
|
|
1114
|
+
const deadline = Date.now() + this.maxPauseMs;
|
|
1115
|
+
const left = () => Math.max(0, deadline - Date.now());
|
|
1116
|
+
const HUNG = Symbol('hung');
|
|
1117
|
+
try { done = await withTimeout(this.turnDetector(this._turnText), left(), HUNG); }
|
|
1118
|
+
catch (e) { this.onEvent({ type: 'error', error: `turnDetector: ${e.message}` }); }
|
|
1119
|
+
if (gen !== this._turnGen) return; // user resumed speaking while we judged → verdict is stale
|
|
1120
|
+
if (this._speaking) return; // already talking again → let the next end-of-speech decide
|
|
1121
|
+
// Report the hang only once it's established the verdict still MATTERED (stale checks above):
|
|
1122
|
+
// a detector abandoned because the user resumed isn't an error worth surfacing.
|
|
1123
|
+
if (done === HUNG) { this.onEvent({ type: 'error', error: `turnDetector: no verdict in ${this.maxPauseMs}ms — closing turn` }); done = true; }
|
|
1124
|
+
if (done) { close(); return; }
|
|
1125
|
+
// Keep listening: the pause was mid-thought. Failsafe so a turn the detector never closes (user
|
|
1126
|
+
// walked away mid-sentence) can't hang forever — commit anyway at the deadline.
|
|
1127
|
+
this._maxPause = setTimeout(() => { if (gen === this._turnGen && !this._speaking && this._turnText) close(); }, left());
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
// ── VAD ───────────────────────────────────────────────────────────────
|
|
1131
|
+
// Shared speech-start/end handling: called by BOTH the bootstrap energy-VAD (_bootstrapVadTick)
|
|
1132
|
+
// and Silero (onSpeechStart/onSpeechEnd below) — a detector swap never changes turn semantics.
|
|
1133
|
+
_onSpeechStart() {
|
|
1134
|
+
this._speaking = true; this._turnGen = (this._turnGen || 0) + 1; clearTimeout(this._maxPause);
|
|
1135
|
+
if (!this._turnText && !this._turnClosing) this.stt.reset?.();
|
|
1136
|
+
this.onEvent({ type: 'vad', active: true });
|
|
1137
|
+
}
|
|
1138
|
+
_onSpeechEnd() { this._speaking = false; this.onEvent({ type: 'vad', active: false }); this._endOfSpeech(); }
|
|
1139
|
+
|
|
1140
|
+
async _startVad(gen) {
|
|
1141
|
+
// Self-serve the UMD deps (onnxruntime + vad-web) if the host never called prewarmVoice() —
|
|
1142
|
+
// prewarming is an optimization, not a setup requirement.
|
|
1143
|
+
if (!window.vad) { const { loadVoiceDeps } = await import('./deps.js'); await loadVoiceDeps(); }
|
|
1144
|
+
const d = window.vad;
|
|
1145
|
+
const vad = await d.MicVAD.new({
|
|
1146
|
+
model: 'v5',
|
|
1147
|
+
// MicVAD.new() auto-starts listening itself when startOnLoad (default true) — we need it INERT
|
|
1148
|
+
// until we explicitly .start() it below (either right away, or later at a safe utterance
|
|
1149
|
+
// boundary via _swapToSileroWhenReady/_bootstrapVadTick) so it never runs concurrently with the
|
|
1150
|
+
// bootstrap energy-VAD.
|
|
1151
|
+
startOnLoad: false,
|
|
1152
|
+
// Listen on the SAME stream we opened in start() (the chosen mic) — not a second
|
|
1153
|
+
// default-mic getUserMedia. Keeps VAD and STT on one input, and one permission prompt.
|
|
1154
|
+
getStream: async () => this._stream,
|
|
1155
|
+
// Same asset paths warmVad() pre-fetched/compiled, so this reuses the cached v5 model + WASM.
|
|
1156
|
+
onnxWASMBasePath: VAD_WASM_PATH,
|
|
1157
|
+
baseAssetPath: VAD_ASSET_PATH,
|
|
1158
|
+
...TUNING.VAD, // speech-detection sensitivity (positive/negativeSpeechThreshold, minSpeechFrames, redemptionFrames, preSpeechPadFrames) — see tuning.js
|
|
1159
|
+
...this.vadOptions, // host tuning overrides the defaults above
|
|
1160
|
+
// NB: barge-in is NOT triggered here anymore — VAD energy alone (the AI's own echo, noise) must not
|
|
1161
|
+
// cut a reply; the first TRANSCRIBED word does, in onPartial. The uplink STT runs through the reply.
|
|
1162
|
+
onSpeechStart: () => this._onSpeechStart(),
|
|
1163
|
+
onSpeechEnd: () => this._onSpeechEnd(),
|
|
1164
|
+
});
|
|
1165
|
+
// A stale build (the pipeline was superseded/torn down while MicVAD.new() resolved) must not
|
|
1166
|
+
// stomp the current session's _vad — destroy its own instance and bail (the caller's .then()
|
|
1167
|
+
// gen-guard skips the swap too; this guard covers the assignment itself).
|
|
1168
|
+
if (gen !== undefined && (gen !== this._pipelineGen || this._closed)) { vad.destroy?.(); throw new Error('superseded'); }
|
|
1169
|
+
this._vad = vad;
|
|
1170
|
+
// Only actually start listening on Silero right away if the bootstrap VAD has already been
|
|
1171
|
+
// swapped out (or never engaged) — otherwise _swapToSileroWhenReady (called after this resolves)
|
|
1172
|
+
// arms it at the next safe utterance boundary instead of running both detectors at once.
|
|
1173
|
+
if (!this._useBootstrap) this._vad.start();
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
// Silero finished loading (this._vad is ready) while the bootstrap energy-VAD was standing in.
|
|
1177
|
+
// Swap over ONLY at a safe boundary — never mid-utterance, so we don't yank the detector while the
|
|
1178
|
+
// user is mid-sentence (which could double-fire or miss the true end-of-speech). If speech is in
|
|
1179
|
+
// progress right now, defer: _onSpeechEnd() (bootstrap path) checks _sileroPending and swaps once
|
|
1180
|
+
// the current utterance closes.
|
|
1181
|
+
_swapToSileroWhenReady() {
|
|
1182
|
+
this._sileroPending = false;
|
|
1183
|
+
if (this._speaking) return; // mid-utterance — _bootstrapVadTick's speech-end path swaps after
|
|
1184
|
+
this._useBootstrap = false; this._bsSpeechFrames = 0; this._bsSilenceFrames = 0;
|
|
1185
|
+
this._vad?.start();
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
// Zero-load RMS-energy stand-in for Silero, active only during `this._useBootstrap` (cold-start
|
|
1189
|
+
// window before Silero has loaded). Deliberately crude — it only needs to survive the first
|
|
1190
|
+
// second(s) of a session without losing/chopping the user's opening words; see tuning.js
|
|
1191
|
+
// BOOTSTRAP_VAD for the thresholds. Ticks once per ~256ms worklet capture chunk (_feed below).
|
|
1192
|
+
_bootstrapVadTick(f32) {
|
|
1193
|
+
const cfg = TUNING.BOOTSTRAP_VAD;
|
|
1194
|
+
let sum = 0; for (let i = 0; i < f32.length; i++) sum += f32[i] * f32[i];
|
|
1195
|
+
const rms = Math.sqrt(sum / f32.length);
|
|
1196
|
+
if (rms >= cfg.energyThreshold) {
|
|
1197
|
+
this._bsSilenceFrames = 0;
|
|
1198
|
+
if (!this._speaking && ++this._bsSpeechFrames >= cfg.minSpeechFrames) this._onSpeechStart();
|
|
1199
|
+
} else {
|
|
1200
|
+
this._bsSpeechFrames = 0;
|
|
1201
|
+
if (this._speaking && ++this._bsSilenceFrames >= cfg.redemptionFrames) {
|
|
1202
|
+
this._bsSilenceFrames = 0;
|
|
1203
|
+
this._onSpeechEnd();
|
|
1204
|
+
// The utterance that was masking the swap (see _swapToSileroWhenReady) just closed — hand
|
|
1205
|
+
// off to Silero now, before the NEXT one starts, instead of waiting for another tick.
|
|
1206
|
+
if (!this._sileroPending && this._vad) { this._useBootstrap = false; this._vad.start(); }
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
// ── STT (VAD-gated) — capture mic, gate on VAD, forward to the active provider ──────────
|
|
1212
|
+
// The provider (this.stt, chosen by `sttProvider`) owns its socket + wire protocol and reports
|
|
1213
|
+
// transcripts via the callbacks wired in the constructor. Here we only convert float→int16,
|
|
1214
|
+
// keep a small preroll so word-starts aren't clipped, and feed/commit on VAD boundaries.
|
|
1215
|
+
_feed(f32) {
|
|
1216
|
+
if (this._useBootstrap) this._bootstrapVadTick(f32);
|
|
1217
|
+
const i16 = new Int16Array(f32.length);
|
|
1218
|
+
for (let i = 0; i < f32.length; i++) i16[i] = Math.max(-1, Math.min(1, f32[i])) * 32767;
|
|
1219
|
+
this._tapPush(i16); // diagnostic ring buffer of the exact frames handed to STT (see dumpAudio)
|
|
1220
|
+
// Continuous provider (e.g. Deepgram Flux): its end-of-turn model needs to hear the real
|
|
1221
|
+
// silences, so feed the FULL stream — no VAD gating, no preroll replay. VAD still runs above
|
|
1222
|
+
// (state events + speech-start reset + the nativeEOT failsafe), it just doesn't gate the audio.
|
|
1223
|
+
if (this.stt.continuous) { this.stt.feed(i16); return; }
|
|
1224
|
+
if (!this._speaking) { this._preroll.push(i16); if (this._preroll.length > this.prerollMax) this._preroll.shift(); this._wasSpeaking = false; return; }
|
|
1225
|
+
if (!this._wasSpeaking) {
|
|
1226
|
+
for (const c of this._preroll) this.stt.feed(c); this._preroll = []; this._wasSpeaking = true;
|
|
1227
|
+
}
|
|
1228
|
+
this.stt.feed(i16);
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
// ── diagnostic tap ─────────────────────────────────────────────────────
|
|
1232
|
+
// Ring-buffer the frame + detect delivery stalls: a wall-clock gap between _feed() calls beyond
|
|
1233
|
+
// ~1.5 chunks. With the worklet capturing off-thread that's usually DELAYED delivery rather than
|
|
1234
|
+
// lost samples, but it timestamps exactly where a corrupted run got starved. atSample is an
|
|
1235
|
+
// ABSOLUTE stream offset (samples since pipeline build), so gaps stay addressable after trimming.
|
|
1236
|
+
_tapPush(i16) {
|
|
1237
|
+
const now = (globalThis.performance ?? Date).now();
|
|
1238
|
+
const chunkMs = (i16.length / 16000) * 1000;
|
|
1239
|
+
if (this._tapLastMs && now - this._tapLastMs > chunkMs * 1.5) {
|
|
1240
|
+
this._tapGaps.push({ atSample: this._tapTotal, gapMs: Math.round(now - this._tapLastMs) });
|
|
1241
|
+
}
|
|
1242
|
+
this._tapLastMs = now;
|
|
1243
|
+
this._tap.push(i16); this._tapSamples += i16.length; this._tapTotal += i16.length;
|
|
1244
|
+
const max = 16000 * 30;
|
|
1245
|
+
while (this._tapSamples - this._tap[0].length >= max) this._tapSamples -= this._tap.shift().length;
|
|
1246
|
+
// Keep only gaps still inside (or near) the retained window so the list can't grow unbounded.
|
|
1247
|
+
const windowStart = this._tapTotal - this._tapSamples;
|
|
1248
|
+
if (this._tapGaps.length && this._tapGaps[0].atSample < windowStart) this._tapGaps = this._tapGaps.filter(g => g.atSample >= windowStart);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
// Dump the tap as a 16kHz mono WAV Blob + detected stall gaps: { wav, gaps, seconds }. Play it back
|
|
1252
|
+
// after a failed run — gapped/corrupt audio → capture layer; clean audio → provider-side.
|
|
1253
|
+
dumpAudio() {
|
|
1254
|
+
const total = this._tapSamples;
|
|
1255
|
+
const windowStart = this._tapTotal - total; // absolute offset of the WAV's first sample
|
|
1256
|
+
const pcm = new Int16Array(total);
|
|
1257
|
+
let off = 0; for (const c of this._tap) { pcm.set(c, off); off += c.length; }
|
|
1258
|
+
const bytes = pcm.length * 2, buf = new ArrayBuffer(44 + bytes), dv = new DataView(buf);
|
|
1259
|
+
const str = (o, s) => { for (let i = 0; i < s.length; i++) dv.setUint8(o + i, s.charCodeAt(i)); };
|
|
1260
|
+
str(0, 'RIFF'); dv.setUint32(4, 36 + bytes, true); str(8, 'WAVE'); str(12, 'fmt ');
|
|
1261
|
+
dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, 1, true);
|
|
1262
|
+
dv.setUint32(24, 16000, true); dv.setUint32(28, 16000 * 2, true); dv.setUint16(32, 2, true); dv.setUint16(34, 16, true);
|
|
1263
|
+
str(36, 'data'); dv.setUint32(40, bytes, true);
|
|
1264
|
+
for (let i = 0; i < pcm.length; i++) dv.setInt16(44 + i * 2, pcm[i], true);
|
|
1265
|
+
// Gaps re-based to WAV-relative sample offsets (atSec for quick seeking during playback).
|
|
1266
|
+
const gaps = this._tapGaps.filter(g => g.atSample >= windowStart)
|
|
1267
|
+
.map(g => ({ atSample: g.atSample - windowStart, atSec: (g.atSample - windowStart) / 16000, gapMs: g.gapMs }));
|
|
1268
|
+
return { wav: new Blob([buf], { type: 'audio/wav' }), gaps, seconds: total / 16000 };
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
|
|
1273
|
+
// Strip markdown to plain prose for the EAR: the persona prompt already forbids markdown, so this is
|
|
1274
|
+
// a safety net — if formatting ever slips into a reply, the synthesizer must not voice the syntax
|
|
1275
|
+
// ("star star", "hash", "backtick"). We only feed this to _synth; the tape/cursor keep the RAW text
|
|
1276
|
+
// so tap-to-seek alignment is untouched. Drops fences/inline-code, image/link syntax (keeping link
|
|
1277
|
+
// text), heading/quote/list markers, and emphasis/strikethrough runs.
|
|
1278
|
+
const stripMd = s => s
|
|
1279
|
+
.replace(/```[^\n]*\n?([\s\S]*?)```/g, '$1') // fenced code -> its contents
|
|
1280
|
+
.replace(/`([^`]+)`/g, '$1') // inline code
|
|
1281
|
+
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1') // images -> alt text
|
|
1282
|
+
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') // links -> link text
|
|
1283
|
+
.replace(/^\s{0,3}#{1,6}\s+/gm, '') // ATX headings
|
|
1284
|
+
.replace(/^\s{0,3}>\s?/gm, '') // blockquotes
|
|
1285
|
+
.replace(/^\s*([-*+]|\d+[.)])\s+/gm, '') // list markers
|
|
1286
|
+
.replace(/(\*\*\*|___)(.+?)\1/g, '$2') // bold+italic
|
|
1287
|
+
.replace(/(\*\*|__)(.+?)\1/g, '$2') // bold
|
|
1288
|
+
.replace(/(\*|_)(.+?)\1/g, '$2') // italic
|
|
1289
|
+
.replace(/~~(.+?)~~/g, '$1') // strikethrough
|
|
1290
|
+
// Streaming splits a reply into word/clause chunks BEFORE stripMd runs, so an emphasis span can
|
|
1291
|
+
// straddle two chunks, leaving one chunk with an UNPAIRED marker the paired rules above can't match.
|
|
1292
|
+
// Sweep any stray emphasis/code markers so the synth never voices "star"/"backtick".
|
|
1293
|
+
.replace(/[*_~`]/g, '')
|
|
1294
|
+
.trim();
|
|
1295
|
+
|
|
1296
|
+
// ── Self-echo detection ─────────────────────────────────────────────────────────────────────────
|
|
1297
|
+
// Is `heard` (transcribed while the agent speaks) just its own voice leaking speakers → mic? We know
|
|
1298
|
+
// the audible reply prefix, so word-match `heard`'s RECENT tail (last `window` words — old echo
|
|
1299
|
+
// followed by a fresh real interruption must still cut in) against the reply's word set: mostly
|
|
1300
|
+
// reply words → echo. Word-set based (unordered, unicode-aware for accented text): tolerant of
|
|
1301
|
+
// STT dropping/reordering echoed words, NOT of substituting them — a badly misheard echo can slip
|
|
1302
|
+
// through (acceptable; it just means the old behavior for that turn). `minHits` = minimum matched
|
|
1303
|
+
// words required (the barge-in suppression check is lenient at 1; the final-drop check demands 3,
|
|
1304
|
+
// so a user briefly QUOTING the reply — "pricing?" — is never swallowed as echo).
|
|
1305
|
+
// Threshold TUNING.ECHO_WORD_MATCH; > 1 disables the filter.
|
|
1306
|
+
// ECHO_GRACE_MS: STT lags the audio, so the tail of the echo can be transcribed just AFTER playback
|
|
1307
|
+
// ended — keep classifying against the finished reply for this long past _replyDoneAt.
|
|
1308
|
+
const ECHO_GRACE_MS = 2000;
|
|
1309
|
+
const wordsOf = (s) => (s.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? []);
|
|
1310
|
+
// Bounded Levenshtein — true if editDistance(a,b) ≤ max. Early-exits a row whose minimum already
|
|
1311
|
+
// exceeds max, so the cost stays tiny on the short words we compare.
|
|
1312
|
+
const editLe = (a, b, max) => {
|
|
1313
|
+
if (a === b) return true;
|
|
1314
|
+
if (Math.abs(a.length - b.length) > max) return false;
|
|
1315
|
+
let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
|
|
1316
|
+
for (let i = 1; i <= a.length; i++) {
|
|
1317
|
+
const cur = [i]; let best = i;
|
|
1318
|
+
for (let j = 1; j <= b.length; j++) {
|
|
1319
|
+
cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
|
|
1320
|
+
if (cur[j] < best) best = cur[j];
|
|
1321
|
+
}
|
|
1322
|
+
if (best > max) return false;
|
|
1323
|
+
prev = cur;
|
|
1324
|
+
}
|
|
1325
|
+
return prev[b.length] <= max;
|
|
1326
|
+
};
|
|
1327
|
+
// FUZZY word hit: STT frequently MIS-hears the synthetic TTS voice ("sure i will create" heard as
|
|
1328
|
+
// "shure i wil kreate"), and an exact word-set match then scores the echo as novel speech — the
|
|
1329
|
+
// filter's main failure mode. A heard word counts as a reply word if it matches exactly OR within an
|
|
1330
|
+
// edit distance scaled by the LONGER of the pair (≥4 chars → 1 edit, ≥8 → 2) — so "wil" still
|
|
1331
|
+
// matches "will", while tiny words ("no" vs "now": longer side 3) must match exactly, or real
|
|
1332
|
+
// interruptions would get swallowed.
|
|
1333
|
+
const fuzzyHas = (replyWords, w) => {
|
|
1334
|
+
for (const r of replyWords) {
|
|
1335
|
+
if (r === w) return true;
|
|
1336
|
+
const len = Math.max(w.length, r.length);
|
|
1337
|
+
const max = len >= 8 ? 2 : len >= 4 ? 1 : 0;
|
|
1338
|
+
if (max && editLe(w, r, max)) return true;
|
|
1339
|
+
}
|
|
1340
|
+
return false;
|
|
1341
|
+
};
|
|
1342
|
+
export function isSelfEcho(heard, reply, { threshold = TUNING.ECHO_WORD_MATCH, window = 8, minHits = 1 } = {}) {
|
|
1343
|
+
if (!(threshold <= 1) || !reply) return false;
|
|
1344
|
+
const hw = window === Infinity ? wordsOf(heard) : wordsOf(heard).slice(-window);
|
|
1345
|
+
if (!hw.length) return false;
|
|
1346
|
+
const rw = wordsOf(reply);
|
|
1347
|
+
let hit = 0; for (const w of hw) if (fuzzyHas(rw, w)) hit++;
|
|
1348
|
+
return hit >= minHits && hit / hw.length >= threshold;
|
|
1349
|
+
}
|
|
1350
|
+
// Chars of `heard` NOT attributable (fuzzily) to `reply` — the barge-in evidence. While the agent
|
|
1351
|
+
// speaks, only these count toward the barge-in threshold: echo words (even misheard ones, via the
|
|
1352
|
+
// fuzzy match) contribute ZERO, so a leaked echo that dodged the ratio-based latch still can't
|
|
1353
|
+
// accumulate enough "user speech" to cut the reply. Real interruptions are made of words the reply
|
|
1354
|
+
// doesn't contain, so they hit the threshold as fast as before. No reply text → everything is novel.
|
|
1355
|
+
export function novelChars(heard, reply) {
|
|
1356
|
+
const rw = wordsOf(reply ?? '');
|
|
1357
|
+
let n = 0;
|
|
1358
|
+
for (const w of wordsOf(heard)) if (!rw.length || !fuzzyHas(rw, w)) n += w.length;
|
|
1359
|
+
return n;
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
// Count non-whitespace chars — the whitespace-invariant position key shared by the cursor and seek().
|
|
1363
|
+
const nw = s => { let n = 0; for (const c of s) if (!/\s/.test(c)) n++; return n; };
|
|
1364
|
+
// Prefix of `s` holding exactly `n` non-whitespace chars (keeps the original whitespace) — the inverse
|
|
1365
|
+
// of nw(), used to turn a non-whitespace cursor position back into a real text slice.
|
|
1366
|
+
const sliceNw = (s, n) => { if (n <= 0) return ''; let seen = 0, i = 0; for (; i < s.length; i++) { if (!/\s/.test(s[i])) { if (++seen === n) { i++; break; } } } return s.slice(0, i); };
|
|
1367
|
+
|
|
1368
|
+
// Earliest break to end the FIRST spoken chunk — minimizing first-audio latency (we don't wait for a
|
|
1369
|
+
// whole sentence to start talking; voice replies are often ONE sentence, so a sentence-only break would
|
|
1370
|
+
// stall TTS until generation finishes). Priority:
|
|
1371
|
+
// 1. a real sentence ender (. ! ? …) or a dash/colon clause break → break immediately;
|
|
1372
|
+
// 2. else, once we have >= firstChunkMinChars, the first comma/semicolon clause pause → break there;
|
|
1373
|
+
// 3. else, if the opener runs past firstChunkMaxChars with no punctuation, break at the last word
|
|
1374
|
+
// boundary anyway so a long unpunctuated start still begins playing.
|
|
1375
|
+
// The VERY first audible unit: break on the first complete word so playback starts the instant the
|
|
1376
|
+
// LLM has emitted one word (Piper synthesizes a whole chunk at once, so a one-word chunk is the
|
|
1377
|
+
// smallest possible first-audio latency). Requires a trailing space — a word still being typed
|
|
1378
|
+
// ("Hel") doesn't break; "Hello " does, returning the index after "Hello". -1 until a word completes.
|
|
1379
|
+
// On free local Piper a slightly choppy opener is worth the instant start; the next chunks below get
|
|
1380
|
+
// whole-clause/sentence prosody, and one-ahead synthesis hides their cost.
|
|
1381
|
+
function firstWordEnd(s) {
|
|
1382
|
+
// Extend across words until the chunk holds >= firstWordMinChars non-space chars, so a tiny opener
|
|
1383
|
+
// ("I", "Hi", "A ...") glues to the next word instead of becoming a choppy 1-2 char micro-clip.
|
|
1384
|
+
const re = /\S\s/g;
|
|
1385
|
+
let m, end = -1, chars = 0;
|
|
1386
|
+
while ((m = re.exec(s))) {
|
|
1387
|
+
end = m.index + 1;
|
|
1388
|
+
chars = nw(s.slice(0, end));
|
|
1389
|
+
if (chars >= TUNING.TTS.firstWordMinChars) return end; // long enough → break here
|
|
1390
|
+
}
|
|
1391
|
+
return -1; // no completed word yet, or all completed words still under the floor (wait for more)
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
// All require a trailing space/end so we don't split inside "3.14", "pl.", or "1,000".
|
|
1395
|
+
function firstSentenceEnd(s) {
|
|
1396
|
+
const hard = /[.!?…]+(?=\s|$)|\s[–—-](?=\s)|:(?=\s)/.exec(s);
|
|
1397
|
+
if (hard) return hard.index + hard[0].trimEnd().length;
|
|
1398
|
+
if (s.length >= TUNING.TTS.firstChunkMinChars) {
|
|
1399
|
+
const soft = /[,;](?=\s)/.exec(s);
|
|
1400
|
+
if (soft && soft.index + 1 >= TUNING.TTS.firstChunkMinChars) return soft.index + 1;
|
|
1401
|
+
}
|
|
1402
|
+
if (s.length >= TUNING.TTS.firstChunkMaxChars) {
|
|
1403
|
+
const sp = s.lastIndexOf(' ', TUNING.TTS.firstChunkMaxChars);
|
|
1404
|
+
if (sp > 0) return sp;
|
|
1405
|
+
}
|
|
1406
|
+
return -1;
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
// First sentence ender (. ! ? …, or a trailing-space colon) in `s`, or -1 — used for every
|
|
1410
|
+
// sentence after the first, where we prefer whole sentences (fewer cuts, better prosody) over
|
|
1411
|
+
// the earliest break. A colon ends a clause often enough to be a good, natural TTS break.
|
|
1412
|
+
// Capped: a long run with no ender (lists, dashed prose) breaks at the last word boundary by
|
|
1413
|
+
// sentenceMaxChars so audio keeps flowing instead of stalling on one monster chunk.
|
|
1414
|
+
function sentenceEnd(s) {
|
|
1415
|
+
const m = /[.!?…]+(?=\s|$)|:(?=\s)/.exec(s);
|
|
1416
|
+
if (m) return m.index + m[0].trimEnd().length;
|
|
1417
|
+
if (s.length >= TUNING.TTS.sentenceMaxChars) {
|
|
1418
|
+
const sp = s.lastIndexOf(' ', TUNING.TTS.sentenceMaxChars);
|
|
1419
|
+
if (sp > 0) return sp;
|
|
1420
|
+
}
|
|
1421
|
+
return -1;
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
// ── streaming TTS base: sentence pipeline + Web Audio playback. Providers (Piper, …)
|
|
1425
|
+
// extend it and implement only `_synth(text, signal) → audio Blob` (and
|
|
1426
|
+
// optionally `voices()`); `speak` below does the rest:
|
|
1427
|
+
// Takes an async iterator of text deltas. Streams sentence-by-sentence for low latency
|
|
1428
|
+
// AND small inter-sentence gaps on long answers:
|
|
1429
|
+
// • re-chunk the deltas into sentences (sentence 1 on the earliest natural break for
|
|
1430
|
+
// fast first audio; later ones on whole-sentence enders for better prosody).
|
|
1431
|
+
// • synth ONE sentence ahead while the current one plays, so the next clip is ready (or
|
|
1432
|
+
// nearly) the moment the current finishes — no growing gap as the answer gets longer.
|
|
1433
|
+
// • play STRICTLY in order, one clip at a time (never two voices at once).
|
|
1434
|
+
// Whole clips (no char timings) → on barge-in the heard prefix is estimated proportionally
|
|
1435
|
+
// within the playing sentence.
|
|
1436
|
+
// ── Web Audio playback ───────────────────────────────────────────────────────
|
|
1437
|
+
// We play TTS via the Web Audio API (AudioContext + AudioBufferSourceNode), NOT an <audio> element:
|
|
1438
|
+
// a buffer source starts the instant its data is decoded — no blob-URL assignment, no
|
|
1439
|
+
// loadedmetadata round-trip — shaving per-clip start latency off EVERY sentence. The tradeoff is no
|
|
1440
|
+
// pitch-preserving rate change, so playback runs at native 1.0x (see StreamingTTS default speed).
|
|
1441
|
+
//
|
|
1442
|
+
// iOS Safari starts the AudioContext "suspended" and only lets it resume() inside a user gesture.
|
|
1443
|
+
// TTS audio arrives async (after STT→LLM), far outside any gesture, so we create + resume ONE shared
|
|
1444
|
+
// context from the host's mic-button click (unlockAudio) and reuse it for all later playback.
|
|
1445
|
+
let sharedCtx = null;
|
|
1446
|
+
const audioCtx = () => (sharedCtx ??= new (window.AudioContext || window.webkitAudioContext)());
|
|
1447
|
+
|
|
1448
|
+
// ── AEC loopback sink ────────────────────────────────────────────────────────
|
|
1449
|
+
// THE self-interruption root cause: the browser's echo canceller (getUserMedia
|
|
1450
|
+
// echoCancellation:true) does NOT use raw Web-Audio output (node.connect(ctx.destination)) as its
|
|
1451
|
+
// cancellation reference — Chrome only cancels audio that plays out via a WebRTC remote track (or a
|
|
1452
|
+
// media element fed by one). So on speakers the mic captures our own TTS verbatim, STT transcribes
|
|
1453
|
+
// it (often MIS-hears the synthetic voice, which defeats the word-match echo filter), and the agent
|
|
1454
|
+
// barge-ins on itself. Fix: route ALL TTS playback through a MediaStreamAudioDestinationNode, pipe
|
|
1455
|
+
// that stream through a loopback RTCPeerConnection pair (pc1→pc2, same page), and play the REMOTE
|
|
1456
|
+
// track in a hidden <audio> element. To the browser that playout is now "WebRTC remote audio", so
|
|
1457
|
+
// the mic-side AEC subtracts it — the echo never reaches VAD/STT at all. Costs a few tens of ms of
|
|
1458
|
+
// playout latency (jitter buffer), which the spoken-word cursor tolerates.
|
|
1459
|
+
// Setup is kicked from unlockAudio() (the mic-button gesture: <audio>.play() needs the same
|
|
1460
|
+
// transient activation the AudioContext resume does). On ANY failure (no RTCPeerConnection —
|
|
1461
|
+
// WebKitGTK builds, negotiation error, autoplay-blocked play()) we fall back to direct
|
|
1462
|
+
// ctx.destination playback — old behavior, still guarded by the isSelfEcho text filter — and allow
|
|
1463
|
+
// a retry on the next unlockAudio() call.
|
|
1464
|
+
let ttsSink = null; // MediaStreamAudioDestinationNode all clips connect to (null → ctx.destination)
|
|
1465
|
+
let sinkAudioEl = null; // keeps the playout element referenced (GC would silence the loopback)
|
|
1466
|
+
let sinkPcs = null; // keeps the loopback peer connections referenced (a GC'd pc closes → silence)
|
|
1467
|
+
let sinkStarting = false;
|
|
1468
|
+
let sinkBuild = null; // resolves when the in-flight loopback build settles — _playBuf awaits it (bounded) so the FIRST clip doesn't race the build and play direct/un-cancelled
|
|
1469
|
+
async function initAecLoopback(ctx) {
|
|
1470
|
+
if (ttsSink || sinkStarting || typeof RTCPeerConnection === 'undefined') return;
|
|
1471
|
+
sinkStarting = true;
|
|
1472
|
+
let settle; sinkBuild = new Promise((r) => (settle = r));
|
|
1473
|
+
const pc1 = new RTCPeerConnection(), pc2 = new RTCPeerConnection();
|
|
1474
|
+
try {
|
|
1475
|
+
const dest = ctx.createMediaStreamDestination();
|
|
1476
|
+
pc1.onicecandidate = (e) => { if (e.candidate) pc2.addIceCandidate(e.candidate).catch(() => {}); };
|
|
1477
|
+
pc2.onicecandidate = (e) => { if (e.candidate) pc1.addIceCandidate(e.candidate).catch(() => {}); };
|
|
1478
|
+
const played = new Promise((res, rej) => {
|
|
1479
|
+
const t = setTimeout(() => rej(new Error('loopback timeout')), 3000);
|
|
1480
|
+
pc2.ontrack = (e) => {
|
|
1481
|
+
const el = new Audio();
|
|
1482
|
+
el.srcObject = e.streams[0] ?? new MediaStream([e.track]);
|
|
1483
|
+
el.play().then(() => { clearTimeout(t); res(el); }, (err) => { clearTimeout(t); rej(err); });
|
|
1484
|
+
};
|
|
1485
|
+
});
|
|
1486
|
+
dest.stream.getTracks().forEach((t) => pc1.addTrack(t, dest.stream));
|
|
1487
|
+
const offer = await pc1.createOffer();
|
|
1488
|
+
await pc1.setLocalDescription(offer);
|
|
1489
|
+
await pc2.setRemoteDescription(offer);
|
|
1490
|
+
const answer = await pc2.createAnswer();
|
|
1491
|
+
await pc2.setLocalDescription(answer);
|
|
1492
|
+
await pc1.setRemoteDescription(answer);
|
|
1493
|
+
sinkAudioEl = await played;
|
|
1494
|
+
sinkPcs = [pc1, pc2];
|
|
1495
|
+
// If the loopback ever dies mid-session (pc failed/closed), clips routed into the dead sink
|
|
1496
|
+
// would play SILENCE — unpublish it so playback falls back to direct output (and a later
|
|
1497
|
+
// unlockAudio() gesture can rebuild it).
|
|
1498
|
+
const onDead = () => {
|
|
1499
|
+
if (pc1.connectionState === 'failed' || pc2.connectionState === 'failed' ||
|
|
1500
|
+
pc1.connectionState === 'closed' || pc2.connectionState === 'closed') {
|
|
1501
|
+
ttsSink = null; sinkAudioEl = null; sinkPcs = null;
|
|
1502
|
+
try { pc1.close(); pc2.close(); } catch {}
|
|
1503
|
+
}
|
|
1504
|
+
};
|
|
1505
|
+
pc1.onconnectionstatechange = pc2.onconnectionstatechange = onDead;
|
|
1506
|
+
ttsSink = dest; // publish only once playout is confirmed — clips route through it from now on
|
|
1507
|
+
} catch {
|
|
1508
|
+
try { pc1.close(); pc2.close(); } catch {}
|
|
1509
|
+
sinkAudioEl = null; // failed (no WebRTC / autoplay-blocked) → direct playback; retry on next gesture
|
|
1510
|
+
} finally {
|
|
1511
|
+
sinkStarting = false;
|
|
1512
|
+
settle(); sinkBuild = null;
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
export function unlockAudio() {
|
|
1516
|
+
try { const ctx = audioCtx(); ctx.resume(); initAecLoopback(ctx); } catch {} // gesture context → blesses/resumes the shared AudioContext + builds the AEC sink
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
// A pre-decoded clip playing through a single AudioBufferSourceNode, exposing the small slice of the
|
|
1520
|
+
// HTMLAudioElement surface that _playBuf / stop() / seek() rely on: paused, currentTime, duration,
|
|
1521
|
+
// pause(), and the onended/onpause callbacks. currentTime is computed from the context clock and the
|
|
1522
|
+
// offset the clip started at, so the live cursor and tap-to-seek math are unchanged. A buffer source
|
|
1523
|
+
// can't be restarted, so a seek/replay just creates a new one (seek() already re-plays clips anyway).
|
|
1524
|
+
class WebAudioClip {
|
|
1525
|
+
constructor(ctx, buffer, offsetSec = 0) {
|
|
1526
|
+
this.ctx = ctx; this.buffer = buffer; this.duration = buffer.duration;
|
|
1527
|
+
this.paused = false; this.ended = false; this.onended = null; this.onpause = null;
|
|
1528
|
+
this._fired = false; this._stopped = false;
|
|
1529
|
+
this._startCtxTime = ctx.currentTime; this._offset = offsetSec;
|
|
1530
|
+
this._node = ctx.createBufferSource();
|
|
1531
|
+
this._node.buffer = buffer;
|
|
1532
|
+
this._node.connect(ttsSink ?? ctx.destination); // via the AEC loopback sink when available (see initAecLoopback)
|
|
1533
|
+
// The native onended fires once — on natural end AND on our stop(). Classify with _stopped, but
|
|
1534
|
+
// treat a stop() at/near the buffer's end as a natural end (a pause() racing the final samples
|
|
1535
|
+
// shouldn't be reported as a barge-in). _fired guards the single-shot callback (not `ended`, which
|
|
1536
|
+
// stays false for a stop()).
|
|
1537
|
+
this._node.onended = () => {
|
|
1538
|
+
if (this._fired) return; this._fired = true;
|
|
1539
|
+
const natural = !this._stopped || this.currentTime >= this.duration - 0.02;
|
|
1540
|
+
this.ended = natural; this.paused = true;
|
|
1541
|
+
try { this._node.disconnect(); } catch {}
|
|
1542
|
+
(natural ? this.onended : this.onpause)?.();
|
|
1543
|
+
};
|
|
1544
|
+
this._node.start(0, offsetSec);
|
|
1545
|
+
}
|
|
1546
|
+
get currentTime() { return Math.min(this._offset + (this.ctx.currentTime - this._startCtxTime), this.duration); }
|
|
1547
|
+
pause() { if (this.paused || this._fired) return; this._stopped = true; this.paused = true; try { this._node.stop(); } catch {} }
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
// rAF with a setTimeout fallback (non-browser/SSR/test runtimes lack requestAnimationFrame).
|
|
1551
|
+
const raf = globalThis.requestAnimationFrame?.bind(globalThis) ?? (fn => setTimeout(() => fn(Date.now()), 16));
|
|
1552
|
+
const caf = globalThis.cancelAnimationFrame?.bind(globalThis) ?? clearTimeout;
|
|
1553
|
+
|
|
1554
|
+
export class StreamingTTS {
|
|
1555
|
+
// `speed` is retained for API compatibility but NO LONGER applied: Web Audio playback runs at the
|
|
1556
|
+
// synthesized native rate (1.0) — a buffer source can't change rate without altering pitch. Tune
|
|
1557
|
+
// delivery speed in the synth engine instead if needed.
|
|
1558
|
+
// `onProgress(spokenText)` (set via setOnProgress) fires while a clip plays with the chars heard
|
|
1559
|
+
// SO FAR across the whole reply — drives the host's live "spoken cursor".
|
|
1560
|
+
constructor(voiceId, speed = 1.0) { this.voiceId = voiceId; this.speed = speed; this._audio = null; this._onProgress = null;
|
|
1561
|
+
this._tape = null; this._speaking = false; this._seekTarget = null; this._seekPending = false; this._curIdx = -1;
|
|
1562
|
+
this._synthQ = Promise.resolve(); this._preClip = null; }
|
|
1563
|
+
|
|
1564
|
+
// Serialize EVERY synth through one instance-level queue: a single ONNX session cannot run two
|
|
1565
|
+
// predict()s, and presynth() below may fire while a previous reply's chain is still draining.
|
|
1566
|
+
// Returns a promise that RESOLVES (wav or null-on-error) — callers surface errors per-entry.
|
|
1567
|
+
_enqueueSynth(text, signal) {
|
|
1568
|
+
const run = this._synthQ.then(() => this._synth(stripMd(text) || text, signal), () => null); // || raw: a pure-syntax chunk strips to empty — voice the raw rather than abort
|
|
1569
|
+
this._synthQ = run.catch(() => null);
|
|
1570
|
+
return run;
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
// SPECULATIVE first-clip synth (see VoiceAgent._presynthFirstClip): pre-render the expected first
|
|
1574
|
+
// chunk of the NEXT reply during the end-of-turn debounce, so speak() finds the clip already made.
|
|
1575
|
+
// Single-slot cache keyed by exact chunk text; a mismatched or unused clip is simply dropped.
|
|
1576
|
+
presynth(text) {
|
|
1577
|
+
if (!text || this._preClip?.text === text) return;
|
|
1578
|
+
this.onEvent?.({ type: 'diag', diag: 'presynth', text });
|
|
1579
|
+
this._preClip = { text, blobP: this._enqueueSynth(text).catch(() => null) };
|
|
1580
|
+
}
|
|
1581
|
+
setSpeed(speed) { this.speed = speed; }
|
|
1582
|
+
setVoice(voiceId) { this.voiceId = voiceId; }
|
|
1583
|
+
setOnProgress(fn) { this._onProgress = fn; }
|
|
1584
|
+
// Eagerly prepare the synth engine (load models, JIT the inference path) so the FIRST utterance
|
|
1585
|
+
// doesn't pay cold-start latency mid-reply. No-op by default; providers with a heavy warm-up
|
|
1586
|
+
// (e.g. Piper's ONNX model download + WASM compile) override it. Safe to call repeatedly.
|
|
1587
|
+
async warm() {}
|
|
1588
|
+
|
|
1589
|
+
// Play a pre-synthesized clip via Web Audio (decode → AudioBufferSourceNode); resolve with the chars
|
|
1590
|
+
// of `text` actually heard. A buffer source starts the instant the blob is decoded (no <audio> src
|
|
1591
|
+
// assignment / loadedmetadata wait), cutting per-clip start latency on every sentence — at the cost
|
|
1592
|
+
// of a pitch-preserving rate change, so playback is native 1.0x (StreamingTTS default speed).
|
|
1593
|
+
// `prefix` is everything already spoken in earlier sentences of this reply; we emit `prefix + heard`
|
|
1594
|
+
// on a rAF tick so the host gets the live spoken-so-far text for its cursor.
|
|
1595
|
+
// Resolves with { reason, heard }: reason is why the clip stopped — 'ended' (natural finish),
|
|
1596
|
+
// 'abort' (signal/barge-in), or 'seek' (a tap repositioned the playhead, see seek()); `heard` is the
|
|
1597
|
+
// spoken-so-far text (prefix + this clip's heard part). `startNw` starts the clip mid-sentence (a
|
|
1598
|
+
// forward/backward tap inside it), as a non-whitespace offset into `text`. abort always dominates seek.
|
|
1599
|
+
async _playBuf(wav, text, signal, prefix = '', startNw = 0) {
|
|
1600
|
+
if (signal?.aborted || !wav) return { reason: 'abort', heard: prefix };
|
|
1601
|
+
const ctx = audioCtx();
|
|
1602
|
+
// Decode first (works while suspended) so we don't block synthesis on resume.
|
|
1603
|
+
let buffer;
|
|
1604
|
+
try { buffer = await ctx.decodeAudioData(await wav.arrayBuffer()); }
|
|
1605
|
+
catch (e) { throw new Error(`Audio decode: ${e?.message || e}`); }
|
|
1606
|
+
// A tap or barge-in during decode has no clip to act on yet — honor it now, before we start audio.
|
|
1607
|
+
if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
|
|
1608
|
+
if (this._seekPending) return { reason: 'seek', heard: prefix + sliceNw(text, startNw) };
|
|
1609
|
+
// Now ensure the context can actually run; resume() must happen in/after a gesture (unlockAudio).
|
|
1610
|
+
// Bound it so a policy-blocked resume can't hang the turn forever — if it won't run, surface it.
|
|
1611
|
+
if (ctx.state !== 'running') {
|
|
1612
|
+
try { await Promise.race([ctx.resume(), new Promise((_, rej) => setTimeout(() => rej(new Error('resume timeout')), 1500))]); } catch {}
|
|
1613
|
+
if (ctx.state !== 'running') throw new Error('AudioContext is not running (needs a user gesture)');
|
|
1614
|
+
}
|
|
1615
|
+
// The AEC loopback build (kicked from unlockAudio) races the first reply: if the sink isn't
|
|
1616
|
+
// published yet, this clip would wire to ctx.destination and play UN-cancelled — the mic hears
|
|
1617
|
+
// it and the agent can self-interrupt on its opening sentence. Wait (bounded) for the in-flight
|
|
1618
|
+
// build; on timeout/failure fall through to direct playback exactly as before.
|
|
1619
|
+
if (!ttsSink && sinkBuild) { await Promise.race([sinkBuild, new Promise((r) => setTimeout(r, 1500))]); }
|
|
1620
|
+
// NO silent fallback: if this clip is about to play direct (un-cancelled — the mic will hear
|
|
1621
|
+
// it), tell the host once per session so the user sees WHY self-echo may happen.
|
|
1622
|
+
if (!ttsSink && !this._aecWarned) {
|
|
1623
|
+
this._aecWarned = true;
|
|
1624
|
+
this.onEvent?.({ type: 'diag', diag: 'aec-fallback', message: 'AEC loopback unavailable — TTS plays un-cancelled; echo is only text-filtered' });
|
|
1625
|
+
}
|
|
1626
|
+
if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
|
|
1627
|
+
const dur = buffer.duration;
|
|
1628
|
+
const offsetSec = startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0; // tap-into-clip → start offset
|
|
1629
|
+
const clip = new WebAudioClip(ctx, buffer, offsetSec);
|
|
1630
|
+
this._audio = clip;
|
|
1631
|
+
return new Promise((resolve) => {
|
|
1632
|
+
// Proportional position in NON-WHITESPACE space (the same domain seek() uses) → slice `text` to that
|
|
1633
|
+
// many non-whitespace chars, so a seek to nw-index N lands the cursor exactly on N.
|
|
1634
|
+
const heard = () => sliceNw(text, Math.round(nw(text) * clip.currentTime / (dur || 1)));
|
|
1635
|
+
// 2nd arg = the AUDIBLE-SCOPE bound: prefix + the WHOLE clip now playing. The proportional
|
|
1636
|
+
// cursor above LAGS the real audio (linear time→text estimate, rAF granularity, mid-word
|
|
1637
|
+
// slicing), so an STT echo partial can contain words just AHEAD of it — judged against the
|
|
1638
|
+
// cursor alone those words look novel and un-latch the echo filter (measured self-interruption
|
|
1639
|
+
// on the echo bench). Echo can never be ahead of the clip being played, so the clip's full text
|
|
1640
|
+
// is a safe upper bound for the echo reference; the UI cursor keeps the precise estimate.
|
|
1641
|
+
const scope = prefix + text;
|
|
1642
|
+
let settled = false, frame = 0;
|
|
1643
|
+
const tick = () => { if (settled) return; this._onProgress?.(prefix + heard(), scope); frame = raf(tick); };
|
|
1644
|
+
const cleanup = () => { if (frame) caf(frame); signal?.removeEventListener('abort', onAbort); clip.onpause = clip.onended = null; };
|
|
1645
|
+
const finish = (reason) => { if (settled) return; settled = true; cleanup(); resolve({ reason, heard: prefix + heard() }); };
|
|
1646
|
+
const onAbort = () => { clip.pause(); finish('abort'); }; // barge-in wins over a pending seek
|
|
1647
|
+
// Natural end → onended; stop()/seek() pause() → onpause. A pending seek flag distinguishes a
|
|
1648
|
+
// reposition from a real interruption; abort got there first if it fired.
|
|
1649
|
+
clip.onpause = () => finish((this._seekPending && !signal?.aborted) ? 'seek' : 'abort'); // a barge-in (signal) that races a pending seek must still win → 'abort'
|
|
1650
|
+
clip.onended = () => finish('ended');
|
|
1651
|
+
signal?.addEventListener('abort', onAbort);
|
|
1652
|
+
if (signal?.aborted) return onAbort(); // aborted between start() and listener
|
|
1653
|
+
if (this._onProgress) frame = raf(tick);
|
|
1654
|
+
// Emit the spoken-so-far prefix the moment this clip starts, so the host's cursor takes over at
|
|
1655
|
+
// playback start — no window where the whole answer shows solid.
|
|
1656
|
+
this._onProgress?.(prefix + sliceNw(text, startNw), scope);
|
|
1657
|
+
});
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
// subclass contract — synthesize one sentence → audio Blob (null when aborted/empty)
|
|
1661
|
+
async _synth(_text, _signal) { throw new Error('not implemented'); }
|
|
1662
|
+
|
|
1663
|
+
// Re-chunk an async iterable of text deltas into sentences. The FIRST sentence breaks on
|
|
1664
|
+
// the earliest natural boundary (fast first audio); the rest on whole-sentence enders
|
|
1665
|
+
// (better prosody). Flushes whatever's left when the stream ends.
|
|
1666
|
+
async *_sentences(src, signal) {
|
|
1667
|
+
const it = src[Symbol.asyncIterator]();
|
|
1668
|
+
const next = async () => { try { return await it.next(); } catch (e) { if (e.name === 'AbortError') return { done: true }; throw e; } };
|
|
1669
|
+
// Three-stage break progression for minimum first-audio latency on local (whole-chunk) TTS:
|
|
1670
|
+
// stage 0 → first complete WORD (start talking the instant one word exists);
|
|
1671
|
+
// stage 1 → rest of the first clause/sentence (earliest natural break);
|
|
1672
|
+
// stage 2+ → whole sentences (best prosody; one-ahead synth hides their cost).
|
|
1673
|
+
const breakers = [firstWordEnd, firstSentenceEnd, sentenceEnd];
|
|
1674
|
+
let acc = '', stage = 0;
|
|
1675
|
+
for (let r = await next(); !r.done; r = await next()) {
|
|
1676
|
+
if (signal?.aborted) return;
|
|
1677
|
+
acc += r.value;
|
|
1678
|
+
let end;
|
|
1679
|
+
while ((end = breakers[Math.min(stage, breakers.length - 1)](acc)) >= 0) {
|
|
1680
|
+
const s = acc.slice(0, end).trim();
|
|
1681
|
+
acc = acc.slice(end).trimStart();
|
|
1682
|
+
if (s) { yield s; stage++; }
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
const tail = acc.trim();
|
|
1686
|
+
if (tail) yield tail;
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
// `input` is an async iterable of text deltas (or a plain string).
|
|
1690
|
+
// Builds a TAPE — one entry per sentence ({ text, blobP, nwStart }) appended at playback pace —
|
|
1691
|
+
// and walks it by a playhead index. Forward play is the same low-latency, one-sentence-ahead
|
|
1692
|
+
// pipeline as before (synth the next clip while the current plays, one voice at a time). The tape
|
|
1693
|
+
// adds tap-to-seek (see seek()): a tap repositions the playhead to any sentence ALREADY on the
|
|
1694
|
+
// tape and plays forward from there — backward replays the cached clip (no re-synth), forward
|
|
1695
|
+
// skips ahead. Resolves with the heard text (high-water in non-whitespace space → history): a full
|
|
1696
|
+
// play returns the whole reply, a barge-in returns the prefix heard before the user cut in.
|
|
1697
|
+
// `fromNw` (non-whitespace offset) begins playback partway through — used to REPLAY a finished reply
|
|
1698
|
+
// from a tapped word. The whole reply still tapes (we synth the skipped prefix too, lazily), so a
|
|
1699
|
+
// backward tap within the replay hits cached clips; replies are short so the extra synth is cheap.
|
|
1700
|
+
async speak(input, signal, fromNw = 0) {
|
|
1701
|
+
if (signal?.aborted) return '';
|
|
1702
|
+
const src = typeof input === 'string' ? (async function* () { yield input; })() : input;
|
|
1703
|
+
const it = this._sentences(src, signal)[Symbol.asyncIterator]();
|
|
1704
|
+
|
|
1705
|
+
const tape = this._tape = []; // { text, blobP, nwStart } — nwStart = non-whitespace chars before it
|
|
1706
|
+
// Generation token: an aborted speak() DETACHES its producer (see the finally block) rather than
|
|
1707
|
+
// waiting for a host stream that may never yield again. The detached producer stays parked in
|
|
1708
|
+
// it.next() and can wake up long after a NEWER speak() owns this instance — so every step that
|
|
1709
|
+
// touches shared state re-checks that it's still the current generation.
|
|
1710
|
+
const gen = this._speakGen = (this._speakGen || 0) + 1;
|
|
1711
|
+
const current = () => gen === this._speakGen && !signal?.aborted;
|
|
1712
|
+
let nwTotal = 0, streamErr = null, streamDone = false, wake = null;
|
|
1713
|
+
// Single waiter slot; woken when the tape grows, the stream ends, OR we abort — so a consumer
|
|
1714
|
+
// parked in awaitEntry() never hangs past an abort/barge-in even if the LLM stream is slow to react.
|
|
1715
|
+
const bump = () => { const w = wake; wake = null; w?.(); };
|
|
1716
|
+
this._ttsWake = bump; // let seek() wake a consumer parked in a stream gap (see seek())
|
|
1717
|
+
// Synth is serial: ONE ONNX session can't run concurrent predict()s. Every _synth goes through
|
|
1718
|
+
// the instance-level _synthQ (see _enqueueSynth) so audio is produced one-at-a-time, in order —
|
|
1719
|
+
// but the TEXT stream (it.next(), which drives the LLM) is drained as fast as tokens arrive,
|
|
1720
|
+
// decoupled from both synth pace AND playback pace: generation never stalls sentence-by-sentence.
|
|
1721
|
+
const pull = async () => {
|
|
1722
|
+
const r = await it.next();
|
|
1723
|
+
// Host code just returned. If we were abandoned while parked in it.next(), stop HERE — before
|
|
1724
|
+
// touching _preClip (which now belongs to a newer reply), the synth queue or the event tap.
|
|
1725
|
+
if (!current() || r.done) return null;
|
|
1726
|
+
const entry = { text: r.value, nwStart: nwTotal };
|
|
1727
|
+
// Speculative hit: the first chunk was already rendered during the end-of-turn debounce
|
|
1728
|
+
// (presynth) with EXACTLY this text — reuse the clip instead of predicting again. Single-shot:
|
|
1729
|
+
// hit or miss, the slot is cleared (a stale speculation must not leak into a later reply).
|
|
1730
|
+
const pre = tape.length === 0 ? this._preClip : null;
|
|
1731
|
+
if (pre) this.onEvent?.({ type: 'diag', diag: pre.text === r.value ? 'presynth-hit' : 'presynth-miss', text: r.value, pre: pre.text });
|
|
1732
|
+
this._preClip = null;
|
|
1733
|
+
// blobP always RESOLVES — to the wav, or null carrying the error on the entry — so an
|
|
1734
|
+
// ahead-of-playback synth failure never raises unhandledrejection; the queue keeps going for
|
|
1735
|
+
// the rest of the reply, and the consumer re-throws entry.err only if it actually reaches it.
|
|
1736
|
+
const p = pre && pre.text === r.value ? pre.blobP : this._enqueueSynth(r.value, signal);
|
|
1737
|
+
entry.blobP = p.catch((e) => { entry.err = e; return null; });
|
|
1738
|
+
tape.push(entry);
|
|
1739
|
+
nwTotal += nw(r.value);
|
|
1740
|
+
return entry;
|
|
1741
|
+
};
|
|
1742
|
+
// Background PRODUCER: keep draining the LLM stream independent of playback so token generation
|
|
1743
|
+
// never stalls sentence-by-sentence behind audio. It runs ahead, synth queues serially behind it.
|
|
1744
|
+
const produce = (async () => {
|
|
1745
|
+
try { while (!signal?.aborted) { if (!(await pull())) break; bump(); } }
|
|
1746
|
+
catch (e) { if (e?.name !== 'AbortError') streamErr = e; } // LLM/iterator failure → surface via awaitEntry, not a silent truncation
|
|
1747
|
+
finally { streamDone = true; bump(); }
|
|
1748
|
+
})();
|
|
1749
|
+
if (signal) signal.addEventListener('abort', bump, { once: true }); // wake a parked consumer the instant we abort
|
|
1750
|
+
// Wait until the tape has an entry at `i` — or the stream ends, we abort, or a seek arrives (a
|
|
1751
|
+
// backward tap while we're parked waiting for a FUTURE clip must be honored now, not after the
|
|
1752
|
+
// stream advances). Producer runs ahead, so this usually returns immediately; it only blocks when
|
|
1753
|
+
// playback has caught up to generation.
|
|
1754
|
+
const awaitEntry = async (i) => {
|
|
1755
|
+
while (i >= tape.length && !streamDone && !signal?.aborted && !this._seekPending) await new Promise((r) => { wake = r; });
|
|
1756
|
+
if (streamErr) throw streamErr;
|
|
1757
|
+
};
|
|
1758
|
+
|
|
1759
|
+
let idx = 0, startNw = 0, heardMax = 0;
|
|
1760
|
+
// A tap that lands while we're awaiting (synthesis or the LLM stream) has no clip to pause, so it
|
|
1761
|
+
// only sets _seekTarget — honor it at the next safe point. Returns true if it moved the playhead.
|
|
1762
|
+
const takePending = () => {
|
|
1763
|
+
if (!this._seekPending || !this._seekTarget || signal?.aborted) return false;
|
|
1764
|
+
idx = this._seekTarget.idx; startNw = this._seekTarget.startNw;
|
|
1765
|
+
this._seekPending = false; this._seekTarget = null;
|
|
1766
|
+
return true;
|
|
1767
|
+
};
|
|
1768
|
+
try {
|
|
1769
|
+
this._speaking = true;
|
|
1770
|
+
// Replay-from-word: tape forward until fromNw falls inside a sentence, then start playing there
|
|
1771
|
+
// (startNw = the offset within that sentence). heardMax seeds to fromNw so the replayed prefix
|
|
1772
|
+
// already counts as heard (a forward replay never un-hears earlier text).
|
|
1773
|
+
if (fromNw > 0) {
|
|
1774
|
+
while (!signal?.aborted) {
|
|
1775
|
+
const last = tape[tape.length - 1];
|
|
1776
|
+
if (last && fromNw < last.nwStart + nw(last.text)) break;
|
|
1777
|
+
if (streamDone) break; // streamed everything and fromNw is past the end → play from the last
|
|
1778
|
+
await awaitEntry(tape.length); // let the producer append the next sentence
|
|
1779
|
+
}
|
|
1780
|
+
const i = tape.findIndex(e => fromNw < e.nwStart + nw(e.text));
|
|
1781
|
+
if (i >= 0) { idx = i; startNw = Math.max(0, fromNw - tape[i].nwStart); heardMax = fromNw; }
|
|
1782
|
+
}
|
|
1783
|
+
while (true) {
|
|
1784
|
+
if (idx >= tape.length) { await awaitEntry(idx); if (takePending()) continue; if (idx >= tape.length) break; } // wait for the producer to reach this sentence (stream end → done)
|
|
1785
|
+
const e = tape[idx];
|
|
1786
|
+
const buf = await e.blobP;
|
|
1787
|
+
if (e.err) throw e.err; // this clip's synth failed → surface it (only when we actually reach it)
|
|
1788
|
+
if (takePending()) continue; // a tap during synth/stream gap → reposition
|
|
1789
|
+
this._curIdx = idx;
|
|
1790
|
+
const prefix = tape.slice(0, idx).map(x => x.text).join(' ') + (idx ? ' ' : '');
|
|
1791
|
+
this._seekPending = false; this._seekTarget = null; // the clip about to play owns the seek channel
|
|
1792
|
+
const playP = this._playBuf(buf, e.text, signal, prefix, startNw); // start (don't await yet)
|
|
1793
|
+
const { reason, heard } = await playP; // producer keeps synthesizing ahead while this clip plays
|
|
1794
|
+
heardMax = Math.max(heardMax, nw(heard)); // monotonic: a backward replay never un-hears
|
|
1795
|
+
if (reason === 'abort' || signal?.aborted) break; // real barge-in → stop, record heard
|
|
1796
|
+
if (reason === 'seek') { idx = this._seekTarget.idx; startNw = this._seekTarget.startNw; this._seekPending = false; this._seekTarget = null; continue; }
|
|
1797
|
+
idx++; startNw = 0; // natural end → next sentence
|
|
1798
|
+
}
|
|
1799
|
+
} finally {
|
|
1800
|
+
signal?.removeEventListener('abort', bump);
|
|
1801
|
+
// A finished stream is joined (it's already done — this just collects a trailing synth). An
|
|
1802
|
+
// ABORTED one is DETACHED: the producer sits in it.next() on the HOST's llm generator, which
|
|
1803
|
+
// may ignore the abort signal and never yield again, so awaiting it here would never return
|
|
1804
|
+
// from speak() — wedging _speakTurn and every turn queued behind it.
|
|
1805
|
+
//
|
|
1806
|
+
// Detaching is safe on both counts the old join protected:
|
|
1807
|
+
// • concurrent predict() — every synth chains on the instance-level _synthQ (_enqueueSynth is
|
|
1808
|
+
// the ONLY path to _synth), so the next turn's synth queues behind an in-flight one instead
|
|
1809
|
+
// of racing it. It waits on its own entry.blobP, not on the whole turn loop.
|
|
1810
|
+
// • stale writes — pull() returns at the generation check above before touching any shared
|
|
1811
|
+
// state, so a producer that wakes up late can't clear a newer reply's _preClip or enqueue.
|
|
1812
|
+
// return() asks a well-behaved generator to wind down (it can't interrupt a pending next(), and
|
|
1813
|
+
// a custom iterator may throw synchronously — hence the guard).
|
|
1814
|
+
if (signal?.aborted) { try { Promise.resolve(it.return?.()).catch(() => {}); } catch {} }
|
|
1815
|
+
else { await produce.catch(() => {}); await this._synthQ.catch(() => {}); }
|
|
1816
|
+
this._ttsWake = null;
|
|
1817
|
+
this._speaking = false; this._tape = null; this._seekTarget = null; this._seekPending = false; this._curIdx = -1;
|
|
1818
|
+
}
|
|
1819
|
+
return sliceNw(tape.map(x => x.text).join(' '), heardMax); // heard prefix in non-whitespace space
|
|
1820
|
+
}
|
|
1821
|
+
stop() { this._seekPending = false; this._seekTarget = null; try { this._audio?.pause(); } catch {} }
|
|
1822
|
+
|
|
1823
|
+
// Move the playhead to a position in the whole reply (a non-whitespace char count — the same
|
|
1824
|
+
// whitespace-invariant key the host's cursor uses, so the rendered answer and the clips agree
|
|
1825
|
+
// regardless of the model's spacing) and play forward from there. Any tap (even inside the clip now
|
|
1826
|
+
// playing) interrupts the current clip (reason 'seek') so speak() repositions and re-plays from the
|
|
1827
|
+
// tapped offset — a Web Audio buffer source can't seek in place. Backward hits the
|
|
1828
|
+
// cached blob, forward skips ahead. Returns false when the target isn't on the tape yet
|
|
1829
|
+
// (not-yet-streamed/partial text) — the host leaves playback as-is.
|
|
1830
|
+
seek(nwIndex) {
|
|
1831
|
+
const tape = this._tape;
|
|
1832
|
+
if (!this._speaking || !tape || !tape.length || nwIndex < 0) return false;
|
|
1833
|
+
const i = tape.findIndex(e => nwIndex < e.nwStart + nw(e.text));
|
|
1834
|
+
if (i < 0) return false; // beyond streamed text
|
|
1835
|
+
const e = tape[i], localNw = Math.max(0, nwIndex - e.nwStart);
|
|
1836
|
+
const a = this._audio;
|
|
1837
|
+
// Every seek (even within the clip now playing) interrupts + repositions: a Web Audio buffer
|
|
1838
|
+
// source can't be re-positioned in place, so we re-play the target clip from the tapped offset
|
|
1839
|
+
// (startNw). speak() catches the 'seek' reason, sets idx/startNw, and _playBuf starts at that offset.
|
|
1840
|
+
this._seekTarget = { idx: i, startNw: localNw };
|
|
1841
|
+
this._seekPending = true;
|
|
1842
|
+
try { a?.pause(); } catch {} // → onpause resolves current clip as 'seek'
|
|
1843
|
+
this._ttsWake?.(); // a tap during a stream gap (no clip playing) → wake the parked consumer to honor it
|
|
1844
|
+
return true;
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
// ── Piper TTS (local WASM, free, has Hungarian) via @mintplex-labs/piper-tts-web.
|
|
1849
|
+
// Default voice: en_US-joe-medium.
|
|
1850
|
+
const CDN_PIPER = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-tts-web.js`;
|
|
1851
|
+
// The espeak phonemizer chunk (createPiperPhonemize) — a build artifact of the pinned 1.0.4, not a
|
|
1852
|
+
// public export; imported directly because the lib only exposes it via TtsSession.predict(), whose
|
|
1853
|
+
// per-call module setup is exactly what we bypass (see PiperTTS._session).
|
|
1854
|
+
const CDN_PIPER_PHONEMIZE = `${CDN}@mintplex-labs/piper-tts-web@1.0.4/dist/piper-o91UDS6e.js`;
|
|
1855
|
+
const CDN_PIPER_WASM = `${CDN}@diffusionstudio/piper-wasm@1.0.0/build/piper_phonemize`;
|
|
1856
|
+
|
|
1857
|
+
// OPFS blob cache — same 'piper' directory and url-basename filenames as the lib's own cache, so
|
|
1858
|
+
// models already downloaded through the lib are reused (and vice versa).
|
|
1859
|
+
async function opfsCachedFetch(url) {
|
|
1860
|
+
const name = url.split('/').at(-1);
|
|
1861
|
+
const dir = () => navigator.storage.getDirectory().then((r) => r.getDirectoryHandle('piper', { create: true }));
|
|
1862
|
+
try { return await (await (await dir()).getFileHandle(name)).getFile(); } catch {}
|
|
1863
|
+
const res = await fetch(url);
|
|
1864
|
+
if (!res.ok) throw new Error(`fetch ${url}: ${res.status}`);
|
|
1865
|
+
const blob = await res.blob();
|
|
1866
|
+
try { const w = await (await (await dir()).getFileHandle(name, { create: true })).createWritable(); await w.write(blob); await w.close(); } catch {}
|
|
1867
|
+
return blob;
|
|
1868
|
+
}
|
|
1869
|
+
|
|
1870
|
+
// float32 PCM → 16-bit mono WAV (the lib's pcm2wav isn't exported).
|
|
1871
|
+
function pcm2wav(f32, sampleRate) {
|
|
1872
|
+
const n = f32.length, v = new DataView(new ArrayBuffer(44 + n * 2));
|
|
1873
|
+
const str = (o, s) => { for (let i = 0; i < s.length; i++) v.setUint8(o + i, s.charCodeAt(i)); };
|
|
1874
|
+
str(0, 'RIFF'); v.setUint32(4, 36 + n * 2, true); str(8, 'WAVEfmt ');
|
|
1875
|
+
v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
|
|
1876
|
+
v.setUint32(24, sampleRate, true); v.setUint32(28, sampleRate * 2, true);
|
|
1877
|
+
v.setUint16(32, 2, true); v.setUint16(34, 16, true);
|
|
1878
|
+
str(36, 'data'); v.setUint32(40, n * 2, true);
|
|
1879
|
+
for (let i = 0, p = 44; i < n; i++, p += 2) v.setInt16(p, Math.max(-1, Math.min(1, f32[i])) * 32767 | 0, true);
|
|
1880
|
+
return v.buffer;
|
|
1881
|
+
}
|
|
1882
|
+
export class PiperTTS extends StreamingTTS {
|
|
1883
|
+
constructor(voiceId = 'en_US-joe-medium', speed = 1.0) { super(voiceId, speed); }
|
|
1884
|
+
// webpackIgnore is honored by webpack AND Turbopack (vite needs its own marker) — without it
|
|
1885
|
+
// bundlers rewrite this into their runtime require, which can't load a URL
|
|
1886
|
+
// ("turbopack_context.x is not a function").
|
|
1887
|
+
async _tts() {
|
|
1888
|
+
return this._lib ??= (async () => {
|
|
1889
|
+
// Piper imports a bare "onnxruntime-web" specifier resolved via the importmap that
|
|
1890
|
+
// loadVoiceDeps() installs — ensure it exists before the module loads.
|
|
1891
|
+
if (typeof document !== 'undefined' && !document.getElementById('vl-importmap')) {
|
|
1892
|
+
const { loadVoiceDeps } = await import('./deps.js'); await loadVoiceDeps();
|
|
1893
|
+
}
|
|
1894
|
+
return import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER);
|
|
1895
|
+
})();
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
// List installable Piper voices: [{ id, name, language }]. `id` is what you pass as
|
|
1899
|
+
// `voiceId` (lib calls it `key`). Lets the UI build a voice picker; switch with setVoice.
|
|
1900
|
+
async voices() {
|
|
1901
|
+
const list = await (await this._tts()).voices(); // [{ key, name, language: { code, … }, … }]
|
|
1902
|
+
return list.map(v => ({ id: v.key, name: v.name || v.key, language: v.language?.code || v.language || '' }));
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
// Our OWN synth pipeline instead of lib.TtsSession.predict(): the lib instantiates a FRESH espeak
|
|
1906
|
+
// phonemizer WASM module (fetch + compile + espeak-data mount) on EVERY predict() — measured
|
|
1907
|
+
// 220–360ms per call, dwarfing the ~10ms phonemize itself. We build the phonemizer ONCE per
|
|
1908
|
+
// session and reuse it (callMain is re-entrant — verified over repeated calls), then run the ONNX
|
|
1909
|
+
// inference directly. Measured first-word synth: 500–575ms (lib) → ~185ms (this path).
|
|
1910
|
+
// Cached as a PROMISE keyed by voice so concurrent callers build one session, not two. Only
|
|
1911
|
+
// called from _synth (which every path — speak, presynth, warm — reaches through the serialized
|
|
1912
|
+
// _synthQ), so when a voice change builds a new session the old one is idle → release it.
|
|
1913
|
+
async _session() {
|
|
1914
|
+
const voiceId = this.voiceId; // snapshot: setVoice() mid-build must not mix voices
|
|
1915
|
+
if (this._sessP && this._sessVoice === voiceId) return this._sessP;
|
|
1916
|
+
this._sessVoice = voiceId;
|
|
1917
|
+
const prev = this._sessP;
|
|
1918
|
+
const p = this._sessP = (async () => {
|
|
1919
|
+
prev?.then((s) => s.onnx.release()).catch(() => {}); // free the old voice's model (worker-side memory isn't GC'd with the JS ref)
|
|
1920
|
+
const lib = await this._tts(); // also installs the importmap that resolves 'onnxruntime-web' below
|
|
1921
|
+
const path = lib.PATH_MAP[voiceId];
|
|
1922
|
+
if (!path) throw new Error(`unknown Piper voice: ${voiceId}`);
|
|
1923
|
+
const [ort, phonMod, cfgBlob, modelBlob] = await Promise.all([
|
|
1924
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'onnxruntime-web'),
|
|
1925
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ CDN_PIPER_PHONEMIZE),
|
|
1926
|
+
opfsCachedFetch(`${lib.HF_BASE}/${path}.json`),
|
|
1927
|
+
opfsCachedFetch(`${lib.HF_BASE}/${path}`),
|
|
1928
|
+
]);
|
|
1929
|
+
const cfg = JSON.parse(await cfgBlob.text());
|
|
1930
|
+
// Match the importmap's onnxruntime-web version: the lib's baked-in default points at a
|
|
1931
|
+
// 1.18.0 CDN folder missing 1.22.0's threaded loader files (404 → "no available backend").
|
|
1932
|
+
ort.env.allowLocalModels = false;
|
|
1933
|
+
ort.env.wasm.numThreads = navigator.hardwareConcurrency || 4; // >1 needs crossOriginIsolated; ort falls back to 1 otherwise
|
|
1934
|
+
ort.env.wasm.proxy = true; // run inference in a worker: session.run computes ON the calling thread otherwise, so an ahead-of-playback synth would starve the clip that's trying to start (measured ~450ms added first-audio delay)
|
|
1935
|
+
ort.env.wasm.wasmPaths = `${CDN}onnxruntime-web@1.22.0/dist/`;
|
|
1936
|
+
const onnx = await ort.InferenceSession.create(await modelBlob.arrayBuffer());
|
|
1937
|
+
let ids = null;
|
|
1938
|
+
const phonemizer = await phonMod.createPiperPhonemize({
|
|
1939
|
+
print: (d) => { ids ??= JSON.parse(d).phoneme_ids; }, // first line only (parity with the lib)
|
|
1940
|
+
printErr: (e) => { throw new Error(e); },
|
|
1941
|
+
locateFile: (u) => u.endsWith('.wasm') ? `${CDN_PIPER_WASM}.wasm` : u.endsWith('.data') ? `${CDN_PIPER_WASM}.data` : u,
|
|
1942
|
+
});
|
|
1943
|
+
// Synchronous; safe to reuse because _synthQ already serializes every synth on this instance.
|
|
1944
|
+
const phonemize = (text) => {
|
|
1945
|
+
ids = null;
|
|
1946
|
+
phonemizer.callMain(['-l', cfg.espeak.voice, '--input', JSON.stringify([{ text }]), '--espeak_data', '/espeak-ng-data']);
|
|
1947
|
+
if (!ids) throw new Error('phonemize produced no output');
|
|
1948
|
+
return ids;
|
|
1949
|
+
};
|
|
1950
|
+
const sid = Object.keys(cfg.speaker_id_map ?? {}).length ? [0] : null;
|
|
1951
|
+
return { ort, onnx, cfg, phonemize, sid };
|
|
1952
|
+
})();
|
|
1953
|
+
p.catch(() => { if (this._sessP === p) this._sessP = null; }); // failed init must not poison the cache — next call retries (only clear if still current: a stale failure must not evict a newer voice's build)
|
|
1954
|
+
return p;
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
// Eagerly download+compile the ONNX model and JIT the WASM inference path, so the first real reply
|
|
1958
|
+
// isn't stalled by Piper's cold start (model fetch + compile + first-inference warm-up) AFTER the LLM
|
|
1959
|
+
// has already produced text. Called when the agent starts listening — the model loads while the user
|
|
1960
|
+
// is still speaking. A throwaway predict primes the inference path; failures are non-fatal (the first
|
|
1961
|
+
// real _synth retries). Idempotent: _session() caches, so a second warm() is cheap.
|
|
1962
|
+
async warm() {
|
|
1963
|
+
try { await this._enqueueSynth('.'); } catch {} // through the queue: a late warm-up must not run concurrently with (or ahead of) a real first clip
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
// Synthesize text → WAV Blob (the slow part; run it ahead of playback).
|
|
1967
|
+
async _synth(text, signal) {
|
|
1968
|
+
text = text?.trim();
|
|
1969
|
+
if (signal?.aborted || !text) return null;
|
|
1970
|
+
const s = await this._session();
|
|
1971
|
+
if (signal?.aborted) return null; // barge-in during cold model download/compile → skip the wasted inference
|
|
1972
|
+
const phonemeIds = s.phonemize(text);
|
|
1973
|
+
const { noise_scale, length_scale, noise_w } = s.cfg.inference;
|
|
1974
|
+
const feeds = {
|
|
1975
|
+
input: new s.ort.Tensor('int64', phonemeIds, [1, phonemeIds.length]),
|
|
1976
|
+
input_lengths: new s.ort.Tensor('int64', [phonemeIds.length]),
|
|
1977
|
+
scales: new s.ort.Tensor('float32', [noise_scale, length_scale, noise_w]),
|
|
1978
|
+
};
|
|
1979
|
+
if (s.sid) feeds.sid = new s.ort.Tensor('int64', s.sid);
|
|
1980
|
+
const { output } = await s.onnx.run(feeds);
|
|
1981
|
+
if (signal?.aborted) return null;
|
|
1982
|
+
return new Blob([pcm2wav(output.data, s.cfg.audio.sample_rate)], { type: 'audio/x-wav' });
|
|
1983
|
+
}
|
|
1984
|
+
}
|