@todoforai/voiceloop 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/package.json +1 -1
- package/src/index.d.ts +28 -5
- package/src/index.js +1 -1
- package/src/llm-openai.js +64 -13
- package/src/voice-agent.js +66 -68
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ A zero-dependency JavaScript library that runs the full voice loop — **VAD →
|
|
|
17
17
|
- **Real barge-in** — triggers on transcribed *novel words*, not mic energy, so the agent's own voice never cuts it off.
|
|
18
18
|
- **Self-echo filtering** — 0 self-interruptions with AEC off (Pipecat cut itself 20/30, OpenAI Realtime 17/30 [bench](bench/results/RESULTS.md#scenario-echo-smalltalk--speakermic-coupling-15db--30ms)).
|
|
19
19
|
- **First audio <1s** — TTS speaks sentence 1 while the LLM writes sentence 2, and the LLM call starts speculatively during your end-of-turn pause.
|
|
20
|
-
- **Serialized turns** — rapid-fire turns, tool results, holds and replays can never talk over each other. Locked in by
|
|
20
|
+
- **Serialized turns** — rapid-fire turns, tool results, holds and replays can never talk over each other. Locked in by 184 tests.
|
|
21
21
|
- **Local-first** — Silero VAD and Piper TTS run as WASM in the tab: free, no cloud round-trip (CDN, then cache).
|
|
22
22
|
|
|
23
23
|
Extra:
|
|
@@ -169,7 +169,8 @@ const agent = new VoiceAgent({
|
|
|
169
169
|
// toolGate: only for adapters that run tools themselves — see below; ignore it otherwise
|
|
170
170
|
for await (const delta of myProvider.stream({ history, system, signal })) {
|
|
171
171
|
yield { text: delta }; // speech text → streamed into TTS
|
|
172
|
-
// yield { tool: 'name', args: {...} }; // tool call →
|
|
172
|
+
// yield { tool: 'name', args: {...} }; // tool call → the agent runs it during playback;
|
|
173
|
+
// // the NEXT user turn sees its tool_result
|
|
173
174
|
}
|
|
174
175
|
},
|
|
175
176
|
});
|
|
@@ -190,13 +191,13 @@ const agent = new VoiceAgent({
|
|
|
190
191
|
});
|
|
191
192
|
```
|
|
192
193
|
|
|
193
|
-
Tool calls execute while the agent is still talking.
|
|
194
|
+
Tool calls execute while the agent is still talking. The default adapter is a loop: it feeds each result back to the model inside the same reply, so the model keeps talking with the answer in hand (bounded to 6 model messages per turn). Results are recorded natively (tool_use / tool_result) so the model never re-fires the same call, and `agent.notify('[TOOL RESULT get_weather] 22°C sunny')` relays an async outcome back for a spoken follow-up — bursts of results collapse into one reply instead of three interrupting monologues.
|
|
194
195
|
|
|
195
196
|
Tools are **dispatched, not awaited**: the turn completes when the agent stops speaking, whatever the tool is still doing. A slow or hung tool can never stall the conversation, and a tool is free to `await agent.notify(...)` with its own result. Long work belongs in a tool that returns promptly ("checking that now") and delivers the outcome later via `notify()`.
|
|
196
197
|
|
|
197
198
|
### Adapters that run tools themselves
|
|
198
199
|
|
|
199
|
-
|
|
200
|
+
The default `makeOpenAILLM` and any agent loop execute tools **inside** the generator, feeding results back to the model so it keeps talking with the answer in hand — all within one turn. A custom loop declares two flags so voiceloop adapts, and yields `{ newMessage: true }` where the model's next message starts:
|
|
200
201
|
|
|
201
202
|
```js
|
|
202
203
|
async function* myAgentLoop(history, system, signal, { toolGate } = {}) {
|
|
@@ -204,6 +205,7 @@ async function* myAgentLoop(history, system, signal, { toolGate } = {}) {
|
|
|
204
205
|
if (ev.type === 'text') yield { text: ev.delta };
|
|
205
206
|
if (ev.type === 'tool_start') yield { tool: ev.name, id: ev.id, args: ev.args, running: true };
|
|
206
207
|
if (ev.type === 'tool_end') yield { tool: ev.name, id: ev.id, args: ev.args, result: ev.result };
|
|
208
|
+
if (ev.type === 'message_start' && !first) yield { newMessage: true };
|
|
207
209
|
}
|
|
208
210
|
}
|
|
209
211
|
myAgentLoop.executesTools = true; // chunks carry the outcome — voiceloop reports, never re-runs
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.15",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -59,9 +59,10 @@ export type HistoryBlock =
|
|
|
59
59
|
| { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean };
|
|
60
60
|
export type HistoryMessage = { role: string; content: string | HistoryBlock[] };
|
|
61
61
|
// Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
|
|
62
|
-
// The default (makeOpenAILLM)
|
|
63
|
-
//
|
|
64
|
-
//
|
|
62
|
+
// The default (makeOpenAILLM) is a tool LOOP: it runs each call inside the generator, feeds the
|
|
63
|
+
// result back to the model and marks the model's next message with { newMessage }. A plain adapter
|
|
64
|
+
// (no flags) yields bare tool chunks that the AGENT executes and records; the NEXT user turn sees
|
|
65
|
+
// the tool_result (no reaction turn is started).
|
|
65
66
|
export interface LLM {
|
|
66
67
|
(
|
|
67
68
|
history: HistoryMessage[],
|
|
@@ -71,12 +72,12 @@ export interface LLM {
|
|
|
71
72
|
): AsyncIterable<LLMChunk>;
|
|
72
73
|
/** This generator is a tool LOOP: it runs every call itself and feeds the result back inside the
|
|
73
74
|
* same reply (chunks carry `result`/`error`, optionally preceded by a `running` announcement).
|
|
74
|
-
* The agent never executes its calls
|
|
75
|
+
* The agent never executes its calls. */
|
|
75
76
|
executesTools?: boolean;
|
|
76
77
|
/** This generator awaits `options.toolGate` before ANY tool executes. Required to keep
|
|
77
78
|
* speculative prefetch on: prefetch starts from an uncommitted interim, and the gate is what
|
|
78
79
|
* stops a speculation from firing a real side effect. `executesTools` without this silently
|
|
79
|
-
* disables prefetch (see voice-agent.js
|
|
80
|
+
* disables prefetch (see _startPrefetch in voice-agent.js). */
|
|
80
81
|
acceptsToolGate?: boolean;
|
|
81
82
|
}
|
|
82
83
|
|
|
@@ -100,7 +101,29 @@ export interface VoiceTTS {
|
|
|
100
101
|
setOnProgress?(fn: TtsProgress | null): void;
|
|
101
102
|
}
|
|
102
103
|
|
|
104
|
+
/** Host-supplied mic capture (React Native, Electron, tests) — replaces getUserMedia + AudioWorklet +
|
|
105
|
+
* Silero. Frames: 16 kHz mono Float32, ~256 ms (4096 samples). The bootstrap energy-VAD runs for
|
|
106
|
+
* the whole session; pair with a continuous STT (Deepgram Flux) for server-side end-of-turn. */
|
|
107
|
+
export interface AudioIO {
|
|
108
|
+
startCapture(onFrame: (f32: Float32Array) => void): Promise<{ stop(): void; setMuted?(muted: boolean): void }>;
|
|
109
|
+
}
|
|
110
|
+
/** Minimal clip surface StreamingTTS drives: position for the spoken cursor, pause() for barge-in,
|
|
111
|
+
* onended (natural end) / onpause (stopped) callbacks. */
|
|
112
|
+
export interface PlaybackClip {
|
|
113
|
+
readonly currentTime: number; readonly duration: number; paused: boolean; ended: boolean;
|
|
114
|
+
onended: (() => void) | null; onpause: (() => void) | null;
|
|
115
|
+
pause(): void;
|
|
116
|
+
}
|
|
117
|
+
/** Host-supplied TTS playout — replaces the shared Web Audio context + AEC loopback sink. */
|
|
118
|
+
export interface PlaybackIO {
|
|
119
|
+
load(blob: Blob): Promise<{ duration: number; start(offsetSec: number): PlaybackClip }>;
|
|
120
|
+
}
|
|
121
|
+
/** Route ALL StreamingTTS playback through `io` (process-wide); null restores Web Audio. */
|
|
122
|
+
export function setPlaybackIO(io: PlaybackIO | null): void;
|
|
123
|
+
|
|
103
124
|
export interface VoiceAgentOptions {
|
|
125
|
+
/** Native mic capture; omit for the browser pipeline. See AudioIO. */
|
|
126
|
+
audioIO?: AudioIO;
|
|
104
127
|
sysmsg?: string;
|
|
105
128
|
/** Persona/system prompt prepended to the conversation context. Defaults to the built-in
|
|
106
129
|
* voice persona (VOICE_SYSMSG); pass your own agent's system message to override it. */
|
package/src/index.js
CHANGED
package/src/llm-openai.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
// ── Default LLM: any OpenAI-compatible /chat/completions endpoint ───────────────────────────────
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
2
|
+
// A LOOP adapter (executesTools + acceptsToolGate): one turn may span several model messages. Each
|
|
3
|
+
// completion streams { text } deltas straight to the TTS; a COMPLETED tool call (arguments accumulate
|
|
4
|
+
// across deltas; a call whose arguments never parse as JSON is DROPPED with a warning — firing a
|
|
5
|
+
// side-effecting tool with guessed args would be worse) is announced ({ running: true }), executed
|
|
6
|
+
// here, reported with its outcome, fed back to the model as a tool message, and the model's next
|
|
7
|
+
// message starts after a { newMessage } boundary — bounded by MAX_ROUNDS per turn.
|
|
7
8
|
//
|
|
8
9
|
// Works against OpenAI, Groq, Cerebras, Together, OpenRouter, Ollama, vLLM, LiteLLM… anything that
|
|
9
10
|
// speaks /chat/completions SSE. NEVER ship a provider secret key to a public page — in production
|
|
@@ -37,6 +38,19 @@ export const toOpenAIMessages = (history) => history.flatMap((m) => {
|
|
|
37
38
|
return text ? [...results, { role: m.role, content: text }] : results; // OpenAI has no is_error → text-marked
|
|
38
39
|
});
|
|
39
40
|
|
|
41
|
+
// Render a host tool's return value as model-facing text (the tool_result content in history, and
|
|
42
|
+
// what adapters feed back to the model). Never throws: the side effect already RAN, so an
|
|
43
|
+
// unserializable result must not retro-fail the call and invite a retry.
|
|
44
|
+
export const toolResultText = (result) => {
|
|
45
|
+
if (result === undefined) return 'ok';
|
|
46
|
+
if (typeof result === 'string') return result;
|
|
47
|
+
if (result && typeof result.text === 'string' && typeof result.ok === 'boolean')
|
|
48
|
+
return result.ok ? result.text : `[failed] ${result.text}`;
|
|
49
|
+
try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const MAX_ROUNDS = 6; // model messages per user turn — a model that keeps calling tools then waits for the user
|
|
53
|
+
|
|
40
54
|
export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 1024, tools = {}, fetchFn, extraBody = {} } = {}) {
|
|
41
55
|
const toolSpecs = Object.entries(tools).map(([name, t]) => ({
|
|
42
56
|
type: 'function',
|
|
@@ -47,15 +61,15 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
|
|
|
47
61
|
},
|
|
48
62
|
}));
|
|
49
63
|
|
|
50
|
-
|
|
51
|
-
|
|
64
|
+
// One completion: streams { text } and COMPLETED { tool, args, id } chunks for a wire-shaped message list.
|
|
65
|
+
async function* streamOnce(messages, signal) {
|
|
52
66
|
const doFetch = fetchFn ?? fetch;
|
|
53
67
|
const r = await doFetch(llmUrl, {
|
|
54
68
|
method: 'POST', signal,
|
|
55
69
|
headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
|
|
56
70
|
body: JSON.stringify({
|
|
57
71
|
stream: true,
|
|
58
|
-
messages
|
|
72
|
+
messages,
|
|
59
73
|
max_tokens: maxTokens,
|
|
60
74
|
...(model ? { model } : {}),
|
|
61
75
|
...(toolSpecs.length ? { tools: toolSpecs } : {}),
|
|
@@ -76,9 +90,6 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
|
|
|
76
90
|
for (const c of pending.values()) {
|
|
77
91
|
if (!c.name) continue;
|
|
78
92
|
let args;
|
|
79
|
-
// Truncated/invalid argument JSON (aborted stream, provider bug) → DROP the call. The agent
|
|
80
|
-
// dispatches tools immediately, so guessing `{}` could fire a real side effect with the
|
|
81
|
-
// wrong arguments.
|
|
82
93
|
try { args = c.args ? JSON.parse(c.args) : {}; }
|
|
83
94
|
catch { console.warn(`makeOpenAILLM: dropping tool call "${c.name}" — arguments never parsed: ${c.args}`); continue; }
|
|
84
95
|
yield { tool: c.name, args, ...(c.id ? { id: c.id } : {}) };
|
|
@@ -86,9 +97,8 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
|
|
|
86
97
|
pending.clear();
|
|
87
98
|
};
|
|
88
99
|
|
|
89
|
-
// One SSE data payload → zero or more chunks.
|
|
100
|
+
// One SSE data payload → zero or more chunks.
|
|
90
101
|
const handle = function* (payload) {
|
|
91
|
-
if (payload === '[DONE]') return; // caller checks the sentinel itself
|
|
92
102
|
let m; try { m = JSON.parse(payload); } catch { return; } // tolerate keep-alive junk
|
|
93
103
|
if (m.error) throw new Error(m.error.message ?? String(m.error));
|
|
94
104
|
const choice = m.choices?.[0];
|
|
@@ -136,5 +146,46 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
|
|
|
136
146
|
} finally {
|
|
137
147
|
reader.cancel().catch(() => {});
|
|
138
148
|
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const gen = async function* (history, system, signal, { toolGate } = {}) {
|
|
152
|
+
if (!llmUrl) throw new Error('VoiceAgent: no `llm` generator and no `llmUrl` configured — pass one of them');
|
|
153
|
+
// Anthropic-shaped working history (what VoiceAgent keeps); re-serialized to the wire every round
|
|
154
|
+
// so tool_use/tool_result pairing stays in ONE place (toOpenAIMessages).
|
|
155
|
+
const hist = [...history];
|
|
156
|
+
for (let round = 0; round < MAX_ROUNDS; round++) {
|
|
157
|
+
let text = '';
|
|
158
|
+
const calls = []; // { id, tool, args, outcome: Promise<{result}|{error}> }
|
|
159
|
+
const seen = new Set();
|
|
160
|
+
for await (const c of streamOnce([{ role: 'system', content: system }, ...toOpenAIMessages(hist)], signal)) {
|
|
161
|
+
if (c.text) { text += c.text; yield c; continue; }
|
|
162
|
+
const id = c.id || `call-${round}-${calls.length + 1}`; // announcements need an id to be replaced in place
|
|
163
|
+
if (seen.has(id)) continue;
|
|
164
|
+
seen.add(id);
|
|
165
|
+
yield { tool: c.tool, id, args: c.args, running: true };
|
|
166
|
+
// Execute NOW, in parallel with TTS. A speculative run waits at the gate: it rejects when the
|
|
167
|
+
// speculation is dropped, so the tool never fires for a turn the user was still amending.
|
|
168
|
+
const outcome = (async () => {
|
|
169
|
+
if (toolGate) await toolGate;
|
|
170
|
+
try { return { result: await tools[c.tool]?.run?.(c.args) }; }
|
|
171
|
+
catch (e) { return { error: e?.message || String(e ?? 'failed') }; }
|
|
172
|
+
})();
|
|
173
|
+
calls.push({ id, tool: c.tool, args: c.args, outcome });
|
|
174
|
+
}
|
|
175
|
+
if (!calls.length) return;
|
|
176
|
+
const results = [];
|
|
177
|
+
for (const c of calls) {
|
|
178
|
+
const o = await c.outcome;
|
|
179
|
+
yield { tool: c.tool, id: c.id, args: c.args, ...o };
|
|
180
|
+
const failed = 'error' in o || (o.result && typeof o.result === 'object' && o.result.ok === false);
|
|
181
|
+
results.push({ type: 'tool_result', tool_use_id: c.id, content: 'error' in o ? String(o.error) : toolResultText(o.result), ...(failed ? { is_error: true } : {}) });
|
|
182
|
+
}
|
|
183
|
+
hist.push({ role: 'assistant', content: [...(text ? [{ type: 'text', text }] : []), ...calls.map((c) => ({ type: 'tool_use', id: c.id, name: c.tool, input: c.args ?? {} }))] });
|
|
184
|
+
hist.push({ role: 'user', content: results });
|
|
185
|
+
yield { newMessage: true };
|
|
186
|
+
}
|
|
139
187
|
};
|
|
188
|
+
gen.executesTools = true;
|
|
189
|
+
gen.acceptsToolGate = true;
|
|
190
|
+
return gen;
|
|
140
191
|
}
|
package/src/voice-agent.js
CHANGED
|
@@ -19,7 +19,8 @@
|
|
|
19
19
|
|
|
20
20
|
import { STT_PROVIDERS, resolveSttProvider } from './stt.js';
|
|
21
21
|
import { TUNING } from './tuning.js';
|
|
22
|
-
import { makeOpenAILLM } from './llm-openai.js';
|
|
22
|
+
import { makeOpenAILLM, toolResultText } from './llm-openai.js';
|
|
23
|
+
export { toolResultText };
|
|
23
24
|
|
|
24
25
|
// Default persona: a minimal voice-first system prompt. Hosts replace it via `persona`
|
|
25
26
|
// (who the agent is) and append live context via `sysmsg` / setSysmsg().
|
|
@@ -79,21 +80,10 @@ const withTimeout = (p, ms, fallback) => {
|
|
|
79
80
|
const callArgs = (c) => JSON.stringify(c.args ?? {}, (_k, v) =>
|
|
80
81
|
v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, v[k]])) : v);
|
|
81
82
|
const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
|
|
82
|
-
// Render a host tool's return value as model-facing text (the tool_result content in history, and
|
|
83
|
-
// what LLM adapters feed back to the model). Never throws: the side effect already RAN,
|
|
84
|
-
// so an unserializable result must not retro-fail the call and invite a retry.
|
|
85
|
-
export const toolResultText = (result) => {
|
|
86
|
-
if (result === undefined) return 'ok';
|
|
87
|
-
if (typeof result === 'string') return result;
|
|
88
|
-
if (result && typeof result.text === 'string' && typeof result.ok === 'boolean')
|
|
89
|
-
return result.ok ? result.text : `[failed] ${result.text}`;
|
|
90
|
-
try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
|
|
91
|
-
};
|
|
92
83
|
// One model-facing history record for a settled/pending tool call: the native tool_result block that
|
|
93
84
|
// pairs with its tool_use. Bounded — history is spoken-conversation-sized, not a log. A call whose
|
|
94
85
|
// outcome has not landed yet says so; _runTool patches the SAME block in place when it does.
|
|
95
86
|
const RESULT_MAX = 4000;
|
|
96
|
-
const MAX_TOOL_ROUNDS = 6;
|
|
97
87
|
const PENDING_RESULT = 'still running — the result will follow in a later turn';
|
|
98
88
|
const bound = (text) => text.length > RESULT_MAX ? `${text.slice(0, RESULT_MAX)}… (${text.length} chars)` : text;
|
|
99
89
|
const toolResultBlock = (c) => {
|
|
@@ -182,7 +172,13 @@ export class VoiceAgent {
|
|
|
182
172
|
tts, speed, sttLang = 'en', micDeviceId = '', sttProvider = 'webspeech', sttEotThreshold,
|
|
183
173
|
sttTokenUrl = '', getSttToken, sttUsageUrl = '', sttUrl = '', sttModel = '',
|
|
184
174
|
tools = {}, keyterms = [], preroll = TUNING.PREROLL_CHUNKS, vadOptions = {}, turnDetector = null,
|
|
185
|
-
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, onEvent = () => {} } = {}) {
|
|
175
|
+
maxPauseMs = TUNING.MAX_PAUSE_MS, bargeInMinChars, audioIO = null, onEvent = () => {} } = {}) {
|
|
176
|
+
// Host-supplied mic capture (React Native, Electron main, tests): `startCapture(onFrame)` delivers
|
|
177
|
+
// 16 kHz mono Float32 frames (~256 ms) and resolves a handle with stop()/setMuted(). Replaces the
|
|
178
|
+
// whole getUserMedia + AudioWorklet + Silero path — the bootstrap energy-VAD then runs for the
|
|
179
|
+
// session (no ONNX/WASM on native), which is fine with a continuous STT (Deepgram Flux) that
|
|
180
|
+
// does end-of-turn itself; VAD only drives state events + barge-in reset there.
|
|
181
|
+
this.audioIO = audioIO; this._capture = null;
|
|
186
182
|
// Each agent gets its own PiperTTS (it owns an AudioContext + playing node) — a shared
|
|
187
183
|
// module-level default would make two agents fight over one audio output. `speed` scales
|
|
188
184
|
// playback rate (1.2 → 20% faster) on the default TTS; ignored if a custom `tts` is passed.
|
|
@@ -314,8 +310,6 @@ export class VoiceAgent {
|
|
|
314
310
|
this.state = 'idle'; // idle | listening | thinking | speaking
|
|
315
311
|
this._abort = null; // aborts the in-flight LLM
|
|
316
312
|
this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
|
|
317
|
-
this._toolRounds = 0; // tool-result reactions since the user last spoke (see _runTurn react)
|
|
318
|
-
this._unseenResults = false; // a tool_result was patched after the last request was built (see _runTurn react)
|
|
319
313
|
this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
|
|
320
314
|
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
321
315
|
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
@@ -368,6 +362,7 @@ export class VoiceAgent {
|
|
|
368
362
|
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
|
|
369
363
|
return;
|
|
370
364
|
}
|
|
365
|
+
if (this.audioIO) return this._startNative();
|
|
371
366
|
// WARM RESUME: stop() pauses (not tears down) the pipeline, so a resume of the SAME agent finds the
|
|
372
367
|
// mic stream + audio graph + VAD model still alive — just re-arm them. This skips the getUserMedia
|
|
373
368
|
// permission round-trip AND the MicVAD.new() Silero/ONNX cold start (the bulk of "open→listening"
|
|
@@ -475,6 +470,27 @@ export class VoiceAgent {
|
|
|
475
470
|
})();
|
|
476
471
|
}
|
|
477
472
|
|
|
473
|
+
// Native capture path (see `audioIO`): same optimistic 'listening' flip + gen guard as the browser
|
|
474
|
+
// path, minus the audio graph and Silero. Warm resume reuses the live capture handle.
|
|
475
|
+
async _startNative() {
|
|
476
|
+
if (this._capture) { // warm resume — capture kept alive across stop()
|
|
477
|
+
this._capture.setMuted?.(this._muted);
|
|
478
|
+
this.stt.open?.(); this.tts.warm?.().catch(() => {});
|
|
479
|
+
this._set('listening');
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const gen = ++this._pipelineGen;
|
|
483
|
+
this._set('listening');
|
|
484
|
+
let cap;
|
|
485
|
+
try { cap = await this.audioIO.startCapture((f32) => { if (gen === this._pipelineGen && !this._closed) this._onCapture(f32); }); }
|
|
486
|
+
catch (e) { if (gen === this._pipelineGen) { this.stop(); this.onEvent({ type: 'error', error: e?.message || 'Microphone access failed' }); } return; }
|
|
487
|
+
if (gen !== this._pipelineGen || this._closed) { cap.stop(); return; }
|
|
488
|
+
this._capture = cap; cap.setMuted?.(this._muted);
|
|
489
|
+
this._useBootstrap = true; this._sileroPending = false; // energy-VAD for the whole session
|
|
490
|
+
this.stt.open?.();
|
|
491
|
+
setTimeout(() => { if (gen === this._pipelineGen && !this._closed) this.tts.warm?.().catch(() => {}); }, TUNING.TTS_WARM_DELAY_MS);
|
|
492
|
+
}
|
|
493
|
+
|
|
478
494
|
// True once the pipeline has been torn down (stop() / a fatal STT error). The host reconciles its
|
|
479
495
|
// own "running" flag against this after an error so a self-stopped agent isn't treated as live.
|
|
480
496
|
get closed() { return this._closed; }
|
|
@@ -502,6 +518,7 @@ export class VoiceAgent {
|
|
|
502
518
|
// session) and before a cold rebuild (mic device switch). After this, start() pays the full cold
|
|
503
519
|
// cost again. Split out so stop() can pause cheaply without duplicating this.
|
|
504
520
|
_teardownPipeline() {
|
|
521
|
+
this._capture?.stop(); this._capture = null;
|
|
505
522
|
this._vad?.destroy?.(); this._vad = null;
|
|
506
523
|
this._node?.disconnect(); this._node = null;
|
|
507
524
|
this._ctx?.close(); this._ctx = null;
|
|
@@ -590,7 +607,7 @@ export class VoiceAgent {
|
|
|
590
607
|
// VAD and STT, so nothing the user says reaches the agent — without dropping the mic permission
|
|
591
608
|
// or tearing down the pipeline. Stored so a mute toggled while paused applies on the next start().
|
|
592
609
|
setMuted(muted) {
|
|
593
|
-
this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted);
|
|
610
|
+
this._muted = muted; this._stream?.getAudioTracks().forEach(t => t.enabled = !muted); this._capture?.setMuted?.(muted);
|
|
594
611
|
if (muted) this._dropPrefetch(); // the utterance that seeded it just got cut off — no final will claim it
|
|
595
612
|
// A self-capturing STT (Web Speech) has no agent stream to gate — it listens on its OWN mic, so
|
|
596
613
|
// stop it explicitly via setEnabled or a "muted" agent keeps transcribing.
|
|
@@ -637,7 +654,6 @@ export class VoiceAgent {
|
|
|
637
654
|
// history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
|
|
638
655
|
// release (_flush) owns starting the single turn, and waits for all of these to settle first.
|
|
639
656
|
_pushHeld(text) {
|
|
640
|
-
this._toolRounds = 0;
|
|
641
657
|
const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
|
|
642
658
|
this._heldPushes.add(p);
|
|
643
659
|
p.finally(() => this._heldPushes.delete(p));
|
|
@@ -700,7 +716,6 @@ export class VoiceAgent {
|
|
|
700
716
|
// caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
|
|
701
717
|
_onUserTurn(text, { speech = false } = {}) {
|
|
702
718
|
if (!text.trim()) return Promise.resolve();
|
|
703
|
-
this._toolRounds = 0;
|
|
704
719
|
return this._runTurn(text, { speech });
|
|
705
720
|
}
|
|
706
721
|
|
|
@@ -737,8 +752,8 @@ export class VoiceAgent {
|
|
|
737
752
|
_startPrefetch(text) {
|
|
738
753
|
// Speculating is only safe if pulling the first chunk cannot fire a real side effect on a
|
|
739
754
|
// transcript the user is still speaking and may yet change — abort cannot undo a sent email.
|
|
740
|
-
//
|
|
741
|
-
//
|
|
755
|
+
// An LLM whose chunks are inert (no tool execution) is always safe. One that executes tools
|
|
756
|
+
// INSIDE its generator (llm.executesTools) is safe only if it also honors `toolGate`
|
|
742
757
|
// (llm.acceptsToolGate): we hand it a promise it must await before ANY tool runs, so text
|
|
743
758
|
// streams speculatively while tools wait for the turn to commit. Neither → don't speculate.
|
|
744
759
|
if (this.llm.executesTools && !this.llm.acceptsToolGate) return;
|
|
@@ -817,7 +832,7 @@ export class VoiceAgent {
|
|
|
817
832
|
// if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
|
|
818
833
|
// sees this message in history and answers over the fuller picture — so a burst of tool results
|
|
819
834
|
// collapses into ONE reply instead of one interrupting monologue per result.
|
|
820
|
-
_runTurn(text, { supersede = true, speech = false
|
|
835
|
+
_runTurn(text, { supersede = true, speech = false } = {}) {
|
|
821
836
|
const prev = this._turn;
|
|
822
837
|
if (supersede) {
|
|
823
838
|
this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
|
|
@@ -843,24 +858,9 @@ export class VoiceAgent {
|
|
|
843
858
|
// Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
|
|
844
859
|
const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
|
|
845
860
|
if (text) this.history.push({ role: 'user', content: text });
|
|
846
|
-
// A reaction turn owes a reply only for results no request has carried yet: either the
|
|
847
|
-
// tool_result record is still the last message, or a late outcome was patched in after the
|
|
848
|
-
// last request was built (_unseenResults; cleared when a request starts) — then a short
|
|
849
|
-
// [EVENT] user turn, the host's channel for out-of-band facts, points the model at the patched
|
|
850
|
-
// block (a request can't end on an assistant message). Bounded per user utterance
|
|
851
|
-
// (_toolRounds, reset on every accepted user turn): a model that keeps calling tools gets
|
|
852
|
-
// MAX_TOOL_ROUNDS reactions, then waits for the user.
|
|
853
|
-
if (react) {
|
|
854
|
-
const last = this.history[this.history.length - 1];
|
|
855
|
-
const owed = (last?.role === 'user' && Array.isArray(last.content)) || this._unseenResults;
|
|
856
|
-
if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this._unseenResults = false; return; }
|
|
857
|
-
this._toolRounds++;
|
|
858
|
-
if (last?.role !== 'user') this.history.push({ role: 'user', content: '[EVENT tool_result] a pending tool finished — see its tool_result above.' });
|
|
859
|
-
}
|
|
860
861
|
// Reply only if the conversation now ends on a user turn (a raced turn may have already
|
|
861
862
|
// replied). pf is always null here: if we pushed a user turn it IS the last entry.
|
|
862
863
|
if (this.history[this.history.length - 1]?.role !== 'user') return;
|
|
863
|
-
this._unseenResults = false; // this request carries every result landed so far
|
|
864
864
|
this._set('thinking'); // LLM stage
|
|
865
865
|
this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
|
|
866
866
|
await this._speakTurn(this._abort.signal, pf?.gen, seq); // _speakTurn never throws — errors surface via onEvent
|
|
@@ -888,9 +888,9 @@ export class VoiceAgent {
|
|
|
888
888
|
// notation it could imitate. `this.llm(...)` is a lazy generator — its fetch fires only when
|
|
889
889
|
// tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
|
|
890
890
|
//
|
|
891
|
-
// Round boundary:
|
|
892
|
-
// the
|
|
893
|
-
//
|
|
891
|
+
// Round boundary: a loop adapter (executesTools) feeds results back inside the stream and marks
|
|
892
|
+
// where the model's NEXT message starts with a { newMessage } chunk. A plain adapter's stream is
|
|
893
|
+
// ONE model message; the agent runs its calls and the NEXT user turn sees the results.
|
|
894
894
|
let answer = '';
|
|
895
895
|
const rounds = [], calls = []; // calls: every act this turn saw, for dedup — outlives recording
|
|
896
896
|
const loop = !!this.llm.executesTools;
|
|
@@ -898,27 +898,23 @@ export class VoiceAgent {
|
|
|
898
898
|
const newRound = () => { const r = rounds[rounds.length - 1]; if (r && (r.text || r.calls.length)) rounds.push({ text: '', calls: [] }); };
|
|
899
899
|
let n = 0;
|
|
900
900
|
// Accept a call once: identity is the provider id, else name+args (an id-less repeat is a
|
|
901
|
-
// provider re-emitting its buffered call).
|
|
902
|
-
// back inside its reply; anything else is agent-run (or agent-reported when the chunk already
|
|
903
|
-
// carries an outcome) and gets a reaction turn.
|
|
901
|
+
// provider re-emitting its buffered call).
|
|
904
902
|
const accept = (item) => {
|
|
905
903
|
const key = callKey(item);
|
|
906
904
|
let c = calls.find((x) => x.key === key);
|
|
907
905
|
if (c) return c;
|
|
908
|
-
c = { ...item, key, id: item.id || `call-${turn}-${++n}
|
|
906
|
+
c = { ...item, key, id: item.id || `call-${turn}-${++n}` };
|
|
909
907
|
calls.push(c); round().calls.push(c);
|
|
910
908
|
return c;
|
|
911
909
|
};
|
|
912
910
|
const outcome = (c, item) => { if ('result' in item) c.result = item.result; else if ('error' in item) c.error = item.error; };
|
|
913
911
|
const hasOutcome = (c) => 'result' in c || 'error' in c;
|
|
914
912
|
// An outcome may land AFTER the turn recorded the call (a hung tool, a barge-in mid-execution):
|
|
915
|
-
// its tool_result block is patched IN PLACE so the next request carries the real result
|
|
916
|
-
// model gets to react to it (see _continue).
|
|
913
|
+
// its tool_result block is patched IN PLACE so the next request carries the real result.
|
|
917
914
|
const settle = (c) => this._runTool(c).then(() => {
|
|
918
915
|
if (!c.block) return; // not recorded yet — record() picks the outcome up
|
|
919
916
|
if (!c.consumed) { Object.assign(c.block, toolResultBlock(c)); c.consumed = true; }
|
|
920
917
|
this._dropPrefetch(); // a speculation serialized the pending block — stale
|
|
921
|
-
this._unseenResults = true; this._continue();
|
|
922
918
|
});
|
|
923
919
|
const record = (keptLen) => {
|
|
924
920
|
let budget = keptLen;
|
|
@@ -936,11 +932,6 @@ export class VoiceAgent {
|
|
|
936
932
|
}
|
|
937
933
|
rounds.length = 0;
|
|
938
934
|
};
|
|
939
|
-
// Follow-up only for calls the AGENT ran: a loop adapter fed its results back inside the reply.
|
|
940
|
-
const recordAndContinue = (keptLen) => {
|
|
941
|
-
record(keptLen);
|
|
942
|
-
if (calls.some((c) => c.owner === 'agent' && c.consumed)) this._continue();
|
|
943
|
-
};
|
|
944
935
|
// Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
|
|
945
936
|
// announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
|
|
946
937
|
// that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
|
|
@@ -979,7 +970,7 @@ export class VoiceAgent {
|
|
|
979
970
|
if (finalized && !hasOutcome(item)) continue; // late bare call: the user cut this reply off
|
|
980
971
|
const c = accept(item);
|
|
981
972
|
outcome(c, item);
|
|
982
|
-
if (finalized) record(0); // late outcome of an unseen call: give it a record of its own
|
|
973
|
+
if (finalized) record(0); // late outcome of an unseen call: give it a record of its own
|
|
983
974
|
settle(c);
|
|
984
975
|
continue;
|
|
985
976
|
}
|
|
@@ -1043,7 +1034,7 @@ export class VoiceAgent {
|
|
|
1043
1034
|
// Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
|
|
1044
1035
|
// Record what was done and heard so far — dropping it would let the next turn call the
|
|
1045
1036
|
// same tool again.
|
|
1046
|
-
|
|
1037
|
+
record(signal.keepFull ? answer.length : this._replyText.length);
|
|
1047
1038
|
finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
|
|
1048
1039
|
if (this.state !== 'idle') this._set('listening');
|
|
1049
1040
|
return;
|
|
@@ -1067,7 +1058,7 @@ export class VoiceAgent {
|
|
|
1067
1058
|
// tail aligns). keepFull/normal-finish → whole answer (no tail); a barge-in → the heard prefix
|
|
1068
1059
|
// solid + the unspoken remainder dimmed, showing exactly where TTS got.
|
|
1069
1060
|
const kept = signal.keepFull ? answer : rawPrefix(spoken);
|
|
1070
|
-
|
|
1061
|
+
record(kept.length);
|
|
1071
1062
|
if (kept) {
|
|
1072
1063
|
this.onEvent({ type: 'assistant', text: kept, full: answer, final: true, turn });
|
|
1073
1064
|
} else if (answer) {
|
|
@@ -1085,19 +1076,6 @@ export class VoiceAgent {
|
|
|
1085
1076
|
if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
|
|
1086
1077
|
}
|
|
1087
1078
|
|
|
1088
|
-
// Native tool semantics: once a tool_result is in history the model gets to react to it. Queues a
|
|
1089
|
-
// SOFT reaction turn (behind the running one; dropped if a newer user turn supersedes it — that
|
|
1090
|
-
// turn sees the result anyway) after the agent ran a call for the default adapter (its stream
|
|
1091
|
-
// ends at the call; the result was never fed back), or when a LATE outcome (hung/slow tool,
|
|
1092
|
-
// barge-in mid-execution) landed in a record. Whether a reply is still owed is decided when the
|
|
1093
|
-
// turn RUNS (see _runTurn `react`), since the predecessor may already have carried the result.
|
|
1094
|
-
// Loop adapters fed the result back inside the reply, so nothing fires for them. Silent while
|
|
1095
|
-
// the user is speaking — their utterance will see the result in history anyway.
|
|
1096
|
-
_continue() {
|
|
1097
|
-
if (this._closed || this._speaking) return;
|
|
1098
|
-
this._runTurn('', { supersede: false, react: true });
|
|
1099
|
-
}
|
|
1100
|
-
|
|
1101
1079
|
// Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
|
|
1102
1080
|
// parallel with TTS, and NEVER joined: the turn records the call (tool_result pending) and finishes
|
|
1103
1081
|
// when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
|
|
@@ -1527,6 +1505,11 @@ function sentenceEnd(s) {
|
|
|
1527
1505
|
// context from the host's mic-button click (unlockAudio) and reuse it for all later playback.
|
|
1528
1506
|
let sharedCtx = null;
|
|
1529
1507
|
const audioCtx = () => (sharedCtx ??= new (window.AudioContext || window.webkitAudioContext)());
|
|
1508
|
+
// Host-supplied TTS playout (React Native etc.): `load(blob) → { duration, start(offsetSec) → clip }`
|
|
1509
|
+
// where clip has the WebAudioClip surface below (currentTime/duration/paused/ended/onended/onpause/pause).
|
|
1510
|
+
// Set once per process via setPlaybackIO(); null → shared Web Audio context + AEC loopback sink.
|
|
1511
|
+
let playbackIO = null;
|
|
1512
|
+
export function setPlaybackIO(io) { playbackIO = io; }
|
|
1530
1513
|
|
|
1531
1514
|
// ── AEC loopback sink ────────────────────────────────────────────────────────
|
|
1532
1515
|
// THE self-interruption root cause: the browser's echo canceller (getUserMedia
|
|
@@ -1681,6 +1664,7 @@ export class StreamingTTS {
|
|
|
1681
1664
|
// forward/backward tap inside it), as a non-whitespace offset into `text`. abort always dominates seek.
|
|
1682
1665
|
async _playBuf(wav, text, signal, prefix = '', startNw = 0) {
|
|
1683
1666
|
if (signal?.aborted || !wav) return { reason: 'abort', heard: prefix };
|
|
1667
|
+
if (playbackIO) return this._playNative(wav, text, signal, prefix, startNw);
|
|
1684
1668
|
const ctx = audioCtx();
|
|
1685
1669
|
// Decode first (works while suspended) so we don't block synthesis on resume.
|
|
1686
1670
|
let buffer;
|
|
@@ -1709,7 +1693,21 @@ export class StreamingTTS {
|
|
|
1709
1693
|
if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
|
|
1710
1694
|
const dur = buffer.duration;
|
|
1711
1695
|
const offsetSec = startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0; // tap-into-clip → start offset
|
|
1712
|
-
|
|
1696
|
+
return this._awaitClip(new WebAudioClip(ctx, buffer, offsetSec), text, signal, prefix, startNw, dur);
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
async _playNative(wav, text, signal, prefix, startNw) {
|
|
1700
|
+
let src;
|
|
1701
|
+
try { src = await playbackIO.load(wav); }
|
|
1702
|
+
catch (e) { throw new Error(`Audio decode: ${e?.message || e}`); }
|
|
1703
|
+
if (signal?.aborted) return { reason: 'abort', heard: prefix + sliceNw(text, startNw) };
|
|
1704
|
+
if (this._seekPending) return { reason: 'seek', heard: prefix + sliceNw(text, startNw) };
|
|
1705
|
+
const dur = src.duration;
|
|
1706
|
+
return this._awaitClip(src.start(startNw > 0 ? (startNw / (nw(text) || 1)) * dur : 0), text, signal, prefix, startNw, dur);
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
// Drive one started clip to a settled {reason, heard}: abort/seek/ended + the spoken-cursor ticks.
|
|
1710
|
+
_awaitClip(clip, text, signal, prefix, startNw, dur) {
|
|
1713
1711
|
this._audio = clip;
|
|
1714
1712
|
return new Promise((resolve) => {
|
|
1715
1713
|
// Proportional position in NON-WHITESPACE space (the same domain seek() uses) → slice `text` to that
|