@todoforai/voiceloop 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/index.d.ts +20 -6
- package/src/index.js +1 -1
- package/src/llm-openai.js +19 -1
- package/src/voice-agent.js +171 -149
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.12",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"index.html"
|
|
22
22
|
],
|
|
23
23
|
"scripts": {
|
|
24
|
-
"test": "node --test src/*.test.js"
|
|
24
|
+
"test": "node --test src/*.test.js bench/*.test.js bench/blackbox/energy.test.js"
|
|
25
25
|
},
|
|
26
26
|
"keywords": [
|
|
27
27
|
"voice",
|
package/src/index.d.ts
CHANGED
|
@@ -44,19 +44,31 @@ export interface VoiceTool {
|
|
|
44
44
|
export type LLMChunk =
|
|
45
45
|
| { text: string }
|
|
46
46
|
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
|
|
47
|
+
// The conversation as the model sees it, Anthropic-shaped. Plain turns are strings; a turn in which
|
|
48
|
+
// the assistant called tools is recorded NATIVELY: an assistant message of text + tool_use blocks,
|
|
49
|
+
// then a user message holding the matching tool_result blocks (content = the result text the model
|
|
50
|
+
// reads; `is_error` on a failure). The agent never writes a textual notation of a tool call into
|
|
51
|
+
// history, so the model has nothing to imitate. Adapters project this onto their wire format
|
|
52
|
+
// (see toOpenAIMessages).
|
|
53
|
+
export type HistoryBlock =
|
|
54
|
+
| { type: 'text'; text: string }
|
|
55
|
+
| { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
|
|
56
|
+
| { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean };
|
|
57
|
+
export type HistoryMessage = { role: string; content: string | HistoryBlock[] };
|
|
47
58
|
// Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
|
|
48
|
-
// The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes;
|
|
49
|
-
//
|
|
59
|
+
// The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; their results are
|
|
60
|
+
// recorded as tool_result and the agent starts a follow-up turn so the model reacts to them.
|
|
50
61
|
// An agent LOOP that runs tools inside its own generator declares the two flags below.
|
|
51
62
|
export interface LLM {
|
|
52
63
|
(
|
|
53
|
-
history:
|
|
64
|
+
history: HistoryMessage[],
|
|
54
65
|
system: string,
|
|
55
66
|
signal: AbortSignal,
|
|
56
67
|
options?: { toolGate?: Promise<unknown> },
|
|
57
68
|
): AsyncIterable<LLMChunk>;
|
|
58
|
-
/** This generator
|
|
59
|
-
*
|
|
69
|
+
/** This generator is a tool LOOP: it runs every call itself and feeds the result back inside the
|
|
70
|
+
* same reply (chunks carry `result`/`error`, optionally preceded by a `running` announcement).
|
|
71
|
+
* The agent never executes its calls and never starts a reaction turn for them. */
|
|
60
72
|
executesTools?: boolean;
|
|
61
73
|
/** This generator awaits `options.toolGate` before ANY tool executes. Required to keep
|
|
62
74
|
* speculative prefetch on: prefetch starts from an uncommitted interim, and the gate is what
|
|
@@ -190,7 +202,7 @@ export class VoiceAgent {
|
|
|
190
202
|
constructor(options?: VoiceAgentOptions);
|
|
191
203
|
/** The live conversation the LLM sees. A mutable reference the host may read and push into —
|
|
192
204
|
* carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
|
|
193
|
-
history:
|
|
205
|
+
history: HistoryMessage[];
|
|
194
206
|
start(deviceId?: string): Promise<void>;
|
|
195
207
|
/** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
|
|
196
208
|
* start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
|
|
@@ -344,6 +356,8 @@ export function makeOpenAILLM(options?: {
|
|
|
344
356
|
llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
|
|
345
357
|
tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
|
|
346
358
|
}): LLM;
|
|
359
|
+
/** VoiceAgent history → OpenAI chat messages (tool_use → tool_calls, tool_result → role:"tool"). */
|
|
360
|
+
export function toOpenAIMessages(history: HistoryMessage[]): Array<Record<string, unknown>>;
|
|
347
361
|
|
|
348
362
|
// ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
|
|
349
363
|
/** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
|
package/src/index.js
CHANGED
|
@@ -11,5 +11,5 @@ export {
|
|
|
11
11
|
} from './stt.js';
|
|
12
12
|
export { ElevenLabsTTS } from './tts-elevenlabs.js';
|
|
13
13
|
export { loadVoiceDeps, prewarmVoice } from './deps.js';
|
|
14
|
-
export { makeOpenAILLM } from './llm-openai.js';
|
|
14
|
+
export { makeOpenAILLM, toOpenAIMessages } from './llm-openai.js';
|
|
15
15
|
export { TUNING } from './tuning.js';
|
package/src/llm-openai.js
CHANGED
|
@@ -19,6 +19,24 @@
|
|
|
19
19
|
// keys, defaulting to all of them)
|
|
20
20
|
// fetchFn — fetch override (tests, custom agents/headers); defaults to global fetch
|
|
21
21
|
// extraBody— merged into the request body (temperature, provider-specific knobs, …)
|
|
22
|
+
// VoiceAgent history → OpenAI chat messages. Plain turns pass through; the agent's native tool
|
|
23
|
+
// records (assistant text + tool_use blocks, then a user turn of tool_result blocks — the same
|
|
24
|
+
// Anthropic-shaped blocks it keeps for every adapter) become `tool_calls` and `role:"tool"` messages.
|
|
25
|
+
export const toOpenAIMessages = (history) => history.flatMap((m) => {
|
|
26
|
+
if (!Array.isArray(m.content)) return [m];
|
|
27
|
+
const text = m.content.filter((b) => b.type === 'text').map((b) => b.text).join('');
|
|
28
|
+
if (m.role === 'assistant') {
|
|
29
|
+
const tool_calls = m.content.filter((b) => b.type === 'tool_use')
|
|
30
|
+
.map((b) => ({ id: b.id, type: 'function', function: { name: b.name, arguments: JSON.stringify(b.input ?? {}) } }));
|
|
31
|
+
return [{ role: 'assistant', content: text || null, ...(tool_calls.length ? { tool_calls } : {}) }];
|
|
32
|
+
}
|
|
33
|
+
const results = m.content.filter((b) => b.type === 'tool_result').map((b) => {
|
|
34
|
+
const content = typeof b.content === 'string' ? b.content : String(b.content ?? '');
|
|
35
|
+
return { role: 'tool', tool_call_id: b.tool_use_id, content: b.is_error && !content.startsWith('[failed]') ? `[failed] ${content}` : content };
|
|
36
|
+
});
|
|
37
|
+
return text ? [...results, { role: m.role, content: text }] : results; // OpenAI has no is_error → text-marked
|
|
38
|
+
});
|
|
39
|
+
|
|
22
40
|
export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 1024, tools = {}, fetchFn, extraBody = {} } = {}) {
|
|
23
41
|
const toolSpecs = Object.entries(tools).map(([name, t]) => ({
|
|
24
42
|
type: 'function',
|
|
@@ -37,7 +55,7 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
|
|
|
37
55
|
headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
|
|
38
56
|
body: JSON.stringify({
|
|
39
57
|
stream: true,
|
|
40
|
-
messages: [{ role: 'system', content: system }, ...history],
|
|
58
|
+
messages: [{ role: 'system', content: system }, ...toOpenAIMessages(history)],
|
|
41
59
|
max_tokens: maxTokens,
|
|
42
60
|
...(model ? { model } : {}),
|
|
43
61
|
...(toolSpecs.length ? { tools: toolSpecs } : {}),
|
package/src/voice-agent.js
CHANGED
|
@@ -60,7 +60,7 @@ const langRule = (lang) => {
|
|
|
60
60
|
return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
|
|
61
61
|
};
|
|
62
62
|
// A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
|
|
63
|
-
// what order a provider streamed the keys in (used
|
|
63
|
+
// what order a provider streamed the keys in (used for dedup).
|
|
64
64
|
// Identity: the provider call id when present (two calls with identical args are DISTINCT acts,
|
|
65
65
|
// e.g. a legit retry after an error result); name+args otherwise (an id-less identical repeat is a
|
|
66
66
|
// provider re-emitting its buffered call).
|
|
@@ -78,8 +78,8 @@ const withTimeout = (p, ms, fallback) => {
|
|
|
78
78
|
|
|
79
79
|
const callArgs = (c) => JSON.stringify(c.args ?? {}, Object.keys(c.args ?? {}).sort());
|
|
80
80
|
const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
|
|
81
|
-
// Render a host tool's return value as model-facing text (
|
|
82
|
-
//
|
|
81
|
+
// Render a host tool's return value as model-facing text (the tool_result content in history, and
|
|
82
|
+
// what LLM adapters feed back to the model). Never throws: the side effect already RAN,
|
|
83
83
|
// so an unserializable result must not retro-fail the call and invite a retry.
|
|
84
84
|
export const toolResultText = (result) => {
|
|
85
85
|
if (result === undefined) return 'ok';
|
|
@@ -88,87 +88,22 @@ export const toolResultText = (result) => {
|
|
|
88
88
|
return result.ok ? result.text : `[failed] ${result.text}`;
|
|
89
89
|
try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
|
|
90
90
|
};
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
const
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
// The model sees `[TOOL CALL name] {...}` ledger lines in its own history turns, and sometimes
|
|
105
|
-
// IMITATES the format instead of calling the tool natively: it types the line as reply text.
|
|
106
|
-
// Untreated, the raw markup leaks into the spoken/visible reply and the tool never runs. This
|
|
107
|
-
// transform strips such lines from the text stream: a bare call (no → outcome) is what the model
|
|
108
|
-
// MEANT to do → re-emitted as a real { tool, args } chunk (the turn's dedup collapses it if the
|
|
109
|
-
// native call also arrived); a line WITH an → outcome is a hallucinated memory of a past call →
|
|
110
|
-
// dropped. Text before/after the line streams through untouched, held back only while a possible
|
|
111
|
-
// marker/line is still forming so TTS latency is unaffected on normal replies.
|
|
112
|
-
const MIMIC_MARK = '[TOOL CALL ';
|
|
113
|
-
const parseMimicLine = (line) => {
|
|
114
|
-
const m = /^\[TOOL CALL ([\w.-]+)\]\s*(.*)$/.exec(line.trim());
|
|
115
|
-
if (!m) return null; // not the format after all — caller emits it as text
|
|
116
|
-
const [, tool, rest] = m;
|
|
117
|
-
if (!rest) return { tool, args: {} };
|
|
118
|
-
if (!rest.startsWith('{')) return { drop: true }; // `→ outcome` (or junk) with no args — a remembered call
|
|
119
|
-
// Balanced-brace scan (string-aware): the args JSON may be followed by a ` → outcome` tail.
|
|
120
|
-
let depth = 0, inStr = false, escaped = false, end = -1;
|
|
121
|
-
for (let i = 0; i < rest.length; i++) {
|
|
122
|
-
const ch = rest[i];
|
|
123
|
-
if (escaped) { escaped = false; continue; }
|
|
124
|
-
if (inStr) { if (ch === '\\') escaped = true; else if (ch === '"') inStr = false; continue; }
|
|
125
|
-
if (ch === '"') inStr = true;
|
|
126
|
-
else if (ch === '{') depth++;
|
|
127
|
-
else if (ch === '}' && --depth === 0) { end = i + 1; break; }
|
|
91
|
+
// One model-facing history record for a settled/pending tool call: the native tool_result block that
|
|
92
|
+
// pairs with its tool_use. Bounded — history is spoken-conversation-sized, not a log. A call whose
|
|
93
|
+
// outcome has not landed yet says so; _runTool patches the SAME block in place when it does.
|
|
94
|
+
const RESULT_MAX = 4000;
|
|
95
|
+
const MAX_TOOL_ROUNDS = 6;
|
|
96
|
+
const PENDING_RESULT = 'still running — the result will follow in a later turn';
|
|
97
|
+
const bound = (text) => text.length > RESULT_MAX ? `${text.slice(0, RESULT_MAX)}… (${text.length} chars)` : text;
|
|
98
|
+
const toolResultBlock = (c) => {
|
|
99
|
+
const block = { type: 'tool_result', tool_use_id: c.id, content: PENDING_RESULT };
|
|
100
|
+
if ('error' in c) { block.content = bound(String(c.error) || 'failed'); block.is_error = true; }
|
|
101
|
+
else if ('result' in c) {
|
|
102
|
+
block.content = bound(toolResultText(c.result));
|
|
103
|
+
if (c.result && typeof c.result === 'object' && c.result.ok === false) block.is_error = true;
|
|
128
104
|
}
|
|
129
|
-
|
|
130
|
-
if (rest.slice(end).trim()) return { drop: true }; // has an → outcome: a memory, not a new call
|
|
131
|
-
try { return { tool, args: JSON.parse(rest.slice(0, end)) }; } catch { return { drop: true }; }
|
|
105
|
+
return block;
|
|
132
106
|
};
|
|
133
|
-
async function* stripToolCallMimicry(src) {
|
|
134
|
-
let carry = '';
|
|
135
|
-
// Longest tail of `carry` that could still grow into the marker — held back; the rest is safe to emit.
|
|
136
|
-
const heldLen = () => {
|
|
137
|
-
for (let n = Math.min(carry.length, MIMIC_MARK.length - 1); n > 0; n--)
|
|
138
|
-
if (MIMIC_MARK.startsWith(carry.slice(-n))) return n;
|
|
139
|
-
return 0;
|
|
140
|
-
};
|
|
141
|
-
function* pump(final) {
|
|
142
|
-
for (;;) {
|
|
143
|
-
const at = carry.indexOf(MIMIC_MARK);
|
|
144
|
-
if (at < 0) {
|
|
145
|
-
const hold = final ? 0 : heldLen();
|
|
146
|
-
const out = carry.slice(0, carry.length - hold);
|
|
147
|
-
carry = carry.slice(carry.length - hold);
|
|
148
|
-
if (out) yield { text: out };
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
const nl = carry.indexOf('\n', at);
|
|
152
|
-
if (nl < 0 && !final) { // the line is still streaming — hold from the marker on
|
|
153
|
-
if (at > 0) { yield { text: carry.slice(0, at) }; carry = carry.slice(at); }
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
const end = nl < 0 ? carry.length : nl;
|
|
157
|
-
const before = carry.slice(0, at), line = carry.slice(at, end);
|
|
158
|
-
carry = nl < 0 ? '' : carry.slice(nl + 1); // the line (and its newline) leave the text stream
|
|
159
|
-
if (before) yield { text: before };
|
|
160
|
-
const parsed = parseMimicLine(line);
|
|
161
|
-
if (!parsed) yield { text: nl < 0 ? line : `${line}\n` }; // false alarm — pass it through untouched (newline included)
|
|
162
|
-
else if (parsed.tool) yield { tool: parsed.tool, args: parsed.args };
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
for await (const item of src) {
|
|
166
|
-
if (item.tool) { yield item; continue; }
|
|
167
|
-
carry += item.text ?? '';
|
|
168
|
-
yield* pump(false);
|
|
169
|
-
}
|
|
170
|
-
yield* pump(true);
|
|
171
|
-
}
|
|
172
107
|
|
|
173
108
|
// Persona (who the agent is) + language rule + the host's live context block (`sysmsg`, delimited
|
|
174
109
|
// so the model never mistakes injected context for its own conversation).
|
|
@@ -378,6 +313,8 @@ export class VoiceAgent {
|
|
|
378
313
|
this.state = 'idle'; // idle | listening | thinking | speaking
|
|
379
314
|
this._abort = null; // aborts the in-flight LLM
|
|
380
315
|
this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
|
|
316
|
+
this._toolRounds = 0; // tool-result reactions since the user last spoke (see _runTurn react)
|
|
317
|
+
this._late = []; // outcomes landed after the last request was built (see _runTurn react)
|
|
381
318
|
this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
|
|
382
319
|
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
383
320
|
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
@@ -699,6 +636,7 @@ export class VoiceAgent {
|
|
|
699
636
|
// history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
|
|
700
637
|
// release (_flush) owns starting the single turn, and waits for all of these to settle first.
|
|
701
638
|
_pushHeld(text) {
|
|
639
|
+
this._toolRounds = 0;
|
|
702
640
|
const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
|
|
703
641
|
this._heldPushes.add(p);
|
|
704
642
|
p.finally(() => this._heldPushes.delete(p));
|
|
@@ -720,15 +658,14 @@ export class VoiceAgent {
|
|
|
720
658
|
}
|
|
721
659
|
|
|
722
660
|
// Inject an out-of-band turn: feed `text` to the LLM as if the user had said it, then speak the
|
|
723
|
-
// reply. Used by the host to surface external events (a
|
|
661
|
+
// reply. Used by the host to surface external events (a background job finishing, a board change)
|
|
724
662
|
// — only meaningful while the agent is running, so it's a no-op once stopped.
|
|
725
663
|
//
|
|
726
664
|
// SOFT turn (supersede: false): unlike a user utterance this must NOT cut the reply in flight.
|
|
727
|
-
//
|
|
728
|
-
//
|
|
729
|
-
//
|
|
730
|
-
//
|
|
731
|
-
// [TOOL CALL] it made, so the model sees the work is already done.
|
|
665
|
+
// An event can arrive WHILE a turn is still streaming/speaking, so aborting here would kill that
|
|
666
|
+
// turn mid-answer and re-run the LLM over a history whose last entry is the request again — the
|
|
667
|
+
// model then re-calls the same tool (duplicate-action spam). Instead it queues behind the running
|
|
668
|
+
// turn, which by then has recorded its answer and its tool calls natively.
|
|
732
669
|
async notify(text) { if (!this._closed) await this._runTurn(text, { supersede: false }); }
|
|
733
670
|
|
|
734
671
|
// Send a TYPED user turn: like speaking, but the text comes from the host's input instead of STT.
|
|
@@ -762,6 +699,7 @@ export class VoiceAgent {
|
|
|
762
699
|
// caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
|
|
763
700
|
_onUserTurn(text, { speech = false } = {}) {
|
|
764
701
|
if (!text.trim()) return Promise.resolve();
|
|
702
|
+
this._toolRounds = 0;
|
|
765
703
|
return this._runTurn(text, { speech });
|
|
766
704
|
}
|
|
767
705
|
|
|
@@ -878,7 +816,7 @@ export class VoiceAgent {
|
|
|
878
816
|
// if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
|
|
879
817
|
// sees this message in history and answers over the fuller picture — so a burst of tool results
|
|
880
818
|
// collapses into ONE reply instead of one interrupting monologue per result.
|
|
881
|
-
_runTurn(text, { supersede = true, speech = false } = {}) {
|
|
819
|
+
_runTurn(text, { supersede = true, speech = false, react = false } = {}) {
|
|
882
820
|
const prev = this._turn;
|
|
883
821
|
if (supersede) {
|
|
884
822
|
this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
|
|
@@ -904,9 +842,24 @@ export class VoiceAgent {
|
|
|
904
842
|
// Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
|
|
905
843
|
const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
|
|
906
844
|
if (text) this.history.push({ role: 'user', content: text });
|
|
845
|
+
// A reaction turn owes a reply only for results no request has carried yet: either the
|
|
846
|
+
// tool_result record is still the last message, or a late outcome landed after the last
|
|
847
|
+
// request was built (_late; cleared when a request starts) — then a short [EVENT] user turn,
|
|
848
|
+
// the host's channel for out-of-band facts, points the model at the patched block (a request
|
|
849
|
+
// can't end on an assistant message). Bounded per user utterance (_toolRounds, reset on
|
|
850
|
+
// every accepted user turn): a model that keeps calling tools gets MAX_TOOL_ROUNDS
|
|
851
|
+
// reactions, then waits for the user.
|
|
852
|
+
if (react) {
|
|
853
|
+
const last = this.history[this.history.length - 1];
|
|
854
|
+
const owed = (last?.role === 'user' && Array.isArray(last.content)) || this._late.length;
|
|
855
|
+
if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this._late.length = 0; return; }
|
|
856
|
+
this._toolRounds++;
|
|
857
|
+
if (last?.role !== 'user') this.history.push({ role: 'user', content: `[EVENT tool_result] ${this._late.map((c) => c.tool).join(', ')} finished — see the tool_result above.` });
|
|
858
|
+
}
|
|
907
859
|
// Reply only if the conversation now ends on a user turn (a raced turn may have already
|
|
908
860
|
// replied). pf is always null here: if we pushed a user turn it IS the last entry.
|
|
909
861
|
if (this.history[this.history.length - 1]?.role !== 'user') return;
|
|
862
|
+
this._late.length = 0; // this request carries every result landed so far
|
|
910
863
|
this._set('thinking'); // LLM stage
|
|
911
864
|
this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
|
|
912
865
|
await this._speakTurn(this._abort.signal, pf?.gen, seq); // _speakTurn never throws — errors surface via onEvent
|
|
@@ -914,7 +867,7 @@ export class VoiceAgent {
|
|
|
914
867
|
}
|
|
915
868
|
|
|
916
869
|
// One assistant turn: stream the LLM → TTS, then record what was actually HEARD (the barge-in
|
|
917
|
-
// prefix, not the full generated answer) plus
|
|
870
|
+
// prefix, not the full generated answer) plus the tools it called, natively. Stored as
|
|
918
871
|
// `this._turn` so the next user turn can await it. Never throws (errors surface via onEvent) so
|
|
919
872
|
// the await is safe — and never awaits a tool, so a hung one can't hold the next turn hostage.
|
|
920
873
|
// `turn` tags every assistant event with the reply it belongs to, so a host can tell a LATE
|
|
@@ -927,31 +880,82 @@ export class VoiceAgent {
|
|
|
927
880
|
// repopulates _replyText with the audible prefix as playback advances.
|
|
928
881
|
if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText;
|
|
929
882
|
this._replyText = ''; this._replyEcho = '';
|
|
930
|
-
// Tap the LLM stream once: yield speech text to TTS, and record the
|
|
931
|
-
//
|
|
932
|
-
//
|
|
883
|
+
// Tap the LLM stream once: yield speech text to TTS, and record what the model did as ROUNDS —
|
|
884
|
+
// each round is one assistant message (text + the tool_use blocks it emitted) followed by the
|
|
885
|
+
// user-role tool_result blocks for those calls. Recorded into history in that order on EVERY
|
|
886
|
+
// exit path, so the next request replays the model's own native tool history — never a text
|
|
887
|
+
// notation it could imitate. `this.llm(...)` is a lazy generator — its fetch fires only when
|
|
933
888
|
// tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
|
|
934
|
-
|
|
935
|
-
//
|
|
936
|
-
//
|
|
937
|
-
//
|
|
938
|
-
|
|
939
|
-
// calls:
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
889
|
+
//
|
|
890
|
+
// Round boundary: the default adapter's stream is ONE model message (text + calls together;
|
|
891
|
+
// the reaction to the results comes from _continue). A loop adapter feeds results back inside
|
|
892
|
+
// the stream, so anything after an outcome landed is the model's NEXT message.
|
|
893
|
+
let answer = '';
|
|
894
|
+
const rounds = [], calls = []; // calls: every act this turn saw, for dedup — outlives recording
|
|
895
|
+
const loop = !!this.llm.executesTools;
|
|
896
|
+
const round = () => {
|
|
897
|
+
const r = rounds[rounds.length - 1];
|
|
898
|
+
if (r && !(loop && r.settled)) return r;
|
|
899
|
+
rounds.push({ text: '', calls: [] }); return rounds[rounds.length - 1];
|
|
900
|
+
};
|
|
901
|
+
let n = 0;
|
|
902
|
+
// Accept a call once: identity is the provider id, else name+args (an id-less repeat is a
|
|
903
|
+
// provider re-emitting its buffered call). `owner` says who executes it — the adapter (the
|
|
904
|
+
// chunk announces or carries the outcome: it ran the tool itself and feeds the result back
|
|
905
|
+
// inside its own reply) or the agent (_runTool runs it) — fixed at acceptance.
|
|
906
|
+
const accept = (item) => {
|
|
907
|
+
const key = callKey(item);
|
|
908
|
+
let c = calls.find((x) => x.key === key);
|
|
909
|
+
if (c) return c;
|
|
910
|
+
const adapterRan = loop || item.running || 'result' in item || 'error' in item;
|
|
911
|
+
c = { ...item, key, id: item.id || `call-${turn}-${++n}`, owner: adapterRan ? 'adapter' : 'agent' };
|
|
912
|
+
calls.push(c); round().calls.push(c);
|
|
913
|
+
return c;
|
|
914
|
+
};
|
|
915
|
+
const outcome = (c, item) => {
|
|
916
|
+
if ('result' in item) c.result = item.result; else if ('error' in item) c.error = item.error;
|
|
917
|
+
const r = rounds.find((r) => r.calls.includes(c)); if (r) r.settled = true; // already recorded → no round to close
|
|
918
|
+
};
|
|
919
|
+
// An outcome may land AFTER the turn recorded the call (a hung tool, a barge-in mid-execution):
|
|
920
|
+
// its tool_result block is patched IN PLACE so the next request carries the real result, and the
|
|
921
|
+
// model gets to react to it (see _continue).
|
|
922
|
+
const settle = (c) => this._runTool(c).then(() => {
|
|
923
|
+
if (!c.block) return; // not recorded yet — record() picks the outcome up
|
|
924
|
+
if (!c.consumed) { Object.assign(c.block, toolResultBlock(c)); c.consumed = true; }
|
|
925
|
+
this._dropPrefetch(); // a speculation serialized the pending block — stale
|
|
926
|
+
this._late.push(c); this._continue();
|
|
927
|
+
});
|
|
928
|
+
const record = (keptLen) => {
|
|
929
|
+
let budget = keptLen;
|
|
930
|
+
for (const r of rounds) {
|
|
931
|
+
const text = r.text.slice(0, Math.max(0, budget)).trimEnd(); budget -= r.text.length;
|
|
932
|
+
if (!text && !r.calls.length) continue;
|
|
933
|
+
if (!r.calls.length) { this.history.push({ role: 'assistant', content: text }); continue; }
|
|
934
|
+
const blocks = text ? [{ type: 'text', text }] : [];
|
|
935
|
+
for (const c of r.calls) blocks.push({ type: 'tool_use', id: c.id, name: c.tool, input: c.args ?? {} });
|
|
936
|
+
this.history.push({ role: 'assistant', content: blocks });
|
|
937
|
+
this.history.push({ role: 'user', content: r.calls.map((c) => {
|
|
938
|
+
c.consumed = 'result' in c || 'error' in c;
|
|
939
|
+
return (c.block = toolResultBlock(c));
|
|
940
|
+
}) });
|
|
941
|
+
}
|
|
942
|
+
rounds.length = 0;
|
|
943
|
+
};
|
|
944
|
+
// Follow-up only for calls the AGENT ran: a loop adapter fed its results back inside the reply.
|
|
945
|
+
const recordAndContinue = (keptLen) => {
|
|
946
|
+
record(keptLen);
|
|
947
|
+
if (calls.some((c) => c.owner === 'agent' && c.consumed)) this._continue();
|
|
948
|
+
};
|
|
945
949
|
// Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
|
|
946
950
|
// announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
|
|
947
951
|
// that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
|
|
948
|
-
// outcome arrives finalizes
|
|
949
|
-
//
|
|
952
|
+
// outcome arrives finalizes the CHIP as interrupted; the history keeps the call with a pending
|
|
953
|
+
// tool_result, patched in place if the outcome still lands.
|
|
950
954
|
const announced = new Map();
|
|
951
955
|
// Latched once this turn has finalized its chips (finishInterrupted ran). A barge-in detaches the
|
|
952
956
|
// LLM producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
|
|
953
|
-
// announcement would open a spinner nothing will ever close
|
|
954
|
-
//
|
|
957
|
+
// announcement would open a spinner nothing will ever close, and a late UNEXECUTED call is work
|
|
958
|
+
// the user cut off. Past the latch only outcomes flow (a tool the adapter really ran is truth).
|
|
955
959
|
let finalized = false;
|
|
956
960
|
const finishInterrupted = () => {
|
|
957
961
|
finalized = true;
|
|
@@ -964,29 +968,31 @@ export class VoiceAgent {
|
|
|
964
968
|
// tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
|
|
965
969
|
// act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
|
|
966
970
|
if (item.tool) {
|
|
967
|
-
|
|
971
|
+
const hasOutcome = 'result' in item || 'error' in item;
|
|
972
|
+
const known = calls.find((x) => x.key === callKey(item));
|
|
973
|
+
if (item.running || (loop && !hasOutcome)) { // announcement; the outcome chunk below does the work
|
|
968
974
|
// An id-less announcement can never be replaced in place NOR finalized (both key on id),
|
|
969
975
|
// so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
|
|
970
|
-
if (finalized || !item.id) continue;
|
|
971
|
-
|
|
972
|
-
// reopen a spinner over a finished chip, and the duplicate outcome is deduped below and
|
|
973
|
-
// never re-reported — so it would hang there, then finalize as falsely "interrupted".
|
|
974
|
-
if (calls.some((c) => c.id === item.id)) continue;
|
|
975
|
-
announced.set(item.id, item);
|
|
976
|
+
if (finalized || !item.id || known) continue;
|
|
977
|
+
announced.set(item.id, item); accept(item);
|
|
976
978
|
self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
|
|
977
979
|
continue;
|
|
978
980
|
}
|
|
979
981
|
if (item.id) announced.delete(item.id);
|
|
980
|
-
//
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
if (
|
|
982
|
+
if (known) { // the announced call's outcome, or a re-emit
|
|
983
|
+
if (known.ran || !hasOutcome) continue;
|
|
984
|
+
outcome(known, item); settle(known);
|
|
985
|
+
continue;
|
|
986
|
+
}
|
|
987
|
+
if (finalized && !hasOutcome) continue; // late bare call: the user cut this reply off
|
|
988
|
+
const c = accept(item);
|
|
989
|
+
if (hasOutcome) outcome(c, item);
|
|
990
|
+
if (finalized) record(0); // late outcome of an unseen call: give it a record of its own (settle schedules the reaction)
|
|
991
|
+
settle(c);
|
|
986
992
|
continue;
|
|
987
993
|
}
|
|
988
994
|
if (self.state === 'thinking') self._set('speaking'); // first token → TTS stage
|
|
989
|
-
answer += item.text;
|
|
995
|
+
answer += item.text; round().text += item.text;
|
|
990
996
|
// While audio plays, the TTS progress hook owns the draft (text = spoken-so-far, full = answer,
|
|
991
997
|
// read live each tick), so streaming the whole answer here would yank the cursor to the end.
|
|
992
998
|
// Once the cursor is live we only grow `answer` (the hook reads it); until then (muted, or the
|
|
@@ -994,7 +1000,7 @@ export class VoiceAgent {
|
|
|
994
1000
|
if (!self._cursorLive) self.onEvent({ type: 'assistant', text: answer, final: false, turn });
|
|
995
1001
|
yield item.text;
|
|
996
1002
|
}
|
|
997
|
-
})(this,
|
|
1003
|
+
})(this, prefetched ?? this.llm(this.history, this.sysmsg, signal)); // prefetched stream already carries this turn's user text (see _startPrefetch)
|
|
998
1004
|
|
|
999
1005
|
let spoken = '';
|
|
1000
1006
|
// Live spoken cursor: while a clip plays the TTS reports chars heard so far, which we mirror as a
|
|
@@ -1043,9 +1049,9 @@ export class VoiceAgent {
|
|
|
1043
1049
|
// follows), and a host tracking bubbles per turn needs to know which one just died.
|
|
1044
1050
|
this.onEvent({ type: 'error', error: e.message, turn });
|
|
1045
1051
|
// Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
|
|
1046
|
-
// Record
|
|
1047
|
-
//
|
|
1048
|
-
|
|
1052
|
+
// Record what was done and heard so far — dropping it would let the next turn call the
|
|
1053
|
+
// same tool again.
|
|
1054
|
+
recordAndContinue(signal.keepFull ? answer.length : this._replyText.length);
|
|
1049
1055
|
finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
|
|
1050
1056
|
if (this.state !== 'idle') this._set('listening');
|
|
1051
1057
|
return;
|
|
@@ -1065,16 +1071,13 @@ export class VoiceAgent {
|
|
|
1065
1071
|
// Normally we record only the HEARD prefix (barge-in: the user reacted to what they heard). But a
|
|
1066
1072
|
// turn aborted on hold-entry was tagged signal.keepFull, so we keep the whole streamed answer —
|
|
1067
1073
|
// on release the LLM sees everything it had formed. Per-turn tag → a quick Resume can't flip it.
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1074
|
+
// Both history and the EVENT use the raw prefix of `answer` (the host's full.slice(text.length)
|
|
1075
|
+
// tail aligns). keepFull/normal-finish → whole answer (no tail); a barge-in → the heard prefix
|
|
1076
|
+
// solid + the unspoken remainder dimmed, showing exactly where TTS got.
|
|
1077
|
+
const kept = signal.keepFull ? answer : rawPrefix(spoken);
|
|
1078
|
+
recordAndContinue(kept.length);
|
|
1072
1079
|
if (kept) {
|
|
1073
|
-
|
|
1074
|
-
// full.slice(text.length) tail aligns. keepFull/normal-finish → whole answer (no tail); a barge-in
|
|
1075
|
-
// → the heard prefix solid + the unspoken remainder dimmed, showing exactly where TTS got.
|
|
1076
|
-
const shown = signal.keepFull ? answer : rawPrefix(spoken);
|
|
1077
|
-
this.onEvent({ type: 'assistant', text: shown, full: answer, final: true, turn });
|
|
1080
|
+
this.onEvent({ type: 'assistant', text: kept, full: answer, final: true, turn });
|
|
1078
1081
|
} else if (answer) {
|
|
1079
1082
|
// Interrupted before anything was heard (nothing voiced yet): drop the streamed draft so the
|
|
1080
1083
|
// transcript doesn't keep a dangling unfinished bubble. Empty text → host clears the draft.
|
|
@@ -1090,8 +1093,21 @@ export class VoiceAgent {
|
|
|
1090
1093
|
if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
|
|
1091
1094
|
}
|
|
1092
1095
|
|
|
1096
|
+
// Native tool semantics: once a tool_result is in history the model gets to react to it. Queues a
|
|
1097
|
+
// SOFT reaction turn (behind the running one; dropped if a newer user turn supersedes it — that
|
|
1098
|
+
// turn sees the result anyway) after the agent ran a call for the default adapter (its stream
|
|
1099
|
+
// ends at the call; the result was never fed back), or when a LATE outcome (hung/slow tool,
|
|
1100
|
+
// barge-in mid-execution) landed in a record. Whether a reply is still owed is decided when the
|
|
1101
|
+
// turn RUNS (see _runTurn `react`), since the predecessor may already have carried the result.
|
|
1102
|
+
// Loop adapters fed the result back inside the reply, so nothing fires for them. Silent while
|
|
1103
|
+
// the user is speaking — their utterance will see the result in history anyway.
|
|
1104
|
+
_continue() {
|
|
1105
|
+
if (this._closed || this._speaking) return;
|
|
1106
|
+
this._runTurn('', { supersede: false, react: true });
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1093
1109
|
// Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
|
|
1094
|
-
// parallel with TTS, and NEVER joined: the turn records the
|
|
1110
|
+
// parallel with TTS, and NEVER joined: the turn records the call (tool_result pending) and finishes
|
|
1095
1111
|
// when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
|
|
1096
1112
|
// else. Its outcome reaches the host via onEvent; the model learns it via notify(). Must not
|
|
1097
1113
|
// reject — nothing awaits it, so a rejection here would be unhandled (hence the catch-all).
|
|
@@ -1107,16 +1123,15 @@ export class VoiceAgent {
|
|
|
1107
1123
|
// runtime/transport failures, not a handled domain-level tool miss.
|
|
1108
1124
|
// `id` rides along on every tool event so a host can replace THIS call's running chip in
|
|
1109
1125
|
// place, and two same-args calls (a legit retry) stay distinct acts in the transcript.
|
|
1110
|
-
if ('result' in call || 'error' in call) {
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1126
|
+
if (!('result' in call || 'error' in call)) {
|
|
1127
|
+
// Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
|
|
1128
|
+
// a deadline could only stop WAITING — reporting "failed" for an operation still on its way
|
|
1129
|
+
// to committing its side effect, inviting a duplicate retry. Nothing awaits this call
|
|
1130
|
+
// (see _speakTurn), so a hung tool costs its own result, never the turn loop.
|
|
1131
|
+
try { call.result = await this.tools[call.tool]?.run?.(call.args); }
|
|
1132
|
+
catch (e) { call.error = e?.message || String(e ?? 'failed'); }
|
|
1114
1133
|
}
|
|
1115
|
-
|
|
1116
|
-
// a deadline could only stop WAITING — reporting "failed" for an operation still on its way
|
|
1117
|
-
// to committing its side effect, inviting a duplicate retry. Nothing awaits this call
|
|
1118
|
-
// (see _speakTurn), so a hung tool costs its own result, never the turn loop.
|
|
1119
|
-
const result = await this.tools[call.tool]?.run?.(call.args);
|
|
1134
|
+
const result = call.error ? { text: call.error, ok: false } : call.result;
|
|
1120
1135
|
this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
|
|
1121
1136
|
} catch (e) {
|
|
1122
1137
|
this.onEvent({ type: 'error', error: `tool ${call.tool}: ${e.message}` });
|
|
@@ -1641,7 +1656,7 @@ export class StreamingTTS {
|
|
|
1641
1656
|
// predict()s, and presynth() below may fire while a previous reply's chain is still draining.
|
|
1642
1657
|
// Returns a promise that RESOLVES (wav or null-on-error) — callers surface errors per-entry.
|
|
1643
1658
|
_enqueueSynth(text, signal) {
|
|
1644
|
-
const run = this._synthQ.then(() => this._synth(stripMd(text) || text, signal), () => null); // || raw: a pure-syntax chunk strips to empty — voice the raw rather than abort
|
|
1659
|
+
const run = this._synthQ.then(() => signal?.aborted ? null : this._synth(stripMd(text) || text, signal), () => null); // || raw: a pure-syntax chunk strips to empty — voice the raw rather than abort
|
|
1645
1660
|
this._synthQ = run.catch(() => null);
|
|
1646
1661
|
return run;
|
|
1647
1662
|
}
|
|
@@ -1822,7 +1837,14 @@ export class StreamingTTS {
|
|
|
1822
1837
|
catch (e) { if (e?.name !== 'AbortError') streamErr = e; } // LLM/iterator failure → surface via awaitEntry, not a silent truncation
|
|
1823
1838
|
finally { streamDone = true; bump(); }
|
|
1824
1839
|
})();
|
|
1825
|
-
|
|
1840
|
+
// Stop waiting for an in-flight synth on abort, without releasing the serial synth queue:
|
|
1841
|
+
// Piper inference (and speculative clips without a signal) can finish only in their own time.
|
|
1842
|
+
let onAbort;
|
|
1843
|
+
const aborted = signal && new Promise((resolve) => { onAbort = () => { resolve(null); bump(); }; });
|
|
1844
|
+
if (signal) {
|
|
1845
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
1846
|
+
if (signal.aborted) onAbort();
|
|
1847
|
+
}
|
|
1826
1848
|
// Wait until the tape has an entry at `i` — or the stream ends, we abort, or a seek arrives (a
|
|
1827
1849
|
// backward tap while we're parked waiting for a FUTURE clip must be honored now, not after the
|
|
1828
1850
|
// stream advances). Producer runs ahead, so this usually returns immediately; it only blocks when
|
|
@@ -1859,7 +1881,7 @@ export class StreamingTTS {
|
|
|
1859
1881
|
while (true) {
|
|
1860
1882
|
if (idx >= tape.length) { await awaitEntry(idx); if (takePending()) continue; if (idx >= tape.length) break; } // wait for the producer to reach this sentence (stream end → done)
|
|
1861
1883
|
const e = tape[idx];
|
|
1862
|
-
const buf = await e.blobP;
|
|
1884
|
+
const buf = await (aborted ? Promise.race([e.blobP, aborted]) : e.blobP);
|
|
1863
1885
|
// Barge-in while this clip was mid-synth: the abort cancels the synth fetch, which rejects with
|
|
1864
1886
|
// AbortError ("signal is aborted without reason"). That's the cancellation we asked for, not a
|
|
1865
1887
|
// failed render — stop quietly, exactly like an abort during playback.
|
|
@@ -1888,7 +1910,7 @@ export class StreamingTTS {
|
|
|
1888
1910
|
idx++; startNw = 0; // natural end → next sentence
|
|
1889
1911
|
}
|
|
1890
1912
|
} finally {
|
|
1891
|
-
signal?.removeEventListener('abort',
|
|
1913
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1892
1914
|
// A finished stream is joined (it's already done — this just collects a trailing synth). An
|
|
1893
1915
|
// ABORTED one is DETACHED: the producer sits in it.next() on the HOST's llm generator, which
|
|
1894
1916
|
// may ignore the abort signal and never yield again, so awaiting it here would never return
|