@todoforai/voiceloop 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.d.ts +24 -7
- package/src/index.js +1 -1
- package/src/llm-openai.js +19 -1
- package/src/voice-agent.js +159 -160
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -43,20 +43,35 @@ export interface VoiceTool {
|
|
|
43
43
|
// reports it instead of running it again.
|
|
44
44
|
export type LLMChunk =
|
|
45
45
|
| { text: string }
|
|
46
|
-
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true }
|
|
46
|
+
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true }
|
|
47
|
+
/** Loop adapters only: the model's NEXT message starts here (after the results it fed back), so
|
|
48
|
+
* the agent records text/calls before and after as separate assistant messages. */
|
|
49
|
+
| { newMessage: true };
|
|
50
|
+
// The conversation as the model sees it, Anthropic-shaped. Plain turns are strings; a turn in which
|
|
51
|
+
// the assistant called tools is recorded NATIVELY: an assistant message of text + tool_use blocks,
|
|
52
|
+
// then a user message holding the matching tool_result blocks (content = the result text the model
|
|
53
|
+
// reads; `is_error` on a failure). The agent never writes a textual notation of a tool call into
|
|
54
|
+
// history, so the model has nothing to imitate. Adapters project this onto their wire format
|
|
55
|
+
// (see toOpenAIMessages).
|
|
56
|
+
export type HistoryBlock =
|
|
57
|
+
| { type: 'text'; text: string }
|
|
58
|
+
| { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
|
|
59
|
+
| { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean };
|
|
60
|
+
export type HistoryMessage = { role: string; content: string | HistoryBlock[] };
|
|
47
61
|
// Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
|
|
48
|
-
// The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes;
|
|
49
|
-
//
|
|
62
|
+
// The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; their results are
|
|
63
|
+
// recorded as tool_result and the agent starts a follow-up turn so the model reacts to them.
|
|
50
64
|
// An agent LOOP that runs tools inside its own generator declares the two flags below.
|
|
51
65
|
export interface LLM {
|
|
52
66
|
(
|
|
53
|
-
history:
|
|
67
|
+
history: HistoryMessage[],
|
|
54
68
|
system: string,
|
|
55
69
|
signal: AbortSignal,
|
|
56
70
|
options?: { toolGate?: Promise<unknown> },
|
|
57
71
|
): AsyncIterable<LLMChunk>;
|
|
58
|
-
/** This generator
|
|
59
|
-
*
|
|
72
|
+
/** This generator is a tool LOOP: it runs every call itself and feeds the result back inside the
|
|
73
|
+
* same reply (chunks carry `result`/`error`, optionally preceded by a `running` announcement).
|
|
74
|
+
* The agent never executes its calls and never starts a reaction turn for them. */
|
|
60
75
|
executesTools?: boolean;
|
|
61
76
|
/** This generator awaits `options.toolGate` before ANY tool executes. Required to keep
|
|
62
77
|
* speculative prefetch on: prefetch starts from an uncommitted interim, and the gate is what
|
|
@@ -190,7 +205,7 @@ export class VoiceAgent {
|
|
|
190
205
|
constructor(options?: VoiceAgentOptions);
|
|
191
206
|
/** The live conversation the LLM sees. A mutable reference the host may read and push into —
|
|
192
207
|
* carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
|
|
193
|
-
history:
|
|
208
|
+
history: HistoryMessage[];
|
|
194
209
|
start(deviceId?: string): Promise<void>;
|
|
195
210
|
/** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
|
|
196
211
|
* start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
|
|
@@ -344,6 +359,8 @@ export function makeOpenAILLM(options?: {
|
|
|
344
359
|
llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
|
|
345
360
|
tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
|
|
346
361
|
}): LLM;
|
|
362
|
+
/** VoiceAgent history → OpenAI chat messages (tool_use → tool_calls, tool_result → role:"tool"). */
|
|
363
|
+
export function toOpenAIMessages(history: HistoryMessage[]): Array<Record<string, unknown>>;
|
|
347
364
|
|
|
348
365
|
// ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
|
|
349
366
|
/** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
|
package/src/index.js
CHANGED
|
@@ -11,5 +11,5 @@ export {
|
|
|
11
11
|
} from './stt.js';
|
|
12
12
|
export { ElevenLabsTTS } from './tts-elevenlabs.js';
|
|
13
13
|
export { loadVoiceDeps, prewarmVoice } from './deps.js';
|
|
14
|
-
export { makeOpenAILLM } from './llm-openai.js';
|
|
14
|
+
export { makeOpenAILLM, toOpenAIMessages } from './llm-openai.js';
|
|
15
15
|
export { TUNING } from './tuning.js';
|
package/src/llm-openai.js
CHANGED
|
@@ -19,6 +19,24 @@
|
|
|
19
19
|
// keys, defaulting to all of them)
|
|
20
20
|
// fetchFn — fetch override (tests, custom agents/headers); defaults to global fetch
|
|
21
21
|
// extraBody— merged into the request body (temperature, provider-specific knobs, …)
|
|
22
|
+
// VoiceAgent history → OpenAI chat messages. Plain turns pass through; the agent's native tool
|
|
23
|
+
// records (assistant text + tool_use blocks, then a user turn of tool_result blocks — the same
|
|
24
|
+
// Anthropic-shaped blocks it keeps for every adapter) become `tool_calls` and `role:"tool"` messages.
|
|
25
|
+
export const toOpenAIMessages = (history) => history.flatMap((m) => {
|
|
26
|
+
if (!Array.isArray(m.content)) return [m];
|
|
27
|
+
const text = m.content.filter((b) => b.type === 'text').map((b) => b.text).join('');
|
|
28
|
+
if (m.role === 'assistant') {
|
|
29
|
+
const tool_calls = m.content.filter((b) => b.type === 'tool_use')
|
|
30
|
+
.map((b) => ({ id: b.id, type: 'function', function: { name: b.name, arguments: JSON.stringify(b.input ?? {}) } }));
|
|
31
|
+
return [{ role: 'assistant', content: text || null, ...(tool_calls.length ? { tool_calls } : {}) }];
|
|
32
|
+
}
|
|
33
|
+
const results = m.content.filter((b) => b.type === 'tool_result').map((b) => {
|
|
34
|
+
const content = typeof b.content === 'string' ? b.content : String(b.content ?? '');
|
|
35
|
+
return { role: 'tool', tool_call_id: b.tool_use_id, content: b.is_error && !content.startsWith('[failed]') ? `[failed] ${content}` : content };
|
|
36
|
+
});
|
|
37
|
+
return text ? [...results, { role: m.role, content: text }] : results; // OpenAI has no is_error → text-marked
|
|
38
|
+
});
|
|
39
|
+
|
|
22
40
|
export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 1024, tools = {}, fetchFn, extraBody = {} } = {}) {
|
|
23
41
|
const toolSpecs = Object.entries(tools).map(([name, t]) => ({
|
|
24
42
|
type: 'function',
|
|
@@ -37,7 +55,7 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
|
|
|
37
55
|
headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
|
|
38
56
|
body: JSON.stringify({
|
|
39
57
|
stream: true,
|
|
40
|
-
messages: [{ role: 'system', content: system }, ...history],
|
|
58
|
+
messages: [{ role: 'system', content: system }, ...toOpenAIMessages(history)],
|
|
41
59
|
max_tokens: maxTokens,
|
|
42
60
|
...(model ? { model } : {}),
|
|
43
61
|
...(toolSpecs.length ? { tools: toolSpecs } : {}),
|
package/src/voice-agent.js
CHANGED
|
@@ -60,7 +60,7 @@ const langRule = (lang) => {
|
|
|
60
60
|
return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
|
|
61
61
|
};
|
|
62
62
|
// A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
|
|
63
|
-
// what order a provider streamed the keys in (used
|
|
63
|
+
// what order a provider streamed the keys in (used for dedup).
|
|
64
64
|
// Identity: the provider call id when present (two calls with identical args are DISTINCT acts,
|
|
65
65
|
// e.g. a legit retry after an error result); name+args otherwise (an id-less identical repeat is a
|
|
66
66
|
// provider re-emitting its buffered call).
|
|
@@ -76,10 +76,11 @@ const withTimeout = (p, ms, fallback) => {
|
|
|
76
76
|
]);
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
-
const callArgs = (c) => JSON.stringify(c.args ?? {},
|
|
79
|
+
const callArgs = (c) => JSON.stringify(c.args ?? {}, (_k, v) =>
|
|
80
|
+
v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, v[k]])) : v);
|
|
80
81
|
const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
|
|
81
|
-
// Render a host tool's return value as model-facing text (
|
|
82
|
-
//
|
|
82
|
+
// Render a host tool's return value as model-facing text (the tool_result content in history, and
|
|
83
|
+
// what LLM adapters feed back to the model). Never throws: the side effect already RAN,
|
|
83
84
|
// so an unserializable result must not retro-fail the call and invite a retry.
|
|
84
85
|
export const toolResultText = (result) => {
|
|
85
86
|
if (result === undefined) return 'ok';
|
|
@@ -88,87 +89,22 @@ export const toolResultText = (result) => {
|
|
|
88
89
|
return result.ok ? result.text : `[failed] ${result.text}`;
|
|
89
90
|
try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
|
|
90
91
|
};
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
const
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
// The model sees `[TOOL CALL name] {...}` ledger lines in its own history turns, and sometimes
|
|
105
|
-
// IMITATES the format instead of calling the tool natively: it types the line as reply text.
|
|
106
|
-
// Untreated, the raw markup leaks into the spoken/visible reply and the tool never runs. This
|
|
107
|
-
// transform strips such lines from the text stream: a bare call (no → outcome) is what the model
|
|
108
|
-
// MEANT to do → re-emitted as a real { tool, args } chunk (the turn's dedup collapses it if the
|
|
109
|
-
// native call also arrived); a line WITH an → outcome is a hallucinated memory of a past call →
|
|
110
|
-
// dropped. Text before/after the line streams through untouched, held back only while a possible
|
|
111
|
-
// marker/line is still forming so TTS latency is unaffected on normal replies.
|
|
112
|
-
const MIMIC_MARK = '[TOOL CALL ';
|
|
113
|
-
const parseMimicLine = (line) => {
|
|
114
|
-
const m = /^\[TOOL CALL ([\w.-]+)\]\s*(.*)$/.exec(line.trim());
|
|
115
|
-
if (!m) return null; // not the format after all — caller emits it as text
|
|
116
|
-
const [, tool, rest] = m;
|
|
117
|
-
if (!rest) return { tool, args: {} };
|
|
118
|
-
if (!rest.startsWith('{')) return { drop: true }; // `→ outcome` (or junk) with no args — a remembered call
|
|
119
|
-
// Balanced-brace scan (string-aware): the args JSON may be followed by a ` → outcome` tail.
|
|
120
|
-
let depth = 0, inStr = false, escaped = false, end = -1;
|
|
121
|
-
for (let i = 0; i < rest.length; i++) {
|
|
122
|
-
const ch = rest[i];
|
|
123
|
-
if (escaped) { escaped = false; continue; }
|
|
124
|
-
if (inStr) { if (ch === '\\') escaped = true; else if (ch === '"') inStr = false; continue; }
|
|
125
|
-
if (ch === '"') inStr = true;
|
|
126
|
-
else if (ch === '{') depth++;
|
|
127
|
-
else if (ch === '}' && --depth === 0) { end = i + 1; break; }
|
|
92
|
+
// One model-facing history record for a settled/pending tool call: the native tool_result block that
|
|
93
|
+
// pairs with its tool_use. Bounded — history is spoken-conversation-sized, not a log. A call whose
|
|
94
|
+
// outcome has not landed yet says so; _runTool patches the SAME block in place when it does.
|
|
95
|
+
const RESULT_MAX = 4000;
|
|
96
|
+
const MAX_TOOL_ROUNDS = 6;
|
|
97
|
+
const PENDING_RESULT = 'still running — the result will follow in a later turn';
|
|
98
|
+
const bound = (text) => text.length > RESULT_MAX ? `${text.slice(0, RESULT_MAX)}… (${text.length} chars)` : text;
|
|
99
|
+
const toolResultBlock = (c) => {
|
|
100
|
+
const block = { type: 'tool_result', tool_use_id: c.id, content: PENDING_RESULT };
|
|
101
|
+
if ('error' in c) { block.content = bound(String(c.error) || 'failed'); block.is_error = true; }
|
|
102
|
+
else if ('result' in c) {
|
|
103
|
+
block.content = bound(toolResultText(c.result));
|
|
104
|
+
if (c.result && typeof c.result === 'object' && c.result.ok === false) block.is_error = true;
|
|
128
105
|
}
|
|
129
|
-
|
|
130
|
-
if (rest.slice(end).trim()) return { drop: true }; // has an → outcome: a memory, not a new call
|
|
131
|
-
try { return { tool, args: JSON.parse(rest.slice(0, end)) }; } catch { return { drop: true }; }
|
|
106
|
+
return block;
|
|
132
107
|
};
|
|
133
|
-
async function* stripToolCallMimicry(src) {
|
|
134
|
-
let carry = '';
|
|
135
|
-
// Longest tail of `carry` that could still grow into the marker — held back; the rest is safe to emit.
|
|
136
|
-
const heldLen = () => {
|
|
137
|
-
for (let n = Math.min(carry.length, MIMIC_MARK.length - 1); n > 0; n--)
|
|
138
|
-
if (MIMIC_MARK.startsWith(carry.slice(-n))) return n;
|
|
139
|
-
return 0;
|
|
140
|
-
};
|
|
141
|
-
function* pump(final) {
|
|
142
|
-
for (;;) {
|
|
143
|
-
const at = carry.indexOf(MIMIC_MARK);
|
|
144
|
-
if (at < 0) {
|
|
145
|
-
const hold = final ? 0 : heldLen();
|
|
146
|
-
const out = carry.slice(0, carry.length - hold);
|
|
147
|
-
carry = carry.slice(carry.length - hold);
|
|
148
|
-
if (out) yield { text: out };
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
const nl = carry.indexOf('\n', at);
|
|
152
|
-
if (nl < 0 && !final) { // the line is still streaming — hold from the marker on
|
|
153
|
-
if (at > 0) { yield { text: carry.slice(0, at) }; carry = carry.slice(at); }
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
const end = nl < 0 ? carry.length : nl;
|
|
157
|
-
const before = carry.slice(0, at), line = carry.slice(at, end);
|
|
158
|
-
carry = nl < 0 ? '' : carry.slice(nl + 1); // the line (and its newline) leave the text stream
|
|
159
|
-
if (before) yield { text: before };
|
|
160
|
-
const parsed = parseMimicLine(line);
|
|
161
|
-
if (!parsed) yield { text: nl < 0 ? line : `${line}\n` }; // false alarm — pass it through untouched (newline included)
|
|
162
|
-
else if (parsed.tool) yield { tool: parsed.tool, args: parsed.args };
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
for await (const item of src) {
|
|
166
|
-
if (item.tool) { yield item; continue; }
|
|
167
|
-
carry += item.text ?? '';
|
|
168
|
-
yield* pump(false);
|
|
169
|
-
}
|
|
170
|
-
yield* pump(true);
|
|
171
|
-
}
|
|
172
108
|
|
|
173
109
|
// Persona (who the agent is) + language rule + the host's live context block (`sysmsg`, delimited
|
|
174
110
|
// so the model never mistakes injected context for its own conversation).
|
|
@@ -378,6 +314,8 @@ export class VoiceAgent {
|
|
|
378
314
|
this.state = 'idle'; // idle | listening | thinking | speaking
|
|
379
315
|
this._abort = null; // aborts the in-flight LLM
|
|
380
316
|
this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
|
|
317
|
+
this._toolRounds = 0; // tool-result reactions since the user last spoke (see _runTurn react)
|
|
318
|
+
this._unseenResults = false; // a tool_result was patched after the last request was built (see _runTurn react)
|
|
381
319
|
this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
|
|
382
320
|
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
383
321
|
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
@@ -699,6 +637,7 @@ export class VoiceAgent {
|
|
|
699
637
|
// history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
|
|
700
638
|
// release (_flush) owns starting the single turn, and waits for all of these to settle first.
|
|
701
639
|
_pushHeld(text) {
|
|
640
|
+
this._toolRounds = 0;
|
|
702
641
|
const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
|
|
703
642
|
this._heldPushes.add(p);
|
|
704
643
|
p.finally(() => this._heldPushes.delete(p));
|
|
@@ -720,15 +659,14 @@ export class VoiceAgent {
|
|
|
720
659
|
}
|
|
721
660
|
|
|
722
661
|
// Inject an out-of-band turn: feed `text` to the LLM as if the user had said it, then speak the
|
|
723
|
-
// reply. Used by the host to surface external events (a
|
|
662
|
+
// reply. Used by the host to surface external events (a background job finishing, a board change)
|
|
724
663
|
// — only meaningful while the agent is running, so it's a no-op once stopped.
|
|
725
664
|
//
|
|
726
665
|
// SOFT turn (supersede: false): unlike a user utterance this must NOT cut the reply in flight.
|
|
727
|
-
//
|
|
728
|
-
//
|
|
729
|
-
//
|
|
730
|
-
//
|
|
731
|
-
// [TOOL CALL] it made, so the model sees the work is already done.
|
|
666
|
+
// An event can arrive WHILE a turn is still streaming/speaking, so aborting here would kill that
|
|
667
|
+
// turn mid-answer and re-run the LLM over a history whose last entry is the request again — the
|
|
668
|
+
// model then re-calls the same tool (duplicate-action spam). Instead it queues behind the running
|
|
669
|
+
// turn, which by then has recorded its answer and its tool calls natively.
|
|
732
670
|
async notify(text) { if (!this._closed) await this._runTurn(text, { supersede: false }); }
|
|
733
671
|
|
|
734
672
|
// Send a TYPED user turn: like speaking, but the text comes from the host's input instead of STT.
|
|
@@ -762,6 +700,7 @@ export class VoiceAgent {
|
|
|
762
700
|
// caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
|
|
763
701
|
_onUserTurn(text, { speech = false } = {}) {
|
|
764
702
|
if (!text.trim()) return Promise.resolve();
|
|
703
|
+
this._toolRounds = 0;
|
|
765
704
|
return this._runTurn(text, { speech });
|
|
766
705
|
}
|
|
767
706
|
|
|
@@ -878,7 +817,7 @@ export class VoiceAgent {
|
|
|
878
817
|
// if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
|
|
879
818
|
// sees this message in history and answers over the fuller picture — so a burst of tool results
|
|
880
819
|
// collapses into ONE reply instead of one interrupting monologue per result.
|
|
881
|
-
_runTurn(text, { supersede = true, speech = false } = {}) {
|
|
820
|
+
_runTurn(text, { supersede = true, speech = false, react = false } = {}) {
|
|
882
821
|
const prev = this._turn;
|
|
883
822
|
if (supersede) {
|
|
884
823
|
this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
|
|
@@ -904,9 +843,24 @@ export class VoiceAgent {
|
|
|
904
843
|
// Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
|
|
905
844
|
const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
|
|
906
845
|
if (text) this.history.push({ role: 'user', content: text });
|
|
846
|
+
// A reaction turn owes a reply only for results no request has carried yet: either the
|
|
847
|
+
// tool_result record is still the last message, or a late outcome was patched in after the
|
|
848
|
+
// last request was built (_unseenResults; cleared when a request starts) — then a short
|
|
849
|
+
// [EVENT] user turn, the host's channel for out-of-band facts, points the model at the patched
|
|
850
|
+
// block (a request can't end on an assistant message). Bounded per user utterance
|
|
851
|
+
// (_toolRounds, reset on every accepted user turn): a model that keeps calling tools gets
|
|
852
|
+
// MAX_TOOL_ROUNDS reactions, then waits for the user.
|
|
853
|
+
if (react) {
|
|
854
|
+
const last = this.history[this.history.length - 1];
|
|
855
|
+
const owed = (last?.role === 'user' && Array.isArray(last.content)) || this._unseenResults;
|
|
856
|
+
if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this._unseenResults = false; return; }
|
|
857
|
+
this._toolRounds++;
|
|
858
|
+
if (last?.role !== 'user') this.history.push({ role: 'user', content: '[EVENT tool_result] a pending tool finished — see its tool_result above.' });
|
|
859
|
+
}
|
|
907
860
|
// Reply only if the conversation now ends on a user turn (a raced turn may have already
|
|
908
861
|
// replied). pf is always null here: if we pushed a user turn it IS the last entry.
|
|
909
862
|
if (this.history[this.history.length - 1]?.role !== 'user') return;
|
|
863
|
+
this._unseenResults = false; // this request carries every result landed so far
|
|
910
864
|
this._set('thinking'); // LLM stage
|
|
911
865
|
this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
|
|
912
866
|
await this._speakTurn(this._abort.signal, pf?.gen, seq); // _speakTurn never throws — errors surface via onEvent
|
|
@@ -914,7 +868,7 @@ export class VoiceAgent {
|
|
|
914
868
|
}
|
|
915
869
|
|
|
916
870
|
// One assistant turn: stream the LLM → TTS, then record what was actually HEARD (the barge-in
|
|
917
|
-
// prefix, not the full generated answer) plus
|
|
871
|
+
// prefix, not the full generated answer) plus the tools it called, natively. Stored as
|
|
918
872
|
// `this._turn` so the next user turn can await it. Never throws (errors surface via onEvent) so
|
|
919
873
|
// the await is safe — and never awaits a tool, so a hung one can't hold the next turn hostage.
|
|
920
874
|
// `turn` tags every assistant event with the reply it belongs to, so a host can tell a LATE
|
|
@@ -927,74 +881,110 @@ export class VoiceAgent {
|
|
|
927
881
|
// repopulates _replyText with the audible prefix as playback advances.
|
|
928
882
|
if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText;
|
|
929
883
|
this._replyText = ''; this._replyEcho = '';
|
|
930
|
-
// Tap the LLM stream once: yield speech text to TTS, and record the
|
|
931
|
-
//
|
|
932
|
-
//
|
|
884
|
+
// Tap the LLM stream once: yield speech text to TTS, and record what the model did as ROUNDS —
|
|
885
|
+
// each round is one assistant message (text + the tool_use blocks it emitted) followed by the
|
|
886
|
+
// user-role tool_result blocks for those calls. Recorded into history in that order on EVERY
|
|
887
|
+
// exit path, so the next request replays the model's own native tool history — never a text
|
|
888
|
+
// notation it could imitate. `this.llm(...)` is a lazy generator — its fetch fires only when
|
|
933
889
|
// tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
|
|
934
|
-
|
|
935
|
-
//
|
|
936
|
-
//
|
|
937
|
-
//
|
|
938
|
-
|
|
939
|
-
// calls:
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
890
|
+
//
|
|
891
|
+
// Round boundary: the default adapter's stream is ONE model message (text + calls together;
|
|
892
|
+
// the reaction to the results comes from _continue). A loop adapter feeds results back inside
|
|
893
|
+
// the stream and marks where the model's NEXT message starts with a { newMessage } chunk.
|
|
894
|
+
let answer = '';
|
|
895
|
+
const rounds = [], calls = []; // calls: every act this turn saw, for dedup — outlives recording
|
|
896
|
+
const loop = !!this.llm.executesTools;
|
|
897
|
+
const round = () => rounds[rounds.length - 1] ?? (rounds.push({ text: '', calls: [] }), rounds[rounds.length - 1]);
|
|
898
|
+
const newRound = () => { const r = rounds[rounds.length - 1]; if (r && (r.text || r.calls.length)) rounds.push({ text: '', calls: [] }); };
|
|
899
|
+
let n = 0;
|
|
900
|
+
// Accept a call once: identity is the provider id, else name+args (an id-less repeat is a
|
|
901
|
+
// provider re-emitting its buffered call). A loop adapter owns execution AND feeds the result
|
|
902
|
+
// back inside its reply; anything else is agent-run (or agent-reported when the chunk already
|
|
903
|
+
// carries an outcome) and gets a reaction turn.
|
|
904
|
+
const accept = (item) => {
|
|
905
|
+
const key = callKey(item);
|
|
906
|
+
let c = calls.find((x) => x.key === key);
|
|
907
|
+
if (c) return c;
|
|
908
|
+
c = { ...item, key, id: item.id || `call-${turn}-${++n}`, owner: loop ? 'adapter' : 'agent' };
|
|
909
|
+
calls.push(c); round().calls.push(c);
|
|
910
|
+
return c;
|
|
911
|
+
};
|
|
912
|
+
const outcome = (c, item) => { if ('result' in item) c.result = item.result; else if ('error' in item) c.error = item.error; };
|
|
913
|
+
const hasOutcome = (c) => 'result' in c || 'error' in c;
|
|
914
|
+
// An outcome may land AFTER the turn recorded the call (a hung tool, a barge-in mid-execution):
|
|
915
|
+
// its tool_result block is patched IN PLACE so the next request carries the real result, and the
|
|
916
|
+
// model gets to react to it (see _continue).
|
|
917
|
+
const settle = (c) => this._runTool(c).then(() => {
|
|
918
|
+
if (!c.block) return; // not recorded yet — record() picks the outcome up
|
|
919
|
+
if (!c.consumed) { Object.assign(c.block, toolResultBlock(c)); c.consumed = true; }
|
|
920
|
+
this._dropPrefetch(); // a speculation serialized the pending block — stale
|
|
921
|
+
this._unseenResults = true; this._continue();
|
|
922
|
+
});
|
|
923
|
+
const record = (keptLen) => {
|
|
924
|
+
let budget = keptLen;
|
|
925
|
+
for (const r of rounds) {
|
|
926
|
+
const text = r.text.slice(0, Math.max(0, budget)).trimEnd(); budget -= r.text.length;
|
|
927
|
+
if (!text && !r.calls.length) continue;
|
|
928
|
+
if (!r.calls.length) { this.history.push({ role: 'assistant', content: text }); continue; }
|
|
929
|
+
const blocks = text ? [{ type: 'text', text }] : [];
|
|
930
|
+
for (const c of r.calls) blocks.push({ type: 'tool_use', id: c.id, name: c.tool, input: c.args ?? {} });
|
|
931
|
+
this.history.push({ role: 'assistant', content: blocks });
|
|
932
|
+
this.history.push({ role: 'user', content: r.calls.map((c) => {
|
|
933
|
+
c.consumed = 'result' in c || 'error' in c;
|
|
934
|
+
return (c.block = toolResultBlock(c));
|
|
935
|
+
}) });
|
|
936
|
+
}
|
|
937
|
+
rounds.length = 0;
|
|
938
|
+
};
|
|
939
|
+
// Follow-up only for calls the AGENT ran: a loop adapter fed its results back inside the reply.
|
|
940
|
+
const recordAndContinue = (keptLen) => {
|
|
941
|
+
record(keptLen);
|
|
942
|
+
if (calls.some((c) => c.owner === 'agent' && c.consumed)) this._continue();
|
|
943
|
+
};
|
|
945
944
|
// Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
|
|
946
945
|
// announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
|
|
947
946
|
// that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
|
|
948
|
-
// outcome arrives finalizes
|
|
949
|
-
//
|
|
950
|
-
|
|
951
|
-
//
|
|
952
|
-
//
|
|
953
|
-
//
|
|
954
|
-
// dropped — outcomes still flow (a tool the adapter really ran is truth worth reporting).
|
|
947
|
+
// outcome arrives finalizes the CHIP as interrupted; the history keeps the call with a pending
|
|
948
|
+
// tool_result, patched in place if the outcome still lands.
|
|
949
|
+
// `finalized` latches once this turn has finalized its chips. A barge-in detaches the LLM
|
|
950
|
+
// producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
|
|
951
|
+
// announcement would open a spinner nothing will ever close, and a late UNEXECUTED call is work
|
|
952
|
+
// the user cut off. Past the latch only outcomes flow (a tool the adapter really ran is truth).
|
|
955
953
|
let finalized = false;
|
|
956
954
|
const finishInterrupted = () => {
|
|
957
955
|
finalized = true;
|
|
958
|
-
for (const
|
|
959
|
-
this.onEvent({ type: 'tool', name:
|
|
960
|
-
announced.clear();
|
|
956
|
+
for (const c of calls) if (c.announced && !hasOutcome(c))
|
|
957
|
+
this.onEvent({ type: 'tool', name: c.tool, id: c.id, args: c.args, result: { ok: false, text: 'interrupted — did not finish' } });
|
|
961
958
|
};
|
|
962
959
|
const tapped = (async function* (self, src) {
|
|
963
960
|
for await (const item of src) {
|
|
964
961
|
// tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
|
|
965
962
|
// act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
|
|
963
|
+
if (item.newMessage) { newRound(); continue; } // loop adapter: the model's next message starts here
|
|
966
964
|
if (item.tool) {
|
|
967
|
-
|
|
965
|
+
const known = calls.find((x) => x.key === callKey(item));
|
|
966
|
+
if (item.running || (loop && !hasOutcome(item))) { // announcement; the outcome chunk below does the work
|
|
968
967
|
// An id-less announcement can never be replaced in place NOR finalized (both key on id),
|
|
969
968
|
// so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
|
|
970
|
-
if (finalized || !item.id) continue;
|
|
971
|
-
|
|
972
|
-
// reopen a spinner over a finished chip, and the duplicate outcome is deduped below and
|
|
973
|
-
// never re-reported — so it would hang there, then finalize as falsely "interrupted".
|
|
974
|
-
if (calls.some((c) => c.id === item.id)) continue;
|
|
975
|
-
announced.set(item.id, item);
|
|
969
|
+
if (finalized || !item.id || known) continue;
|
|
970
|
+
accept(item).announced = true;
|
|
976
971
|
self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
|
|
977
972
|
continue;
|
|
978
973
|
}
|
|
979
|
-
if (
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
const dup = calls.some((c) => callKey(c) === callKey(item) ||
|
|
984
|
-
((!c.id || !item.id) && c.tool === item.tool && callArgs(c) === callArgs(item)));
|
|
985
|
-
if (!dup) {
|
|
986
|
-
calls.push(item); self._runTool(item);
|
|
987
|
-
// Late outcome: the turn already closed (barge-in mid-execution) and its ledger is
|
|
988
|
-
// written, so this result would reach the transcript chip but never the model. Relay it
|
|
989
|
-
// as a soft [TOOL RESULT] turn — the same channel out-of-band receipts use — so the next
|
|
990
|
-
// reply knows what the tool returned instead of seeing a call that vanished.
|
|
991
|
-
if (finalized && ('result' in item || 'error' in item))
|
|
992
|
-
self.notify(`[TOOL RESULT ${item.tool}] ${callArgs(item)}${ledgerOutcome(item)}`);
|
|
974
|
+
if (known) { // the announced call's outcome, or a re-emit
|
|
975
|
+
if (known.ran || !hasOutcome(item)) continue;
|
|
976
|
+
outcome(known, item); settle(known);
|
|
977
|
+
continue;
|
|
993
978
|
}
|
|
979
|
+
if (finalized && !hasOutcome(item)) continue; // late bare call: the user cut this reply off
|
|
980
|
+
const c = accept(item);
|
|
981
|
+
outcome(c, item);
|
|
982
|
+
if (finalized) record(0); // late outcome of an unseen call: give it a record of its own (settle schedules the reaction)
|
|
983
|
+
settle(c);
|
|
994
984
|
continue;
|
|
995
985
|
}
|
|
996
986
|
if (self.state === 'thinking') self._set('speaking'); // first token → TTS stage
|
|
997
|
-
answer += item.text;
|
|
987
|
+
answer += item.text; round().text += item.text;
|
|
998
988
|
// While audio plays, the TTS progress hook owns the draft (text = spoken-so-far, full = answer,
|
|
999
989
|
// read live each tick), so streaming the whole answer here would yank the cursor to the end.
|
|
1000
990
|
// Once the cursor is live we only grow `answer` (the hook reads it); until then (muted, or the
|
|
@@ -1002,7 +992,7 @@ export class VoiceAgent {
|
|
|
1002
992
|
if (!self._cursorLive) self.onEvent({ type: 'assistant', text: answer, final: false, turn });
|
|
1003
993
|
yield item.text;
|
|
1004
994
|
}
|
|
1005
|
-
})(this,
|
|
995
|
+
})(this, prefetched ?? this.llm(this.history, this.sysmsg, signal)); // prefetched stream already carries this turn's user text (see _startPrefetch)
|
|
1006
996
|
|
|
1007
997
|
let spoken = '';
|
|
1008
998
|
// Live spoken cursor: while a clip plays the TTS reports chars heard so far, which we mirror as a
|
|
@@ -1051,9 +1041,9 @@ export class VoiceAgent {
|
|
|
1051
1041
|
// follows), and a host tracking bubbles per turn needs to know which one just died.
|
|
1052
1042
|
this.onEvent({ type: 'error', error: e.message, turn });
|
|
1053
1043
|
// Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
|
|
1054
|
-
// Record
|
|
1055
|
-
//
|
|
1056
|
-
|
|
1044
|
+
// Record what was done and heard so far — dropping it would let the next turn call the
|
|
1045
|
+
// same tool again.
|
|
1046
|
+
recordAndContinue(signal.keepFull ? answer.length : this._replyText.length);
|
|
1057
1047
|
finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
|
|
1058
1048
|
if (this.state !== 'idle') this._set('listening');
|
|
1059
1049
|
return;
|
|
@@ -1073,16 +1063,13 @@ export class VoiceAgent {
|
|
|
1073
1063
|
// Normally we record only the HEARD prefix (barge-in: the user reacted to what they heard). But a
|
|
1074
1064
|
// turn aborted on hold-entry was tagged signal.keepFull, so we keep the whole streamed answer —
|
|
1075
1065
|
// on release the LLM sees everything it had formed. Per-turn tag → a quick Resume can't flip it.
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1066
|
+
// Both history and the EVENT use the raw prefix of `answer` (the host's full.slice(text.length)
|
|
1067
|
+
// tail aligns). keepFull/normal-finish → whole answer (no tail); a barge-in → the heard prefix
|
|
1068
|
+
// solid + the unspoken remainder dimmed, showing exactly where TTS got.
|
|
1069
|
+
const kept = signal.keepFull ? answer : rawPrefix(spoken);
|
|
1070
|
+
recordAndContinue(kept.length);
|
|
1080
1071
|
if (kept) {
|
|
1081
|
-
|
|
1082
|
-
// full.slice(text.length) tail aligns. keepFull/normal-finish → whole answer (no tail); a barge-in
|
|
1083
|
-
// → the heard prefix solid + the unspoken remainder dimmed, showing exactly where TTS got.
|
|
1084
|
-
const shown = signal.keepFull ? answer : rawPrefix(spoken);
|
|
1085
|
-
this.onEvent({ type: 'assistant', text: shown, full: answer, final: true, turn });
|
|
1072
|
+
this.onEvent({ type: 'assistant', text: kept, full: answer, final: true, turn });
|
|
1086
1073
|
} else if (answer) {
|
|
1087
1074
|
// Interrupted before anything was heard (nothing voiced yet): drop the streamed draft so the
|
|
1088
1075
|
// transcript doesn't keep a dangling unfinished bubble. Empty text → host clears the draft.
|
|
@@ -1098,8 +1085,21 @@ export class VoiceAgent {
|
|
|
1098
1085
|
if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
|
|
1099
1086
|
}
|
|
1100
1087
|
|
|
1088
|
+
// Native tool semantics: once a tool_result is in history the model gets to react to it. Queues a
|
|
1089
|
+
// SOFT reaction turn (behind the running one; dropped if a newer user turn supersedes it — that
|
|
1090
|
+
// turn sees the result anyway) after the agent ran a call for the default adapter (its stream
|
|
1091
|
+
// ends at the call; the result was never fed back), or when a LATE outcome (hung/slow tool,
|
|
1092
|
+
// barge-in mid-execution) landed in a record. Whether a reply is still owed is decided when the
|
|
1093
|
+
// turn RUNS (see _runTurn `react`), since the predecessor may already have carried the result.
|
|
1094
|
+
// Loop adapters fed the result back inside the reply, so nothing fires for them. Silent while
|
|
1095
|
+
// the user is speaking — their utterance will see the result in history anyway.
|
|
1096
|
+
_continue() {
|
|
1097
|
+
if (this._closed || this._speaking) return;
|
|
1098
|
+
this._runTurn('', { supersede: false, react: true });
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
1101
|
// Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
|
|
1102
|
-
// parallel with TTS, and NEVER joined: the turn records the
|
|
1102
|
+
// parallel with TTS, and NEVER joined: the turn records the call (tool_result pending) and finishes
|
|
1103
1103
|
// when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
|
|
1104
1104
|
// else. Its outcome reaches the host via onEvent; the model learns it via notify(). Must not
|
|
1105
1105
|
// reject — nothing awaits it, so a rejection here would be unhandled (hence the catch-all).
|
|
@@ -1115,16 +1115,15 @@ export class VoiceAgent {
|
|
|
1115
1115
|
// runtime/transport failures, not a handled domain-level tool miss.
|
|
1116
1116
|
// `id` rides along on every tool event so a host can replace THIS call's running chip in
|
|
1117
1117
|
// place, and two same-args calls (a legit retry) stay distinct acts in the transcript.
|
|
1118
|
-
if ('result' in call || 'error' in call) {
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1118
|
+
if (!('result' in call || 'error' in call)) {
|
|
1119
|
+
// Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
|
|
1120
|
+
// a deadline could only stop WAITING — reporting "failed" for an operation still on its way
|
|
1121
|
+
// to committing its side effect, inviting a duplicate retry. Nothing awaits this call
|
|
1122
|
+
// (see _speakTurn), so a hung tool costs its own result, never the turn loop.
|
|
1123
|
+
try { call.result = await this.tools[call.tool]?.run?.(call.args); }
|
|
1124
|
+
catch (e) { call.error = e?.message || String(e ?? 'failed'); }
|
|
1122
1125
|
}
|
|
1123
|
-
|
|
1124
|
-
// a deadline could only stop WAITING — reporting "failed" for an operation still on its way
|
|
1125
|
-
// to committing its side effect, inviting a duplicate retry. Nothing awaits this call
|
|
1126
|
-
// (see _speakTurn), so a hung tool costs its own result, never the turn loop.
|
|
1127
|
-
const result = await this.tools[call.tool]?.run?.(call.args);
|
|
1126
|
+
const result = call.error ? { text: call.error, ok: false } : call.result;
|
|
1128
1127
|
this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
|
|
1129
1128
|
} catch (e) {
|
|
1130
1129
|
this.onEvent({ type: 'error', error: `tool ${call.tool}: ${e.message}` });
|