@todoforai/voiceloop 0.1.11 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@todoforai/voiceloop",
3
- "version": "0.1.11",
3
+ "version": "0.1.12",
4
4
  "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
package/src/index.d.ts CHANGED
@@ -44,19 +44,31 @@ export interface VoiceTool {
44
44
  export type LLMChunk =
45
45
  | { text: string }
46
46
  | { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
47
+ // The conversation as the model sees it, Anthropic-shaped. Plain turns are strings; a turn in which
48
+ // the assistant called tools is recorded NATIVELY: an assistant message of text + tool_use blocks,
49
+ // then a user message holding the matching tool_result blocks (content = the result text the model
50
+ // reads; `is_error` on a failure). The agent never writes a textual notation of a tool call into
51
+ // history, so the model has nothing to imitate. Adapters project this onto their wire format
52
+ // (see toOpenAIMessages).
53
+ export type HistoryBlock =
54
+ | { type: 'text'; text: string }
55
+ | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
56
+ | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean };
57
+ export type HistoryMessage = { role: string; content: string | HistoryBlock[] };
47
58
  // Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
48
- // The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; the results are not
49
- // fed back into it mid-turn, so such calls are terminal for the turn. That is the low-level seam.
59
+ // The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; their results are
60
+ // recorded as tool_result and the agent starts a follow-up turn so the model reacts to them.
50
61
  // An agent LOOP that runs tools inside its own generator declares the two flags below.
51
62
  export interface LLM {
52
63
  (
53
- history: Array<{ role: string; content: string }>,
64
+ history: HistoryMessage[],
54
65
  system: string,
55
66
  signal: AbortSignal,
56
67
  options?: { toolGate?: Promise<unknown> },
57
68
  ): AsyncIterable<LLMChunk>;
58
- /** This generator ran the tool itself: chunks carry `result`/`error` and the agent REPORTS them
59
- * instead of executing again. Without it, an outcome-bearing chunk is re-run. */
69
+ /** This generator is a tool LOOP: it runs every call itself and feeds the result back inside the
70
+ * same reply (chunks carry `result`/`error`, optionally preceded by a `running` announcement).
71
+ * The agent never executes its calls and never starts a reaction turn for them. */
60
72
  executesTools?: boolean;
61
73
  /** This generator awaits `options.toolGate` before ANY tool executes. Required to keep
62
74
  * speculative prefetch on: prefetch starts from an uncommitted interim, and the gate is what
@@ -190,7 +202,7 @@ export class VoiceAgent {
190
202
  constructor(options?: VoiceAgentOptions);
191
203
  /** The live conversation the LLM sees. A mutable reference the host may read and push into —
192
204
  * carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
193
- history: Array<{ role: string; content: string }>;
205
+ history: HistoryMessage[];
194
206
  start(deviceId?: string): Promise<void>;
195
207
  /** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
196
208
  * start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
@@ -344,6 +356,8 @@ export function makeOpenAILLM(options?: {
344
356
  llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
345
357
  tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
346
358
  }): LLM;
359
+ /** VoiceAgent history → OpenAI chat messages (tool_use → tool_calls, tool_result → role:"tool"). */
360
+ export function toOpenAIMessages(history: HistoryMessage[]): Array<Record<string, unknown>>;
347
361
 
348
362
  // ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
349
363
  /** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
package/src/index.js CHANGED
@@ -11,5 +11,5 @@ export {
11
11
  } from './stt.js';
12
12
  export { ElevenLabsTTS } from './tts-elevenlabs.js';
13
13
  export { loadVoiceDeps, prewarmVoice } from './deps.js';
14
- export { makeOpenAILLM } from './llm-openai.js';
14
+ export { makeOpenAILLM, toOpenAIMessages } from './llm-openai.js';
15
15
  export { TUNING } from './tuning.js';
package/src/llm-openai.js CHANGED
@@ -19,6 +19,24 @@
19
19
  // keys, defaulting to all of them)
20
20
  // fetchFn — fetch override (tests, custom agents/headers); defaults to global fetch
21
21
  // extraBody— merged into the request body (temperature, provider-specific knobs, …)
22
+ // VoiceAgent history → OpenAI chat messages. Plain turns pass through; the agent's native tool
23
+ // records (assistant text + tool_use blocks, then a user turn of tool_result blocks — the same
24
+ // Anthropic-shaped blocks it keeps for every adapter) become `tool_calls` and `role:"tool"` messages.
25
+ export const toOpenAIMessages = (history) => history.flatMap((m) => {
26
+ if (!Array.isArray(m.content)) return [m];
27
+ const text = m.content.filter((b) => b.type === 'text').map((b) => b.text).join('');
28
+ if (m.role === 'assistant') {
29
+ const tool_calls = m.content.filter((b) => b.type === 'tool_use')
30
+ .map((b) => ({ id: b.id, type: 'function', function: { name: b.name, arguments: JSON.stringify(b.input ?? {}) } }));
31
+ return [{ role: 'assistant', content: text || null, ...(tool_calls.length ? { tool_calls } : {}) }];
32
+ }
33
+ const results = m.content.filter((b) => b.type === 'tool_result').map((b) => {
34
+ const content = typeof b.content === 'string' ? b.content : String(b.content ?? '');
35
+ return { role: 'tool', tool_call_id: b.tool_use_id, content: b.is_error && !content.startsWith('[failed]') ? `[failed] ${content}` : content };
36
+ });
37
+ return text ? [...results, { role: m.role, content: text }] : results; // OpenAI has no is_error → text-marked
38
+ });
39
+
22
40
  export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 1024, tools = {}, fetchFn, extraBody = {} } = {}) {
23
41
  const toolSpecs = Object.entries(tools).map(([name, t]) => ({
24
42
  type: 'function',
@@ -37,7 +55,7 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
37
55
  headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
38
56
  body: JSON.stringify({
39
57
  stream: true,
40
- messages: [{ role: 'system', content: system }, ...history],
58
+ messages: [{ role: 'system', content: system }, ...toOpenAIMessages(history)],
41
59
  max_tokens: maxTokens,
42
60
  ...(model ? { model } : {}),
43
61
  ...(toolSpecs.length ? { tools: toolSpecs } : {}),
@@ -60,7 +60,7 @@ const langRule = (lang) => {
60
60
  return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
61
61
  };
62
62
  // A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
63
- // what order a provider streamed the keys in (used both for dedup and for the [TOOL CALL] ledger).
63
+ // what order a provider streamed the keys in (used for dedup).
64
64
  // Identity: the provider call id when present (two calls with identical args are DISTINCT acts,
65
65
  // e.g. a legit retry after an error result); name+args otherwise (an id-less identical repeat is a
66
66
  // provider re-emitting its buffered call).
@@ -78,8 +78,8 @@ const withTimeout = (p, ms, fallback) => {
78
78
 
79
79
  const callArgs = (c) => JSON.stringify(c.args ?? {}, Object.keys(c.args ?? {}).sort());
80
80
  const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
81
- // Render a host tool's return value as model-facing text (used in the history ledger, and by LLM
82
- // adapters when feeding tool results back to the model). Never throws: the side effect already RAN,
81
+ // Render a host tool's return value as model-facing text (the tool_result content in history, and
82
+ // what LLM adapters feed back to the model). Never throws: the side effect already RAN,
83
83
  // so an unserializable result must not retro-fail the call and invite a retry.
84
84
  export const toolResultText = (result) => {
85
85
  if (result === undefined) return 'ok';
@@ -88,87 +88,22 @@ export const toolResultText = (result) => {
88
88
  return result.ok ? result.text : `[failed] ${result.text}`;
89
89
  try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
90
90
  };
91
- // Ledger suffix for a call whose outcome is attached to the chunk: keeps the RESULT in the
92
- // plain-text history so the NEXT turn still knows what the tool said. Bounded — history is
93
- // spoken-conversation-sized, not a log. Calls without an outcome field contribute nothing.
94
- const LEDGER_RESULT_MAX = 4000;
95
- const ledgerOutcome = (c) => {
96
- if (c.error) return ` → error: ${String(c.error).slice(0, LEDGER_RESULT_MAX)}`;
97
- if (!('result' in c)) return '';
98
- let text = toolResultText(c.result);
99
- if (text.length > LEDGER_RESULT_MAX) text = `${text.slice(0, LEDGER_RESULT_MAX)}… (${text.length} chars)`;
100
- return ` → ${text}`;
101
- };
102
-
103
- // ── [TOOL CALL] mimicry filter ──────────────────────────────────────────────────────────────────
104
- // The model sees `[TOOL CALL name] {...}` ledger lines in its own history turns, and sometimes
105
- // IMITATES the format instead of calling the tool natively: it types the line as reply text.
106
- // Untreated, the raw markup leaks into the spoken/visible reply and the tool never runs. This
107
- // transform strips such lines from the text stream: a bare call (no → outcome) is what the model
108
- // MEANT to do → re-emitted as a real { tool, args } chunk (the turn's dedup collapses it if the
109
- // native call also arrived); a line WITH an → outcome is a hallucinated memory of a past call →
110
- // dropped. Text before/after the line streams through untouched, held back only while a possible
111
- // marker/line is still forming so TTS latency is unaffected on normal replies.
112
- const MIMIC_MARK = '[TOOL CALL ';
113
- const parseMimicLine = (line) => {
114
- const m = /^\[TOOL CALL ([\w.-]+)\]\s*(.*)$/.exec(line.trim());
115
- if (!m) return null; // not the format after all — caller emits it as text
116
- const [, tool, rest] = m;
117
- if (!rest) return { tool, args: {} };
118
- if (!rest.startsWith('{')) return { drop: true }; // `→ outcome` (or junk) with no args — a remembered call
119
- // Balanced-brace scan (string-aware): the args JSON may be followed by a ` → outcome` tail.
120
- let depth = 0, inStr = false, escaped = false, end = -1;
121
- for (let i = 0; i < rest.length; i++) {
122
- const ch = rest[i];
123
- if (escaped) { escaped = false; continue; }
124
- if (inStr) { if (ch === '\\') escaped = true; else if (ch === '"') inStr = false; continue; }
125
- if (ch === '"') inStr = true;
126
- else if (ch === '{') depth++;
127
- else if (ch === '}' && --depth === 0) { end = i + 1; break; }
91
+ // One model-facing history record for a settled/pending tool call: the native tool_result block that
92
+ // pairs with its tool_use. Bounded — history is spoken-conversation-sized, not a log. A call whose
93
+ // outcome has not landed yet says so; _runTool patches the SAME block in place when it does.
94
+ const RESULT_MAX = 4000;
95
+ const MAX_TOOL_ROUNDS = 6;
96
+ const PENDING_RESULT = 'still running — the result will follow in a later turn';
97
+ const bound = (text) => text.length > RESULT_MAX ? `${text.slice(0, RESULT_MAX)}… (${text.length} chars)` : text;
98
+ const toolResultBlock = (c) => {
99
+ const block = { type: 'tool_result', tool_use_id: c.id, content: PENDING_RESULT };
100
+ if ('error' in c) { block.content = bound(String(c.error) || 'failed'); block.is_error = true; }
101
+ else if ('result' in c) {
102
+ block.content = bound(toolResultText(c.result));
103
+ if (c.result && typeof c.result === 'object' && c.result.ok === false) block.is_error = true;
128
104
  }
129
- if (end < 0) return { drop: true }; // truncated JSON — can't run it faithfully
130
- if (rest.slice(end).trim()) return { drop: true }; // has an → outcome: a memory, not a new call
131
- try { return { tool, args: JSON.parse(rest.slice(0, end)) }; } catch { return { drop: true }; }
105
+ return block;
132
106
  };
133
- async function* stripToolCallMimicry(src) {
134
- let carry = '';
135
- // Longest tail of `carry` that could still grow into the marker — held back; the rest is safe to emit.
136
- const heldLen = () => {
137
- for (let n = Math.min(carry.length, MIMIC_MARK.length - 1); n > 0; n--)
138
- if (MIMIC_MARK.startsWith(carry.slice(-n))) return n;
139
- return 0;
140
- };
141
- function* pump(final) {
142
- for (;;) {
143
- const at = carry.indexOf(MIMIC_MARK);
144
- if (at < 0) {
145
- const hold = final ? 0 : heldLen();
146
- const out = carry.slice(0, carry.length - hold);
147
- carry = carry.slice(carry.length - hold);
148
- if (out) yield { text: out };
149
- return;
150
- }
151
- const nl = carry.indexOf('\n', at);
152
- if (nl < 0 && !final) { // the line is still streaming — hold from the marker on
153
- if (at > 0) { yield { text: carry.slice(0, at) }; carry = carry.slice(at); }
154
- return;
155
- }
156
- const end = nl < 0 ? carry.length : nl;
157
- const before = carry.slice(0, at), line = carry.slice(at, end);
158
- carry = nl < 0 ? '' : carry.slice(nl + 1); // the line (and its newline) leave the text stream
159
- if (before) yield { text: before };
160
- const parsed = parseMimicLine(line);
161
- if (!parsed) yield { text: nl < 0 ? line : `${line}\n` }; // false alarm — pass it through untouched (newline included)
162
- else if (parsed.tool) yield { tool: parsed.tool, args: parsed.args };
163
- }
164
- }
165
- for await (const item of src) {
166
- if (item.tool) { yield item; continue; }
167
- carry += item.text ?? '';
168
- yield* pump(false);
169
- }
170
- yield* pump(true);
171
- }
172
107
 
173
108
  // Persona (who the agent is) + language rule + the host's live context block (`sysmsg`, delimited
174
109
  // so the model never mistakes injected context for its own conversation).
@@ -378,6 +313,8 @@ export class VoiceAgent {
378
313
  this.state = 'idle'; // idle | listening | thinking | speaking
379
314
  this._abort = null; // aborts the in-flight LLM
380
315
  this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
316
+ this._toolRounds = 0; // tool-result reactions since the user last spoke (see _runTurn react)
317
+ this._late = []; // outcomes landed after the last request was built (see _runTurn react)
381
318
  this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
382
319
  // Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
383
320
  // dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
@@ -699,6 +636,7 @@ export class VoiceAgent {
699
636
  // history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
700
637
  // release (_flush) owns starting the single turn, and waits for all of these to settle first.
701
638
  _pushHeld(text) {
639
+ this._toolRounds = 0;
702
640
  const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
703
641
  this._heldPushes.add(p);
704
642
  p.finally(() => this._heldPushes.delete(p));
@@ -720,15 +658,14 @@ export class VoiceAgent {
720
658
  }
721
659
 
722
660
  // Inject an out-of-band turn: feed `text` to the LLM as if the user had said it, then speak the
723
- // reply. Used by the host to surface external events (a tool result, a background job finishing)
661
+ // reply. Used by the host to surface external events (a background job finishing, a board change)
724
662
  // — only meaningful while the agent is running, so it's a no-op once stopped.
725
663
  //
726
664
  // SOFT turn (supersede: false): unlike a user utterance this must NOT cut the reply in flight.
727
- // A tool result arrives WHILE the turn that called the tool is still streaming/speaking, so
728
- // aborting here would kill that turn mid-answer and re-run the LLM over a history whose last
729
- // entry is the request again — the model then re-calls the same tool (duplicate-action spam).
730
- // Instead it queues behind the running turn, which by then has recorded both its answer and the
731
- // [TOOL CALL] it made, so the model sees the work is already done.
665
+ // An event can arrive WHILE a turn is still streaming/speaking, so aborting here would kill that
666
+ // turn mid-answer and re-run the LLM over a history whose last entry is the request again — the
667
+ // model then re-calls the same tool (duplicate-action spam). Instead it queues behind the running
668
+ // turn, which by then has recorded its answer and its tool calls natively.
732
669
  async notify(text) { if (!this._closed) await this._runTurn(text, { supersede: false }); }
733
670
 
734
671
  // Send a TYPED user turn: like speaking, but the text comes from the host's input instead of STT.
@@ -762,6 +699,7 @@ export class VoiceAgent {
762
699
  // caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
763
700
  _onUserTurn(text, { speech = false } = {}) {
764
701
  if (!text.trim()) return Promise.resolve();
702
+ this._toolRounds = 0;
765
703
  return this._runTurn(text, { speech });
766
704
  }
767
705
 
@@ -878,7 +816,7 @@ export class VoiceAgent {
878
816
  // if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
879
817
  // sees this message in history and answers over the fuller picture — so a burst of tool results
880
818
  // collapses into ONE reply instead of one interrupting monologue per result.
881
- _runTurn(text, { supersede = true, speech = false } = {}) {
819
+ _runTurn(text, { supersede = true, speech = false, react = false } = {}) {
882
820
  const prev = this._turn;
883
821
  if (supersede) {
884
822
  this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
@@ -904,9 +842,24 @@ export class VoiceAgent {
904
842
  // Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
905
843
  const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
906
844
  if (text) this.history.push({ role: 'user', content: text });
845
+ // A reaction turn owes a reply only for results no request has carried yet: either the
846
+ // tool_result record is still the last message, or a late outcome landed after the last
847
+ // request was built (_late; cleared when a request starts) — then a short [EVENT] user turn,
848
+ // the host's channel for out-of-band facts, points the model at the patched block (a request
849
+ // can't end on an assistant message). Bounded per user utterance (_toolRounds, reset on
850
+ // every accepted user turn): a model that keeps calling tools gets MAX_TOOL_ROUNDS
851
+ // reactions, then waits for the user.
852
+ if (react) {
853
+ const last = this.history[this.history.length - 1];
854
+ const owed = (last?.role === 'user' && Array.isArray(last.content)) || this._late.length;
855
+ if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this._late.length = 0; return; }
856
+ this._toolRounds++;
857
+ if (last?.role !== 'user') this.history.push({ role: 'user', content: `[EVENT tool_result] ${this._late.map((c) => c.tool).join(', ')} finished — see the tool_result above.` });
858
+ }
907
859
  // Reply only if the conversation now ends on a user turn (a raced turn may have already
908
860
  // replied). pf is always null here: if we pushed a user turn it IS the last entry.
909
861
  if (this.history[this.history.length - 1]?.role !== 'user') return;
862
+ this._late.length = 0; // this request carries every result landed so far
910
863
  this._set('thinking'); // LLM stage
911
864
  this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
912
865
  await this._speakTurn(this._abort.signal, pf?.gen, seq); // _speakTurn never throws — errors surface via onEvent
@@ -914,7 +867,7 @@ export class VoiceAgent {
914
867
  }
915
868
 
916
869
  // One assistant turn: stream the LLM → TTS, then record what was actually HEARD (the barge-in
917
- // prefix, not the full generated answer) plus a ledger of the tools it dispatched. Stored as
870
+ // prefix, not the full generated answer) plus the tools it called, natively. Stored as
918
871
  // `this._turn` so the next user turn can await it. Never throws (errors surface via onEvent) so
919
872
  // the await is safe — and never awaits a tool, so a hung one can't hold the next turn hostage.
920
873
  // `turn` tags every assistant event with the reply it belongs to, so a host can tell a LATE
@@ -927,31 +880,82 @@ export class VoiceAgent {
927
880
  // repopulates _replyText with the audible prefix as playback advances.
928
881
  if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText;
929
882
  this._replyText = ''; this._replyEcho = '';
930
- // Tap the LLM stream once: yield speech text to TTS, and record the tool calls it makes (they
931
- // fire on arrival; `calls` is the ledger). First delta flips us to 'speaking'; accumulate for the
932
- // heard-vs-unheard report. `this.llm(...)` is a lazy generator — its fetch fires only when
883
+ // Tap the LLM stream once: yield speech text to TTS, and record what the model did as ROUNDS —
884
+ // each round is one assistant message (text + the tool_use blocks it emitted) followed by the
885
+ // user-role tool_result blocks for those calls. Recorded into history in that order on EVERY
886
+ // exit path, so the next request replays the model's own native tool history — never a text
887
+ // notation it could imitate. `this.llm(...)` is a lazy generator — its fetch fires only when
933
888
  // tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
934
- let answer = '', calls = [];
935
- // The tools this turn dispatched, as a ledger line recorded INTO its assistant message. The
936
- // runtime's tool calls are fire-and-forget (no tool_use/tool_result pair on the wire, and history
937
- // is provider-neutral {role, content} text), so without this the history reads as if it never
938
- // called anything and the next turn happily calls it AGAIN — duplicate tool-call spam. Relayed
939
- // calls: the outcome arrives separately as the host's `[TOOL RESULT <name>]` notify() turn.
940
- // Custom LLMs that execute tools inside their own generator (llm.executesTools) attach the
941
- // outcome directly, so the ledger line carries it — bounded — or the next turn would see the call with no result and lose what
942
- // the model learned (a barge-in cuts the spoken follow-up but not the fact). Read late (the
943
- // stream may still be appending) and on EVERY exit path, including a failed reply.
944
- const toolNote = () => calls.map((c) => `[TOOL CALL ${c.tool}] ${callArgs(c)}${ledgerOutcome(c)}`).join('\n');
889
+ //
890
+ // Round boundary: the default adapter's stream is ONE model message (text + calls together;
891
+ // the reaction to the results comes from _continue). A loop adapter feeds results back inside
892
+ // the stream, so anything after an outcome landed is the model's NEXT message.
893
+ let answer = '';
894
+ const rounds = [], calls = []; // calls: every act this turn saw, for dedup — outlives recording
895
+ const loop = !!this.llm.executesTools;
896
+ const round = () => {
897
+ const r = rounds[rounds.length - 1];
898
+ if (r && !(loop && r.settled)) return r;
899
+ rounds.push({ text: '', calls: [] }); return rounds[rounds.length - 1];
900
+ };
901
+ let n = 0;
902
+ // Accept a call once: identity is the provider id, else name+args (an id-less repeat is a
903
+ // provider re-emitting its buffered call). `owner` says who executes it — the adapter (the
904
+ // chunk announces or carries the outcome: it ran the tool itself and feeds the result back
905
+ // inside its own reply) or the agent (_runTool runs it) — fixed at acceptance.
906
+ const accept = (item) => {
907
+ const key = callKey(item);
908
+ let c = calls.find((x) => x.key === key);
909
+ if (c) return c;
910
+ const adapterRan = loop || item.running || 'result' in item || 'error' in item;
911
+ c = { ...item, key, id: item.id || `call-${turn}-${++n}`, owner: adapterRan ? 'adapter' : 'agent' };
912
+ calls.push(c); round().calls.push(c);
913
+ return c;
914
+ };
915
+ const outcome = (c, item) => {
916
+ if ('result' in item) c.result = item.result; else if ('error' in item) c.error = item.error;
917
+ const r = rounds.find((r) => r.calls.includes(c)); if (r) r.settled = true; // already recorded → no round to close
918
+ };
919
+ // An outcome may land AFTER the turn recorded the call (a hung tool, a barge-in mid-execution):
920
+ // its tool_result block is patched IN PLACE so the next request carries the real result, and the
921
+ // model gets to react to it (see _continue).
922
+ const settle = (c) => this._runTool(c).then(() => {
923
+ if (!c.block) return; // not recorded yet — record() picks the outcome up
924
+ if (!c.consumed) { Object.assign(c.block, toolResultBlock(c)); c.consumed = true; }
925
+ this._dropPrefetch(); // a speculation serialized the pending block — stale
926
+ this._late.push(c); this._continue();
927
+ });
928
+ const record = (keptLen) => {
929
+ let budget = keptLen;
930
+ for (const r of rounds) {
931
+ const text = r.text.slice(0, Math.max(0, budget)).trimEnd(); budget -= r.text.length;
932
+ if (!text && !r.calls.length) continue;
933
+ if (!r.calls.length) { this.history.push({ role: 'assistant', content: text }); continue; }
934
+ const blocks = text ? [{ type: 'text', text }] : [];
935
+ for (const c of r.calls) blocks.push({ type: 'tool_use', id: c.id, name: c.tool, input: c.args ?? {} });
936
+ this.history.push({ role: 'assistant', content: blocks });
937
+ this.history.push({ role: 'user', content: r.calls.map((c) => {
938
+ c.consumed = 'result' in c || 'error' in c;
939
+ return (c.block = toolResultBlock(c));
940
+ }) });
941
+ }
942
+ rounds.length = 0;
943
+ };
944
+ // Follow-up only for calls the AGENT ran: a loop adapter fed its results back inside the reply.
945
+ const recordAndContinue = (keptLen) => {
946
+ record(keptLen);
947
+ if (calls.some((c) => c.owner === 'agent' && c.consumed)) this._continue();
948
+ };
945
949
  // Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
946
950
  // announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
947
951
  // that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
948
- // outcome arrives finalizes it as interrupted — DISPLAY truth only: an announcement never reaches
949
- // the ledger (`calls`), so the model treats it as never having happened.
952
+ // outcome arrives finalizes the CHIP as interrupted; the history keeps the call with a pending
953
+ // tool_result, patched in place if the outcome still lands.
950
954
  const announced = new Map();
951
955
  // Latched once this turn has finalized its chips (finishInterrupted ran). A barge-in detaches the
952
956
  // LLM producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
953
- // announcement would open a spinner nothing will ever close. Past the latch, announcements are
954
- // dropped — outcomes still flow (a tool the adapter really ran is truth worth reporting).
957
+ // announcement would open a spinner nothing will ever close, and a late UNEXECUTED call is work
958
+ // the user cut off. Past the latch only outcomes flow (a tool the adapter really ran is truth).
955
959
  let finalized = false;
956
960
  const finishInterrupted = () => {
957
961
  finalized = true;
@@ -964,37 +968,31 @@ export class VoiceAgent {
964
968
  // tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
965
969
  // act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
966
970
  if (item.tool) {
967
- if (item.running) { // display-only announcement; the outcome chunk below does the work
971
+ const hasOutcome = 'result' in item || 'error' in item;
972
+ const known = calls.find((x) => x.key === callKey(item));
973
+ if (item.running || (loop && !hasOutcome)) { // announcement; the outcome chunk below does the work
968
974
  // An id-less announcement can never be replaced in place NOR finalized (both key on id),
969
975
  // so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
970
- if (finalized || !item.id) continue;
971
- // Already settled this turn (adapter re-emitting its buffered pair): re-announcing would
972
- // reopen a spinner over a finished chip, and the duplicate outcome is deduped below and
973
- // never re-reported — so it would hang there, then finalize as falsely "interrupted".
974
- if (calls.some((c) => c.id === item.id)) continue;
975
- announced.set(item.id, item);
976
+ if (finalized || !item.id || known) continue;
977
+ announced.set(item.id, item); accept(item);
976
978
  self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
977
979
  continue;
978
980
  }
979
981
  if (item.id) announced.delete(item.id);
980
- // Same key = same act (provider re-emit / model asking twice). A MIMIC chunk carries no id,
981
- // so also match by name+args across the id boundary — a typed echo of an id-carrying native
982
- // call (llm.executesTools mode) must collapse too, or it would execute the tool a second time.
983
- const dup = calls.some((c) => callKey(c) === callKey(item) ||
984
- ((!c.id || !item.id) && c.tool === item.tool && callArgs(c) === callArgs(item)));
985
- if (!dup) {
986
- calls.push(item); self._runTool(item);
987
- // Late outcome: the turn already closed (barge-in mid-execution) and its ledger is
988
- // written, so this result would reach the transcript chip but never the model. Relay it
989
- // as a soft [TOOL RESULT] turn — the same channel out-of-band receipts use — so the next
990
- // reply knows what the tool returned instead of seeing a call that vanished.
991
- if (finalized && ('result' in item || 'error' in item))
992
- self.notify(`[TOOL RESULT ${item.tool}] ${callArgs(item)}${ledgerOutcome(item)}`);
982
+ if (known) { // the announced call's outcome, or a re-emit
983
+ if (known.ran || !hasOutcome) continue;
984
+ outcome(known, item); settle(known);
985
+ continue;
993
986
  }
987
+ if (finalized && !hasOutcome) continue; // late bare call: the user cut this reply off
988
+ const c = accept(item);
989
+ if (hasOutcome) outcome(c, item);
990
+ if (finalized) record(0); // late outcome of an unseen call: give it a record of its own (settle schedules the reaction)
991
+ settle(c);
994
992
  continue;
995
993
  }
996
994
  if (self.state === 'thinking') self._set('speaking'); // first token → TTS stage
997
- answer += item.text;
995
+ answer += item.text; round().text += item.text;
998
996
  // While audio plays, the TTS progress hook owns the draft (text = spoken-so-far, full = answer,
999
997
  // read live each tick), so streaming the whole answer here would yank the cursor to the end.
1000
998
  // Once the cursor is live we only grow `answer` (the hook reads it); until then (muted, or the
@@ -1002,7 +1000,7 @@ export class VoiceAgent {
1002
1000
  if (!self._cursorLive) self.onEvent({ type: 'assistant', text: answer, final: false, turn });
1003
1001
  yield item.text;
1004
1002
  }
1005
- })(this, stripToolCallMimicry(prefetched ?? this.llm(this.history, this.sysmsg, signal))); // prefetched stream already carries this turn's user text (see _startPrefetch)
1003
+ })(this, prefetched ?? this.llm(this.history, this.sysmsg, signal)); // prefetched stream already carries this turn's user text (see _startPrefetch)
1006
1004
 
1007
1005
  let spoken = '';
1008
1006
  // Live spoken cursor: while a clip plays the TTS reports chars heard so far, which we mirror as a
@@ -1051,9 +1049,9 @@ export class VoiceAgent {
1051
1049
  // follows), and a host tracking bubbles per turn needs to know which one just died.
1052
1050
  this.onEvent({ type: 'error', error: e.message, turn });
1053
1051
  // Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
1054
- // Record it anyway: the ledger is about what was DONE, not about what got spoken — dropping
1055
- // it would let the next turn call the same tool again.
1056
- if (toolNote()) this.history.push({ role: 'assistant', content: toolNote() });
1052
+ // Record what was done and heard so far — dropping it would let the next turn call the
1053
+ // same tool again.
1054
+ recordAndContinue(signal.keepFull ? answer.length : this._replyText.length);
1057
1055
  finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
1058
1056
  if (this.state !== 'idle') this._set('listening');
1059
1057
  return;
@@ -1073,16 +1071,13 @@ export class VoiceAgent {
1073
1071
  // Normally we record only the HEARD prefix (barge-in: the user reacted to what they heard). But a
1074
1072
  // turn aborted on hold-entry was tagged signal.keepFull, so we keep the whole streamed answer —
1075
1073
  // on release the LLM sees everything it had formed. Per-turn tag → a quick Resume can't flip it.
1076
- const kept = signal.keepFull ? answer : spoken;
1077
- if (kept || toolNote()) {
1078
- this.history.push({ role: 'assistant', content: [kept, toolNote()].filter(Boolean).join('\n') });
1079
- }
1074
+ // Both history and the EVENT use the raw prefix of `answer` (the host's full.slice(text.length)
1075
+ // tail aligns). keepFull/normal-finish → whole answer (no tail); a barge-in → the heard prefix
1076
+ // solid + the unspoken remainder dimmed, showing exactly where TTS got.
1077
+ const kept = signal.keepFull ? answer : rawPrefix(spoken);
1078
+ recordAndContinue(kept.length);
1080
1079
  if (kept) {
1081
- // History keeps the normalized `kept`; the EVENT emits a raw prefix of `answer` so the host's
1082
- // full.slice(text.length) tail aligns. keepFull/normal-finish → whole answer (no tail); a barge-in
1083
- // → the heard prefix solid + the unspoken remainder dimmed, showing exactly where TTS got.
1084
- const shown = signal.keepFull ? answer : rawPrefix(spoken);
1085
- this.onEvent({ type: 'assistant', text: shown, full: answer, final: true, turn });
1080
+ this.onEvent({ type: 'assistant', text: kept, full: answer, final: true, turn });
1086
1081
  } else if (answer) {
1087
1082
  // Interrupted before anything was heard (nothing voiced yet): drop the streamed draft so the
1088
1083
  // transcript doesn't keep a dangling unfinished bubble. Empty text → host clears the draft.
@@ -1098,8 +1093,21 @@ export class VoiceAgent {
1098
1093
  if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
1099
1094
  }
1100
1095
 
1096
+ // Native tool semantics: once a tool_result is in history the model gets to react to it. Queues a
1097
+ // SOFT reaction turn (behind the running one; dropped if a newer user turn supersedes it — that
1098
+ // turn sees the result anyway) after the agent ran a call for the default adapter (its stream
1099
+ // ends at the call; the result was never fed back), or when a LATE outcome (hung/slow tool,
1100
+ // barge-in mid-execution) landed in a record. Whether a reply is still owed is decided when the
1101
+ // turn RUNS (see _runTurn `react`), since the predecessor may already have carried the result.
1102
+ // Loop adapters fed the result back inside the reply, so nothing fires for them. Silent while
1103
+ // the user is speaking — their utterance will see the result in history anyway.
1104
+ _continue() {
1105
+ if (this._closed || this._speaking) return;
1106
+ this._runTurn('', { supersede: false, react: true });
1107
+ }
1108
+
1101
1109
  // Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
1102
- // parallel with TTS, and NEVER joined: the turn records the dispatch in its ledger and finishes
1110
+ // parallel with TTS, and NEVER joined: the turn records the call (tool_result pending) and finishes
1103
1111
  // when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
1104
1112
  // else. Its outcome reaches the host via onEvent; the model learns it via notify(). Must not
1105
1113
  // reject — nothing awaits it, so a rejection here would be unhandled (hence the catch-all).
@@ -1115,16 +1123,15 @@ export class VoiceAgent {
1115
1123
  // runtime/transport failures, not a handled domain-level tool miss.
1116
1124
  // `id` rides along on every tool event so a host can replace THIS call's running chip in
1117
1125
  // place, and two same-args calls (a legit retry) stay distinct acts in the transcript.
1118
- if ('result' in call || 'error' in call) {
1119
- const result = call.error ? { text: call.error, ok: false } : call.result;
1120
- this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
1121
- return;
1126
+ if (!('result' in call || 'error' in call)) {
1127
+ // Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
1128
+ // a deadline could only stop WAITING — reporting "failed" for an operation still on its way
1129
+ // to committing its side effect, inviting a duplicate retry. Nothing awaits this call
1130
+ // (see _speakTurn), so a hung tool costs its own result, never the turn loop.
1131
+ try { call.result = await this.tools[call.tool]?.run?.(call.args); }
1132
+ catch (e) { call.error = e?.message || String(e ?? 'failed'); }
1122
1133
  }
1123
- // Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
1124
- // a deadline could only stop WAITING — reporting "failed" for an operation still on its way
1125
- // to committing its side effect, inviting a duplicate retry. Nothing awaits this call
1126
- // (see _speakTurn), so a hung tool costs its own result, never the turn loop.
1127
- const result = await this.tools[call.tool]?.run?.(call.args);
1134
+ const result = call.error ? { text: call.error, ok: false } : call.result;
1128
1135
  this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
1129
1136
  } catch (e) {
1130
1137
  this.onEvent({ type: 'error', error: `tool ${call.tool}: ${e.message}` });