@todoforai/voiceloop 0.1.11 → 0.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@todoforai/voiceloop",
3
- "version": "0.1.11",
3
+ "version": "0.1.13",
4
4
  "description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
package/src/index.d.ts CHANGED
@@ -43,20 +43,35 @@ export interface VoiceTool {
43
43
  // reports it instead of running it again.
44
44
  export type LLMChunk =
45
45
  | { text: string }
46
- | { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true };
46
+ | { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true }
47
+ /** Loop adapters only: the model's NEXT message starts here (after the results it fed back), so
48
+ * the agent records text/calls before and after as separate assistant messages. */
49
+ | { newMessage: true };
50
+ // The conversation as the model sees it, Anthropic-shaped. Plain turns are strings; a turn in which
51
+ // the assistant called tools is recorded NATIVELY: an assistant message of text + tool_use blocks,
52
+ // then a user message holding the matching tool_result blocks (content = the result text the model
53
+ // reads; `is_error` on a failure). The agent never writes a textual notation of a tool call into
54
+ // history, so the model has nothing to imitate. Adapters project this onto their wire format
55
+ // (see toOpenAIMessages).
56
+ export type HistoryBlock =
57
+ | { type: 'text'; text: string }
58
+ | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
59
+ | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean };
60
+ export type HistoryMessage = { role: string; content: string | HistoryBlock[] };
47
61
  // Pluggable LLM: lazy async generator over the chat history (fetch fires on first pull).
48
- // The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; the results are not
49
- // fed back into it mid-turn, so such calls are terminal for the turn. That is the low-level seam.
62
+ // The default (makeOpenAILLM) yields bare tool chunks that the AGENT executes; their results are
63
+ // recorded as tool_result and the agent starts a follow-up turn so the model reacts to them.
50
64
  // An agent LOOP that runs tools inside its own generator declares the two flags below.
51
65
  export interface LLM {
52
66
  (
53
- history: Array<{ role: string; content: string }>,
67
+ history: HistoryMessage[],
54
68
  system: string,
55
69
  signal: AbortSignal,
56
70
  options?: { toolGate?: Promise<unknown> },
57
71
  ): AsyncIterable<LLMChunk>;
58
- /** This generator ran the tool itself: chunks carry `result`/`error` and the agent REPORTS them
59
- * instead of executing again. Without it, an outcome-bearing chunk is re-run. */
72
+ /** This generator is a tool LOOP: it runs every call itself and feeds the result back inside the
73
+ * same reply (chunks carry `result`/`error`, optionally preceded by a `running` announcement).
74
+ * The agent never executes its calls and never starts a reaction turn for them. */
60
75
  executesTools?: boolean;
61
76
  /** This generator awaits `options.toolGate` before ANY tool executes. Required to keep
62
77
  * speculative prefetch on: prefetch starts from an uncommitted interim, and the gate is what
@@ -190,7 +205,7 @@ export class VoiceAgent {
190
205
  constructor(options?: VoiceAgentOptions);
191
206
  /** The live conversation the LLM sees. A mutable reference the host may read and push into —
192
207
  * carrying it over is how a hot-restart (e.g. a language switch) keeps the conversation. */
193
- history: Array<{ role: string; content: string }>;
208
+ history: HistoryMessage[];
194
209
  start(deviceId?: string): Promise<void>;
195
210
  /** Pause: stop replying/listening but KEEP the mic stream + audio graph + VAD model warm, so a later
196
211
  * start() of this same agent resumes instantly (no mic prompt, no VAD cold start). Use destroy() to
@@ -344,6 +359,8 @@ export function makeOpenAILLM(options?: {
344
359
  llmUrl?: string; apiKey?: string; model?: string; maxTokens?: number;
345
360
  tools?: Record<string, VoiceTool>; fetchFn?: typeof fetch; extraBody?: Record<string, unknown>;
346
361
  }): LLM;
362
+ /** VoiceAgent history → OpenAI chat messages (tool_use → tool_calls, tool_result → role:"tool"). */
363
+ export function toOpenAIMessages(history: HistoryMessage[]): Array<Record<string, unknown>>;
347
364
 
348
365
  // ── Warmup & helpers ─────────────────────────────────────────────────────────────────────────
349
366
  /** Load the VAD/ONNX runtime bundles (memoized). `prewarmVoice` additionally compiles the VAD
package/src/index.js CHANGED
@@ -11,5 +11,5 @@ export {
11
11
  } from './stt.js';
12
12
  export { ElevenLabsTTS } from './tts-elevenlabs.js';
13
13
  export { loadVoiceDeps, prewarmVoice } from './deps.js';
14
- export { makeOpenAILLM } from './llm-openai.js';
14
+ export { makeOpenAILLM, toOpenAIMessages } from './llm-openai.js';
15
15
  export { TUNING } from './tuning.js';
package/src/llm-openai.js CHANGED
@@ -19,6 +19,24 @@
19
19
  // keys, defaulting to all of them)
20
20
  // fetchFn — fetch override (tests, custom agents/headers); defaults to global fetch
21
21
  // extraBody— merged into the request body (temperature, provider-specific knobs, …)
22
+ // VoiceAgent history → OpenAI chat messages. Plain turns pass through; the agent's native tool
23
+ // records (assistant text + tool_use blocks, then a user turn of tool_result blocks — the same
24
+ // Anthropic-shaped blocks it keeps for every adapter) become `tool_calls` and `role:"tool"` messages.
25
+ export const toOpenAIMessages = (history) => history.flatMap((m) => {
26
+ if (!Array.isArray(m.content)) return [m];
27
+ const text = m.content.filter((b) => b.type === 'text').map((b) => b.text).join('');
28
+ if (m.role === 'assistant') {
29
+ const tool_calls = m.content.filter((b) => b.type === 'tool_use')
30
+ .map((b) => ({ id: b.id, type: 'function', function: { name: b.name, arguments: JSON.stringify(b.input ?? {}) } }));
31
+ return [{ role: 'assistant', content: text || null, ...(tool_calls.length ? { tool_calls } : {}) }];
32
+ }
33
+ const results = m.content.filter((b) => b.type === 'tool_result').map((b) => {
34
+ const content = typeof b.content === 'string' ? b.content : String(b.content ?? '');
35
+ return { role: 'tool', tool_call_id: b.tool_use_id, content: b.is_error && !content.startsWith('[failed]') ? `[failed] ${content}` : content };
36
+ });
37
+ return text ? [...results, { role: m.role, content: text }] : results; // OpenAI has no is_error → text-marked
38
+ });
39
+
22
40
  export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 1024, tools = {}, fetchFn, extraBody = {} } = {}) {
23
41
  const toolSpecs = Object.entries(tools).map(([name, t]) => ({
24
42
  type: 'function',
@@ -37,7 +55,7 @@ export function makeOpenAILLM({ llmUrl, apiKey = '', model = '', maxTokens = 102
37
55
  headers: { 'Content-Type': 'application/json', ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) },
38
56
  body: JSON.stringify({
39
57
  stream: true,
40
- messages: [{ role: 'system', content: system }, ...history],
58
+ messages: [{ role: 'system', content: system }, ...toOpenAIMessages(history)],
41
59
  max_tokens: maxTokens,
42
60
  ...(model ? { model } : {}),
43
61
  ...(toolSpecs.length ? { tools: toolSpecs } : {}),
@@ -60,7 +60,7 @@ const langRule = (lang) => {
60
60
  return name ? `ALWAYS reply in ${name} — including tool-call acknowledgements.` : '';
61
61
  };
62
62
  // A tool call's arguments, key-sorted — so the SAME act always renders/compares identically no matter
63
- // what order a provider streamed the keys in (used both for dedup and for the [TOOL CALL] ledger).
63
+ // what order a provider streamed the keys in (used for dedup).
64
64
  // Identity: the provider call id when present (two calls with identical args are DISTINCT acts,
65
65
  // e.g. a legit retry after an error result); name+args otherwise (an id-less identical repeat is a
66
66
  // provider re-emitting its buffered call).
@@ -76,10 +76,11 @@ const withTimeout = (p, ms, fallback) => {
76
76
  ]);
77
77
  };
78
78
 
79
- const callArgs = (c) => JSON.stringify(c.args ?? {}, Object.keys(c.args ?? {}).sort());
79
+ const callArgs = (c) => JSON.stringify(c.args ?? {}, (_k, v) =>
80
+ v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, v[k]])) : v);
80
81
  const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
81
- // Render a host tool's return value as model-facing text (used in the history ledger, and by LLM
82
- // adapters when feeding tool results back to the model). Never throws: the side effect already RAN,
82
+ // Render a host tool's return value as model-facing text (the tool_result content in history, and
83
+ // what LLM adapters feed back to the model). Never throws: the side effect already RAN,
83
84
  // so an unserializable result must not retro-fail the call and invite a retry.
84
85
  export const toolResultText = (result) => {
85
86
  if (result === undefined) return 'ok';
@@ -88,87 +89,22 @@ export const toolResultText = (result) => {
88
89
  return result.ok ? result.text : `[failed] ${result.text}`;
89
90
  try { return JSON.stringify(result) ?? 'ok'; } catch { return String(result); }
90
91
  };
91
- // Ledger suffix for a call whose outcome is attached to the chunk: keeps the RESULT in the
92
- // plain-text history so the NEXT turn still knows what the tool said. Bounded — history is
93
- // spoken-conversation-sized, not a log. Calls without an outcome field contribute nothing.
94
- const LEDGER_RESULT_MAX = 4000;
95
- const ledgerOutcome = (c) => {
96
- if (c.error) return ` → error: ${String(c.error).slice(0, LEDGER_RESULT_MAX)}`;
97
- if (!('result' in c)) return '';
98
- let text = toolResultText(c.result);
99
- if (text.length > LEDGER_RESULT_MAX) text = `${text.slice(0, LEDGER_RESULT_MAX)}… (${text.length} chars)`;
100
- return ` → ${text}`;
101
- };
102
-
103
- // ── [TOOL CALL] mimicry filter ──────────────────────────────────────────────────────────────────
104
- // The model sees `[TOOL CALL name] {...}` ledger lines in its own history turns, and sometimes
105
- // IMITATES the format instead of calling the tool natively: it types the line as reply text.
106
- // Untreated, the raw markup leaks into the spoken/visible reply and the tool never runs. This
107
- // transform strips such lines from the text stream: a bare call (no → outcome) is what the model
108
- // MEANT to do → re-emitted as a real { tool, args } chunk (the turn's dedup collapses it if the
109
- // native call also arrived); a line WITH an → outcome is a hallucinated memory of a past call →
110
- // dropped. Text before/after the line streams through untouched, held back only while a possible
111
- // marker/line is still forming so TTS latency is unaffected on normal replies.
112
- const MIMIC_MARK = '[TOOL CALL ';
113
- const parseMimicLine = (line) => {
114
- const m = /^\[TOOL CALL ([\w.-]+)\]\s*(.*)$/.exec(line.trim());
115
- if (!m) return null; // not the format after all — caller emits it as text
116
- const [, tool, rest] = m;
117
- if (!rest) return { tool, args: {} };
118
- if (!rest.startsWith('{')) return { drop: true }; // `→ outcome` (or junk) with no args — a remembered call
119
- // Balanced-brace scan (string-aware): the args JSON may be followed by a ` → outcome` tail.
120
- let depth = 0, inStr = false, escaped = false, end = -1;
121
- for (let i = 0; i < rest.length; i++) {
122
- const ch = rest[i];
123
- if (escaped) { escaped = false; continue; }
124
- if (inStr) { if (ch === '\\') escaped = true; else if (ch === '"') inStr = false; continue; }
125
- if (ch === '"') inStr = true;
126
- else if (ch === '{') depth++;
127
- else if (ch === '}' && --depth === 0) { end = i + 1; break; }
92
+ // One model-facing history record for a settled/pending tool call: the native tool_result block that
93
+ // pairs with its tool_use. Bounded — history is spoken-conversation-sized, not a log. A call whose
94
+ // outcome has not landed yet says so; _runTool patches the SAME block in place when it does.
95
+ const RESULT_MAX = 4000;
96
+ const MAX_TOOL_ROUNDS = 6;
97
+ const PENDING_RESULT = 'still running — the result will follow in a later turn';
98
+ const bound = (text) => text.length > RESULT_MAX ? `${text.slice(0, RESULT_MAX)}… (${text.length} chars)` : text;
99
+ const toolResultBlock = (c) => {
100
+ const block = { type: 'tool_result', tool_use_id: c.id, content: PENDING_RESULT };
101
+ if ('error' in c) { block.content = bound(String(c.error) || 'failed'); block.is_error = true; }
102
+ else if ('result' in c) {
103
+ block.content = bound(toolResultText(c.result));
104
+ if (c.result && typeof c.result === 'object' && c.result.ok === false) block.is_error = true;
128
105
  }
129
- if (end < 0) return { drop: true }; // truncated JSON — can't run it faithfully
130
- if (rest.slice(end).trim()) return { drop: true }; // has an → outcome: a memory, not a new call
131
- try { return { tool, args: JSON.parse(rest.slice(0, end)) }; } catch { return { drop: true }; }
106
+ return block;
132
107
  };
133
- async function* stripToolCallMimicry(src) {
134
- let carry = '';
135
- // Longest tail of `carry` that could still grow into the marker — held back; the rest is safe to emit.
136
- const heldLen = () => {
137
- for (let n = Math.min(carry.length, MIMIC_MARK.length - 1); n > 0; n--)
138
- if (MIMIC_MARK.startsWith(carry.slice(-n))) return n;
139
- return 0;
140
- };
141
- function* pump(final) {
142
- for (;;) {
143
- const at = carry.indexOf(MIMIC_MARK);
144
- if (at < 0) {
145
- const hold = final ? 0 : heldLen();
146
- const out = carry.slice(0, carry.length - hold);
147
- carry = carry.slice(carry.length - hold);
148
- if (out) yield { text: out };
149
- return;
150
- }
151
- const nl = carry.indexOf('\n', at);
152
- if (nl < 0 && !final) { // the line is still streaming — hold from the marker on
153
- if (at > 0) { yield { text: carry.slice(0, at) }; carry = carry.slice(at); }
154
- return;
155
- }
156
- const end = nl < 0 ? carry.length : nl;
157
- const before = carry.slice(0, at), line = carry.slice(at, end);
158
- carry = nl < 0 ? '' : carry.slice(nl + 1); // the line (and its newline) leave the text stream
159
- if (before) yield { text: before };
160
- const parsed = parseMimicLine(line);
161
- if (!parsed) yield { text: nl < 0 ? line : `${line}\n` }; // false alarm — pass it through untouched (newline included)
162
- else if (parsed.tool) yield { tool: parsed.tool, args: parsed.args };
163
- }
164
- }
165
- for await (const item of src) {
166
- if (item.tool) { yield item; continue; }
167
- carry += item.text ?? '';
168
- yield* pump(false);
169
- }
170
- yield* pump(true);
171
- }
172
108
 
173
109
  // Persona (who the agent is) + language rule + the host's live context block (`sysmsg`, delimited
174
110
  // so the model never mistakes injected context for its own conversation).
@@ -378,6 +314,8 @@ export class VoiceAgent {
378
314
  this.state = 'idle'; // idle | listening | thinking | speaking
379
315
  this._abort = null; // aborts the in-flight LLM
380
316
  this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
317
+ this._toolRounds = 0; // tool-result reactions since the user last spoke (see _runTurn react)
318
+ this._unseenResults = false; // a tool_result was patched after the last request was built (see _runTurn react)
381
319
  this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
382
320
  // Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
383
321
  // dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
@@ -699,6 +637,7 @@ export class VoiceAgent {
699
637
  // history stays ordered (…, assistant-so-far, user, user). Held pushes NEVER start an LLM turn —
700
638
  // release (_flush) owns starting the single turn, and waits for all of these to settle first.
701
639
  _pushHeld(text) {
640
+ this._toolRounds = 0;
702
641
  const p = (async () => { await this._holdBarrier; this.history.push({ role: 'user', content: text }); })();
703
642
  this._heldPushes.add(p);
704
643
  p.finally(() => this._heldPushes.delete(p));
@@ -720,15 +659,14 @@ export class VoiceAgent {
720
659
  }
721
660
 
722
661
  // Inject an out-of-band turn: feed `text` to the LLM as if the user had said it, then speak the
723
- // reply. Used by the host to surface external events (a tool result, a background job finishing)
662
+ // reply. Used by the host to surface external events (a background job finishing, a board change)
724
663
  // — only meaningful while the agent is running, so it's a no-op once stopped.
725
664
  //
726
665
  // SOFT turn (supersede: false): unlike a user utterance this must NOT cut the reply in flight.
727
- // A tool result arrives WHILE the turn that called the tool is still streaming/speaking, so
728
- // aborting here would kill that turn mid-answer and re-run the LLM over a history whose last
729
- // entry is the request again — the model then re-calls the same tool (duplicate-action spam).
730
- // Instead it queues behind the running turn, which by then has recorded both its answer and the
731
- // [TOOL CALL] it made, so the model sees the work is already done.
666
+ // An event can arrive WHILE a turn is still streaming/speaking, so aborting here would kill that
667
+ // turn mid-answer and re-run the LLM over a history whose last entry is the request again — the
668
+ // model then re-calls the same tool (duplicate-action spam). Instead it queues behind the running
669
+ // turn, which by then has recorded its answer and its tool calls natively.
732
670
  async notify(text) { if (!this._closed) await this._runTurn(text, { supersede: false }); }
733
671
 
734
672
  // Send a TYPED user turn: like speaking, but the text comes from the host's input instead of STT.
@@ -762,6 +700,7 @@ export class VoiceAgent {
762
700
  // caller (errors surface via onEvent), so it's safe to call without awaiting/catching.
763
701
  _onUserTurn(text, { speech = false } = {}) {
764
702
  if (!text.trim()) return Promise.resolve();
703
+ this._toolRounds = 0;
765
704
  return this._runTurn(text, { speech });
766
705
  }
767
706
 
@@ -878,7 +817,7 @@ export class VoiceAgent {
878
817
  // if nothing newer has queued behind it, and never cut the reply in flight. Anything newer already
879
818
  // sees this message in history and answers over the fuller picture — so a burst of tool results
880
819
  // collapses into ONE reply instead of one interrupting monologue per result.
881
- _runTurn(text, { supersede = true, speech = false } = {}) {
820
+ _runTurn(text, { supersede = true, speech = false, react = false } = {}) {
882
821
  const prev = this._turn;
883
822
  if (supersede) {
884
823
  this._abort?.abort(); // cut the in-flight reply NOW (barge-in / superseded)
@@ -904,9 +843,24 @@ export class VoiceAgent {
904
843
  // Only a SPOKEN final may adopt (it seeded the speculation); typed/notify/held turns drop it.
905
844
  const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
906
845
  if (text) this.history.push({ role: 'user', content: text });
846
+ // A reaction turn owes a reply only for results no request has carried yet: either the
847
+ // tool_result record is still the last message, or a late outcome was patched in after the
848
+ // last request was built (_unseenResults; cleared when a request starts) — then a short
849
+ // [EVENT] user turn, the host's channel for out-of-band facts, points the model at the patched
850
+ // block (a request can't end on an assistant message). Bounded per user utterance
851
+ // (_toolRounds, reset on every accepted user turn): a model that keeps calling tools gets
852
+ // MAX_TOOL_ROUNDS reactions, then waits for the user.
853
+ if (react) {
854
+ const last = this.history[this.history.length - 1];
855
+ const owed = (last?.role === 'user' && Array.isArray(last.content)) || this._unseenResults;
856
+ if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this._unseenResults = false; return; }
857
+ this._toolRounds++;
858
+ if (last?.role !== 'user') this.history.push({ role: 'user', content: '[EVENT tool_result] a pending tool finished — see its tool_result above.' });
859
+ }
907
860
  // Reply only if the conversation now ends on a user turn (a raced turn may have already
908
861
  // replied). pf is always null here: if we pushed a user turn it IS the last entry.
909
862
  if (this.history[this.history.length - 1]?.role !== 'user') return;
863
+ this._unseenResults = false; // this request carries every result landed so far
910
864
  this._set('thinking'); // LLM stage
911
865
  this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
912
866
  await this._speakTurn(this._abort.signal, pf?.gen, seq); // _speakTurn never throws — errors surface via onEvent
@@ -914,7 +868,7 @@ export class VoiceAgent {
914
868
  }
915
869
 
916
870
  // One assistant turn: stream the LLM → TTS, then record what was actually HEARD (the barge-in
917
- // prefix, not the full generated answer) plus a ledger of the tools it dispatched. Stored as
871
+ // prefix, not the full generated answer) plus the tools it called, natively. Stored as
918
872
  // `this._turn` so the next user turn can await it. Never throws (errors surface via onEvent) so
919
873
  // the await is safe — and never awaits a tool, so a hung one can't hold the next turn hostage.
920
874
  // `turn` tags every assistant event with the reply it belongs to, so a host can tell a LATE
@@ -927,74 +881,110 @@ export class VoiceAgent {
927
881
  // repopulates _replyText with the audible prefix as playback advances.
928
882
  if (this._replyText) this._prevReplyText = this._replyEcho || this._replyText;
929
883
  this._replyText = ''; this._replyEcho = '';
930
- // Tap the LLM stream once: yield speech text to TTS, and record the tool calls it makes (they
931
- // fire on arrival; `calls` is the ledger). First delta flips us to 'speaking'; accumulate for the
932
- // heard-vs-unheard report. `this.llm(...)` is a lazy generator — its fetch fires only when
884
+ // Tap the LLM stream once: yield speech text to TTS, and record what the model did as ROUNDS —
885
+ // each round is one assistant message (text + the tool_use blocks it emitted) followed by the
886
+ // user-role tool_result blocks for those calls. Recorded into history in that order on EVERY
887
+ // exit path, so the next request replays the model's own native tool history — never a text
888
+ // notation it could imitate. `this.llm(...)` is a lazy generator — its fetch fires only when
933
889
  // tts.speak first pulls, by which point any prior turn is aborted; keeps LLM streams serial.
934
- let answer = '', calls = [];
935
- // The tools this turn dispatched, as a ledger line recorded INTO its assistant message. The
936
- // runtime's tool calls are fire-and-forget (no tool_use/tool_result pair on the wire, and history
937
- // is provider-neutral {role, content} text), so without this the history reads as if it never
938
- // called anything and the next turn happily calls it AGAIN — duplicate tool-call spam. Relayed
939
- // calls: the outcome arrives separately as the host's `[TOOL RESULT <name>]` notify() turn.
940
- // Custom LLMs that execute tools inside their own generator (llm.executesTools) attach the
941
- // outcome directly, so the ledger line carries it — bounded — or the next turn would see the call with no result and lose what
942
- // the model learned (a barge-in cuts the spoken follow-up but not the fact). Read late (the
943
- // stream may still be appending) and on EVERY exit path, including a failed reply.
944
- const toolNote = () => calls.map((c) => `[TOOL CALL ${c.tool}] ${callArgs(c)}${ledgerOutcome(c)}`).join('\n');
890
+ //
891
+ // Round boundary: the default adapter's stream is ONE model message (text + calls together;
892
+ // the reaction to the results comes from _continue). A loop adapter feeds results back inside
893
+ // the stream and marks where the model's NEXT message starts with a { newMessage } chunk.
894
+ let answer = '';
895
+ const rounds = [], calls = []; // calls: every act this turn saw, for dedup — outlives recording
896
+ const loop = !!this.llm.executesTools;
897
+ const round = () => rounds[rounds.length - 1] ?? (rounds.push({ text: '', calls: [] }), rounds[rounds.length - 1]);
898
+ const newRound = () => { const r = rounds[rounds.length - 1]; if (r && (r.text || r.calls.length)) rounds.push({ text: '', calls: [] }); };
899
+ let n = 0;
900
+ // Accept a call once: identity is the provider id, else name+args (an id-less repeat is a
901
+ // provider re-emitting its buffered call). A loop adapter owns execution AND feeds the result
902
+ // back inside its reply; anything else is agent-run (or agent-reported when the chunk already
903
+ // carries an outcome) and gets a reaction turn.
904
+ const accept = (item) => {
905
+ const key = callKey(item);
906
+ let c = calls.find((x) => x.key === key);
907
+ if (c) return c;
908
+ c = { ...item, key, id: item.id || `call-${turn}-${++n}`, owner: loop ? 'adapter' : 'agent' };
909
+ calls.push(c); round().calls.push(c);
910
+ return c;
911
+ };
912
+ const outcome = (c, item) => { if ('result' in item) c.result = item.result; else if ('error' in item) c.error = item.error; };
913
+ const hasOutcome = (c) => 'result' in c || 'error' in c;
914
+ // An outcome may land AFTER the turn recorded the call (a hung tool, a barge-in mid-execution):
915
+ // its tool_result block is patched IN PLACE so the next request carries the real result, and the
916
+ // model gets to react to it (see _continue).
917
+ const settle = (c) => this._runTool(c).then(() => {
918
+ if (!c.block) return; // not recorded yet — record() picks the outcome up
919
+ if (!c.consumed) { Object.assign(c.block, toolResultBlock(c)); c.consumed = true; }
920
+ this._dropPrefetch(); // a speculation serialized the pending block — stale
921
+ this._unseenResults = true; this._continue();
922
+ });
923
+ const record = (keptLen) => {
924
+ let budget = keptLen;
925
+ for (const r of rounds) {
926
+ const text = r.text.slice(0, Math.max(0, budget)).trimEnd(); budget -= r.text.length;
927
+ if (!text && !r.calls.length) continue;
928
+ if (!r.calls.length) { this.history.push({ role: 'assistant', content: text }); continue; }
929
+ const blocks = text ? [{ type: 'text', text }] : [];
930
+ for (const c of r.calls) blocks.push({ type: 'tool_use', id: c.id, name: c.tool, input: c.args ?? {} });
931
+ this.history.push({ role: 'assistant', content: blocks });
932
+ this.history.push({ role: 'user', content: r.calls.map((c) => {
933
+ c.consumed = 'result' in c || 'error' in c;
934
+ return (c.block = toolResultBlock(c));
935
+ }) });
936
+ }
937
+ rounds.length = 0;
938
+ };
939
+ // Follow-up only for calls the AGENT ran: a loop adapter fed its results back inside the reply.
940
+ const recordAndContinue = (keptLen) => {
941
+ record(keptLen);
942
+ if (calls.some((c) => c.owner === 'agent' && c.consumed)) this._continue();
943
+ };
945
944
  // Pre-execution markers: an LLM that runs tools inside its own generator (executesTools) can
946
945
  // announce a call with `running: true` BEFORE it executes, so the host shows a live spinner chip
947
946
  // that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
948
- // outcome arrives finalizes it as interrupted — DISPLAY truth only: an announcement never reaches
949
- // the ledger (`calls`), so the model treats it as never having happened.
950
- const announced = new Map();
951
- // Latched once this turn has finalized its chips (finishInterrupted ran). A barge-in detaches the
952
- // LLM producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
953
- // announcement would open a spinner nothing will ever close. Past the latch, announcements are
954
- // dropped — outcomes still flow (a tool the adapter really ran is truth worth reporting).
947
+ // outcome arrives finalizes the CHIP as interrupted; the history keeps the call with a pending
948
+ // tool_result, patched in place if the outcome still lands.
949
+ // `finalized` latches once this turn has finalized its chips. A barge-in detaches the LLM
950
+ // producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
951
+ // announcement would open a spinner nothing will ever close, and a late UNEXECUTED call is work
952
+ // the user cut off. Past the latch only outcomes flow (a tool the adapter really ran is truth).
955
953
  let finalized = false;
956
954
  const finishInterrupted = () => {
957
955
  finalized = true;
958
- for (const r of announced.values())
959
- this.onEvent({ type: 'tool', name: r.tool, id: r.id, args: r.args, result: { ok: false, text: 'interrupted — did not finish' } });
960
- announced.clear();
956
+ for (const c of calls) if (c.announced && !hasOutcome(c))
957
+ this.onEvent({ type: 'tool', name: c.tool, id: c.id, args: c.args, result: { ok: false, text: 'interrupted — did not finish' } });
961
958
  };
962
959
  const tapped = (async function* (self, src) {
963
960
  for await (const item of src) {
964
961
  // tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
965
962
  // act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
963
+ if (item.newMessage) { newRound(); continue; } // loop adapter: the model's next message starts here
966
964
  if (item.tool) {
967
- if (item.running) { // display-only announcement; the outcome chunk below does the work
965
+ const known = calls.find((x) => x.key === callKey(item));
966
+ if (item.running || (loop && !hasOutcome(item))) { // announcement; the outcome chunk below does the work
968
967
  // An id-less announcement can never be replaced in place NOR finalized (both key on id),
969
968
  // so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
970
- if (finalized || !item.id) continue;
971
- // Already settled this turn (adapter re-emitting its buffered pair): re-announcing would
972
- // reopen a spinner over a finished chip, and the duplicate outcome is deduped below and
973
- // never re-reported — so it would hang there, then finalize as falsely "interrupted".
974
- if (calls.some((c) => c.id === item.id)) continue;
975
- announced.set(item.id, item);
969
+ if (finalized || !item.id || known) continue;
970
+ accept(item).announced = true;
976
971
  self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
977
972
  continue;
978
973
  }
979
- if (item.id) announced.delete(item.id);
980
- // Same key = same act (provider re-emit / model asking twice). A MIMIC chunk carries no id,
981
- // so also match by name+args across the id boundary — a typed echo of an id-carrying native
982
- // call (llm.executesTools mode) must collapse too, or it would execute the tool a second time.
983
- const dup = calls.some((c) => callKey(c) === callKey(item) ||
984
- ((!c.id || !item.id) && c.tool === item.tool && callArgs(c) === callArgs(item)));
985
- if (!dup) {
986
- calls.push(item); self._runTool(item);
987
- // Late outcome: the turn already closed (barge-in mid-execution) and its ledger is
988
- // written, so this result would reach the transcript chip but never the model. Relay it
989
- // as a soft [TOOL RESULT] turn — the same channel out-of-band receipts use — so the next
990
- // reply knows what the tool returned instead of seeing a call that vanished.
991
- if (finalized && ('result' in item || 'error' in item))
992
- self.notify(`[TOOL RESULT ${item.tool}] ${callArgs(item)}${ledgerOutcome(item)}`);
974
+ if (known) { // the announced call's outcome, or a re-emit
975
+ if (known.ran || !hasOutcome(item)) continue;
976
+ outcome(known, item); settle(known);
977
+ continue;
993
978
  }
979
+ if (finalized && !hasOutcome(item)) continue; // late bare call: the user cut this reply off
980
+ const c = accept(item);
981
+ outcome(c, item);
982
+ if (finalized) record(0); // late outcome of an unseen call: give it a record of its own (settle schedules the reaction)
983
+ settle(c);
994
984
  continue;
995
985
  }
996
986
  if (self.state === 'thinking') self._set('speaking'); // first token → TTS stage
997
- answer += item.text;
987
+ answer += item.text; round().text += item.text;
998
988
  // While audio plays, the TTS progress hook owns the draft (text = spoken-so-far, full = answer,
999
989
  // read live each tick), so streaming the whole answer here would yank the cursor to the end.
1000
990
  // Once the cursor is live we only grow `answer` (the hook reads it); until then (muted, or the
@@ -1002,7 +992,7 @@ export class VoiceAgent {
1002
992
  if (!self._cursorLive) self.onEvent({ type: 'assistant', text: answer, final: false, turn });
1003
993
  yield item.text;
1004
994
  }
1005
- })(this, stripToolCallMimicry(prefetched ?? this.llm(this.history, this.sysmsg, signal))); // prefetched stream already carries this turn's user text (see _startPrefetch)
995
+ })(this, prefetched ?? this.llm(this.history, this.sysmsg, signal)); // prefetched stream already carries this turn's user text (see _startPrefetch)
1006
996
 
1007
997
  let spoken = '';
1008
998
  // Live spoken cursor: while a clip plays the TTS reports chars heard so far, which we mirror as a
@@ -1051,9 +1041,9 @@ export class VoiceAgent {
1051
1041
  // follows), and a host tracking bubbles per turn needs to know which one just died.
1052
1042
  this.onEvent({ type: 'error', error: e.message, turn });
1053
1043
  // Tools fire DURING streaming, so a reply that dies here may already have dispatched one.
1054
- // Record it anyway: the ledger is about what was DONE, not about what got spoken — dropping
1055
- // it would let the next turn call the same tool again.
1056
- if (toolNote()) this.history.push({ role: 'assistant', content: toolNote() });
1044
+ // Record what was done and heard so far — dropping it would let the next turn call the
1045
+ // same tool again.
1046
+ recordAndContinue(signal.keepFull ? answer.length : this._replyText.length);
1057
1047
  finishInterrupted(); // announced-but-unfinished → failed chip, not a stuck spinner
1058
1048
  if (this.state !== 'idle') this._set('listening');
1059
1049
  return;
@@ -1073,16 +1063,13 @@ export class VoiceAgent {
1073
1063
  // Normally we record only the HEARD prefix (barge-in: the user reacted to what they heard). But a
1074
1064
  // turn aborted on hold-entry was tagged signal.keepFull, so we keep the whole streamed answer —
1075
1065
  // on release the LLM sees everything it had formed. Per-turn tag → a quick Resume can't flip it.
1076
- const kept = signal.keepFull ? answer : spoken;
1077
- if (kept || toolNote()) {
1078
- this.history.push({ role: 'assistant', content: [kept, toolNote()].filter(Boolean).join('\n') });
1079
- }
1066
+ // Both history and the EVENT use the raw prefix of `answer` (the host's full.slice(text.length)
1067
+ // tail aligns). keepFull/normal-finish → whole answer (no tail); a barge-in → the heard prefix
1068
+ // solid + the unspoken remainder dimmed, showing exactly where TTS got.
1069
+ const kept = signal.keepFull ? answer : rawPrefix(spoken);
1070
+ recordAndContinue(kept.length);
1080
1071
  if (kept) {
1081
- // History keeps the normalized `kept`; the EVENT emits a raw prefix of `answer` so the host's
1082
- // full.slice(text.length) tail aligns. keepFull/normal-finish → whole answer (no tail); a barge-in
1083
- // → the heard prefix solid + the unspoken remainder dimmed, showing exactly where TTS got.
1084
- const shown = signal.keepFull ? answer : rawPrefix(spoken);
1085
- this.onEvent({ type: 'assistant', text: shown, full: answer, final: true, turn });
1072
+ this.onEvent({ type: 'assistant', text: kept, full: answer, final: true, turn });
1086
1073
  } else if (answer) {
1087
1074
  // Interrupted before anything was heard (nothing voiced yet): drop the streamed draft so the
1088
1075
  // transcript doesn't keep a dangling unfinished bubble. Empty text → host clears the draft.
@@ -1098,8 +1085,21 @@ export class VoiceAgent {
1098
1085
  if (!signal.aborted && this.state !== 'idle') this._set('listening'); // superseded turns don't touch state
1099
1086
  }
1100
1087
 
1088
+ // Native tool semantics: once a tool_result is in history the model gets to react to it. Queues a
1089
+ // SOFT reaction turn (behind the running one; dropped if a newer user turn supersedes it — that
1090
+ // turn sees the result anyway) after the agent ran a call for the default adapter (its stream
1091
+ // ends at the call; the result was never fed back), or when a LATE outcome (hung/slow tool,
1092
+ // barge-in mid-execution) landed in a record. Whether a reply is still owed is decided when the
1093
+ // turn RUNS (see _runTurn `react`), since the predecessor may already have carried the result.
1094
+ // Loop adapters fed the result back inside the reply, so nothing fires for them. Silent while
1095
+ // the user is speaking — their utterance will see the result in history anyway.
1096
+ _continue() {
1097
+ if (this._closed || this._speaking) return;
1098
+ this._runTurn('', { supersede: false, react: true });
1099
+ }
1100
+
1101
1101
  // Run one chosen tool call and report it. Fired as soon as its tool_use arrives in the stream, in
1102
- // parallel with TTS, and NEVER joined: the turn records the dispatch in its ledger and finishes
1102
+ // parallel with TTS, and NEVER joined: the turn records the call (tool_result pending) and finishes
1103
1103
  // when the speaking does (see _speakTurn), so a tool that hangs costs its own result and nothing
1104
1104
  // else. Its outcome reaches the host via onEvent; the model learns it via notify(). Must not
1105
1105
  // reject — nothing awaits it, so a rejection here would be unhandled (hence the catch-all).
@@ -1115,16 +1115,15 @@ export class VoiceAgent {
1115
1115
  // runtime/transport failures, not a handled domain-level tool miss.
1116
1116
  // `id` rides along on every tool event so a host can replace THIS call's running chip in
1117
1117
  // place, and two same-args calls (a legit retry) stay distinct acts in the transcript.
1118
- if ('result' in call || 'error' in call) {
1119
- const result = call.error ? { text: call.error, ok: false } : call.result;
1120
- this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
1121
- return;
1118
+ if (!('result' in call || 'error' in call)) {
1119
+ // Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
1120
+ // a deadline could only stop WAITING — reporting "failed" for an operation still on its way
1121
+ // to committing its side effect, inviting a duplicate retry. Nothing awaits this call
1122
+ // (see _speakTurn), so a hung tool costs its own result, never the turn loop.
1123
+ try { call.result = await this.tools[call.tool]?.run?.(call.args); }
1124
+ catch (e) { call.error = e?.message || String(e ?? 'failed'); }
1122
1125
  }
1123
- // Host tool code. Deliberately NOT bounded by a timeout: we cannot cancel a host promise, so
1124
- // a deadline could only stop WAITING — reporting "failed" for an operation still on its way
1125
- // to committing its side effect, inviting a duplicate retry. Nothing awaits this call
1126
- // (see _speakTurn), so a hung tool costs its own result, never the turn loop.
1127
- const result = await this.tools[call.tool]?.run?.(call.args);
1126
+ const result = call.error ? { text: call.error, ok: false } : call.result;
1128
1127
  this.onEvent({ type: 'tool', name: call.tool, id: call.id, args: call.args, result });
1129
1128
  } catch (e) {
1130
1129
  this.onEvent({ type: 'error', error: `tool ${call.tool}: ${e.message}` });