@todoforai/voiceloop 0.1.12 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.d.ts +4 -1
- package/src/voice-agent.js +33 -41
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@todoforai/voiceloop",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13",
|
|
4
4
|
"description": "Fast, interruptible browser voice agent: VAD → STT → LLM → TTS with real barge-in, echo filtering, streaming first-sentence TTS and tap-to-seek. Zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/index.d.ts
CHANGED
|
@@ -43,7 +43,10 @@ export interface VoiceTool {
|
|
|
43
43
|
// reports it instead of running it again.
|
|
44
44
|
export type LLMChunk =
|
|
45
45
|
| { text: string }
|
|
46
|
-
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true }
|
|
46
|
+
| { tool: string; id?: string; args: Record<string, unknown>; result?: unknown; error?: string; running?: true }
|
|
47
|
+
/** Loop adapters only: the model's NEXT message starts here (after the results it fed back), so
|
|
48
|
+
* the agent records text/calls before and after as separate assistant messages. */
|
|
49
|
+
| { newMessage: true };
|
|
47
50
|
// The conversation as the model sees it, Anthropic-shaped. Plain turns are strings; a turn in which
|
|
48
51
|
// the assistant called tools is recorded NATIVELY: an assistant message of text + tool_use blocks,
|
|
49
52
|
// then a user message holding the matching tool_result blocks (content = the result text the model
|
package/src/voice-agent.js
CHANGED
|
@@ -76,7 +76,8 @@ const withTimeout = (p, ms, fallback) => {
|
|
|
76
76
|
]);
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
-
const callArgs = (c) => JSON.stringify(c.args ?? {},
|
|
79
|
+
const callArgs = (c) => JSON.stringify(c.args ?? {}, (_k, v) =>
|
|
80
|
+
v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, v[k]])) : v);
|
|
80
81
|
const callKey = (c) => c.id ? `id:${c.id}` : `${c.tool} ${callArgs(c)}`;
|
|
81
82
|
// Render a host tool's return value as model-facing text (the tool_result content in history, and
|
|
82
83
|
// what LLM adapters feed back to the model). Never throws: the side effect already RAN,
|
|
@@ -314,7 +315,7 @@ export class VoiceAgent {
|
|
|
314
315
|
this._abort = null; // aborts the in-flight LLM
|
|
315
316
|
this._replySeq = 0; // bumped per reply-generating turn; a queued turn drops itself if a newer one exists
|
|
316
317
|
this._toolRounds = 0; // tool-result reactions since the user last spoke (see _runTurn react)
|
|
317
|
-
this.
|
|
318
|
+
this._unseenResults = false; // a tool_result was patched after the last request was built (see _runTurn react)
|
|
318
319
|
this._preroll = []; this._wasSpeaking = false; this._closed = false; this._destroyed = false;
|
|
319
320
|
// Diagnostic tap: last ~30s of the exact i16 frames handed to STT + delivery-stall gaps, for
|
|
320
321
|
// dumpAudio() (replay any "it misheard me" run and prove capture clean-or-not).
|
|
@@ -843,23 +844,23 @@ export class VoiceAgent {
|
|
|
843
844
|
const pf = speech && text ? this._takePrefetch(text) : (this._dropPrefetch(), null);
|
|
844
845
|
if (text) this.history.push({ role: 'user', content: text });
|
|
845
846
|
// A reaction turn owes a reply only for results no request has carried yet: either the
|
|
846
|
-
// tool_result record is still the last message, or a late outcome
|
|
847
|
-
// request was built (
|
|
848
|
-
// the host's channel for out-of-band facts, points the model at the patched
|
|
849
|
-
// can't end on an assistant message). Bounded per user utterance
|
|
850
|
-
// every accepted user turn): a model that keeps calling tools gets
|
|
851
|
-
// reactions, then waits for the user.
|
|
847
|
+
// tool_result record is still the last message, or a late outcome was patched in after the
|
|
848
|
+
// last request was built (_unseenResults; cleared when a request starts) — then a short
|
|
849
|
+
// [EVENT] user turn, the host's channel for out-of-band facts, points the model at the patched
|
|
850
|
+
// block (a request can't end on an assistant message). Bounded per user utterance
|
|
851
|
+
// (_toolRounds, reset on every accepted user turn): a model that keeps calling tools gets
|
|
852
|
+
// MAX_TOOL_ROUNDS reactions, then waits for the user.
|
|
852
853
|
if (react) {
|
|
853
854
|
const last = this.history[this.history.length - 1];
|
|
854
|
-
const owed = (last?.role === 'user' && Array.isArray(last.content)) || this.
|
|
855
|
-
if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this.
|
|
855
|
+
const owed = (last?.role === 'user' && Array.isArray(last.content)) || this._unseenResults;
|
|
856
|
+
if (!owed || this._toolRounds >= MAX_TOOL_ROUNDS) { this._unseenResults = false; return; }
|
|
856
857
|
this._toolRounds++;
|
|
857
|
-
if (last?.role !== 'user') this.history.push({ role: 'user', content:
|
|
858
|
+
if (last?.role !== 'user') this.history.push({ role: 'user', content: '[EVENT tool_result] a pending tool finished — see its tool_result above.' });
|
|
858
859
|
}
|
|
859
860
|
// Reply only if the conversation now ends on a user turn (a raced turn may have already
|
|
860
861
|
// replied). pf is always null here: if we pushed a user turn it IS the last entry.
|
|
861
862
|
if (this.history[this.history.length - 1]?.role !== 'user') return;
|
|
862
|
-
this.
|
|
863
|
+
this._unseenResults = false; // this request carries every result landed so far
|
|
863
864
|
this._set('thinking'); // LLM stage
|
|
864
865
|
this._abort = pf?.ctl ?? new AbortController(); // adopted prefetch keeps ITS controller so barge-in aborts the right fetch
|
|
865
866
|
await this._speakTurn(this._abort.signal, pf?.gen, seq); // _speakTurn never throws — errors surface via onEvent
|
|
@@ -889,33 +890,27 @@ export class VoiceAgent {
|
|
|
889
890
|
//
|
|
890
891
|
// Round boundary: the default adapter's stream is ONE model message (text + calls together;
|
|
891
892
|
// the reaction to the results comes from _continue). A loop adapter feeds results back inside
|
|
892
|
-
// the stream
|
|
893
|
+
// the stream and marks where the model's NEXT message starts with a { newMessage } chunk.
|
|
893
894
|
let answer = '';
|
|
894
895
|
const rounds = [], calls = []; // calls: every act this turn saw, for dedup — outlives recording
|
|
895
896
|
const loop = !!this.llm.executesTools;
|
|
896
|
-
const round = () => {
|
|
897
|
-
|
|
898
|
-
if (r && !(loop && r.settled)) return r;
|
|
899
|
-
rounds.push({ text: '', calls: [] }); return rounds[rounds.length - 1];
|
|
900
|
-
};
|
|
897
|
+
const round = () => rounds[rounds.length - 1] ?? (rounds.push({ text: '', calls: [] }), rounds[rounds.length - 1]);
|
|
898
|
+
const newRound = () => { const r = rounds[rounds.length - 1]; if (r && (r.text || r.calls.length)) rounds.push({ text: '', calls: [] }); };
|
|
901
899
|
let n = 0;
|
|
902
900
|
// Accept a call once: identity is the provider id, else name+args (an id-less repeat is a
|
|
903
|
-
// provider re-emitting its buffered call).
|
|
904
|
-
//
|
|
905
|
-
//
|
|
901
|
+
// provider re-emitting its buffered call). A loop adapter owns execution AND feeds the result
|
|
902
|
+
// back inside its reply; anything else is agent-run (or agent-reported when the chunk already
|
|
903
|
+
// carries an outcome) and gets a reaction turn.
|
|
906
904
|
const accept = (item) => {
|
|
907
905
|
const key = callKey(item);
|
|
908
906
|
let c = calls.find((x) => x.key === key);
|
|
909
907
|
if (c) return c;
|
|
910
|
-
|
|
911
|
-
c = { ...item, key, id: item.id || `call-${turn}-${++n}`, owner: adapterRan ? 'adapter' : 'agent' };
|
|
908
|
+
c = { ...item, key, id: item.id || `call-${turn}-${++n}`, owner: loop ? 'adapter' : 'agent' };
|
|
912
909
|
calls.push(c); round().calls.push(c);
|
|
913
910
|
return c;
|
|
914
911
|
};
|
|
915
|
-
const outcome = (c, item) => {
|
|
916
|
-
|
|
917
|
-
const r = rounds.find((r) => r.calls.includes(c)); if (r) r.settled = true; // already recorded → no round to close
|
|
918
|
-
};
|
|
912
|
+
const outcome = (c, item) => { if ('result' in item) c.result = item.result; else if ('error' in item) c.error = item.error; };
|
|
913
|
+
const hasOutcome = (c) => 'result' in c || 'error' in c;
|
|
919
914
|
// An outcome may land AFTER the turn recorded the call (a hung tool, a barge-in mid-execution):
|
|
920
915
|
// its tool_result block is patched IN PLACE so the next request carries the real result, and the
|
|
921
916
|
// model gets to react to it (see _continue).
|
|
@@ -923,7 +918,7 @@ export class VoiceAgent {
|
|
|
923
918
|
if (!c.block) return; // not recorded yet — record() picks the outcome up
|
|
924
919
|
if (!c.consumed) { Object.assign(c.block, toolResultBlock(c)); c.consumed = true; }
|
|
925
920
|
this._dropPrefetch(); // a speculation serialized the pending block — stale
|
|
926
|
-
this.
|
|
921
|
+
this._unseenResults = true; this._continue();
|
|
927
922
|
});
|
|
928
923
|
const record = (keptLen) => {
|
|
929
924
|
let budget = keptLen;
|
|
@@ -951,42 +946,39 @@ export class VoiceAgent {
|
|
|
951
946
|
// that the outcome chunk replaces in place (same id). A turn that dies before an announced call's
|
|
952
947
|
// outcome arrives finalizes the CHIP as interrupted; the history keeps the call with a pending
|
|
953
948
|
// tool_result, patched in place if the outcome still lands.
|
|
954
|
-
|
|
955
|
-
//
|
|
956
|
-
// LLM producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
|
|
949
|
+
// `finalized` latches once this turn has finalized its chips. A barge-in detaches the LLM
|
|
950
|
+
// producer from TTS, so it can wake AFTER we returned and keep pushing chunks: a late
|
|
957
951
|
// announcement would open a spinner nothing will ever close, and a late UNEXECUTED call is work
|
|
958
952
|
// the user cut off. Past the latch only outcomes flow (a tool the adapter really ran is truth).
|
|
959
953
|
let finalized = false;
|
|
960
954
|
const finishInterrupted = () => {
|
|
961
955
|
finalized = true;
|
|
962
|
-
for (const
|
|
963
|
-
this.onEvent({ type: 'tool', name:
|
|
964
|
-
announced.clear();
|
|
956
|
+
for (const c of calls) if (c.announced && !hasOutcome(c))
|
|
957
|
+
this.onEvent({ type: 'tool', name: c.tool, id: c.id, args: c.args, result: { ok: false, text: 'interrupted — did not finish' } });
|
|
965
958
|
};
|
|
966
959
|
const tapped = (async function* (self, src) {
|
|
967
960
|
for await (const item of src) {
|
|
968
961
|
// tool_use → fire NOW, in parallel with TTS. Identical repeats within one turn are the SAME
|
|
969
962
|
// act (a provider re-emitting its buffered call, or the model asking twice) — run it once.
|
|
963
|
+
if (item.newMessage) { newRound(); continue; } // loop adapter: the model's next message starts here
|
|
970
964
|
if (item.tool) {
|
|
971
|
-
const hasOutcome = 'result' in item || 'error' in item;
|
|
972
965
|
const known = calls.find((x) => x.key === callKey(item));
|
|
973
|
-
if (item.running || (loop && !hasOutcome)) { // announcement; the outcome chunk below does the work
|
|
966
|
+
if (item.running || (loop && !hasOutcome(item))) { // announcement; the outcome chunk below does the work
|
|
974
967
|
// An id-less announcement can never be replaced in place NOR finalized (both key on id),
|
|
975
968
|
// so it could only ever become a stuck spinner — drop it. The outcome chunk still lands.
|
|
976
969
|
if (finalized || !item.id || known) continue;
|
|
977
|
-
|
|
970
|
+
accept(item).announced = true;
|
|
978
971
|
self.onEvent({ type: 'tool', name: item.tool, id: item.id, args: item.args, running: true });
|
|
979
972
|
continue;
|
|
980
973
|
}
|
|
981
|
-
if (item.id) announced.delete(item.id);
|
|
982
974
|
if (known) { // the announced call's outcome, or a re-emit
|
|
983
|
-
if (known.ran || !hasOutcome) continue;
|
|
975
|
+
if (known.ran || !hasOutcome(item)) continue;
|
|
984
976
|
outcome(known, item); settle(known);
|
|
985
977
|
continue;
|
|
986
978
|
}
|
|
987
|
-
if (finalized && !hasOutcome) continue;
|
|
979
|
+
if (finalized && !hasOutcome(item)) continue; // late bare call: the user cut this reply off
|
|
988
980
|
const c = accept(item);
|
|
989
|
-
|
|
981
|
+
outcome(c, item);
|
|
990
982
|
if (finalized) record(0); // late outcome of an unseen call: give it a record of its own (settle schedules the reaction)
|
|
991
983
|
settle(c);
|
|
992
984
|
continue;
|